diff options
Diffstat (limited to 'src/core/iomgr')
25 files changed, 769 insertions, 412 deletions
diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 28ed7708f7..632d2a4609 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -74,6 +74,7 @@ static void freelist_fd(grpc_fd *fd) { gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); gpr_mu_unlock(&fd_freelist_mu); } @@ -100,6 +101,7 @@ static grpc_fd *alloc_fd(int fd) { &r->inactive_watcher_root; r->freelist_next = NULL; r->read_watcher = r->write_watcher = NULL; + r->on_done_closure = NULL; return r; } @@ -109,16 +111,36 @@ static void destroy(grpc_fd *fd) { gpr_free(fd); } +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, + gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) static void ref_by(grpc_fd *fd, int n) { +#endif GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); } +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, + gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else static void unref_by(grpc_fd *fd, int n) { - gpr_atm old = gpr_atm_full_fetch_add(&fd->refst, -n); + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); if (old == n) { - grpc_iomgr_add_callback(&fd->on_done_closure); freelist_fd(fd); - grpc_iomgr_unregister_object(&fd->iomgr_object); } else { GPR_ASSERT(old > n); } @@ -135,12 +157,9 @@ void grpc_fd_global_shutdown(void) { gpr_mu_destroy(&fd_freelist_mu); } -static void do_nothing(void *ignored, int success) {} - grpc_fd *grpc_fd_create(int fd, const char *name) { grpc_fd *r = alloc_fd(fd); grpc_iomgr_register_object(&r->iomgr_object, name); - grpc_pollset_add_fd(grpc_backup_pollset(), r); return r; } @@ -178,24 +197,46 @@ static void wake_all_watchers_locked(grpc_fd *fd) { } } -void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_cb_func on_done, void *user_data) { - grpc_iomgr_closure_init(&fd->on_done_closure, on_done ? on_done : do_nothing, - user_data); +static int has_watchers(grpc_fd *fd) { + return fd->read_watcher != NULL || fd->write_watcher != NULL || fd->inactive_watcher_root.next != &fd->inactive_watcher_root; +} + +void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, + const char *reason) { + fd->on_done_closure = on_done; shutdown(fd->fd, SHUT_RDWR); - ref_by(fd, 1); /* remove active status, but keep referenced */ + REF_BY(fd, 1, reason); /* remove active status, but keep referenced */ gpr_mu_lock(&fd->watcher_mu); - wake_all_watchers_locked(fd); + if (!has_watchers(fd)) { + close(fd->fd); + if (fd->on_done_closure) { + grpc_iomgr_add_callback(fd->on_done_closure); + } + } else { + wake_all_watchers_locked(fd); + } gpr_mu_unlock(&fd->watcher_mu); - unref_by(fd, 2); /* drop the reference */ + UNREF_BY(fd, 2, reason); /* drop the reference */ } /* increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +void grpc_fd_ref(grpc_fd *fd, const char *reason, const char *file, int line) { + ref_by(fd, 2, reason, file, line); +} + +void grpc_fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else void grpc_fd_ref(grpc_fd *fd) { ref_by(fd, 2); } void grpc_fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif static void process_callback(grpc_iomgr_closure *closure, int success, - int allow_synchronous_callback) { + int allow_synchronous_callback) { if (allow_synchronous_callback) { closure->cb(closure->cb_arg, success); } else { @@ -235,7 +276,7 @@ static void notify_on(grpc_fd *fd, gpr_atm *st, grpc_iomgr_closure *closure, GPR_ASSERT(gpr_atm_no_barrier_load(st) == READY); gpr_atm_rel_store(st, NOT_READY); process_callback(closure, !gpr_atm_acq_load(&fd->shutdown), - allow_synchronous_callback); + allow_synchronous_callback); return; default: /* WAITING */ /* upcallptr was set to a different closure. This is an error! */ @@ -279,7 +320,7 @@ static void set_ready(grpc_fd *fd, gpr_atm *st, /* only one set_ready can be active at once (but there may be a racing notify_on) */ int success; - grpc_iomgr_closure* closure; + grpc_iomgr_closure *closure; size_t ncb = 0; gpr_mu_lock(&fd->set_state_mu); @@ -319,17 +360,25 @@ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, gpr_uint32 mask = 0; /* keep track of pollers that have requested our events, in case they change */ - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "poll"); gpr_mu_lock(&fd->watcher_mu); + /* if we are shutdown, then don't add to the watcher set */ + if (gpr_atm_no_barrier_load(&fd->shutdown)) { + watcher->fd = NULL; + watcher->pollset = NULL; + gpr_mu_unlock(&fd->watcher_mu); + GRPC_FD_UNREF(fd, "poll"); + return 0; + } /* if there is nobody polling for read, but we need to, then start doing so */ - if (!fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) { + if (read_mask && !fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) { fd->read_watcher = watcher; mask |= read_mask; } /* if there is nobody polling for write, but we need to, then start doing so */ - if (!fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) { + if (write_mask && !fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) { fd->write_watcher = watcher; mask |= write_mask; } @@ -351,6 +400,10 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) { int kick = 0; grpc_fd *fd = watcher->fd; + if (fd == NULL) { + return; + } + gpr_mu_lock(&fd->watcher_mu); if (watcher == fd->read_watcher) { /* remove read watcher, kick if we still need a read */ @@ -372,9 +425,15 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) { if (kick) { maybe_wake_one_watcher_locked(fd); } + if (grpc_fd_is_orphaned(fd) && !has_watchers(fd)) { + close(fd->fd); + if (fd->on_done_closure != NULL) { + grpc_iomgr_add_callback(fd->on_done_closure); + } + } gpr_mu_unlock(&fd->watcher_mu); - grpc_fd_unref(fd); + GRPC_FD_UNREF(fd, "poll"); } void grpc_fd_become_readable(grpc_fd *fd, int allow_synchronous_callback) { diff --git a/src/core/iomgr/fd_posix.h b/src/core/iomgr/fd_posix.h index 0fa71850e3..94d0019fa4 100644 --- a/src/core/iomgr/fd_posix.h +++ b/src/core/iomgr/fd_posix.h @@ -62,12 +62,12 @@ struct grpc_fd { gpr_atm shutdown; /* The watcher list. - + The following watcher related fields are protected by watcher_mu. - + An fd_watcher is an ephemeral object created when an fd wants to begin polling, and destroyed after the poll. - + It denotes the fd's interest in whether to read poll or write poll or both or neither on this fd. @@ -93,7 +93,7 @@ struct grpc_fd { struct grpc_fd *freelist_next; - grpc_iomgr_closure on_done_closure; + grpc_iomgr_closure *on_done_closure; grpc_iomgr_closure *shutdown_closures[2]; grpc_iomgr_object iomgr_object; @@ -109,7 +109,8 @@ grpc_fd *grpc_fd_create(int fd, const char *name); If on_done is NULL, no callback will be made. Requires: *fd initialized; no outstanding notify_on_read or notify_on_write. */ -void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_cb_func on_done, void *user_data); +void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, + const char *reason); /* Begin polling on an fd. Registers that the given pollset is interested in this fd - so that if read @@ -159,10 +160,19 @@ void grpc_fd_become_readable(grpc_fd *fd, int allow_synchronous_callback); void grpc_fd_become_writable(grpc_fd *fd, int allow_synchronous_callback); /* Reference counting for fds */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +void grpc_fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +void grpc_fd_unref(grpc_fd *fd, const char *reason, const char *file, int line); +#define GRPC_FD_REF(fd, reason) grpc_fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) grpc_fd_unref(fd, reason, __FILE__, __LINE__) +#else void grpc_fd_ref(grpc_fd *fd); void grpc_fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) grpc_fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) grpc_fd_unref(fd) +#endif void grpc_fd_global_init(void); void grpc_fd_global_shutdown(void); -#endif /* GRPC_INTERNAL_CORE_IOMGR_FD_POSIX_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_FD_POSIX_H */ diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index fa8dcc5b4a..c47528aa94 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -112,13 +112,20 @@ void grpc_iomgr_shutdown(void) { gpr_timespec shutdown_deadline = gpr_time_add(gpr_now(), gpr_time_from_seconds(10)); - gpr_mu_lock(&g_mu); g_shutdown = 1; - while (g_cbs_head || g_root_object.next != &g_root_object) { - size_t nobjs = count_objects(); - gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed%s", nobjs, - g_cbs_head ? " and executing final callbacks" : ""); + while (g_cbs_head != NULL || g_root_object.next != &g_root_object) { + if (g_cbs_head != NULL && g_root_object.next != &g_root_object) { + gpr_log(GPR_DEBUG, + "Waiting for %d iomgr objects to be destroyed and executing " + "final callbacks", + count_objects()); + } else if (g_cbs_head != NULL) { + gpr_log(GPR_DEBUG, "Executing final iomgr callbacks"); + } else { + gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed", + count_objects()); + } if (g_cbs_head) { do { closure = g_cbs_head; @@ -131,10 +138,14 @@ void grpc_iomgr_shutdown(void) { } while (g_cbs_head); continue; } - if (nobjs > 0) { + if (grpc_alarm_check(&g_mu, gpr_inf_future, NULL)) { + gpr_log(GPR_DEBUG, "got late alarm"); + continue; + } + if (g_root_object.next != &g_root_object) { int timeout = 0; - gpr_timespec short_deadline = gpr_time_add(gpr_now(), - gpr_time_from_millis(100)); + gpr_timespec short_deadline = + gpr_time_add(gpr_now(), gpr_time_from_millis(100)); while (gpr_cv_wait(&g_rcv, &g_mu, short_deadline) && g_cbs_head == NULL) { if (gpr_time_cmp(gpr_now(), shutdown_deadline) > 0) { timeout = 1; @@ -158,15 +169,16 @@ void grpc_iomgr_shutdown(void) { grpc_kick_poller(); gpr_event_wait(&g_background_callback_executor_done, gpr_inf_future); - grpc_iomgr_platform_shutdown(); grpc_alarm_list_shutdown(); + + grpc_iomgr_platform_shutdown(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_rcv); } void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name) { - obj->name = gpr_strdup(name); gpr_mu_lock(&g_mu); + obj->name = gpr_strdup(name); obj->next = &g_root_object; obj->prev = obj->next->prev; obj->next->prev = obj->prev->next = obj; @@ -174,15 +186,14 @@ void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name) { } void grpc_iomgr_unregister_object(grpc_iomgr_object *obj) { - gpr_free(obj->name); gpr_mu_lock(&g_mu); obj->next->prev = obj->prev; obj->prev->next = obj->next; + gpr_free(obj->name); gpr_cv_signal(&g_rcv); gpr_mu_unlock(&g_mu); } - void grpc_iomgr_closure_init(grpc_iomgr_closure *closure, grpc_iomgr_cb_func cb, void *cb_arg) { closure->cb = cb; @@ -200,15 +211,16 @@ void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *closure, int success) { g_cbs_tail->next = closure; g_cbs_tail = closure; } + if (g_shutdown) { + gpr_cv_signal(&g_rcv); + } gpr_mu_unlock(&g_mu); } - void grpc_iomgr_add_callback(grpc_iomgr_closure *closure) { grpc_iomgr_add_delayed_callback(closure, 1 /* GPR_TRUE */); } - int grpc_maybe_call_delayed_callbacks(gpr_mu *drop_mu, int success) { int n = 0; gpr_mu *retake_mu = NULL; diff --git a/src/core/iomgr/iomgr.h b/src/core/iomgr/iomgr.h index a10e481e48..6d4a82917b 100644 --- a/src/core/iomgr/iomgr.h +++ b/src/core/iomgr/iomgr.h @@ -73,4 +73,8 @@ void grpc_iomgr_shutdown(void); * Can be called from within a callback or from anywhere else */ void grpc_iomgr_add_callback(grpc_iomgr_closure *closure); +/** As per grpc_iomgr_add_callback, with the ability to set the success + argument. */ +void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *iocb, int success); + #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_H */ diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 067af87c93..c40188b3c9 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -52,23 +52,24 @@ #include "src/core/iomgr/pollset_windows.h" #endif - void grpc_pollset_init(grpc_pollset *pollset); void grpc_pollset_shutdown(grpc_pollset *pollset, void (*shutdown_done)(void *arg), void *shutdown_done_arg); void grpc_pollset_destroy(grpc_pollset *pollset); - /* Do some work on a pollset. May involve invoking asynchronous callbacks, or actually polling file descriptors. Requires GRPC_POLLSET_MU(pollset) locked. - May unlock GRPC_POLLSET_MU(pollset) during its execution. */ + May unlock GRPC_POLLSET_MU(pollset) during its execution. + + Returns true if some work has been done, and false if the deadline + got attained. */ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline); -/* Break a pollset out of polling work +/* Break one polling thread out of polling work for this pollset. Requires GRPC_POLLSET_MU(pollset) locked. */ void grpc_pollset_kick(grpc_pollset *pollset); -#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */ diff --git a/src/core/iomgr/pollset_kick.c b/src/core/iomgr/pollset_kick_posix.c index f0211b8274..51021784f2 100644 --- a/src/core/iomgr/pollset_kick.c +++ b/src/core/iomgr/pollset_kick_posix.c @@ -34,7 +34,7 @@ #include <grpc/support/port_platform.h> #ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_kick.h" +#include "src/core/iomgr/pollset_kick_posix.h" #include <errno.h> #include <string.h> @@ -73,7 +73,7 @@ static grpc_kick_fd_info *allocate_wfd(void) { return info; } -static void destroy_wfd(grpc_kick_fd_info* wfd) { +static void destroy_wfd(grpc_kick_fd_info *wfd) { grpc_wakeup_fd_destroy(&wfd->wakeup_fd); gpr_free(wfd); } @@ -96,41 +96,49 @@ static void free_wfd(grpc_kick_fd_info *fd_info) { void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state) { gpr_mu_init(&kick_state->mu); kick_state->kicked = 0; - kick_state->fd_info = NULL; + kick_state->fd_list.next = kick_state->fd_list.prev = &kick_state->fd_list; } void grpc_pollset_kick_destroy(grpc_pollset_kick_state *kick_state) { gpr_mu_destroy(&kick_state->mu); - GPR_ASSERT(kick_state->fd_info == NULL); + GPR_ASSERT(kick_state->fd_list.next == &kick_state->fd_list); } -int grpc_pollset_kick_pre_poll(grpc_pollset_kick_state *kick_state) { +grpc_kick_fd_info *grpc_pollset_kick_pre_poll( + grpc_pollset_kick_state *kick_state) { + grpc_kick_fd_info *fd_info; gpr_mu_lock(&kick_state->mu); if (kick_state->kicked) { kick_state->kicked = 0; gpr_mu_unlock(&kick_state->mu); - return -1; + return NULL; } - kick_state->fd_info = allocate_wfd(); + fd_info = allocate_wfd(); + fd_info->next = &kick_state->fd_list; + fd_info->prev = fd_info->next->prev; + fd_info->next->prev = fd_info->prev->next = fd_info; gpr_mu_unlock(&kick_state->mu); - return GRPC_WAKEUP_FD_GET_READ_FD(&kick_state->fd_info->wakeup_fd); + return fd_info; } -void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state) { - grpc_wakeup_fd_consume_wakeup(&kick_state->fd_info->wakeup_fd); +void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state, + grpc_kick_fd_info *fd_info) { + grpc_wakeup_fd_consume_wakeup(&fd_info->wakeup_fd); } -void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state) { +void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state, + grpc_kick_fd_info *fd_info) { gpr_mu_lock(&kick_state->mu); - free_wfd(kick_state->fd_info); - kick_state->fd_info = NULL; + fd_info->next->prev = fd_info->prev; + fd_info->prev->next = fd_info->next; + free_wfd(fd_info); gpr_mu_unlock(&kick_state->mu); } void grpc_pollset_kick_kick(grpc_pollset_kick_state *kick_state) { gpr_mu_lock(&kick_state->mu); - if (kick_state->fd_info != NULL) { - grpc_wakeup_fd_wakeup(&kick_state->fd_info->wakeup_fd); + if (kick_state->fd_list.next != &kick_state->fd_list) { + grpc_wakeup_fd_wakeup(&kick_state->fd_list.next->wakeup_fd); } else { kick_state->kicked = 1; } @@ -157,5 +165,4 @@ void grpc_pollset_kick_global_destroy(void) { gpr_mu_destroy(&fd_freelist_mu); } - -#endif /* GPR_POSIX_SOCKET */ +#endif /* GPR_POSIX_SOCKET */ diff --git a/src/core/iomgr/pollset_kick_posix.h b/src/core/iomgr/pollset_kick_posix.h index 427699198c..77e32a8d51 100644 --- a/src/core/iomgr/pollset_kick_posix.h +++ b/src/core/iomgr/pollset_kick_posix.h @@ -37,15 +37,57 @@ #include "src/core/iomgr/wakeup_fd_posix.h" #include <grpc/support/sync.h> +/* pollset kicking allows breaking a thread out of polling work for + a given pollset. + writing a byte to a pipe is used as a posix-ly portable base + mechanism, and eventfds are utilized on Linux for better performance. */ + typedef struct grpc_kick_fd_info { grpc_wakeup_fd_info wakeup_fd; + /* used for polling list and free list */ struct grpc_kick_fd_info *next; + /* only used when polling */ + struct grpc_kick_fd_info *prev; } grpc_kick_fd_info; typedef struct grpc_pollset_kick_state { gpr_mu mu; int kicked; - struct grpc_kick_fd_info *fd_info; + struct grpc_kick_fd_info fd_list; } grpc_pollset_kick_state; -#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_POSIX_H */ +#define GRPC_POLLSET_KICK_GET_FD(kick_fd_info) \ + GRPC_WAKEUP_FD_GET_READ_FD(&(kick_fd_info)->wakeup_fd) + +/* This is an abstraction around the typical pipe mechanism for waking up a + thread sitting in a poll() style call. */ + +void grpc_pollset_kick_global_init(void); +void grpc_pollset_kick_global_destroy(void); + +void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state); +void grpc_pollset_kick_destroy(grpc_pollset_kick_state *kick_state); + +/* Guarantees a pure posix implementation rather than a specialized one, if + * applicable. Intended for testing. */ +void grpc_pollset_kick_global_init_fallback_fd(void); + +/* Must be called before entering poll(). If return value is NULL, this consumed + an existing kick. Otherwise the return value is an FD to add to the poll set. + */ +grpc_kick_fd_info *grpc_pollset_kick_pre_poll( + grpc_pollset_kick_state *kick_state); + +/* Consume an existing kick. Must be called after poll returns that the fd was + readable, and before calling kick_post_poll. */ +void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state, + grpc_kick_fd_info *fd_info); + +/* Must be called after pre_poll, and after consume if applicable */ +void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state, + grpc_kick_fd_info *fd_info); + +/* Actually kick */ +void grpc_pollset_kick_kick(grpc_pollset_kick_state *kick_state); + +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_POSIX_H */ diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 40b7935a57..1900bbf9e1 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -54,17 +54,25 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset, pollset_hdr *h = pollset->data.ptr; struct epoll_event ev; int err; - - ev.events = EPOLLIN | EPOLLOUT | EPOLLET; - ev.data.ptr = fd; - err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); - if (err < 0) { - /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ - if (errno != EEXIST) { - gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, - strerror(errno)); + grpc_fd_watcher watcher; + + /* We pretend to be polling whilst adding an fd to keep the fd from being + closed during the add. This may result in a spurious wakeup being assigned + to this pollset whilst adding, but that should be benign. */ + GPR_ASSERT(grpc_fd_begin_poll(fd, pollset, 0, 0, &watcher) == 0); + if (watcher.fd != NULL) { + ev.events = EPOLLIN | EPOLLOUT | EPOLLET; + ev.data.ptr = fd; + err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0) { + /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ + if (errno != EEXIST) { + gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, + strerror(errno)); + } } } + grpc_fd_end_poll(&watcher, 0, 0); } static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset, @@ -83,7 +91,7 @@ static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset, /* TODO(klempner): We probably want to turn this down a bit */ #define GRPC_EPOLL_MAX_EVENTS 1000 -static int multipoll_with_epoll_pollset_maybe_work( +static void multipoll_with_epoll_pollset_maybe_work( grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now, int allow_synchronous_callback) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; @@ -97,14 +105,7 @@ static int multipoll_with_epoll_pollset_maybe_work( * here. */ - if (gpr_time_cmp(deadline, gpr_inf_future) == 0) { - timeout_ms = -1; - } else { - timeout_ms = gpr_time_to_millis(gpr_time_sub(deadline, now)); - if (timeout_ms <= 0) { - return 1; - } - } + timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now); pollset->counter += 1; gpr_mu_unlock(&pollset->mu); @@ -140,13 +141,11 @@ static int multipoll_with_epoll_pollset_maybe_work( gpr_mu_lock(&pollset->mu); pollset->counter -= 1; - /* TODO(klempner): This should signal once per event rather than broadcast, - * although it probably doesn't matter because threads will generally be - * blocked in epoll_wait rather than being blocked on the cv. */ - gpr_cv_broadcast(&pollset->cv); - return 1; } +static void multipoll_with_epoll_pollset_finish_shutdown( + grpc_pollset *pollset) {} + static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) { pollset_hdr *h = pollset->data.ptr; grpc_wakeup_fd_destroy(&h->wakeup_fd); @@ -160,8 +159,11 @@ static void epoll_kick(grpc_pollset *pollset) { } static const grpc_pollset_vtable multipoll_with_epoll_pollset = { - multipoll_with_epoll_pollset_add_fd, multipoll_with_epoll_pollset_del_fd, - multipoll_with_epoll_pollset_maybe_work, epoll_kick, + multipoll_with_epoll_pollset_add_fd, + multipoll_with_epoll_pollset_del_fd, + multipoll_with_epoll_pollset_maybe_work, + epoll_kick, + multipoll_with_epoll_pollset_finish_shutdown, multipoll_with_epoll_pollset_destroy}; static void epoll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index d781c9b4bb..7b717bd159 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -78,7 +78,7 @@ static void multipoll_with_poll_pollset_add_fd(grpc_pollset *pollset, h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity); } h->fds[h->fd_count++] = fd; - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "multipoller"); } static void multipoll_with_poll_pollset_del_fd(grpc_pollset *pollset, @@ -90,7 +90,7 @@ static void multipoll_with_poll_pollset_del_fd(grpc_pollset *pollset, h->dels = gpr_realloc(h->dels, sizeof(grpc_fd *) * h->del_capacity); } h->dels[h->del_count++] = fd; - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "multipoller_del"); } static void end_polling(grpc_pollset *pollset) { @@ -103,26 +103,17 @@ static void end_polling(grpc_pollset *pollset) { } } -static int multipoll_with_poll_pollset_maybe_work( +static void multipoll_with_poll_pollset_maybe_work( grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now, int allow_synchronous_callback) { int timeout; int r; size_t i, np, nf, nd; pollset_hdr *h; + grpc_kick_fd_info *kfd; - if (pollset->counter) { - return 0; - } h = pollset->data.ptr; - if (gpr_time_cmp(deadline, gpr_inf_future) == 0) { - timeout = -1; - } else { - timeout = gpr_time_to_millis(gpr_time_sub(deadline, now)); - if (timeout <= 0) { - return 1; - } - } + timeout = grpc_poll_deadline_to_millis_timeout(deadline, now); if (h->pfd_capacity < h->fd_count + 1) { h->pfd_capacity = GPR_MAX(h->pfd_capacity * 3 / 2, h->fd_count + 1); gpr_free(h->pfds); @@ -132,11 +123,12 @@ static int multipoll_with_poll_pollset_maybe_work( } nf = 0; np = 1; - h->pfds[0].fd = grpc_pollset_kick_pre_poll(&pollset->kick_state); - if (h->pfds[0].fd < 0) { + kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state); + if (kfd == NULL) { /* Already kicked */ - return 1; + return; } + h->pfds[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd); h->pfds[0].events = POLLIN; h->pfds[0].revents = POLLOUT; for (i = 0; i < h->fd_count; i++) { @@ -145,7 +137,7 @@ static int multipoll_with_poll_pollset_maybe_work( if (h->fds[i] == h->dels[nd]) remove = 1; } if (remove) { - grpc_fd_unref(h->fds[i]); + GRPC_FD_UNREF(h->fds[i], "multipoller"); } else { h->fds[nf++] = h->fds[i]; h->watchers[np].fd = h->fds[i]; @@ -157,14 +149,14 @@ static int multipoll_with_poll_pollset_maybe_work( h->pfd_count = np; h->fd_count = nf; for (nd = 0; nd < h->del_count; nd++) { - grpc_fd_unref(h->dels[nd]); + GRPC_FD_UNREF(h->dels[nd], "multipoller_del"); } h->del_count = 0; if (h->pfd_count == 0) { end_polling(pollset); - return 0; + return; } - pollset->counter = 1; + pollset->counter++; gpr_mu_unlock(&pollset->mu); for (i = 1; i < np; i++) { @@ -184,9 +176,12 @@ static int multipoll_with_poll_pollset_maybe_work( /* do nothing */ } else { if (h->pfds[0].revents & POLLIN) { - grpc_pollset_kick_consume(&pollset->kick_state); + grpc_pollset_kick_consume(&pollset->kick_state, kfd); } for (i = 1; i < np; i++) { + if (h->watchers[i].fd == NULL) { + continue; + } if (h->pfds[i].revents & (POLLIN | POLLHUP | POLLERR)) { grpc_fd_become_readable(h->watchers[i].fd, allow_synchronous_callback); } @@ -195,28 +190,33 @@ static int multipoll_with_poll_pollset_maybe_work( } } } - grpc_pollset_kick_post_poll(&pollset->kick_state); + grpc_pollset_kick_post_poll(&pollset->kick_state, kfd); gpr_mu_lock(&pollset->mu); - pollset->counter = 0; - gpr_cv_broadcast(&pollset->cv); - return 1; + pollset->counter--; } static void multipoll_with_poll_pollset_kick(grpc_pollset *p) { grpc_pollset_force_kick(p); } -static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { +static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) { size_t i; pollset_hdr *h = pollset->data.ptr; GPR_ASSERT(pollset->counter == 0); for (i = 0; i < h->fd_count; i++) { - grpc_fd_unref(h->fds[i]); + GRPC_FD_UNREF(h->fds[i], "multipoller"); } for (i = 0; i < h->del_count; i++) { - grpc_fd_unref(h->dels[i]); + GRPC_FD_UNREF(h->dels[i], "multipoller_del"); } + h->fd_count = 0; + h->del_count = 0; +} + +static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { + pollset_hdr *h = pollset->data.ptr; + multipoll_with_poll_pollset_finish_shutdown(pollset); gpr_free(h->pfds); gpr_free(h->watchers); gpr_free(h->fds); @@ -225,8 +225,11 @@ static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { } static const grpc_pollset_vtable multipoll_with_poll_pollset = { - multipoll_with_poll_pollset_add_fd, multipoll_with_poll_pollset_del_fd, - multipoll_with_poll_pollset_maybe_work, multipoll_with_poll_pollset_kick, + multipoll_with_poll_pollset_add_fd, + multipoll_with_poll_pollset_del_fd, + multipoll_with_poll_pollset_maybe_work, + multipoll_with_poll_pollset_kick, + multipoll_with_poll_pollset_finish_shutdown, multipoll_with_poll_pollset_destroy}; void grpc_poll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, @@ -247,7 +250,7 @@ void grpc_poll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, h->dels = NULL; for (i = 0; i < nfds; i++) { h->fds[i] = fds[i]; - grpc_fd_ref(fds[i]); + GRPC_FD_REF(fds[i], "multipoller"); } } diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index d2f615271e..15ed8e75e6 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -54,31 +54,8 @@ #include <grpc/support/tls.h> #include <grpc/support/useful.h> -static grpc_pollset g_backup_pollset; -static int g_shutdown_backup_poller; -static gpr_event g_backup_poller_done; -static gpr_event g_backup_pollset_shutdown_done; - GPR_TLS_DECL(g_current_thread_poller); -static void backup_poller(void *p) { - gpr_timespec delta = gpr_time_from_millis(100); - gpr_timespec last_poll = gpr_now(); - - gpr_mu_lock(&g_backup_pollset.mu); - while (g_shutdown_backup_poller == 0) { - gpr_timespec next_poll = gpr_time_add(last_poll, delta); - grpc_pollset_work(&g_backup_pollset, gpr_time_add(gpr_now(), gpr_time_from_seconds(1))); - gpr_mu_unlock(&g_backup_pollset.mu); - gpr_sleep_until(next_poll); - gpr_mu_lock(&g_backup_pollset.mu); - last_poll = next_poll; - } - gpr_mu_unlock(&g_backup_pollset.mu); - - gpr_event_set(&g_backup_poller_done, (void *)1); -} - void grpc_pollset_kick(grpc_pollset *p) { if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p && p->counter) { p->vtable->kick(p); @@ -99,44 +76,14 @@ static void kick_using_pollset_kick(grpc_pollset *p) { /* global state management */ -grpc_pollset *grpc_backup_pollset(void) { return &g_backup_pollset; } - void grpc_pollset_global_init(void) { - gpr_thd_id id; - gpr_tls_init(&g_current_thread_poller); /* Initialize kick fd state */ grpc_pollset_kick_global_init(); - - /* initialize the backup pollset */ - grpc_pollset_init(&g_backup_pollset); - - /* start the backup poller thread */ - g_shutdown_backup_poller = 0; - gpr_event_init(&g_backup_poller_done); - gpr_event_init(&g_backup_pollset_shutdown_done); - gpr_thd_new(&id, backup_poller, NULL, NULL); -} - -static void on_backup_pollset_shutdown_done(void *arg) { - gpr_event_set(&g_backup_pollset_shutdown_done, (void *)1); } void grpc_pollset_global_shutdown(void) { - /* terminate the backup poller thread */ - gpr_mu_lock(&g_backup_pollset.mu); - g_shutdown_backup_poller = 1; - gpr_mu_unlock(&g_backup_pollset.mu); - gpr_event_wait(&g_backup_poller_done, gpr_inf_future); - - grpc_pollset_shutdown(&g_backup_pollset, on_backup_pollset_shutdown_done, - NULL); - gpr_event_wait(&g_backup_pollset_shutdown_done, gpr_inf_future); - - /* destroy the backup pollset */ - grpc_pollset_destroy(&g_backup_pollset); - /* destroy the kick pipes */ grpc_pollset_kick_global_destroy(); @@ -145,36 +92,37 @@ void grpc_pollset_global_shutdown(void) { /* main interface */ -static void become_empty_pollset(grpc_pollset *pollset); -static void become_unary_pollset(grpc_pollset *pollset, grpc_fd *fd); +static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); void grpc_pollset_init(grpc_pollset *pollset) { gpr_mu_init(&pollset->mu); - gpr_cv_init(&pollset->cv); grpc_pollset_kick_init(&pollset->kick_state); pollset->in_flight_cbs = 0; pollset->shutting_down = 0; - become_empty_pollset(pollset); + pollset->called_shutdown = 0; + become_basic_pollset(pollset, NULL); } void grpc_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { gpr_mu_lock(&pollset->mu); pollset->vtable->add_fd(pollset, fd); - gpr_cv_broadcast(&pollset->cv); gpr_mu_unlock(&pollset->mu); } void grpc_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) { gpr_mu_lock(&pollset->mu); pollset->vtable->del_fd(pollset, fd); - gpr_cv_broadcast(&pollset->cv); gpr_mu_unlock(&pollset->mu); } +static void finish_shutdown(grpc_pollset *pollset) { + pollset->vtable->finish_shutdown(pollset); + pollset->shutdown_done_cb(pollset->shutdown_done_arg); +} + int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) { /* pollset->mu already held */ gpr_timespec now = gpr_now(); - int r; if (gpr_time_cmp(now, deadline) > 0) { return 0; } @@ -188,14 +136,15 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) { return 1; } gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset); - r = pollset->vtable->maybe_work(pollset, deadline, now, 1); + pollset->vtable->maybe_work(pollset, deadline, now, 1); gpr_tls_set(&g_current_thread_poller, 0); if (pollset->shutting_down) { if (pollset->counter > 0) { grpc_pollset_kick(pollset); - } else if (pollset->in_flight_cbs == 0) { + } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { + pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); - pollset->shutdown_done_cb(pollset->shutdown_done_arg); + finish_shutdown(pollset); /* Continuing to access pollset here is safe -- it is the caller's * responsibility to not destroy when it has outstanding calls to * grpc_pollset_work. @@ -203,27 +152,30 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) { gpr_mu_lock(&pollset->mu); } } - return r; + return 1; } void grpc_pollset_shutdown(grpc_pollset *pollset, void (*shutdown_done)(void *arg), void *shutdown_done_arg) { - int in_flight_cbs; - int counter; + int call_shutdown = 0; gpr_mu_lock(&pollset->mu); + GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; - in_flight_cbs = pollset->in_flight_cbs; - counter = pollset->counter; + if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && + pollset->counter == 0) { + pollset->called_shutdown = 1; + call_shutdown = 1; + } pollset->shutdown_done_cb = shutdown_done; pollset->shutdown_done_arg = shutdown_done_arg; - if (counter > 0) { + if (pollset->counter > 0) { grpc_pollset_kick(pollset); } gpr_mu_unlock(&pollset->mu); - if (in_flight_cbs == 0 && counter == 0) { - shutdown_done(shutdown_done_arg); + if (call_shutdown) { + finish_shutdown(pollset); } } @@ -233,41 +185,29 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { pollset->vtable->destroy(pollset); grpc_pollset_kick_destroy(&pollset->kick_state); gpr_mu_destroy(&pollset->mu); - gpr_cv_destroy(&pollset->cv); } -/* - * empty_pollset - a vtable that provides polling for NO file descriptors - */ - -static void empty_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { - become_unary_pollset(pollset, fd); -} - -static void empty_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) {} - -static int empty_pollset_maybe_work(grpc_pollset *pollset, - gpr_timespec deadline, gpr_timespec now, - int allow_synchronous_callback) { - return 0; -} - -static void empty_pollset_destroy(grpc_pollset *pollset) {} - -static const grpc_pollset_vtable empty_pollset = { - empty_pollset_add_fd, empty_pollset_del_fd, empty_pollset_maybe_work, - kick_using_pollset_kick, empty_pollset_destroy}; - -static void become_empty_pollset(grpc_pollset *pollset) { - pollset->vtable = &empty_pollset; +int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { + gpr_timespec timeout; + static const int max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future) == 0) { + return -1; + } + if (gpr_time_cmp( + deadline, + gpr_time_add(now, gpr_time_from_micros(max_spin_polling_us))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + return gpr_time_to_millis( + gpr_time_add(timeout, gpr_time_from_nanos(GPR_NS_PER_SEC - 1))); } /* - * unary_poll_pollset - a vtable that provides polling for one file descriptor - * via poll() + * basic_pollset - a vtable that provides polling for zero or one file + * descriptor via poll() */ - typedef struct grpc_unary_promote_args { const grpc_pollset_vtable *original_vtable; grpc_pollset *pollset; @@ -275,7 +215,7 @@ typedef struct grpc_unary_promote_args { grpc_iomgr_closure promotion_closure; } grpc_unary_promote_args; -static void unary_poll_do_promote(void *args, int success) { +static void basic_do_promote(void *args, int success) { grpc_unary_promote_args *up_args = args; const grpc_pollset_vtable *original_vtable = up_args->original_vtable; grpc_pollset *pollset = up_args->pollset; @@ -293,7 +233,7 @@ static void unary_poll_do_promote(void *args, int success) { gpr_mu_lock(&pollset->mu); /* First we need to ensure that nobody is polling concurrently */ - while (pollset->counter != 0) { + if (pollset->counter != 0) { grpc_pollset_kick(pollset); grpc_iomgr_add_callback(&up_args->promotion_closure); gpr_mu_unlock(&pollset->mu); @@ -321,33 +261,33 @@ static void unary_poll_do_promote(void *args, int success) { fds[0] = pollset->data.ptr; fds[1] = fd; - if (!grpc_fd_is_orphaned(fds[0])) { + if (fds[0] && !grpc_fd_is_orphaned(fds[0])) { grpc_platform_become_multipoller(pollset, fds, GPR_ARRAY_SIZE(fds)); - grpc_fd_unref(fds[0]); + GRPC_FD_UNREF(fds[0], "basicpoll"); } else { /* old fd is orphaned and we haven't cleaned it up until now, so remain a * unary poller */ /* Note that it is possible that fds[1] is also orphaned at this point. * That's okay, we'll correct it at the next add or poll. */ - grpc_fd_unref(fds[0]); + if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll"); pollset->data.ptr = fd; - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "basicpoll"); } } - gpr_cv_broadcast(&pollset->cv); gpr_mu_unlock(&pollset->mu); if (do_shutdown_cb) { pollset->shutdown_done_cb(pollset->shutdown_done_arg); } - /* Matching ref in unary_poll_pollset_add_fd */ - grpc_fd_unref(fd); + /* Matching ref in basic_pollset_add_fd */ + GRPC_FD_UNREF(fd, "basicpoll_add"); } -static void unary_poll_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { +static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { grpc_unary_promote_args *up_args; + GPR_ASSERT(fd); if (fd == pollset->data.ptr) return; if (!pollset->counter) { @@ -358,92 +298,100 @@ static void unary_poll_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { fds[0] = pollset->data.ptr; fds[1] = fd; - if (!grpc_fd_is_orphaned(fds[0])) { + if (fds[0] == NULL) { + pollset->data.ptr = fd; + GRPC_FD_REF(fd, "basicpoll"); + } else if (!grpc_fd_is_orphaned(fds[0])) { grpc_platform_become_multipoller(pollset, fds, GPR_ARRAY_SIZE(fds)); - grpc_fd_unref(fds[0]); + GRPC_FD_UNREF(fds[0], "basicpoll"); } else { /* old fd is orphaned and we haven't cleaned it up until now, so remain a * unary poller */ - grpc_fd_unref(fds[0]); + GRPC_FD_UNREF(fds[0], "basicpoll"); pollset->data.ptr = fd; - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "basicpoll"); } return; } /* Now we need to promote. This needs to happen when we're not polling. Since * this may be called from poll, the wait needs to happen asynchronously. */ - grpc_fd_ref(fd); + GRPC_FD_REF(fd, "basicpoll_add"); pollset->in_flight_cbs++; up_args = gpr_malloc(sizeof(*up_args)); up_args->pollset = pollset; up_args->fd = fd; up_args->original_vtable = pollset->vtable; - up_args->promotion_closure.cb = unary_poll_do_promote; + up_args->promotion_closure.cb = basic_do_promote; up_args->promotion_closure.cb_arg = up_args; grpc_iomgr_add_callback(&up_args->promotion_closure); grpc_pollset_kick(pollset); } -static void unary_poll_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) { +static void basic_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) { + GPR_ASSERT(fd); if (fd == pollset->data.ptr) { - grpc_fd_unref(pollset->data.ptr); - become_empty_pollset(pollset); + GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); + pollset->data.ptr = NULL; } } -static int unary_poll_pollset_maybe_work(grpc_pollset *pollset, - gpr_timespec deadline, - gpr_timespec now, - int allow_synchronous_callback) { +static void basic_pollset_maybe_work(grpc_pollset *pollset, + gpr_timespec deadline, gpr_timespec now, + int allow_synchronous_callback) { struct pollfd pfd[2]; grpc_fd *fd; grpc_fd_watcher fd_watcher; + grpc_kick_fd_info *kfd; int timeout; int r; + int nfds; - if (pollset->counter) { - return 0; - } if (pollset->in_flight_cbs) { /* Give do_promote priority so we don't starve it out */ - return 0; + gpr_mu_unlock(&pollset->mu); + gpr_mu_lock(&pollset->mu); + return; } fd = pollset->data.ptr; - if (grpc_fd_is_orphaned(fd)) { - grpc_fd_unref(fd); - become_empty_pollset(pollset); - return 0; - } - if (gpr_time_cmp(deadline, gpr_inf_future) == 0) { - timeout = -1; - } else { - timeout = gpr_time_to_millis(gpr_time_sub(deadline, now)); - if (timeout <= 0) { - return 1; - } + if (fd && grpc_fd_is_orphaned(fd)) { + GRPC_FD_UNREF(fd, "basicpoll"); + fd = pollset->data.ptr = NULL; } - pfd[0].fd = grpc_pollset_kick_pre_poll(&pollset->kick_state); - if (pfd[0].fd < 0) { + timeout = grpc_poll_deadline_to_millis_timeout(deadline, now); + kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state); + if (kfd == NULL) { /* Already kicked */ - return 1; + return; } + pfd[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd); pfd[0].events = POLLIN; pfd[0].revents = 0; - pfd[1].fd = fd->fd; - pfd[1].revents = 0; - pollset->counter = 1; - gpr_mu_unlock(&pollset->mu); - - pfd[1].events = grpc_fd_begin_poll(fd, pollset, POLLIN, POLLOUT, &fd_watcher); + nfds = 1; + pollset->counter++; + if (fd) { + pfd[1].fd = fd->fd; + pfd[1].revents = 0; + gpr_mu_unlock(&pollset->mu); + pfd[1].events = + grpc_fd_begin_poll(fd, pollset, POLLIN, POLLOUT, &fd_watcher); + if (pfd[1].events != 0) { + nfds++; + } + } else { + gpr_mu_unlock(&pollset->mu); + } /* poll fd count (argument 2) is shortened by one if we have no events to poll on - such that it only includes the kicker */ - r = poll(pfd, GPR_ARRAY_SIZE(pfd) - (pfd[1].events == 0), timeout); + r = poll(pfd, nfds, timeout); GRPC_TIMER_MARK(GRPC_PTAG_POLL_FINISHED, r); - grpc_fd_end_poll(&fd_watcher, pfd[1].revents & POLLIN, pfd[1].revents & POLLOUT); + if (fd) { + grpc_fd_end_poll(&fd_watcher, pfd[1].revents & POLLIN, + pfd[1].revents & POLLOUT); + } if (r < 0) { if (errno != EINTR) { @@ -453,39 +401,43 @@ static int unary_poll_pollset_maybe_work(grpc_pollset *pollset, /* do nothing */ } else { if (pfd[0].revents & POLLIN) { - grpc_pollset_kick_consume(&pollset->kick_state); + grpc_pollset_kick_consume(&pollset->kick_state, kfd); } - if (pfd[1].revents & (POLLIN | POLLHUP | POLLERR)) { - grpc_fd_become_readable(fd, allow_synchronous_callback); - } - if (pfd[1].revents & (POLLOUT | POLLHUP | POLLERR)) { - grpc_fd_become_writable(fd, allow_synchronous_callback); + if (nfds > 1) { + if (pfd[1].revents & (POLLIN | POLLHUP | POLLERR)) { + grpc_fd_become_readable(fd, allow_synchronous_callback); + } + if (pfd[1].revents & (POLLOUT | POLLHUP | POLLERR)) { + grpc_fd_become_writable(fd, allow_synchronous_callback); + } } } - grpc_pollset_kick_post_poll(&pollset->kick_state); + grpc_pollset_kick_post_poll(&pollset->kick_state, kfd); gpr_mu_lock(&pollset->mu); - pollset->counter = 0; - gpr_cv_broadcast(&pollset->cv); - return 1; + pollset->counter--; } -static void unary_poll_pollset_destroy(grpc_pollset *pollset) { +static void basic_pollset_destroy(grpc_pollset *pollset) { GPR_ASSERT(pollset->counter == 0); - grpc_fd_unref(pollset->data.ptr); + if (pollset->data.ptr != NULL) { + GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); + pollset->data.ptr = NULL; + } } -static const grpc_pollset_vtable unary_poll_pollset = { - unary_poll_pollset_add_fd, unary_poll_pollset_del_fd, - unary_poll_pollset_maybe_work, kick_using_pollset_kick, - unary_poll_pollset_destroy}; +static const grpc_pollset_vtable basic_pollset = { + basic_pollset_add_fd, basic_pollset_del_fd, basic_pollset_maybe_work, + kick_using_pollset_kick, basic_pollset_destroy, basic_pollset_destroy}; -static void become_unary_pollset(grpc_pollset *pollset, grpc_fd *fd) { - pollset->vtable = &unary_poll_pollset; +static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) { + pollset->vtable = &basic_pollset; pollset->counter = 0; - pollset->data.ptr = fd; - grpc_fd_ref(fd); + pollset->data.ptr = fd_or_null; + if (fd_or_null) { + GRPC_FD_REF(fd_or_null, "basicpoll"); + } } #endif /* GPR_POSIX_POLLSET */ diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index 088ec910c2..53585a2886 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -36,7 +36,7 @@ #include <grpc/support/sync.h> -#include "src/core/iomgr/pollset_kick.h" +#include "src/core/iomgr/pollset_kick_posix.h" typedef struct grpc_pollset_vtable grpc_pollset_vtable; @@ -52,11 +52,11 @@ typedef struct grpc_pollset { few fds, and an epoll() based implementation for many fds */ const grpc_pollset_vtable *vtable; gpr_mu mu; - gpr_cv cv; grpc_pollset_kick_state kick_state; int counter; int in_flight_cbs; int shutting_down; + int called_shutdown; void (*shutdown_done_cb)(void *arg); void *shutdown_done_arg; union { @@ -68,14 +68,14 @@ typedef struct grpc_pollset { struct grpc_pollset_vtable { void (*add_fd)(grpc_pollset *pollset, struct grpc_fd *fd); void (*del_fd)(grpc_pollset *pollset, struct grpc_fd *fd); - int (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline, - gpr_timespec now, int allow_synchronous_callback); + void (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline, + gpr_timespec now, int allow_synchronous_callback); void (*kick)(grpc_pollset *pollset); + void (*finish_shutdown)(grpc_pollset *pollset); void (*destroy)(grpc_pollset *pollset); }; #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) -#define GRPC_POLLSET_CV(pollset) (&(pollset)->cv) /* Add an fd to a pollset */ void grpc_pollset_add_fd(grpc_pollset *pollset, struct grpc_fd *fd); @@ -94,11 +94,14 @@ int grpc_kick_read_fd(grpc_pollset *p); /* Call after polling has been kicked to leave the kicked state */ void grpc_kick_drain(grpc_pollset *p); -/* All fds get added to a backup pollset to ensure that progress is made - regardless of applications listening to events. Relying on this is slow - however (the backup pollset only listens every 100ms or so) - so it's not - to be relied on. */ -grpc_pollset *grpc_backup_pollset(void); +/* Convert a timespec to milliseconds: + - very small or negative poll times are clamped to zero to do a + non-blocking poll (which becomes spin polling) + - other small values are rounded up to one millisecond + - longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - infinite timeouts are converted to -1 */ +int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now); /* turn a pollset into a multipoller: platform specific */ typedef void (*grpc_platform_become_multipoller_type)(grpc_pollset *pollset, diff --git a/src/core/iomgr/pollset_kick.h b/src/core/iomgr/pollset_set.h index cc9357de1f..98e3b552a7 100644 --- a/src/core/iomgr/pollset_kick.h +++ b/src/core/iomgr/pollset_set.h @@ -31,44 +31,29 @@ * */ -#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_H -#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_H +#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_H +#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_H -#include <grpc/support/port_platform.h> +#include "src/core/iomgr/pollset.h" + +/* A grpc_pollset_set is a set of pollsets that are interested in an + action. Adding a pollset to a pollset_set automatically adds any + fd's (etc) that have been registered with the set_set with that pollset. + Registering fd's automatically adds them to all current pollsets. */ #ifdef GPR_POSIX_SOCKET -#include "src/core/iomgr/pollset_kick_posix.h" +#include "src/core/iomgr/pollset_set_posix.h" #endif #ifdef GPR_WIN32 -#include "src/core/iomgr/pollset_kick_windows.h" +#include "src/core/iomgr/pollset_set_windows.h" #endif -/* This is an abstraction around the typical pipe mechanism for waking up a - thread sitting in a poll() style call. */ - -void grpc_pollset_kick_global_init(void); -void grpc_pollset_kick_global_destroy(void); - -void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state); -void grpc_pollset_kick_destroy(grpc_pollset_kick_state *kick_state); - -/* Guarantees a pure posix implementation rather than a specialized one, if - * applicable. Intended for testing. */ -void grpc_pollset_kick_global_init_fallback_fd(void); - -/* Must be called before entering poll(). If return value is -1, this consumed - an existing kick. Otherwise the return value is an FD to add to the poll set. - */ -int grpc_pollset_kick_pre_poll(grpc_pollset_kick_state *kick_state); - -/* Consume an existing kick. Must be called after poll returns that the fd was - readable, and before calling kick_post_poll. */ -void grpc_pollset_kick_consume(grpc_pollset_kick_state *kick_state); - -/* Must be called after pre_poll, and after consume if applicable */ -void grpc_pollset_kick_post_poll(grpc_pollset_kick_state *kick_state); - -void grpc_pollset_kick_kick(grpc_pollset_kick_state *kick_state); +void grpc_pollset_set_init(grpc_pollset_set *pollset_set); +void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); +void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset); +void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset); -#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */ diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c new file mode 100644 index 0000000000..005e938398 --- /dev/null +++ b/src/core/iomgr/pollset_set_posix.c @@ -0,0 +1,125 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/port_platform.h> + +#ifdef GPR_POSIX_SOCKET + +#include <stdlib.h> +#include <string.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/useful.h> + +#include "src/core/iomgr/pollset_set.h" + +void grpc_pollset_set_init(grpc_pollset_set *pollset_set) { + memset(pollset_set, 0, sizeof(*pollset_set)); + gpr_mu_init(&pollset_set->mu); +} + +void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { + size_t i; + gpr_mu_destroy(&pollset_set->mu); + for (i = 0; i < pollset_set->fd_count; i++) { + GRPC_FD_UNREF(pollset_set->fds[i], "pollset"); + } + gpr_free(pollset_set->pollsets); + gpr_free(pollset_set->fds); +} + +void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + if (pollset_set->pollset_count == pollset_set->pollset_capacity) { + pollset_set->pollset_capacity = + GPR_MAX(8, 2 * pollset_set->pollset_capacity); + pollset_set->pollsets = + gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity * + sizeof(*pollset_set->pollsets)); + } + pollset_set->pollsets[pollset_set->pollset_count++] = pollset; + for (i = 0; i < pollset_set->fd_count; i++) { + grpc_pollset_add_fd(pollset, pollset_set->fds[i]); + } + gpr_mu_unlock(&pollset_set->mu); +} + +void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + for (i = 0; i < pollset_set->pollset_count; i++) { + if (pollset_set->pollsets[i] == pollset) { + pollset_set->pollset_count--; + GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i], + pollset_set->pollsets[pollset_set->pollset_count]); + break; + } + } + gpr_mu_unlock(&pollset_set->mu); +} + +void grpc_pollset_set_add_fd(grpc_pollset_set *pollset_set, grpc_fd *fd) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + if (pollset_set->fd_count == pollset_set->fd_capacity) { + pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity); + pollset_set->fds = gpr_realloc( + pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds)); + } + GRPC_FD_REF(fd, "pollset_set"); + pollset_set->fds[pollset_set->fd_count++] = fd; + for (i = 0; i < pollset_set->pollset_count; i++) { + grpc_pollset_add_fd(pollset_set->pollsets[i], fd); + } + gpr_mu_unlock(&pollset_set->mu); +} + +void grpc_pollset_set_del_fd(grpc_pollset_set *pollset_set, grpc_fd *fd) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + for (i = 0; i < pollset_set->fd_count; i++) { + if (pollset_set->fds[i] == fd) { + pollset_set->fd_count--; + GPR_SWAP(grpc_fd *, pollset_set->fds[i], + pollset_set->fds[pollset_set->pollset_count]); + GRPC_FD_UNREF(fd, "pollset_set"); + break; + } + } + gpr_mu_unlock(&pollset_set->mu); +} + +#endif /* GPR_POSIX_SOCKET */ diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h new file mode 100644 index 0000000000..e88740bde1 --- /dev/null +++ b/src/core/iomgr/pollset_set_posix.h @@ -0,0 +1,55 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H +#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_POSIX_H + +#include "src/core/iomgr/fd_posix.h" +#include "src/core/iomgr/pollset_posix.h" + +typedef struct grpc_pollset_set { + gpr_mu mu; + + size_t pollset_count; + size_t pollset_capacity; + grpc_pollset **pollsets; + + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; +} grpc_pollset_set; + +void grpc_pollset_set_add_fd(grpc_pollset_set *pollset_set, grpc_fd *fd); +void grpc_pollset_set_del_fd(grpc_pollset_set *pollset_set, grpc_fd *fd); + +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c new file mode 100644 index 0000000000..b9c209cd2c --- /dev/null +++ b/src/core/iomgr/pollset_set_windows.c @@ -0,0 +1,50 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/port_platform.h> + +#ifdef GPR_WINSOCK_SOCKET + +#include "src/core/iomgr/pollset_set.h" + +void grpc_pollset_set_init(grpc_pollset_set *pollset_set) {} + +void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) {} + +void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset) {} + +void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, + grpc_pollset *pollset) {} + +#endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/pollset_kick_windows.h b/src/core/iomgr/pollset_set_windows.h index c675c119ab..cada0d2b61 100644 --- a/src/core/iomgr/pollset_kick_windows.h +++ b/src/core/iomgr/pollset_set_windows.h @@ -31,18 +31,9 @@ * */ -#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_WINDOWS_H -#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_WINDOWS_H +#ifndef GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H +#define GRPC_INTERNAL_CORE_IOMGR_POLLSET_SET_WINDOWS_H -#include <grpc/support/sync.h> +typedef struct grpc_pollset_set { void *unused; } grpc_pollset_set; -/* There isn't really any such thing as a pollset under Windows, due to the - nature of the IO completion ports. */ - -struct grpc_kick_fd_info; - -typedef struct grpc_pollset_kick_state { - int unused; -} grpc_pollset_kick_state; - -#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_KICK_WINDOWS_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index b1f4c09a2c..8d6bc79c96 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -39,6 +39,7 @@ #include "src/core/iomgr/alarm_internal.h" #include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_windows.h" /* There isn't really any such thing as a pollset under Windows, due to the @@ -47,6 +48,7 @@ won't actually do any polling, and return as quickly as possible. */ void grpc_pollset_init(grpc_pollset *pollset) { + memset(pollset, 0, sizeof(*pollset)); gpr_mu_init(&pollset->mu); gpr_cv_init(&pollset->cv); } @@ -54,6 +56,10 @@ void grpc_pollset_init(grpc_pollset *pollset) { void grpc_pollset_shutdown(grpc_pollset *pollset, void (*shutdown_done)(void *arg), void *shutdown_done_arg) { + gpr_mu_lock(&pollset->mu); + pollset->shutting_down = 1; + gpr_cv_broadcast(&pollset->cv); + gpr_mu_unlock(&pollset->mu); shutdown_done(shutdown_done_arg); } @@ -68,15 +74,20 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) { if (gpr_time_cmp(now, deadline) > 0) { return 0 /* GPR_FALSE */; } - if (grpc_maybe_call_delayed_callbacks(NULL, 1 /* GPR_TRUE */)) { + if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1 /* GPR_TRUE */)) { return 1 /* GPR_TRUE */; } - if (grpc_alarm_check(NULL, now, &deadline)) { + if (grpc_alarm_check(&pollset->mu, now, &deadline)) { return 1 /* GPR_TRUE */; } - return 0 /* GPR_FALSE */; + if (!pollset->shutting_down) { + gpr_cv_wait(&pollset->cv, &pollset->mu, deadline); + } + return 1 /* GPR_TRUE */; } -void grpc_pollset_kick(grpc_pollset *p) { } +void grpc_pollset_kick(grpc_pollset *p) { + gpr_cv_signal(&p->cv); +} -#endif /* GPR_WINSOCK_SOCKET */ +#endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index e1115bac4f..57a2907926 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -37,20 +37,18 @@ #include <windows.h> #include <grpc/support/sync.h> -#include "src/core/iomgr/pollset_kick.h" #include "src/core/iomgr/socket_windows.h" /* There isn't really any such thing as a pollset under Windows, due to the nature of the IO completion ports. A Windows "pollset" is merely a mutex - and a condition variable, as this is the minimal set of features we need - implemented for the rest of grpc. But we won't use them directly. */ + and a condition variable, used to synchronize with the IOCP. */ typedef struct grpc_pollset { gpr_mu mu; gpr_cv cv; + int shutting_down; } grpc_pollset; #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) -#define GRPC_POLLSET_CV(pollset) (&(pollset)->cv) -#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c index 20d8c58eb4..dbf884c769 100644 --- a/src/core/iomgr/resolve_address_posix.c +++ b/src/core/iomgr/resolve_address_posix.c @@ -155,9 +155,9 @@ static void do_request(void *rp) { grpc_resolve_cb cb = r->cb; gpr_free(r->name); gpr_free(r->default_port); + cb(arg, resolved); grpc_iomgr_unregister_object(&r->iomgr_object); gpr_free(r); - cb(arg, resolved); } void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) { diff --git a/src/core/iomgr/tcp_client.h b/src/core/iomgr/tcp_client.h index 2e91497fb7..0fa08b52b0 100644 --- a/src/core/iomgr/tcp_client.h +++ b/src/core/iomgr/tcp_client.h @@ -35,14 +35,18 @@ #define GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H #include "src/core/iomgr/endpoint.h" +#include "src/core/iomgr/pollset_set.h" #include "src/core/iomgr/sockaddr.h" #include <grpc/support/time.h> /* Asynchronously connect to an address (specified as (addr, len)), and call cb with arg and the completed connection when done (or call cb with arg and - NULL on failure) */ + NULL on failure). + interested_parties points to a set of pollsets that would be interested + in this connection being established (in order to continue their work) */ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), - void *arg, const struct sockaddr *addr, - int addr_len, gpr_timespec deadline); + void *arg, grpc_pollset_set *interested_parties, + const struct sockaddr *addr, int addr_len, + gpr_timespec deadline); -#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H */ diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c index 668a651947..bbf7711588 100644 --- a/src/core/iomgr/tcp_client_posix.c +++ b/src/core/iomgr/tcp_client_posix.c @@ -113,8 +113,6 @@ static void on_writable(void *acp, int success) { void (*cb)(void *arg, grpc_endpoint *tcp) = ac->cb; void *cb_arg = ac->cb_arg; - grpc_alarm_cancel(&ac->alarm); - if (success) { do { so_error_size = sizeof(so_error); @@ -167,26 +165,30 @@ static void on_writable(void *acp, int success) { finish: gpr_mu_lock(&ac->mu); if (!ep) { - grpc_fd_orphan(ac->fd, NULL, NULL); + grpc_fd_orphan(ac->fd, NULL, "tcp_client_orphan"); } done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { gpr_mu_destroy(&ac->mu); gpr_free(ac); + } else { + grpc_alarm_cancel(&ac->alarm); } cb(cb_arg, ep); } void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), - void *arg, const struct sockaddr *addr, - int addr_len, gpr_timespec deadline) { + void *arg, grpc_pollset_set *interested_parties, + const struct sockaddr *addr, int addr_len, + gpr_timespec deadline) { int fd; grpc_dualstack_mode dsmode; int err; async_connect *ac; struct sockaddr_in6 addr6_v4mapped; struct sockaddr_in addr4_copy; + grpc_fd *fdobj; char *name; char *addr_str; @@ -218,31 +220,35 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), grpc_sockaddr_to_string(&addr_str, addr, 1); gpr_asprintf(&name, "tcp-client:%s", addr_str); + fdobj = grpc_fd_create(fd, name); + if (err >= 0) { - gpr_log(GPR_DEBUG, "instant connect"); - cb(arg, grpc_tcp_create(grpc_fd_create(fd, name), - GRPC_TCP_DEFAULT_READ_SLICE_SIZE)); + cb(arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE)); goto done; } if (errno != EWOULDBLOCK && errno != EINPROGRESS) { gpr_log(GPR_ERROR, "connect error to '%s': %s", addr_str, strerror(errno)); - close(fd); + grpc_fd_orphan(fdobj, NULL, "tcp_client_connect_error"); cb(arg, NULL); goto done; } + grpc_pollset_set_add_fd(interested_parties, fdobj); + ac = gpr_malloc(sizeof(async_connect)); ac->cb = cb; ac->cb_arg = arg; - ac->fd = grpc_fd_create(fd, name); + ac->fd = fdobj; gpr_mu_init(&ac->mu); ac->refs = 2; ac->write_closure.cb = on_writable; ac->write_closure.cb_arg = ac; + gpr_mu_lock(&ac->mu); grpc_alarm_init(&ac->alarm, deadline, on_alarm, ac, gpr_now()); grpc_fd_notify_on_write(ac->fd, &ac->write_closure); + gpr_mu_unlock(&ac->mu); done: gpr_free(name); diff --git a/src/core/iomgr/tcp_client_windows.c b/src/core/iomgr/tcp_client_windows.c index 2a040ffc4a..b1a169b519 100644 --- a/src/core/iomgr/tcp_client_windows.c +++ b/src/core/iomgr/tcp_client_windows.c @@ -52,7 +52,7 @@ #include "src/core/iomgr/socket_windows.h" typedef struct { - void(*cb)(void *arg, grpc_endpoint *tcp); + void (*cb)(void *arg, grpc_endpoint *tcp); void *cb_arg; gpr_mu mu; grpc_winsocket *socket; @@ -86,7 +86,7 @@ static void on_connect(void *acp, int from_iocp) { SOCKET sock = ac->socket->socket; grpc_endpoint *ep = NULL; grpc_winsocket_callback_info *info = &ac->socket->write_info; - void(*cb)(void *arg, grpc_endpoint *tcp) = ac->cb; + void (*cb)(void *arg, grpc_endpoint *tcp) = ac->cb; void *cb_arg = ac->cb_arg; int aborted; @@ -99,8 +99,7 @@ static void on_connect(void *acp, int from_iocp) { DWORD transfered_bytes = 0; DWORD flags; BOOL wsa_success = WSAGetOverlappedResult(sock, &info->overlapped, - &transfered_bytes, FALSE, - &flags); + &transfered_bytes, FALSE, &flags); info->outstanding = 0; GPR_ASSERT(transfered_bytes == 0); if (!wsa_success) { @@ -138,9 +137,10 @@ static void on_connect(void *acp, int from_iocp) { /* Tries to issue one async connection, then schedules both an IOCP notification request for the connection, and one timeout alert. */ -void grpc_tcp_client_connect(void(*cb)(void *arg, grpc_endpoint *tcp), - void *arg, const struct sockaddr *addr, - int addr_len, gpr_timespec deadline) { +void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), + void *arg, grpc_pollset_set *interested_parties, + const struct sockaddr *addr, int addr_len, + gpr_timespec deadline) { SOCKET sock = INVALID_SOCKET; BOOL success; int status; @@ -175,9 +175,9 @@ void grpc_tcp_client_connect(void(*cb)(void *arg, grpc_endpoint *tcp), /* Grab the function pointer for ConnectEx for that specific socket. It may change depending on the interface. */ - status = WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, - &guid, sizeof(guid), &ConnectEx, sizeof(ConnectEx), - &ioctl_num_bytes, NULL, NULL); + status = + WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &guid, sizeof(guid), + &ConnectEx, sizeof(ConnectEx), &ioctl_num_bytes, NULL, NULL); if (status != 0) { message = "Unable to retrieve ConnectEx pointer: %s"; @@ -186,8 +186,7 @@ void grpc_tcp_client_connect(void(*cb)(void *arg, grpc_endpoint *tcp), grpc_sockaddr_make_wildcard6(0, &local_address); - status = bind(sock, (struct sockaddr *) &local_address, - sizeof(local_address)); + status = bind(sock, (struct sockaddr *)&local_address, sizeof(local_address)); if (status != 0) { message = "Unable to bind socket: %s"; goto failure; @@ -233,4 +232,4 @@ failure: cb(arg, NULL); } -#endif /* GPR_WINSOCK_SOCKET */ +#endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 2f19f9d442..9ad089af66 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -266,7 +266,7 @@ typedef struct { grpc_endpoint base; grpc_fd *em_fd; int fd; - int iov_size; /* Number of slices to allocate per read attempt */ + int iov_size; /* Number of slices to allocate per read attempt */ int finished_edge; size_t slice_size; gpr_refcount refcount; @@ -295,7 +295,7 @@ static void grpc_tcp_shutdown(grpc_endpoint *ep) { static void grpc_tcp_unref(grpc_tcp *tcp) { int refcount_zero = gpr_unref(&tcp->refcount); if (refcount_zero) { - grpc_fd_orphan(tcp->em_fd, NULL, NULL); + grpc_fd_orphan(tcp->em_fd, NULL, "tcp_unref_orphan"); gpr_free(tcp); } } @@ -412,8 +412,7 @@ static void grpc_tcp_continue_read(grpc_tcp *tcp) { ++tcp->iov_size; } GPR_ASSERT(slice_state_has_available(&read_state)); - slice_state_transfer_ownership(&read_state, &final_slices, - &final_nslices); + slice_state_transfer_ownership(&read_state, &final_slices, &final_nslices); call_read_cb(tcp, final_slices, final_nslices, GRPC_ENDPOINT_CB_OK); slice_state_destroy(&read_state); grpc_tcp_unref(tcp); diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index c49f3e1518..5854031c9b 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -85,6 +85,7 @@ typedef struct { } addr; int addr_len; grpc_iomgr_closure read_closure; + grpc_iomgr_closure destroyed_closure; } server_port; static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { @@ -101,13 +102,15 @@ struct grpc_tcp_server { void *cb_arg; gpr_mu mu; - gpr_cv cv; /* active port count: how many ports are actually still listening */ size_t active_ports; /* destroyed port count: how many ports are completely destroyed */ size_t destroyed_ports; + /* is this server shutting down? (boolean) */ + int shutdown; + /* all listening ports */ server_port *ports; size_t nports; @@ -116,14 +119,19 @@ struct grpc_tcp_server { /* shutdown callback */ void (*shutdown_complete)(void *); void *shutdown_complete_arg; + + /* all pollsets interested in new connections */ + grpc_pollset **pollsets; + /* number of pollsets in the pollsets array */ + size_t pollset_count; }; grpc_tcp_server *grpc_tcp_server_create(void) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); gpr_mu_init(&s->mu); - gpr_cv_init(&s->cv); s->active_ports = 0; s->destroyed_ports = 0; + s->shutdown = 0; s->cb = NULL; s->cb_arg = NULL; s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); @@ -136,7 +144,6 @@ static void finish_shutdown(grpc_tcp_server *s) { s->shutdown_complete(s->shutdown_complete_arg); gpr_mu_destroy(&s->mu); - gpr_cv_destroy(&s->cv); gpr_free(s->ports); gpr_free(s); @@ -156,40 +163,60 @@ static void destroyed_port(void *server, int success) { static void dont_care_about_shutdown_completion(void *ignored) {} +/* called when all listening endpoints have been shutdown, so no further + events will be received on them - at this point it's safe to destroy + things */ +static void deactivated_all_ports(grpc_tcp_server *s) { + size_t i; + + /* delete ALL the things */ + gpr_mu_lock(&s->mu); + + if (!s->shutdown) { + gpr_mu_unlock(&s->mu); + return; + } + + if (s->nports) { + for (i = 0; i < s->nports; i++) { + server_port *sp = &s->ports[i]; + if (sp->addr.sockaddr.sa_family == AF_UNIX) { + unlink_if_unix_domain_socket(&sp->addr.un); + } + sp->destroyed_closure.cb = destroyed_port; + sp->destroyed_closure.cb_arg = s; + grpc_fd_orphan(sp->emfd, &sp->destroyed_closure, "tcp_listener_shutdown"); + } + gpr_mu_unlock(&s->mu); + } else { + gpr_mu_unlock(&s->mu); + finish_shutdown(s); + } +} + void grpc_tcp_server_destroy( grpc_tcp_server *s, void (*shutdown_complete)(void *shutdown_complete_arg), void *shutdown_complete_arg) { size_t i; gpr_mu_lock(&s->mu); + GPR_ASSERT(!s->shutdown); + s->shutdown = 1; + s->shutdown_complete = shutdown_complete ? shutdown_complete : dont_care_about_shutdown_completion; s->shutdown_complete_arg = shutdown_complete_arg; /* shutdown all fd's */ - for (i = 0; i < s->nports; i++) { - grpc_fd_shutdown(s->ports[i].emfd); - } - /* wait while that happens */ - /* TODO(ctiller): make this asynchronous also */ - while (s->active_ports) { - gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future); - } - - /* delete ALL the things */ - if (s->nports) { + if (s->active_ports) { for (i = 0; i < s->nports; i++) { - server_port *sp = &s->ports[i]; - if (sp->addr.sockaddr.sa_family == AF_UNIX) { - unlink_if_unix_domain_socket(&sp->addr.un); - } - grpc_fd_orphan(sp->emfd, destroyed_port, s); + grpc_fd_shutdown(s->ports[i].emfd); } gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); - finish_shutdown(s); + deactivated_all_ports(s); } } @@ -274,6 +301,8 @@ error: /* event manager callback when reads are ready */ static void on_read(void *arg, int success) { server_port *sp = arg; + grpc_fd *fdobj; + size_t i; if (!success) { goto error; @@ -306,12 +335,18 @@ static void on_read(void *arg, int success) { grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1); gpr_asprintf(&name, "tcp-server-connection:%s", addr_str); + fdobj = grpc_fd_create(fd, name); + /* TODO(ctiller): revise this when we have server-side sharding + of channels -- we certainly should not be automatically adding every + incoming channel to every pollset owned by the server */ + for (i = 0; i < sp->server->pollset_count; i++) { + grpc_pollset_add_fd(sp->server->pollsets[i], fdobj); + } sp->server->cb(sp->server->cb_arg, - grpc_tcp_create(grpc_fd_create(fd, name), - GRPC_TCP_DEFAULT_READ_SLICE_SIZE)); + grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE)); - gpr_free(addr_str); gpr_free(name); + gpr_free(addr_str); } abort(); @@ -319,9 +354,11 @@ static void on_read(void *arg, int success) { error: gpr_mu_lock(&sp->server->mu); if (0 == --sp->server->active_ports) { - gpr_cv_broadcast(&sp->server->cv); + gpr_mu_unlock(&sp->server->mu); + deactivated_all_ports(sp->server); + } else { + gpr_mu_unlock(&sp->server->mu); } - gpr_mu_unlock(&sp->server->mu); } static int add_socket_to_server(grpc_tcp_server *s, int fd, @@ -452,6 +489,8 @@ void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset **pollsets, GPR_ASSERT(s->active_ports == 0); s->cb = cb; s->cb_arg = cb_arg; + s->pollsets = pollsets; + s->pollset_count = pollset_count; for (i = 0; i < s->nports; i++) { for (j = 0; j < pollset_count; j++) { grpc_pollset_add_fd(pollsets[j], s->ports[i].emfd); diff --git a/src/core/iomgr/tcp_windows.c b/src/core/iomgr/tcp_windows.c index 12dac03080..15759c398a 100644 --- a/src/core/iomgr/tcp_windows.c +++ b/src/core/iomgr/tcp_windows.c @@ -154,7 +154,7 @@ static void on_read(void *tcpp, int from_iocp) { status = GRPC_ENDPOINT_CB_ERROR; } else { if (info->bytes_transfered != 0) { - sub = gpr_slice_sub(tcp->read_slice, 0, info->bytes_transfered); + sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered); status = GRPC_ENDPOINT_CB_OK; slice = ⊂ nslices = 1; |