From e8b5f627ccae2db13e366059c3738e96ac9a5d29 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 2 Nov 2015 14:15:03 -0800 Subject: stream_op cleanup: pollset changes --- src/core/iomgr/exec_ctx.c | 5 +- src/core/iomgr/pollset.h | 5 ++ src/core/iomgr/pollset_multipoller_with_epoll.c | 14 +--- .../iomgr/pollset_multipoller_with_poll_posix.c | 4 +- src/core/iomgr/pollset_posix.c | 88 ++++++++++++++-------- src/core/iomgr/pollset_posix.h | 9 ++- src/core/iomgr/pollset_windows.c | 4 +- src/core/iomgr/workqueue_posix.c | 2 - 8 files changed, 80 insertions(+), 51 deletions(-) diff --git a/src/core/iomgr/exec_ctx.c b/src/core/iomgr/exec_ctx.c index 410b34c521..e95eaf267a 100644 --- a/src/core/iomgr/exec_ctx.c +++ b/src/core/iomgr/exec_ctx.c @@ -44,10 +44,11 @@ int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { grpc_closure *c = exec_ctx->closure_list.head; exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; while (c != NULL) { - grpc_closure *next = c->next; + int success = (int)(c->final_data & 1); + grpc_closure *next = (grpc_closure *)(c->final_data & ~(gpr_uintptr)1); did_something++; GPR_TIMER_BEGIN("grpc_exec_ctx_flush.cb", 0); - c->cb(exec_ctx, c->cb_arg, c->success); + c->cb(exec_ctx, c->cb_arg, success); GPR_TIMER_END("grpc_exec_ctx_flush.cb", 0); c = next; } diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index d15553a12a..c6b0214dea 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -55,8 +55,13 @@ #endif void grpc_pollset_init(grpc_pollset *pollset); +/* Begin shutting down the pollset, and call closure when done. + * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); +/** Reset the pollset to its initial state (perhaps with some cached objects); + * must have been previously shutdown */ +void grpc_pollset_reset(grpc_pollset *pollset); void grpc_pollset_destroy(grpc_pollset *pollset); /* Do some work on a pollset. diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 2aafd21dfb..1f1bf47e98 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -47,21 +47,13 @@ #include "src/core/support/block_annotate.h" #include "src/core/profiling/timers.h" -typedef struct wakeup_fd_hdl { - grpc_wakeup_fd wakeup_fd; - struct wakeup_fd_hdl *next; -} wakeup_fd_hdl; - typedef struct { grpc_pollset *pollset; grpc_fd *fd; grpc_closure closure; } delayed_add; -typedef struct { - int epoll_fd; - wakeup_fd_hdl *free_wakeup_fds; -} pollset_hdr; +typedef struct { int epoll_fd; } pollset_hdr; static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { @@ -174,7 +166,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now); - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfds[0].events = POLLIN; pfds[0].revents = 0; pfds[1].fd = h->epoll_fd; @@ -197,7 +189,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( /* do nothing */ } else { if (pfds[0].revents) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } if (pfds[1].revents) { do { diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index faa6c14491..09f04b64b9 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -124,7 +124,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfds[0].events = POLLIN; pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfds[1].events = POLLIN; pfds[1].revents = 0; for (i = 0; i < h->fd_count; i++) { @@ -174,7 +174,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); } if (pfds[1].revents & POLLIN_CHECK) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } for (i = 2; i < pfd_count; i++) { if (watchers[i].fd == NULL) { diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 6f478ccacb..5d701fa87b 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -111,7 +111,7 @@ void grpc_pollset_kick_ext(grpc_pollset *p, for (specific_worker = p->root_worker.next; specific_worker != &p->root_worker; specific_worker = specific_worker->next) { - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } p->kicked_without_pollers = 1; GPR_TIMER_END("grpc_pollset_kick_ext.broadcast", 0); @@ -122,14 +122,14 @@ void grpc_pollset_kick_ext(grpc_pollset *p, specific_worker->reevaluate_polling_on_wakeup = 1; } specific_worker->kicked_specifically = 1; - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { GPR_TIMER_MARK("kick_yoself", 0); if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { specific_worker->reevaluate_polling_on_wakeup = 1; } specific_worker->kicked_specifically = 1; - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } } else if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) { GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); @@ -151,7 +151,7 @@ void grpc_pollset_kick_ext(grpc_pollset *p, if (specific_worker != NULL) { GPR_TIMER_MARK("finally_kick", 0); push_back_worker(p, specific_worker); - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } } else { GPR_TIMER_MARK("kicked_no_pollers", 0); @@ -177,9 +177,9 @@ void grpc_pollset_global_init(void) { void grpc_pollset_global_shutdown(void) { grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); - grpc_wakeup_fd_global_destroy(); gpr_tls_destroy(&g_current_thread_poller); gpr_tls_destroy(&g_current_thread_worker); + grpc_wakeup_fd_global_destroy(); } void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } @@ -195,6 +195,34 @@ void grpc_pollset_init(grpc_pollset *pollset) { pollset->shutting_down = 0; pollset->called_shutdown = 0; pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; + pollset->local_wakeup_cache = NULL; + pollset->kicked_without_pollers = 0; + become_basic_pollset(pollset, NULL); +} + +void grpc_pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!grpc_pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + gpr_mu_destroy(&pollset->mu); + while (pollset->local_wakeup_cache) { + grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; + grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); + gpr_free(pollset->local_wakeup_cache); + pollset->local_wakeup_cache = next; + } +} + +void grpc_pollset_reset(grpc_pollset *pollset) { + GPR_ASSERT(pollset->shutting_down); + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!grpc_pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + pollset->shutting_down = 0; + pollset->called_shutdown = 0; + pollset->kicked_without_pollers = 0; become_basic_pollset(pollset, NULL); } @@ -244,13 +272,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* this must happen before we (potentially) drop pollset->mu */ worker->next = worker->prev = NULL; worker->reevaluate_polling_on_wakeup = 0; + if (pollset->local_wakeup_cache != NULL) { + worker->wakeup_fd = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker->wakeup_fd->next; + } else { + worker->wakeup_fd = gpr_malloc(sizeof(*worker->wakeup_fd)); + grpc_wakeup_fd_init(&worker->wakeup_fd->fd); + } worker->kicked_specifically = 0; - /* TODO(ctiller): pool these */ - grpc_wakeup_fd_init(&worker->wakeup_fd); /* If there's work waiting for the pollset to be idle, and the pollset is idle, then do that work */ if (!grpc_pollset_has_workers(pollset) && !grpc_closure_list_empty(pollset->idle_jobs)) { + GPR_TIMER_MARK("grpc_pollset_work.idle_jobs", 0); grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); goto done; } @@ -259,16 +293,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, May update deadline to ensure timely wakeups. TODO(ctiller): can this work be localized? */ if (grpc_timer_check(exec_ctx, now, &deadline)) { + GPR_TIMER_MARK("grpc_pollset_work.alarm_triggered", 0); gpr_mu_unlock(&pollset->mu); locked = 0; goto done; } /* If we're shutting down then we don't execute any extended work */ if (pollset->shutting_down) { + GPR_TIMER_MARK("grpc_pollset_work.shutting_down", 0); goto done; } /* Give do_promote priority so we don't starve it out */ if (pollset->in_flight_cbs) { + GPR_TIMER_MARK("grpc_pollset_work.in_flight_cbs", 0); gpr_mu_unlock(&pollset->mu); locked = 0; goto done; @@ -293,6 +330,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, locked = 0; gpr_tls_set(&g_current_thread_poller, 0); } else { + GPR_TIMER_MARK("grpc_pollset_work.kicked_without_pollers", 0); pollset->kicked_without_pollers = 0; } /* Finished execution - start cleaning up. @@ -323,7 +361,10 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, remove_worker(pollset, worker); gpr_tls_set(&g_current_thread_worker, 0); } - grpc_wakeup_fd_destroy(&worker->wakeup_fd); + /* release wakeup fd to the local pool */ + worker->wakeup_fd->next = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker->wakeup_fd; + /* check shutdown conditions */ if (pollset->shutting_down) { if (grpc_pollset_has_workers(pollset)) { grpc_pollset_kick(pollset, NULL); @@ -338,8 +379,8 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ gpr_mu_lock(&pollset->mu); } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { - gpr_mu_unlock(&pollset->mu); grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); + gpr_mu_unlock(&pollset->mu); grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } @@ -349,35 +390,20 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { - int call_shutdown = 0; - gpr_mu_lock(&pollset->mu); GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; - if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && - !grpc_pollset_has_workers(pollset)) { - pollset->called_shutdown = 1; - call_shutdown = 1; - } + pollset->shutdown_done = closure; + grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!grpc_pollset_has_workers(pollset)) { grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); } - pollset->shutdown_done = closure; - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - gpr_mu_unlock(&pollset->mu); - - if (call_shutdown) { + if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && + !grpc_pollset_has_workers(pollset)) { + pollset->called_shutdown = 1; finish_shutdown(exec_ctx, pollset); } } -void grpc_pollset_destroy(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT(pollset->in_flight_cbs == 0); - GPR_ASSERT(!grpc_pollset_has_workers(pollset)); - pollset->vtable->destroy(pollset); - gpr_mu_destroy(&pollset->mu); -} - int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { gpr_timespec timeout; @@ -557,7 +583,7 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfd[0].events = POLLIN; pfd[0].revents = 0; - pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfd[1].events = POLLIN; pfd[1].revents = 0; nfds = 2; @@ -599,7 +625,7 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); } if (pfd[1].revents & POLLIN_CHECK) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } if (nfds > 2) { grpc_fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index 95ebeab1c2..e4593728bd 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -48,8 +48,13 @@ typedef struct grpc_pollset_vtable grpc_pollset_vtable; use the struct tag */ struct grpc_fd; +typedef struct grpc_cached_wakeup_fd { + grpc_wakeup_fd fd; + struct grpc_cached_wakeup_fd *next; +} grpc_cached_wakeup_fd; + typedef struct grpc_pollset_worker { - grpc_wakeup_fd wakeup_fd; + grpc_cached_wakeup_fd *wakeup_fd; int reevaluate_polling_on_wakeup; int kicked_specifically; struct grpc_pollset_worker *next; @@ -74,6 +79,8 @@ typedef struct grpc_pollset { int fd; void *ptr; } data; + /* Local cache of eventfds for workers */ + grpc_cached_wakeup_fd *local_wakeup_cache; } grpc_pollset; struct grpc_pollset_vtable { diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 9f74580273..4443a6a9a7 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -112,7 +112,6 @@ void grpc_pollset_init(grpc_pollset *pollset) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { - gpr_mu_lock(&grpc_polling_mu); pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { @@ -120,11 +119,12 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } else { pollset->on_shutdown = closure; } - gpr_mu_unlock(&grpc_polling_mu); } void grpc_pollset_destroy(grpc_pollset *pollset) {} +void grpc_pollset_reset(grpc_pollset *pollset) {} + void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec now, gpr_timespec deadline) { diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c index 0a0f3c364e..c087b887b8 100644 --- a/src/core/iomgr/workqueue_posix.c +++ b/src/core/iomgr/workqueue_posix.c @@ -129,8 +129,6 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, int success) { void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure, int success) { - closure->success = success; - closure->next = NULL; gpr_mu_lock(&workqueue->mu); if (grpc_closure_list_empty(workqueue->closure_list)) { grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd); -- cgit v1.2.3