diff options
Diffstat (limited to 'src/core/iomgr')
60 files changed, 2743 insertions, 1905 deletions
diff --git a/src/core/iomgr/closure.c b/src/core/iomgr/closure.c new file mode 100644 index 0000000000..b4f1817de4 --- /dev/null +++ b/src/core/iomgr/closure.c @@ -0,0 +1,105 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/iomgr/closure.h" + +#include <grpc/support/alloc.h> + +void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb, + void *cb_arg) { + closure->cb = cb; + closure->cb_arg = cb_arg; + closure->next = NULL; +} + +void grpc_closure_list_add(grpc_closure_list *closure_list, + grpc_closure *closure, int success) { + if (closure == NULL) return; + closure->next = NULL; + closure->success = success; + if (closure_list->head == NULL) { + closure_list->head = closure; + } else { + closure_list->tail->next = closure; + } + closure_list->tail = closure; +} + +int grpc_closure_list_empty(grpc_closure_list closure_list) { + return closure_list.head == NULL; +} + +void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst) { + if (src->head == NULL) { + return; + } + if (dst->head == NULL) { + *dst = *src; + } else { + dst->tail->next = src->head; + dst->tail = src->tail; + } + src->head = src->tail = NULL; +} + +grpc_closure *grpc_closure_list_pop(grpc_closure_list *list) { + grpc_closure *head; + if (list->head == NULL) { + return NULL; + } + head = list->head; + list->head = list->head->next; + return head; +} + +typedef struct { + grpc_iomgr_cb_func cb; + void *cb_arg; + grpc_closure wrapper; +} wrapped_closure; + +static void closure_wrapper(grpc_exec_ctx *exec_ctx, void *arg, int success) { + wrapped_closure *wc = arg; + grpc_iomgr_cb_func cb = wc->cb; + void *cb_arg = wc->cb_arg; + gpr_free(wc); + cb(exec_ctx, cb_arg, success); +} + +grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg) { + wrapped_closure *wc = gpr_malloc(sizeof(*wc)); + wc->cb = cb; + wc->cb_arg = cb_arg; + grpc_closure_init(&wc->wrapper, closure_wrapper, wc); + return &wc->wrapper; +} diff --git a/src/core/iomgr/closure.h b/src/core/iomgr/closure.h new file mode 100644 index 0000000000..7a9f7ccad0 --- /dev/null +++ b/src/core/iomgr/closure.h @@ -0,0 +1,100 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H +#define GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H + +#include <stddef.h> + +struct grpc_closure; +typedef struct grpc_closure grpc_closure; + +/* forward declaration for exec_ctx.h */ +struct grpc_exec_ctx; +typedef struct grpc_exec_ctx grpc_exec_ctx; + +typedef struct grpc_closure_list { + grpc_closure *head; + grpc_closure *tail; +} grpc_closure_list; + +/** gRPC Callback definition. + * + * \param arg Arbitrary input. + * \param success An indication on the state of the iomgr. On false, cleanup + * actions should be taken (eg, shutdown). */ +typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg, + int success); + +/** A closure over a grpc_iomgr_cb_func. */ +struct grpc_closure { + /** Bound callback. */ + grpc_iomgr_cb_func cb; + + /** Arguments to be passed to "cb". */ + void *cb_arg; + + /** Internal. A boolean indication to "cb" on the state of the iomgr. + * For instance, closures created during a shutdown would have this field set + * to false. */ + int success; + + /**< Internal. Do not touch */ + struct grpc_closure *next; +}; + +/** Initializes \a closure with \a cb and \a cb_arg. */ +void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb, + void *cb_arg); + +/* Create a heap allocated closure: try to avoid except for very rare events */ +grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg); + +#define GRPC_CLOSURE_LIST_INIT \ + { NULL, NULL } + +/** add \a closure to the end of \a list and set \a closure's success to \a + * success */ +void grpc_closure_list_add(grpc_closure_list *list, grpc_closure *closure, + int success); + +/** append all closures from \a src to \a dst and empty \a src. */ +void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst); + +/** pop (return and remove) the head closure from \a list. */ +grpc_closure *grpc_closure_list_pop(grpc_closure_list *list); + +/** return whether \a list is empty. */ +int grpc_closure_list_empty(grpc_closure_list list); + +#endif /* GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H */ diff --git a/src/core/iomgr/endpoint.c b/src/core/iomgr/endpoint.c index 8ee14bce9b..bd64707669 100644 --- a/src/core/iomgr/endpoint.c +++ b/src/core/iomgr/endpoint.c @@ -33,32 +33,35 @@ #include "src/core/iomgr/endpoint.h" -void grpc_endpoint_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, - void *user_data) { - ep->vtable->notify_on_read(ep, cb, user_data); +void grpc_endpoint_read(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep, + gpr_slice_buffer* slices, grpc_closure* cb) { + ep->vtable->read(exec_ctx, ep, slices, cb); } -grpc_endpoint_write_status grpc_endpoint_write(grpc_endpoint *ep, - gpr_slice *slices, - size_t nslices, - grpc_endpoint_write_cb cb, - void *user_data) { - return ep->vtable->write(ep, slices, nslices, cb, user_data); +void grpc_endpoint_write(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep, + gpr_slice_buffer* slices, grpc_closure* cb) { + ep->vtable->write(exec_ctx, ep, slices, cb); } -void grpc_endpoint_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) { - ep->vtable->add_to_pollset(ep, pollset); +void grpc_endpoint_add_to_pollset(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep, + grpc_pollset* pollset) { + ep->vtable->add_to_pollset(exec_ctx, ep, pollset); } -void grpc_endpoint_add_to_pollset_set(grpc_endpoint *ep, - grpc_pollset_set *pollset_set) { - ep->vtable->add_to_pollset_set(ep, pollset_set); +void grpc_endpoint_add_to_pollset_set(grpc_exec_ctx* exec_ctx, + grpc_endpoint* ep, + grpc_pollset_set* pollset_set) { + ep->vtable->add_to_pollset_set(exec_ctx, ep, pollset_set); } -void grpc_endpoint_shutdown(grpc_endpoint *ep) { ep->vtable->shutdown(ep); } +void grpc_endpoint_shutdown(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep) { + ep->vtable->shutdown(exec_ctx, ep); +} -void grpc_endpoint_destroy(grpc_endpoint *ep) { ep->vtable->destroy(ep); } +void grpc_endpoint_destroy(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep) { + ep->vtable->destroy(exec_ctx, ep); +} -char *grpc_endpoint_get_peer(grpc_endpoint *ep) { +char* grpc_endpoint_get_peer(grpc_endpoint* ep) { return ep->vtable->get_peer(ep); } diff --git a/src/core/iomgr/endpoint.h b/src/core/iomgr/endpoint.h index ea92a500e8..cbdc947abb 100644 --- a/src/core/iomgr/endpoint.h +++ b/src/core/iomgr/endpoint.h @@ -37,6 +37,7 @@ #include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_set.h" #include <grpc/support/slice.h> +#include <grpc/support/slice_buffer.h> #include <grpc/support/time.h> /* An endpoint caps a streaming channel between two communicating processes. @@ -45,65 +46,53 @@ typedef struct grpc_endpoint grpc_endpoint; typedef struct grpc_endpoint_vtable grpc_endpoint_vtable; -typedef enum grpc_endpoint_cb_status { - GRPC_ENDPOINT_CB_OK = 0, /* Call completed successfully */ - GRPC_ENDPOINT_CB_EOF, /* Call completed successfully, end of file reached */ - GRPC_ENDPOINT_CB_SHUTDOWN, /* Call interrupted by shutdown */ - GRPC_ENDPOINT_CB_ERROR /* Call interrupted by socket error */ -} grpc_endpoint_cb_status; - -typedef enum grpc_endpoint_write_status { - GRPC_ENDPOINT_WRITE_DONE, /* completed immediately, cb won't be called */ - GRPC_ENDPOINT_WRITE_PENDING, /* cb will be called when completed */ - GRPC_ENDPOINT_WRITE_ERROR /* write errored out, cb won't be called */ -} grpc_endpoint_write_status; - -typedef void (*grpc_endpoint_read_cb)(void *user_data, gpr_slice *slices, - size_t nslices, - grpc_endpoint_cb_status error); -typedef void (*grpc_endpoint_write_cb)(void *user_data, - grpc_endpoint_cb_status error); - struct grpc_endpoint_vtable { - void (*notify_on_read)(grpc_endpoint *ep, grpc_endpoint_read_cb cb, - void *user_data); - grpc_endpoint_write_status (*write)(grpc_endpoint *ep, gpr_slice *slices, - size_t nslices, grpc_endpoint_write_cb cb, - void *user_data); - void (*add_to_pollset)(grpc_endpoint *ep, grpc_pollset *pollset); - void (*add_to_pollset_set)(grpc_endpoint *ep, grpc_pollset_set *pollset); - void (*shutdown)(grpc_endpoint *ep); - void (*destroy)(grpc_endpoint *ep); + void (*read)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *slices, grpc_closure *cb); + void (*write)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *slices, grpc_closure *cb); + void (*add_to_pollset)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset *pollset); + void (*add_to_pollset_set)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset_set *pollset); + void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); + void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); char *(*get_peer)(grpc_endpoint *ep); }; -/* When data is available on the connection, calls the callback with slices. */ -void grpc_endpoint_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, - void *user_data); +/* When data is available on the connection, calls the callback with slices. + Callback success indicates that the endpoint can accept more reads, failure + indicates the endpoint is closed. + Valid slices may be placed into \a slices even on callback success == 0. */ +void grpc_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *slices, grpc_closure *cb); char *grpc_endpoint_get_peer(grpc_endpoint *ep); /* Write slices out to the socket. If the connection is ready for more data after the end of the call, it - returns GRPC_ENDPOINT_WRITE_DONE. - Otherwise it returns GRPC_ENDPOINT_WRITE_PENDING and calls cb when the - connection is ready for more data. */ -grpc_endpoint_write_status grpc_endpoint_write(grpc_endpoint *ep, - gpr_slice *slices, - size_t nslices, - grpc_endpoint_write_cb cb, - void *user_data); + returns GRPC_ENDPOINT_DONE. + Otherwise it returns GRPC_ENDPOINT_PENDING and calls cb when the + connection is ready for more data. + \a slices may be mutated at will by the endpoint until cb is called. + No guarantee is made to the content of slices after a write EXCEPT that + it is a valid slice buffer. + */ +void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *slices, grpc_closure *cb); /* Causes any pending read/write callbacks to run immediately with - GRPC_ENDPOINT_CB_SHUTDOWN status */ -void grpc_endpoint_shutdown(grpc_endpoint *ep); -void grpc_endpoint_destroy(grpc_endpoint *ep); + success==0 */ +void grpc_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); +void grpc_endpoint_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); /* Add an endpoint to a pollset, so that when the pollset is polled, events from this endpoint are considered */ -void grpc_endpoint_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset); -void grpc_endpoint_add_to_pollset_set(grpc_endpoint *ep, +void grpc_endpoint_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset *pollset); +void grpc_endpoint_add_to_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_endpoint *ep, grpc_pollset_set *pollset_set); struct grpc_endpoint { diff --git a/src/core/iomgr/exec_ctx.c b/src/core/iomgr/exec_ctx.c new file mode 100644 index 0000000000..410b34c521 --- /dev/null +++ b/src/core/iomgr/exec_ctx.c @@ -0,0 +1,71 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/iomgr/exec_ctx.h" + +#include <grpc/support/log.h> + +#include "src/core/profiling/timers.h" + +int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { + int did_something = 0; + GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); + while (!grpc_closure_list_empty(exec_ctx->closure_list)) { + grpc_closure *c = exec_ctx->closure_list.head; + exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; + while (c != NULL) { + grpc_closure *next = c->next; + did_something++; + GPR_TIMER_BEGIN("grpc_exec_ctx_flush.cb", 0); + c->cb(exec_ctx, c->cb_arg, c->success); + GPR_TIMER_END("grpc_exec_ctx_flush.cb", 0); + c = next; + } + } + GPR_TIMER_END("grpc_exec_ctx_flush", 0); + return did_something; +} + +void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) { + grpc_exec_ctx_flush(exec_ctx); +} + +void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + int success) { + grpc_closure_list_add(&exec_ctx->closure_list, closure, success); +} + +void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx, + grpc_closure_list *list) { + grpc_closure_list_move(list, &exec_ctx->closure_list); +} diff --git a/src/core/iomgr/exec_ctx.h b/src/core/iomgr/exec_ctx.h new file mode 100644 index 0000000000..43df488094 --- /dev/null +++ b/src/core/iomgr/exec_ctx.h @@ -0,0 +1,78 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_EXEC_CTX_H +#define GRPC_INTERNAL_CORE_IOMGR_EXEC_CTX_H + +#include "src/core/iomgr/closure.h" + +/** Execution context. + * A bag of data that collects information along a callstack. + * Generally created at public API entry points, and passed down as + * pointer to child functions that manipulate it. + * + * Specific responsibilities (this may grow in the future): + * - track a list of work that needs to be delayed until the top of the + * call stack (this provides a convenient mechanism to run callbacks + * without worrying about locking issues) + * + * CONVENTIONS: + * Instance of this must ALWAYS be constructed on the stack, never + * heap allocated. Instances and pointers to them must always be called + * exec_ctx. Instances are always passed as the first argument + * to a function that takes it, and always as a pointer (grpc_exec_ctx + * is never copied). + */ +struct grpc_exec_ctx { + grpc_closure_list closure_list; +}; + +#define GRPC_EXEC_CTX_INIT \ + { GRPC_CLOSURE_LIST_INIT } + +/** Flush any work that has been enqueued onto this grpc_exec_ctx. + * Caller must guarantee that no interfering locks are held. + * Returns 1 if work was performed, 0 otherwise. */ +int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx); +/** Finish any pending work for a grpc_exec_ctx. Must be called before + * the instance is destroyed, or work may be lost. */ +void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx); +/** Add a closure to be executed at the next flush/finish point */ +void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + int success); +/** Add a list of closures to be executed at the next flush/finish point. + * Leaves \a list empty. */ +void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx, + grpc_closure_list *list); + +#endif diff --git a/src/core/iomgr/executor.c b/src/core/iomgr/executor.c new file mode 100644 index 0000000000..457e5cdbac --- /dev/null +++ b/src/core/iomgr/executor.c @@ -0,0 +1,148 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/iomgr/executor.h" + +#include <string.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/sync.h> +#include <grpc/support/thd.h> +#include "src/core/iomgr/exec_ctx.h" + +typedef struct grpc_executor_data { + int busy; /**< is the thread currently running? */ + int shutting_down; /**< has \a grpc_shutdown() been invoked? */ + int pending_join; /**< has the thread finished but not been joined? */ + grpc_closure_list closures; /**< collection of pending work */ + gpr_thd_id tid; /**< thread id of the thread, only valid if \a busy or \a + pending_join are true */ + gpr_thd_options options; + gpr_mu mu; +} grpc_executor; + +static grpc_executor g_executor; + +void grpc_executor_init() { + memset(&g_executor, 0, sizeof(grpc_executor)); + gpr_mu_init(&g_executor.mu); + g_executor.options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&g_executor.options); +} + +/* thread body */ +static void closure_exec_thread_func(void *ignored) { + grpc_closure *closure; + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (1) { + gpr_mu_lock(&g_executor.mu); + if (g_executor.shutting_down != 0) { + gpr_mu_unlock(&g_executor.mu); + break; + } + closure = grpc_closure_list_pop(&g_executor.closures); + if (closure == NULL) { + /* no more work, time to die */ + GPR_ASSERT(g_executor.busy == 1); + g_executor.busy = 0; + gpr_mu_unlock(&g_executor.mu); + break; + } + gpr_mu_unlock(&g_executor.mu); + closure->cb(&exec_ctx, closure->cb_arg, closure->success); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); +} + +/* Spawn the thread if new work has arrived a no thread is up */ +static void maybe_spawn_locked() { + if (grpc_closure_list_empty(g_executor.closures) == 1) { + return; + } + if (g_executor.shutting_down == 1) { + return; + } + + if (g_executor.busy != 0) { + /* Thread still working. New work will be picked up by already running + * thread. Not spawning anything. */ + return; + } else if (g_executor.pending_join != 0) { + /* Pickup the remains of the previous incarnations of the thread. */ + gpr_thd_join(g_executor.tid); + g_executor.pending_join = 0; + } + + /* All previous instances of the thread should have been joined at this point. + * Spawn time! */ + g_executor.busy = 1; + gpr_thd_new(&g_executor.tid, closure_exec_thread_func, NULL, + &g_executor.options); + g_executor.pending_join = 1; +} + +void grpc_executor_enqueue(grpc_closure *closure, int success) { + gpr_mu_lock(&g_executor.mu); + if (g_executor.shutting_down == 0) { + grpc_closure_list_add(&g_executor.closures, closure, success); + maybe_spawn_locked(); + } + gpr_mu_unlock(&g_executor.mu); +} + +void grpc_executor_shutdown() { + int pending_join; + grpc_closure *closure; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + gpr_mu_lock(&g_executor.mu); + pending_join = g_executor.pending_join; + g_executor.shutting_down = 1; + gpr_mu_unlock(&g_executor.mu); + /* we can release the lock at this point despite the access to the closure + * list below because we aren't accepting new work */ + + /* Execute pending callbacks, some may be performing cleanups */ + while ((closure = grpc_closure_list_pop(&g_executor.closures)) != NULL) { + closure->cb(&exec_ctx, closure->cb_arg, closure->success); + } + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(grpc_closure_list_empty(g_executor.closures)); + if (pending_join) { + gpr_thd_join(g_executor.tid); + } + gpr_mu_destroy(&g_executor.mu); +} diff --git a/src/core/iomgr/executor.h b/src/core/iomgr/executor.h new file mode 100644 index 0000000000..6da446ae9c --- /dev/null +++ b/src/core/iomgr/executor.h @@ -0,0 +1,53 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H +#define GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H + +#include "src/core/iomgr/closure.h" + +/** Initialize the global executor. + * + * This mechanism is meant to outsource work (grpc_closure instances) to a + * thread, for those cases where blocking isn't an option but there isn't a + * non-blocking solution available. */ +void grpc_executor_init(); + +/** Enqueue \a closure for its eventual execution of \a f(arg) on a separate + * thread */ +void grpc_executor_enqueue(grpc_closure *closure, int success); + +/** Shutdown the executor, running all pending work as part of the call */ +void grpc_executor_shutdown(); + +#endif /* GRPC_INTERNAL_CORE_IOMGR_EXECUTOR_H */ diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 2d08a77a70..7ff80e6cf8 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -45,10 +45,8 @@ #include <grpc/support/log.h> #include <grpc/support/useful.h> -enum descriptor_state { - NOT_READY = 0, - READY = 1 -}; /* or a pointer to a closure to call */ +#define CLOSURE_NOT_READY ((grpc_closure *)0) +#define CLOSURE_READY ((grpc_closure *)1) /* We need to keep a freelist not because of any concerns of malloc performance * but instead so that implementations with multiple threads in (for example) @@ -88,14 +86,13 @@ static grpc_fd *alloc_fd(int fd) { gpr_mu_unlock(&fd_freelist_mu); if (r == NULL) { r = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&r->set_state_mu); - gpr_mu_init(&r->watcher_mu); + gpr_mu_init(&r->mu); } gpr_atm_rel_store(&r->refst, 1); - gpr_atm_rel_store(&r->readst, NOT_READY); - gpr_atm_rel_store(&r->writest, NOT_READY); - gpr_atm_rel_store(&r->shutdown, 0); + r->shutdown = 0; + r->read_closure = CLOSURE_NOT_READY; + r->write_closure = CLOSURE_NOT_READY; r->fd = fd; r->inactive_watcher_root.next = r->inactive_watcher_root.prev = &r->inactive_watcher_root; @@ -107,8 +104,7 @@ static grpc_fd *alloc_fd(int fd) { } static void destroy(grpc_fd *fd) { - gpr_mu_destroy(&fd->set_state_mu); - gpr_mu_destroy(&fd->watcher_mu); + gpr_mu_destroy(&fd->mu); gpr_free(fd); } @@ -150,6 +146,8 @@ static void unref_by(grpc_fd *fd, int n) { void grpc_fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } void grpc_fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; @@ -161,6 +159,9 @@ void grpc_fd_global_shutdown(void) { grpc_fd *grpc_fd_create(int fd, const char *name) { grpc_fd *r = alloc_fd(fd); grpc_iomgr_register_object(&r->iomgr_object, name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name); +#endif return r; } @@ -168,39 +169,35 @@ int grpc_fd_is_orphaned(grpc_fd *fd) { return (gpr_atm_acq_load(&fd->refst) & 1) == 0; } -static void pollset_kick_locked(grpc_pollset *pollset) { - gpr_mu_lock(GRPC_POLLSET_MU(pollset)); - grpc_pollset_kick(pollset, NULL); - gpr_mu_unlock(GRPC_POLLSET_MU(pollset)); +static void pollset_kick_locked(grpc_fd_watcher *watcher) { + gpr_mu_lock(GRPC_POLLSET_MU(watcher->pollset)); + GPR_ASSERT(watcher->worker); + grpc_pollset_kick_ext(watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + gpr_mu_unlock(GRPC_POLLSET_MU(watcher->pollset)); } static void maybe_wake_one_watcher_locked(grpc_fd *fd) { if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next->pollset); + pollset_kick_locked(fd->inactive_watcher_root.next); } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher->pollset); + pollset_kick_locked(fd->read_watcher); } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher->pollset); + pollset_kick_locked(fd->write_watcher); } } -static void maybe_wake_one_watcher(grpc_fd *fd) { - gpr_mu_lock(&fd->watcher_mu); - maybe_wake_one_watcher_locked(fd); - gpr_mu_unlock(&fd->watcher_mu); -} - static void wake_all_watchers_locked(grpc_fd *fd) { grpc_fd_watcher *watcher; for (watcher = fd->inactive_watcher_root.next; watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher->pollset); + pollset_kick_locked(watcher); } if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher->pollset); + pollset_kick_locked(fd->read_watcher); } if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher->pollset); + pollset_kick_locked(fd->write_watcher); } } @@ -209,23 +206,20 @@ static int has_watchers(grpc_fd *fd) { fd->inactive_watcher_root.next != &fd->inactive_watcher_root; } -void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, +void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, const char *reason) { fd->on_done_closure = on_done; shutdown(fd->fd, SHUT_RDWR); + gpr_mu_lock(&fd->mu); REF_BY(fd, 1, reason); /* remove active status, but keep referenced */ - gpr_mu_lock(&fd->watcher_mu); if (!has_watchers(fd)) { - GPR_ASSERT(!fd->closed); fd->closed = 1; close(fd->fd); - if (fd->on_done_closure) { - grpc_iomgr_add_callback(fd->on_done_closure); - } + grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, 1); } else { wake_all_watchers_locked(fd); } - gpr_mu_unlock(&fd->watcher_mu); + gpr_mu_unlock(&fd->mu); UNREF_BY(fd, 2, reason); /* drop the reference */ } @@ -245,169 +239,127 @@ void grpc_fd_ref(grpc_fd *fd) { ref_by(fd, 2); } void grpc_fd_unref(grpc_fd *fd) { unref_by(fd, 2); } #endif -static void process_callback(grpc_iomgr_closure *closure, int success, - int allow_synchronous_callback) { - if (allow_synchronous_callback) { - closure->cb(closure->cb_arg, success); +static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure **st, grpc_closure *closure) { + if (*st == CLOSURE_NOT_READY) { + /* not ready ==> switch to a waiting state by setting the closure */ + *st = closure; + } else if (*st == CLOSURE_READY) { + /* already ready ==> queue the closure to run immediately */ + *st = CLOSURE_NOT_READY; + grpc_exec_ctx_enqueue(exec_ctx, closure, !fd->shutdown); + maybe_wake_one_watcher_locked(fd); } else { - grpc_iomgr_add_delayed_callback(closure, success); - } -} - -static void process_callbacks(grpc_iomgr_closure *callbacks, size_t n, - int success, int allow_synchronous_callback) { - size_t i; - for (i = 0; i < n; i++) { - process_callback(callbacks + i, success, allow_synchronous_callback); - } -} - -static void notify_on(grpc_fd *fd, gpr_atm *st, grpc_iomgr_closure *closure, - int allow_synchronous_callback) { - switch (gpr_atm_acq_load(st)) { - case NOT_READY: - /* There is no race if the descriptor is already ready, so we skip - the interlocked op in that case. As long as the app doesn't - try to set the same upcall twice (which it shouldn't) then - oldval should never be anything other than READY or NOT_READY. We - don't - check for user error on the fast path. */ - if (gpr_atm_rel_cas(st, NOT_READY, (gpr_intptr)closure)) { - /* swap was successful -- the closure will run after the next - set_ready call. NOTE: we don't have an ABA problem here, - since we should never have concurrent calls to the same - notify_on function. */ - maybe_wake_one_watcher(fd); - return; - } - /* swap was unsuccessful due to an intervening set_ready call. - Fall through to the READY code below */ - case READY: - GPR_ASSERT(gpr_atm_no_barrier_load(st) == READY); - gpr_atm_rel_store(st, NOT_READY); - process_callback(closure, !gpr_atm_acq_load(&fd->shutdown), - allow_synchronous_callback); - return; - default: /* WAITING */ - /* upcallptr was set to a different closure. This is an error! */ - gpr_log(GPR_ERROR, - "User called a notify_on function with a previous callback still " - "pending"); - abort(); + /* upcallptr was set to a different closure. This is an error! */ + gpr_log(GPR_ERROR, + "User called a notify_on function with a previous callback still " + "pending"); + abort(); } - gpr_log(GPR_ERROR, "Corrupt memory in &st->state"); - abort(); } -static void set_ready_locked(gpr_atm *st, grpc_iomgr_closure **callbacks, - size_t *ncallbacks) { - gpr_intptr state = gpr_atm_acq_load(st); - - switch (state) { - case READY: - /* duplicate ready, ignore */ - return; - case NOT_READY: - if (gpr_atm_rel_cas(st, NOT_READY, READY)) { - /* swap was successful -- the closure will run after the next - notify_on call. */ - return; - } - /* swap was unsuccessful due to an intervening set_ready call. - Fall through to the WAITING code below */ - state = gpr_atm_acq_load(st); - default: /* waiting */ - GPR_ASSERT(gpr_atm_no_barrier_load(st) != READY && - gpr_atm_no_barrier_load(st) != NOT_READY); - callbacks[(*ncallbacks)++] = (grpc_iomgr_closure *)state; - gpr_atm_rel_store(st, NOT_READY); - return; +/* returns 1 if state becomes not ready */ +static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure **st) { + if (*st == CLOSURE_READY) { + /* duplicate ready ==> ignore */ + return 0; + } else if (*st == CLOSURE_NOT_READY) { + /* not ready, and not waiting ==> flag ready */ + *st = CLOSURE_READY; + return 0; + } else { + /* waiting ==> queue closure */ + grpc_exec_ctx_enqueue(exec_ctx, *st, !fd->shutdown); + *st = CLOSURE_NOT_READY; + return 1; } } -static void set_ready(grpc_fd *fd, gpr_atm *st, - int allow_synchronous_callback) { +static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) { /* only one set_ready can be active at once (but there may be a racing notify_on) */ - int success; - grpc_iomgr_closure *closure; - size_t ncb = 0; - - gpr_mu_lock(&fd->set_state_mu); - set_ready_locked(st, &closure, &ncb); - gpr_mu_unlock(&fd->set_state_mu); - success = !gpr_atm_acq_load(&fd->shutdown); - GPR_ASSERT(ncb <= 1); - if (ncb > 0) { - process_callbacks(closure, ncb, success, allow_synchronous_callback); - } + gpr_mu_lock(&fd->mu); + set_ready_locked(exec_ctx, fd, st); + gpr_mu_unlock(&fd->mu); } -void grpc_fd_shutdown(grpc_fd *fd) { - size_t ncb = 0; - gpr_mu_lock(&fd->set_state_mu); - GPR_ASSERT(!gpr_atm_no_barrier_load(&fd->shutdown)); - gpr_atm_rel_store(&fd->shutdown, 1); - set_ready_locked(&fd->readst, &fd->shutdown_closures[0], &ncb); - set_ready_locked(&fd->writest, &fd->shutdown_closures[0], &ncb); - gpr_mu_unlock(&fd->set_state_mu); - GPR_ASSERT(ncb <= 2); - process_callbacks(fd->shutdown_closures[0], ncb, 0 /* GPR_FALSE */, - 0 /* GPR_FALSE */); +void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + gpr_mu_lock(&fd->mu); + GPR_ASSERT(!fd->shutdown); + fd->shutdown = 1; + set_ready_locked(exec_ctx, fd, &fd->read_closure); + set_ready_locked(exec_ctx, fd, &fd->write_closure); + gpr_mu_unlock(&fd->mu); } -void grpc_fd_notify_on_read(grpc_fd *fd, grpc_iomgr_closure *closure) { - notify_on(fd, &fd->readst, closure, 0); +void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + gpr_mu_lock(&fd->mu); + notify_on_locked(exec_ctx, fd, &fd->read_closure, closure); + gpr_mu_unlock(&fd->mu); } -void grpc_fd_notify_on_write(grpc_fd *fd, grpc_iomgr_closure *closure) { - notify_on(fd, &fd->writest, closure, 0); +void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + gpr_mu_lock(&fd->mu); + notify_on_locked(exec_ctx, fd, &fd->write_closure, closure); + gpr_mu_unlock(&fd->mu); } gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, - gpr_uint32 read_mask, gpr_uint32 write_mask, - grpc_fd_watcher *watcher) { + grpc_pollset_worker *worker, gpr_uint32 read_mask, + gpr_uint32 write_mask, grpc_fd_watcher *watcher) { gpr_uint32 mask = 0; + grpc_closure *cur; + int requested; /* keep track of pollers that have requested our events, in case they change */ GRPC_FD_REF(fd, "poll"); - gpr_mu_lock(&fd->watcher_mu); + gpr_mu_lock(&fd->mu); + /* if we are shutdown, then don't add to the watcher set */ - if (gpr_atm_no_barrier_load(&fd->shutdown)) { + if (fd->shutdown) { watcher->fd = NULL; watcher->pollset = NULL; - gpr_mu_unlock(&fd->watcher_mu); + watcher->worker = NULL; + gpr_mu_unlock(&fd->mu); GRPC_FD_UNREF(fd, "poll"); return 0; } + /* if there is nobody polling for read, but we need to, then start doing so */ - if (read_mask && !fd->read_watcher && - (gpr_uintptr)gpr_atm_acq_load(&fd->readst) > READY) { + cur = fd->read_closure; + requested = cur != CLOSURE_READY; + if (read_mask && fd->read_watcher == NULL && requested) { fd->read_watcher = watcher; mask |= read_mask; } /* if there is nobody polling for write, but we need to, then start doing so */ - if (write_mask && !fd->write_watcher && - (gpr_uintptr)gpr_atm_acq_load(&fd->writest) > READY) { + cur = fd->write_closure; + requested = cur != CLOSURE_READY; + if (write_mask && fd->write_watcher == NULL && requested) { fd->write_watcher = watcher; mask |= write_mask; } /* if not polling, remember this watcher in case we need someone to later */ - if (mask == 0) { + if (mask == 0 && worker != NULL) { watcher->next = &fd->inactive_watcher_root; watcher->prev = watcher->next->prev; watcher->next->prev = watcher->prev->next = watcher; } watcher->pollset = pollset; + watcher->worker = worker; watcher->fd = fd; - gpr_mu_unlock(&fd->watcher_mu); + gpr_mu_unlock(&fd->mu); return mask; } -void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) { +void grpc_fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, + int got_read, int got_write) { int was_polling = 0; int kick = 0; grpc_fd *fd = watcher->fd; @@ -416,45 +368,58 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) { return; } - gpr_mu_lock(&fd->watcher_mu); + gpr_mu_lock(&fd->mu); + if (watcher == fd->read_watcher) { /* remove read watcher, kick if we still need a read */ was_polling = 1; - kick = kick || !got_read; + if (!got_read) { + kick = 1; + } fd->read_watcher = NULL; } if (watcher == fd->write_watcher) { /* remove write watcher, kick if we still need a write */ was_polling = 1; - kick = kick || !got_write; + if (!got_write) { + kick = 1; + } fd->write_watcher = NULL; } - if (!was_polling) { + if (!was_polling && watcher->worker != NULL) { /* remove from inactive list */ watcher->next->prev = watcher->prev; watcher->prev->next = watcher->next; } + if (got_read) { + if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) { + kick = 1; + } + } + if (got_write) { + if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) { + kick = 1; + } + } if (kick) { maybe_wake_one_watcher_locked(fd); } if (grpc_fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { fd->closed = 1; close(fd->fd); - if (fd->on_done_closure != NULL) { - grpc_iomgr_add_callback(fd->on_done_closure); - } + grpc_exec_ctx_enqueue(exec_ctx, fd->on_done_closure, 1); } - gpr_mu_unlock(&fd->watcher_mu); + gpr_mu_unlock(&fd->mu); GRPC_FD_UNREF(fd, "poll"); } -void grpc_fd_become_readable(grpc_fd *fd, int allow_synchronous_callback) { - set_ready(fd, &fd->readst, allow_synchronous_callback); +void grpc_fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + set_ready(exec_ctx, fd, &fd->read_closure); } -void grpc_fd_become_writable(grpc_fd *fd, int allow_synchronous_callback) { - set_ready(fd, &fd->writest, allow_synchronous_callback); +void grpc_fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + set_ready(exec_ctx, fd, &fd->write_closure); } #endif diff --git a/src/core/iomgr/fd_posix.h b/src/core/iomgr/fd_posix.h index 835e9b339a..dc917ebbc0 100644 --- a/src/core/iomgr/fd_posix.h +++ b/src/core/iomgr/fd_posix.h @@ -46,20 +46,21 @@ typedef struct grpc_fd_watcher { struct grpc_fd_watcher *next; struct grpc_fd_watcher *prev; grpc_pollset *pollset; + grpc_pollset_worker *worker; grpc_fd *fd; } grpc_fd_watcher; struct grpc_fd { int fd; /* refst format: - bit0: 1=active/0=orphaned - bit1-n: refcount + bit0: 1=active/0=orphaned + bit1-n: refcount meaning that mostly we ref by two to avoid altering the orphaned bit, and just unref by 1 when we're ready to flag the object as orphaned */ gpr_atm refst; - gpr_mu set_state_mu; - gpr_atm shutdown; + gpr_mu mu; + int shutdown; int closed; /* The watcher list. @@ -84,18 +85,16 @@ struct grpc_fd { If at a later time there becomes need of a poller to poll, one of the inactive pollers may be kicked out of their poll loops to take that responsibility. */ - gpr_mu watcher_mu; grpc_fd_watcher inactive_watcher_root; grpc_fd_watcher *read_watcher; grpc_fd_watcher *write_watcher; - gpr_atm readst; - gpr_atm writest; + grpc_closure *read_closure; + grpc_closure *write_closure; struct grpc_fd *freelist_next; - grpc_iomgr_closure *on_done_closure; - grpc_iomgr_closure *shutdown_closures[2]; + grpc_closure *on_done_closure; grpc_iomgr_object iomgr_object; }; @@ -111,7 +110,7 @@ grpc_fd *grpc_fd_create(int fd, const char *name); Requires: *fd initialized; no outstanding notify_on_read or notify_on_write. MUST NOT be called with a pollset lock taken */ -void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, +void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, const char *reason); /* Begin polling on an fd. @@ -126,17 +125,20 @@ void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done, fd's current interest (such as epoll) do not need to call this function. MUST NOT be called with a pollset lock taken */ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, - gpr_uint32 read_mask, gpr_uint32 write_mask, - grpc_fd_watcher *rec); + grpc_pollset_worker *worker, gpr_uint32 read_mask, + gpr_uint32 write_mask, grpc_fd_watcher *rec); /* Complete polling previously started with grpc_fd_begin_poll - MUST NOT be called with a pollset lock taken */ -void grpc_fd_end_poll(grpc_fd_watcher *rec, int got_read, int got_write); + MUST NOT be called with a pollset lock taken + if got_read or got_write are 1, also does the become_{readable,writable} as + appropriate. */ +void grpc_fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec, + int got_read, int got_write); /* Return 1 if this fd is orphaned, 0 otherwise */ int grpc_fd_is_orphaned(grpc_fd *fd); /* Cause any current callbacks to error out with GRPC_CALLBACK_CANCELLED. */ -void grpc_fd_shutdown(grpc_fd *fd); +void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd); /* Register read interest, causing read_cb to be called once when fd becomes readable, on deadline specified by deadline, or on shutdown triggered by @@ -151,17 +153,19 @@ void grpc_fd_shutdown(grpc_fd *fd); underlying platform. This means that users must drain fd in read_cb before calling notify_on_read again. Users are also expected to handle spurious events, i.e read_cb is called while nothing can be readable from fd */ -void grpc_fd_notify_on_read(grpc_fd *fd, grpc_iomgr_closure *closure); +void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure); /* Exactly the same semantics as above, except based on writable events. */ -void grpc_fd_notify_on_write(grpc_fd *fd, grpc_iomgr_closure *closure); +void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure); /* Notification from the poller to an fd that it has become readable or writable. If allow_synchronous_callback is 1, allow running the fd callback inline in this callstack, otherwise register an asynchronous callback and return */ -void grpc_fd_become_readable(grpc_fd *fd, int allow_synchronous_callback); -void grpc_fd_become_writable(grpc_fd *fd, int allow_synchronous_callback); +void grpc_fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd); +void grpc_fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd); /* Reference counting for fds */ #ifdef GRPC_FD_REF_COUNT_DEBUG diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 09a457dd9a..65da3a9d2d 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -42,7 +42,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/thd.h> -#include "src/core/iomgr/alarm_internal.h" +#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/socket_windows.h" @@ -50,14 +50,28 @@ static ULONG g_iocp_kick_token; static OVERLAPPED g_iocp_custom_overlap; -static gpr_event g_shutdown_iocp; -static gpr_event g_iocp_done; -static gpr_atm g_orphans = 0; static gpr_atm g_custom_events = 0; static HANDLE g_iocp; -static void do_iocp_work() { +static DWORD deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return INFINITE; + } + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + return gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); +} + +void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { BOOL success; DWORD bytes = 0; DWORD flags = 0; @@ -65,13 +79,13 @@ static void do_iocp_work() { LPOVERLAPPED overlapped; grpc_winsocket *socket; grpc_winsocket_callback_info *info; - void (*f)(void *, int) = NULL; - void *opaque = NULL; - success = GetQueuedCompletionStatus(g_iocp, &bytes, &completion_key, - &overlapped, INFINITE); - /* success = 0 and overlapped = NULL means the deadline got attained. - Which is impossible. since our wait time is +inf */ - GPR_ASSERT(success || overlapped); + grpc_closure *closure = NULL; + success = GetQueuedCompletionStatus( + g_iocp, &bytes, &completion_key, &overlapped, + deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type))); + if (success == 0 && overlapped == NULL) { + return; + } GPR_ASSERT(completion_key && overlapped); if (overlapped == &g_iocp_custom_overlap) { gpr_atm_full_fetch_add(&g_custom_events, -1); @@ -92,53 +106,27 @@ static void do_iocp_work() { gpr_log(GPR_ERROR, "Unknown IOCP operation"); abort(); } - GPR_ASSERT(info->outstanding); - if (socket->orphan) { - info->outstanding = 0; - if (!socket->read_info.outstanding && !socket->write_info.outstanding) { - grpc_winsocket_destroy(socket); - gpr_atm_full_fetch_add(&g_orphans, -1); - } - return; - } success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, FALSE, &flags); info->bytes_transfered = bytes; info->wsa_error = success ? 0 : WSAGetLastError(); GPR_ASSERT(overlapped == &info->overlapped); - gpr_mu_lock(&socket->state_mu); GPR_ASSERT(!info->has_pending_iocp); - if (info->cb) { - f = info->cb; - opaque = info->opaque; - info->cb = NULL; + gpr_mu_lock(&socket->state_mu); + if (info->closure) { + closure = info->closure; + info->closure = NULL; } else { info->has_pending_iocp = 1; } gpr_mu_unlock(&socket->state_mu); - if (f) f(opaque, 1); -} - -static void iocp_loop(void *p) { - while (gpr_atm_acq_load(&g_orphans) || gpr_atm_acq_load(&g_custom_events) || - !gpr_event_get(&g_shutdown_iocp)) { - grpc_maybe_call_delayed_callbacks(NULL, 1); - do_iocp_work(); - } - - gpr_event_set(&g_iocp_done, (void *)1); + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); } void grpc_iocp_init(void) { - gpr_thd_id id; - g_iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, (ULONG_PTR)NULL, 0); GPR_ASSERT(g_iocp); - - gpr_event_init(&g_iocp_done); - gpr_event_init(&g_shutdown_iocp); - gpr_thd_new(&id, iocp_loop, NULL, NULL); } void grpc_iocp_kick(void) { @@ -150,13 +138,22 @@ void grpc_iocp_kick(void) { GPR_ASSERT(success); } +void grpc_iocp_flush(void) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + do { + grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); + } while (grpc_exec_ctx_flush(&exec_ctx)); +} + void grpc_iocp_shutdown(void) { - BOOL success; - gpr_event_set(&g_shutdown_iocp, (void *)1); - grpc_iocp_kick(); - gpr_event_wait(&g_iocp_done, gpr_inf_future(GPR_CLOCK_REALTIME)); - success = CloseHandle(g_iocp); - GPR_ASSERT(success); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (gpr_atm_acq_load(&g_custom_events)) { + grpc_iocp_work(&exec_ctx, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(CloseHandle(g_iocp)); } void grpc_iocp_add_socket(grpc_winsocket *socket) { @@ -175,41 +172,35 @@ void grpc_iocp_add_socket(grpc_winsocket *socket) { GPR_ASSERT(ret == g_iocp); } -void grpc_iocp_socket_orphan(grpc_winsocket *socket) { - GPR_ASSERT(!socket->orphan); - gpr_atm_full_fetch_add(&g_orphans, 1); - socket->orphan = 1; -} - /* Calling notify_on_read or write means either of two things: -) The IOCP already completed in the background, and we need to call the callback now. -) The IOCP hasn't completed yet, and we're queuing it for later. */ -static void socket_notify_on_iocp(grpc_winsocket *socket, - void (*cb)(void *, int), void *opaque, +static void socket_notify_on_iocp(grpc_exec_ctx *exec_ctx, + grpc_winsocket *socket, grpc_closure *closure, grpc_winsocket_callback_info *info) { int run_now = 0; - GPR_ASSERT(!info->cb); + GPR_ASSERT(info->closure == NULL); gpr_mu_lock(&socket->state_mu); if (info->has_pending_iocp) { run_now = 1; info->has_pending_iocp = 0; + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); } else { - info->cb = cb; - info->opaque = opaque; + info->closure = closure; } gpr_mu_unlock(&socket->state_mu); - if (run_now) cb(opaque, 1); } -void grpc_socket_notify_on_write(grpc_winsocket *socket, - void (*cb)(void *, int), void *opaque) { - socket_notify_on_iocp(socket, cb, opaque, &socket->write_info); +void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx, + grpc_winsocket *socket, + grpc_closure *closure) { + socket_notify_on_iocp(exec_ctx, socket, closure, &socket->write_info); } -void grpc_socket_notify_on_read(grpc_winsocket *socket, void (*cb)(void *, int), - void *opaque) { - socket_notify_on_iocp(socket, cb, opaque, &socket->read_info); +void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, + grpc_closure *closure) { + socket_notify_on_iocp(exec_ctx, socket, closure, &socket->read_info); } #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h index ee3847a229..75f3ba8477 100644 --- a/src/core/iomgr/iocp_windows.h +++ b/src/core/iomgr/iocp_windows.h @@ -38,16 +38,19 @@ #include "src/core/iomgr/socket_windows.h" +void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); void grpc_iocp_init(void); void grpc_iocp_kick(void); +void grpc_iocp_flush(void); void grpc_iocp_shutdown(void); void grpc_iocp_add_socket(grpc_winsocket *); -void grpc_iocp_socket_orphan(grpc_winsocket *); -void grpc_socket_notify_on_write(grpc_winsocket *, - void (*cb)(void *, int success), void *opaque); +void grpc_socket_notify_on_write(grpc_exec_ctx *exec_ctx, + grpc_winsocket *winsocket, + grpc_closure *closure); -void grpc_socket_notify_on_read(grpc_winsocket *, - void (*cb)(void *, int success), void *opaque); +void grpc_socket_notify_on_read(grpc_exec_ctx *exec_ctx, + grpc_winsocket *winsocket, + grpc_closure *closure); #endif /* GRPC_INTERNAL_CORE_IOMGR_IOCP_WINDOWS_H */ diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index fdc9adf4af..212ce5534d 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -34,69 +34,32 @@ #include "src/core/iomgr/iomgr.h" #include <stdlib.h> +#include <string.h> -#include "src/core/iomgr/iomgr_internal.h" -#include "src/core/iomgr/alarm_internal.h" -#include "src/core/support/string.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> #include <grpc/support/sync.h> #include <grpc/support/thd.h> +#include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/timer_internal.h" +#include "src/core/support/string.h" + static gpr_mu g_mu; static gpr_cv g_rcv; -static grpc_iomgr_closure *g_cbs_head = NULL; -static grpc_iomgr_closure *g_cbs_tail = NULL; static int g_shutdown; -static gpr_event g_background_callback_executor_done; static grpc_iomgr_object g_root_object; -/* Execute followup callbacks continuously. - Other threads may check in and help during pollset_work() */ -static void background_callback_executor(void *ignored) { - gpr_mu_lock(&g_mu); - while (!g_shutdown) { - gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); - gpr_timespec short_deadline = gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_millis(100, GPR_TIMESPAN)); - if (g_cbs_head) { - grpc_iomgr_closure *closure = g_cbs_head; - g_cbs_head = closure->next; - if (!g_cbs_head) g_cbs_tail = NULL; - gpr_mu_unlock(&g_mu); - closure->cb(closure->cb_arg, closure->success); - gpr_mu_lock(&g_mu); - } else if (grpc_alarm_check(&g_mu, gpr_now(GPR_CLOCK_MONOTONIC), - &deadline)) { - } else { - gpr_mu_unlock(&g_mu); - gpr_sleep_until(gpr_time_min(short_deadline, deadline)); - gpr_mu_lock(&g_mu); - } - } - gpr_mu_unlock(&g_mu); - gpr_event_set(&g_background_callback_executor_done, (void *)1); -} - -void grpc_kick_poller(void) { - /* Empty. The background callback executor polls periodically. The activity - * the kicker is trying to draw the executor's attention to will be picked up - * either by one of the periodic wakeups or by one of the polling application - * threads. */ -} - void grpc_iomgr_init(void) { - gpr_thd_id id; g_shutdown = 0; gpr_mu_init(&g_mu); gpr_cv_init(&g_rcv); - grpc_alarm_list_init(gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC)); g_root_object.next = g_root_object.prev = &g_root_object; g_root_object.name = "root"; grpc_iomgr_platform_init(); - gpr_event_init(&g_background_callback_executor_done); - gpr_thd_new(&id, background_callback_executor, NULL, NULL); + grpc_pollset_global_init(); } static size_t count_objects(void) { @@ -108,77 +71,67 @@ static size_t count_objects(void) { return n; } -void grpc_iomgr_shutdown(void) { +static void dump_objects(const char *kind) { grpc_iomgr_object *obj; - grpc_iomgr_closure *closure; + for (obj = g_root_object.next; obj != &g_root_object; obj = obj->next) { + gpr_log(GPR_DEBUG, "%s OBJECT: %s %p", kind, obj->name, obj); + } +} + +void grpc_iomgr_shutdown(void) { gpr_timespec shutdown_deadline = gpr_time_add( gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN)); gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + grpc_iomgr_platform_flush(); gpr_mu_lock(&g_mu); g_shutdown = 1; - while (g_cbs_head != NULL || g_root_object.next != &g_root_object) { + while (g_root_object.next != &g_root_object) { if (gpr_time_cmp( gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), last_warning_time), gpr_time_from_seconds(1, GPR_TIMESPAN)) >= 0) { - if (g_cbs_head != NULL && g_root_object.next != &g_root_object) { - gpr_log(GPR_DEBUG, - "Waiting for %d iomgr objects to be destroyed and executing " - "final callbacks", - count_objects()); - } else if (g_cbs_head != NULL) { - gpr_log(GPR_DEBUG, "Executing final iomgr callbacks"); - } else { + if (g_root_object.next != &g_root_object) { gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed", count_objects()); } last_warning_time = gpr_now(GPR_CLOCK_REALTIME); } - if (g_cbs_head) { - do { - closure = g_cbs_head; - g_cbs_head = closure->next; - if (!g_cbs_head) g_cbs_tail = NULL; - gpr_mu_unlock(&g_mu); - - closure->cb(closure->cb_arg, 0); - gpr_mu_lock(&g_mu); - } while (g_cbs_head); - continue; - } - if (grpc_alarm_check(&g_mu, gpr_inf_future(GPR_CLOCK_MONOTONIC), NULL)) { + if (grpc_timer_check(&exec_ctx, gpr_inf_future(GPR_CLOCK_MONOTONIC), + NULL)) { + gpr_mu_unlock(&g_mu); + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(&g_mu); continue; } if (g_root_object.next != &g_root_object) { - int timeout = 0; gpr_timespec short_deadline = gpr_time_add( gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(100, GPR_TIMESPAN)); - while (gpr_cv_wait(&g_rcv, &g_mu, short_deadline) && g_cbs_head == NULL) { + if (gpr_cv_wait(&g_rcv, &g_mu, short_deadline)) { if (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), shutdown_deadline) > 0) { - timeout = 1; + if (g_root_object.next != &g_root_object) { + gpr_log(GPR_DEBUG, + "Failed to free %d iomgr objects before shutdown deadline: " + "memory leaks are likely", + count_objects()); + dump_objects("LEAKED"); + } break; } } - if (timeout) { - gpr_log(GPR_DEBUG, - "Failed to free %d iomgr objects before shutdown deadline: " - "memory leaks are likely", - count_objects()); - for (obj = g_root_object.next; obj != &g_root_object; obj = obj->next) { - gpr_log(GPR_DEBUG, "LEAKED OBJECT: %s %p", obj->name, obj); - } - break; - } } } gpr_mu_unlock(&g_mu); - grpc_kick_poller(); - gpr_event_wait(&g_background_callback_executor_done, - gpr_inf_future(GPR_CLOCK_REALTIME)); + grpc_timer_list_shutdown(&exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); - grpc_alarm_list_shutdown(); + /* ensure all threads have left g_mu */ + gpr_mu_lock(&g_mu); + gpr_mu_unlock(&g_mu); + grpc_pollset_global_shutdown(); grpc_iomgr_platform_shutdown(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_rcv); @@ -188,7 +141,7 @@ void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name) { obj->name = gpr_strdup(name); gpr_mu_lock(&g_mu); obj->next = &g_root_object; - obj->prev = obj->next->prev; + obj->prev = g_root_object.prev; obj->next->prev = obj->prev->next = obj; gpr_mu_unlock(&g_mu); } @@ -201,74 +154,3 @@ void grpc_iomgr_unregister_object(grpc_iomgr_object *obj) { gpr_mu_unlock(&g_mu); gpr_free(obj->name); } - -void grpc_iomgr_closure_init(grpc_iomgr_closure *closure, grpc_iomgr_cb_func cb, - void *cb_arg) { - closure->cb = cb; - closure->cb_arg = cb_arg; - closure->next = NULL; -} - -static void assert_not_scheduled_locked(grpc_iomgr_closure *closure) { -#ifndef NDEBUG - grpc_iomgr_closure *c; - - for (c = g_cbs_head; c; c = c->next) { - GPR_ASSERT(c != closure); - } -#endif -} - -void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *closure, int success) { - closure->success = success; - GPR_ASSERT(closure->cb); - gpr_mu_lock(&g_mu); - assert_not_scheduled_locked(closure); - closure->next = NULL; - if (!g_cbs_tail) { - g_cbs_head = g_cbs_tail = closure; - } else { - g_cbs_tail->next = closure; - g_cbs_tail = closure; - } - if (g_shutdown) { - gpr_cv_signal(&g_rcv); - } - gpr_mu_unlock(&g_mu); -} - -void grpc_iomgr_add_callback(grpc_iomgr_closure *closure) { - grpc_iomgr_add_delayed_callback(closure, 1 /* GPR_TRUE */); -} - -int grpc_maybe_call_delayed_callbacks(gpr_mu *drop_mu, int success) { - int n = 0; - gpr_mu *retake_mu = NULL; - grpc_iomgr_closure *closure; - for (;;) { - /* check for new work */ - if (!gpr_mu_trylock(&g_mu)) { - break; - } - closure = g_cbs_head; - if (!closure) { - gpr_mu_unlock(&g_mu); - break; - } - g_cbs_head = closure->next; - if (!g_cbs_head) g_cbs_tail = NULL; - gpr_mu_unlock(&g_mu); - /* if we have a mutex to drop, do so before executing work */ - if (drop_mu) { - gpr_mu_unlock(drop_mu); - retake_mu = drop_mu; - drop_mu = NULL; - } - closure->cb(closure->cb_arg, success && closure->success); - n++; - } - if (retake_mu) { - gpr_mu_lock(retake_mu); - } - return n; -} diff --git a/src/core/iomgr/iomgr.h b/src/core/iomgr/iomgr.h index 261c17366a..c9ea84c605 100644 --- a/src/core/iomgr/iomgr.h +++ b/src/core/iomgr/iomgr.h @@ -34,47 +34,10 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_IOMGR_H #define GRPC_INTERNAL_CORE_IOMGR_IOMGR_H -/** gRPC Callback definition. - * - * \param arg Arbitrary input. - * \param success An indication on the state of the iomgr. On false, cleanup - * actions should be taken (eg, shutdown). */ -typedef void (*grpc_iomgr_cb_func)(void *arg, int success); - -/** A closure over a grpc_iomgr_cb_func. */ -typedef struct grpc_iomgr_closure { - /** Bound callback. */ - grpc_iomgr_cb_func cb; - - /** Arguments to be passed to "cb". */ - void *cb_arg; - - /** Internal. A boolean indication to "cb" on the state of the iomgr. - * For instance, closures created during a shutdown would have this field set - * to false. */ - int success; - - /**< Internal. Do not touch */ - struct grpc_iomgr_closure *next; -} grpc_iomgr_closure; - -/** Initializes \a closure with \a cb and \a cb_arg. */ -void grpc_iomgr_closure_init(grpc_iomgr_closure *closure, grpc_iomgr_cb_func cb, - void *cb_arg); - /** Initializes the iomgr. */ void grpc_iomgr_init(void); /** Signals the intention to shutdown the iomgr. */ void grpc_iomgr_shutdown(void); -/** Registers a closure to be invoked at some point in the future. - * - * Can be called from within a callback or from anywhere else */ -void grpc_iomgr_add_callback(grpc_iomgr_closure *closure); - -/** As per grpc_iomgr_add_callback, with the ability to set the success - argument. */ -void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *iocb, int success); - #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_H */ diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/iomgr/iomgr_internal.h index 4cec973ba0..e372c18e8a 100644 --- a/src/core/iomgr/iomgr_internal.h +++ b/src/core/iomgr/iomgr_internal.h @@ -43,13 +43,16 @@ typedef struct grpc_iomgr_object { struct grpc_iomgr_object *prev; } grpc_iomgr_object; -int grpc_maybe_call_delayed_callbacks(gpr_mu *drop_mu, int success); -void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *iocb, int success); +void grpc_pollset_global_init(void); +void grpc_pollset_global_shutdown(void); void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name); void grpc_iomgr_unregister_object(grpc_iomgr_object *obj); void grpc_iomgr_platform_init(void); +/** flush any globally queued work from iomgr */ +void grpc_iomgr_platform_flush(void); +/** tear down all platform specific global iomgr structures */ void grpc_iomgr_platform_shutdown(void); #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/iomgr/iomgr_posix.c b/src/core/iomgr/iomgr_posix.c index 2425e59941..fecb7b9760 100644 --- a/src/core/iomgr/iomgr_posix.c +++ b/src/core/iomgr/iomgr_posix.c @@ -42,13 +42,11 @@ void grpc_iomgr_platform_init(void) { grpc_fd_global_init(); - grpc_pollset_global_init(); grpc_register_tracer("tcp", &grpc_tcp_trace); } -void grpc_iomgr_platform_shutdown(void) { - grpc_pollset_global_shutdown(); - grpc_fd_global_shutdown(); -} +void grpc_iomgr_platform_flush(void) {} + +void grpc_iomgr_platform_shutdown(void) { grpc_fd_global_shutdown(); } #endif /* GRPC_POSIX_SOCKET */ diff --git a/src/core/iomgr/iomgr_posix.h b/src/core/iomgr/iomgr_posix.h index 716fedb636..068a5c6d7c 100644 --- a/src/core/iomgr/iomgr_posix.h +++ b/src/core/iomgr/iomgr_posix.h @@ -36,7 +36,4 @@ #include "src/core/iomgr/iomgr_internal.h" -void grpc_pollset_global_init(void); -void grpc_pollset_global_shutdown(void); - #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_POSIX_H */ diff --git a/src/core/iomgr/iomgr_windows.c b/src/core/iomgr/iomgr_windows.c index b49cb87e97..14775516bb 100644 --- a/src/core/iomgr/iomgr_windows.c +++ b/src/core/iomgr/iomgr_windows.c @@ -63,6 +63,8 @@ void grpc_iomgr_platform_init(void) { grpc_iocp_init(); } +void grpc_iomgr_platform_flush(void) { grpc_iocp_flush(); } + void grpc_iomgr_platform_shutdown(void) { grpc_iocp_shutdown(); winsock_shutdown(); diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 337596cb74..d15553a12a 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -55,9 +55,8 @@ #endif void grpc_pollset_init(grpc_pollset *pollset); -void grpc_pollset_shutdown(grpc_pollset *pollset, - void (*shutdown_done)(void *arg), - void *shutdown_done_arg); +void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure); void grpc_pollset_destroy(grpc_pollset *pollset); /* Do some work on a pollset. @@ -74,9 +73,13 @@ void grpc_pollset_destroy(grpc_pollset *pollset); grpc_pollset_work, and it is guaranteed that GRPC_POLLSET_MU(pollset) will not be released by grpc_pollset_work AFTER worker has been destroyed. - Tries not to block past deadline. */ -void grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline); + Tries not to block past deadline. + May call grpc_closure_list_run on grpc_closure_list, without holding the + pollset + lock */ +void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, gpr_timespec now, + gpr_timespec deadline); /* Break one polling thread out of polling work for this pollset. If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers. diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index fe66ebed77..2aafd21dfb 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -41,9 +41,11 @@ #include <sys/epoll.h> #include <unistd.h> -#include "src/core/iomgr/fd_posix.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/iomgr/fd_posix.h" +#include "src/core/support/block_annotate.h" +#include "src/core/profiling/timers.h" typedef struct wakeup_fd_hdl { grpc_wakeup_fd wakeup_fd; @@ -53,7 +55,7 @@ typedef struct wakeup_fd_hdl { typedef struct { grpc_pollset *pollset; grpc_fd *fd; - grpc_iomgr_closure closure; + grpc_closure closure; } delayed_add; typedef struct { @@ -61,7 +63,8 @@ typedef struct { wakeup_fd_hdl *free_wakeup_fds; } pollset_hdr; -static void finally_add_fd(grpc_pollset *pollset, grpc_fd *fd) { +static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { pollset_hdr *h = pollset->data.ptr; struct epoll_event ev; int err; @@ -70,9 +73,9 @@ static void finally_add_fd(grpc_pollset *pollset, grpc_fd *fd) { /* We pretend to be polling whilst adding an fd to keep the fd from being closed during the add. This may result in a spurious wakeup being assigned to this pollset whilst adding, but that should be benign. */ - GPR_ASSERT(grpc_fd_begin_poll(fd, pollset, 0, 0, &watcher) == 0); + GPR_ASSERT(grpc_fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0); if (watcher.fd != NULL) { - ev.events = EPOLLIN | EPOLLOUT | EPOLLET; + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); ev.data.ptr = fd; err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); if (err < 0) { @@ -83,15 +86,15 @@ static void finally_add_fd(grpc_pollset *pollset, grpc_fd *fd) { } } } - grpc_fd_end_poll(&watcher, 0, 0); + grpc_fd_end_poll(exec_ctx, &watcher, 0, 0); } -static void perform_delayed_add(void *arg, int iomgr_status) { +static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, + int iomgr_status) { delayed_add *da = arg; - int do_shutdown_cb = 0; if (!grpc_fd_is_orphaned(da->fd)) { - finally_add_fd(da->pollset, da->fd); + finally_add_fd(exec_ctx, da->pollset, da->fd); } gpr_mu_lock(&da->pollset->mu); @@ -99,40 +102,37 @@ static void perform_delayed_add(void *arg, int iomgr_status) { if (da->pollset->shutting_down) { /* We don't care about this pollset anymore. */ if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) { - GPR_ASSERT(!grpc_pollset_has_workers(da->pollset)); da->pollset->called_shutdown = 1; - do_shutdown_cb = 1; + grpc_exec_ctx_enqueue(exec_ctx, da->pollset->shutdown_done, 1); } } gpr_mu_unlock(&da->pollset->mu); GRPC_FD_UNREF(da->fd, "delayed_add"); - if (do_shutdown_cb) { - da->pollset->shutdown_done_cb(da->pollset->shutdown_done_arg); - } - gpr_free(da); } -static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset, +static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, int and_unlock_pollset) { if (and_unlock_pollset) { gpr_mu_unlock(&pollset->mu); - finally_add_fd(pollset, fd); + finally_add_fd(exec_ctx, pollset, fd); } else { delayed_add *da = gpr_malloc(sizeof(*da)); da->pollset = pollset; da->fd = fd; GRPC_FD_REF(fd, "delayed_add"); - grpc_iomgr_closure_init(&da->closure, perform_delayed_add, da); + grpc_closure_init(&da->closure, perform_delayed_add, da); pollset->in_flight_cbs++; - grpc_iomgr_add_callback(&da->closure); + grpc_exec_ctx_enqueue(exec_ctx, &da->closure, 1); } } -static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset, +static void multipoll_with_epoll_pollset_del_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, int and_unlock_pollset) { pollset_hdr *h = pollset->data.ptr; @@ -154,9 +154,9 @@ static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset, /* TODO(klempner): We probably want to turn this down a bit */ #define GRPC_EPOLL_MAX_EVENTS 1000 -static void multipoll_with_epoll_pollset_maybe_work( - grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec deadline, - gpr_timespec now, int allow_synchronous_callback) { +static void multipoll_with_epoll_pollset_maybe_work_and_unlock( + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; int poll_rv; @@ -181,7 +181,13 @@ static void multipoll_with_epoll_pollset_maybe_work( pfds[1].events = POLLIN; pfds[1].revents = 0; + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ + GPR_TIMER_BEGIN("poll", 0); + GRPC_SCHEDULING_START_BLOCKING_REGION; poll_rv = grpc_poll_function(pfds, 2, timeout_ms); + GRPC_SCHEDULING_END_BLOCKING_REGION; + GPR_TIMER_END("poll", 0); if (poll_rv < 0) { if (errno != EINTR) { @@ -195,6 +201,7 @@ static void multipoll_with_epoll_pollset_maybe_work( } if (pfds[1].revents) { do { + /* The following epoll_wait never blocks; it has a timeout of 0 */ ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); if (ep_rv < 0) { if (errno != EINTR) { @@ -207,21 +214,23 @@ static void multipoll_with_epoll_pollset_maybe_work( /* TODO(klempner): We might want to consider making err and pri * separate events */ int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write = ep_ev[i].events & EPOLLOUT; - if (read || cancel) { - grpc_fd_become_readable(fd, allow_synchronous_callback); - } - if (write || cancel) { - grpc_fd_become_writable(fd, allow_synchronous_callback); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (fd == NULL) { + grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); + } else { + if (read_ev || cancel) { + grpc_fd_become_readable(exec_ctx, fd); + } + if (write_ev || cancel) { + grpc_fd_become_writable(exec_ctx, fd); + } } } } } while (ep_rv == GRPC_EPOLL_MAX_EVENTS); } } - - gpr_mu_lock(&pollset->mu); } static void multipoll_with_epoll_pollset_finish_shutdown( @@ -235,14 +244,17 @@ static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) { static const grpc_pollset_vtable multipoll_with_epoll_pollset = { multipoll_with_epoll_pollset_add_fd, multipoll_with_epoll_pollset_del_fd, - multipoll_with_epoll_pollset_maybe_work, + multipoll_with_epoll_pollset_maybe_work_and_unlock, multipoll_with_epoll_pollset_finish_shutdown, multipoll_with_epoll_pollset_destroy}; -static void epoll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, +static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd **fds, size_t nfds) { size_t i; pollset_hdr *h = gpr_malloc(sizeof(pollset_hdr)); + struct epoll_event ev; + int err; pollset->vtable = &multipoll_with_epoll_pollset; pollset->data.ptr = h; @@ -252,8 +264,19 @@ static void epoll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno)); abort(); } + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = NULL; + err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev); + if (err < 0) { + gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", + GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), + strerror(errno)); + } + for (i = 0; i < nfds; i++) { - multipoll_with_epoll_pollset_add_fd(pollset, fds[i], 0); + multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0); } } diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 30ee6e24db..faa6c14491 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -44,6 +44,7 @@ #include "src/core/iomgr/fd_posix.h" #include "src/core/iomgr/iomgr_internal.h" +#include "src/core/support/block_annotate.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/useful.h> @@ -59,7 +60,8 @@ typedef struct { grpc_fd **dels; } pollset_hdr; -static void multipoll_with_poll_pollset_add_fd(grpc_pollset *pollset, +static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, int and_unlock_pollset) { size_t i; @@ -80,7 +82,8 @@ exit: } } -static void multipoll_with_poll_pollset_del_fd(grpc_pollset *pollset, +static void multipoll_with_poll_pollset_del_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, int and_unlock_pollset) { /* will get removed next poll cycle */ @@ -96,12 +99,16 @@ static void multipoll_with_poll_pollset_del_fd(grpc_pollset *pollset, } } -static void multipoll_with_poll_pollset_maybe_work( - grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec deadline, - gpr_timespec now, int allow_synchronous_callback) { +static void multipoll_with_poll_pollset_maybe_work_and_unlock( + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now) { +#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) +#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) + int timeout; int r; - size_t i, j, pfd_count, fd_count; + size_t i, j, fd_count; + nfds_t pfd_count; pollset_hdr *h; /* TODO(ctiller): inline some elements to avoid an allocation */ grpc_fd_watcher *watchers; @@ -110,13 +117,16 @@ static void multipoll_with_poll_pollset_maybe_work( h = pollset->data.ptr; timeout = grpc_poll_deadline_to_millis_timeout(deadline, now); /* TODO(ctiller): perform just one malloc here if we exceed the inline case */ - pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 1)); - watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 1)); + pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2)); + watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2)); fd_count = 0; - pfd_count = 1; - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfd_count = 2; + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfds[0].events = POLLIN; - pfds[0].revents = POLLOUT; + pfds[0].revents = 0; + pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfds[1].events = POLLIN; + pfds[1].revents = 0; for (i = 0; i < h->fd_count; i++) { int remove = grpc_fd_is_orphaned(h->fds[i]); for (j = 0; !remove && j < h->del_count; j++) { @@ -139,45 +149,45 @@ static void multipoll_with_poll_pollset_maybe_work( h->fd_count = fd_count; gpr_mu_unlock(&pollset->mu); - for (i = 1; i < pfd_count; i++) { - pfds[i].events = grpc_fd_begin_poll(watchers[i].fd, pollset, POLLIN, - POLLOUT, &watchers[i]); + for (i = 2; i < pfd_count; i++) { + pfds[i].events = (short)grpc_fd_begin_poll(watchers[i].fd, pollset, worker, + POLLIN, POLLOUT, &watchers[i]); } + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ + GRPC_SCHEDULING_START_BLOCKING_REGION; r = grpc_poll_function(pfds, pfd_count, timeout); - - for (i = 1; i < pfd_count; i++) { - grpc_fd_end_poll(&watchers[i], pfds[i].revents & POLLIN, - pfds[i].revents & POLLOUT); - } + GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) { - if (errno != EINTR) { - gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno)); + gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno)); + for (i = 2; i < pfd_count; i++) { + grpc_fd_end_poll(exec_ctx, &watchers[i], 0, 0); } } else if (r == 0) { - /* do nothing */ + for (i = 2; i < pfd_count; i++) { + grpc_fd_end_poll(exec_ctx, &watchers[i], 0, 0); + } } else { - if (pfds[0].revents & POLLIN) { + if (pfds[0].revents & POLLIN_CHECK) { + grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); + } + if (pfds[1].revents & POLLIN_CHECK) { grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); } - for (i = 1; i < pfd_count; i++) { + for (i = 2; i < pfd_count; i++) { if (watchers[i].fd == NULL) { + grpc_fd_end_poll(exec_ctx, &watchers[i], 0, 0); continue; } - if (pfds[i].revents & (POLLIN | POLLHUP | POLLERR)) { - grpc_fd_become_readable(watchers[i].fd, allow_synchronous_callback); - } - if (pfds[i].revents & (POLLOUT | POLLHUP | POLLERR)) { - grpc_fd_become_writable(watchers[i].fd, allow_synchronous_callback); - } + grpc_fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK, + pfds[i].revents & POLLOUT_CHECK); } } gpr_free(pfds); gpr_free(watchers); - - gpr_mu_lock(&pollset->mu); } static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) { @@ -203,11 +213,12 @@ static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { static const grpc_pollset_vtable multipoll_with_poll_pollset = { multipoll_with_poll_pollset_add_fd, multipoll_with_poll_pollset_del_fd, - multipoll_with_poll_pollset_maybe_work, + multipoll_with_poll_pollset_maybe_work_and_unlock, multipoll_with_poll_pollset_finish_shutdown, multipoll_with_poll_pollset_destroy}; -void grpc_poll_become_multipoller(grpc_pollset *pollset, grpc_fd **fds, +void grpc_poll_become_multipoller(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd **fds, size_t nfds) { size_t i; pollset_hdr *h = gpr_malloc(sizeof(pollset_hdr)); diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 6bd1b61f24..6f478ccacb 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -42,11 +42,12 @@ #include <string.h> #include <unistd.h> -#include "src/core/iomgr/alarm_internal.h" +#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/fd_posix.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/socket_utils_posix.h" #include "src/core/profiling/timers.h" +#include "src/core/support/block_annotate.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/thd.h> @@ -56,8 +57,16 @@ GPR_TLS_DECL(g_current_thread_poller); GPR_TLS_DECL(g_current_thread_worker); +/** Default poll() function - a pointer so that it can be overridden by some + * tests */ grpc_poll_function_type grpc_poll_function = poll; +/** The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). + * This wakeup_fd gives us something to alert on when such a case occurs. */ +grpc_wakeup_fd grpc_global_wakeup_fd; + static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->prev->next = worker->next; worker->next->prev = worker->prev; @@ -89,43 +98,92 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { worker->prev->next = worker->next->prev = worker; } -void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { +void grpc_pollset_kick_ext(grpc_pollset *p, + grpc_pollset_worker *specific_worker, + gpr_uint32 flags) { + GPR_TIMER_BEGIN("grpc_pollset_kick_ext", 0); + /* pollset->mu already held */ if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { + GPR_TIMER_BEGIN("grpc_pollset_kick_ext.broadcast", 0); + GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); for (specific_worker = p->root_worker.next; specific_worker != &p->root_worker; specific_worker = specific_worker->next) { grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); } p->kicked_without_pollers = 1; + GPR_TIMER_END("grpc_pollset_kick_ext.broadcast", 0); } else if (gpr_tls_get(&g_current_thread_worker) != (gpr_intptr)specific_worker) { + GPR_TIMER_MARK("different_thread_worker", 0); + if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { + specific_worker->reevaluate_polling_on_wakeup = 1; + } + specific_worker->kicked_specifically = 1; + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { + GPR_TIMER_MARK("kick_yoself", 0); + if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { + specific_worker->reevaluate_polling_on_wakeup = 1; + } + specific_worker->kicked_specifically = 1; grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); } } else if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) { + GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); + GPR_TIMER_MARK("kick_anonymous", 0); specific_worker = pop_front_worker(p); if (specific_worker != NULL) { - push_back_worker(p, specific_worker); - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + if (gpr_tls_get(&g_current_thread_worker) == + (gpr_intptr)specific_worker) { + GPR_TIMER_MARK("kick_anonymous_not_self", 0); + push_back_worker(p, specific_worker); + specific_worker = pop_front_worker(p); + if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 && + gpr_tls_get(&g_current_thread_worker) == + (gpr_intptr)specific_worker) { + push_back_worker(p, specific_worker); + specific_worker = NULL; + } + } + if (specific_worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, specific_worker); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + } } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); p->kicked_without_pollers = 1; } } + + GPR_TIMER_END("grpc_pollset_kick_ext", 0); +} + +void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { + grpc_pollset_kick_ext(p, specific_worker, 0); } /* global state management */ void grpc_pollset_global_init(void) { gpr_tls_init(&g_current_thread_poller); + gpr_tls_init(&g_current_thread_worker); grpc_wakeup_fd_global_init(); + grpc_wakeup_fd_init(&grpc_global_wakeup_fd); } void grpc_pollset_global_shutdown(void) { - gpr_tls_destroy(&g_current_thread_poller); + grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); grpc_wakeup_fd_global_destroy(); + gpr_tls_destroy(&g_current_thread_poller); + gpr_tls_destroy(&g_current_thread_worker); } +void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } + /* main interface */ static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); @@ -136,12 +194,14 @@ void grpc_pollset_init(grpc_pollset *pollset) { pollset->in_flight_cbs = 0; pollset->shutting_down = 0; pollset->called_shutdown = 0; + pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; become_basic_pollset(pollset, NULL); } -void grpc_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { +void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { gpr_mu_lock(&pollset->mu); - pollset->vtable->add_fd(pollset, fd, 1); + pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); /* the following (enabled only in debug) will reacquire and then release our lock - meaning that if the unlocking flag passed to del_fd above is not respected, the code will deadlock (in a way that we have a chance of @@ -152,9 +212,10 @@ void grpc_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd) { #endif } -void grpc_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) { +void grpc_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { gpr_mu_lock(&pollset->mu); - pollset->vtable->del_fd(pollset, fd, 1); + pollset->vtable->del_fd(exec_ctx, pollset, fd, 1); /* the following (enabled only in debug) will reacquire and then release our lock - meaning that if the unlocking flag passed to del_fd above is not respected, the code will deadlock (in a way that we have a chance of @@ -165,61 +226,129 @@ void grpc_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) { #endif } -static void finish_shutdown(grpc_pollset *pollset) { +static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs)); pollset->vtable->finish_shutdown(pollset); - pollset->shutdown_done_cb(pollset->shutdown_done_arg); + grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, 1); } -void grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline) { +void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, gpr_timespec now, + gpr_timespec deadline) { /* pollset->mu already held */ int added_worker = 0; + int locked = 1; + int queued_work = 0; + int keep_polling = 0; + GPR_TIMER_BEGIN("grpc_pollset_work", 0); /* this must happen before we (potentially) drop pollset->mu */ worker->next = worker->prev = NULL; + worker->reevaluate_polling_on_wakeup = 0; + worker->kicked_specifically = 0; /* TODO(ctiller): pool these */ grpc_wakeup_fd_init(&worker->wakeup_fd); - if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1)) { + /* If there's work waiting for the pollset to be idle, and the + pollset is idle, then do that work */ + if (!grpc_pollset_has_workers(pollset) && + !grpc_closure_list_empty(pollset->idle_jobs)) { + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); goto done; } - if (grpc_alarm_check(&pollset->mu, now, &deadline)) { + /* Check alarms - these are a global resource so we just ping + each time through on every pollset. + May update deadline to ensure timely wakeups. + TODO(ctiller): can this work be localized? */ + if (grpc_timer_check(exec_ctx, now, &deadline)) { + gpr_mu_unlock(&pollset->mu); + locked = 0; goto done; } + /* If we're shutting down then we don't execute any extended work */ if (pollset->shutting_down) { goto done; } - if (!pollset->kicked_without_pollers) { - push_front_worker(pollset, worker); - added_worker = 1; - gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset); - pollset->vtable->maybe_work(pollset, worker, deadline, now, 1); - gpr_tls_set(&g_current_thread_poller, 0); - } else { - pollset->kicked_without_pollers = 0; + /* Give do_promote priority so we don't starve it out */ + if (pollset->in_flight_cbs) { + gpr_mu_unlock(&pollset->mu); + locked = 0; + goto done; + } + /* Start polling, and keep doing so while we're being asked to + re-evaluate our pollers (this allows poll() based pollers to + ensure they don't miss wakeups) */ + keep_polling = 1; + while (keep_polling) { + keep_polling = 0; + if (!pollset->kicked_without_pollers) { + if (!added_worker) { + push_front_worker(pollset, worker); + added_worker = 1; + gpr_tls_set(&g_current_thread_worker, (gpr_intptr)worker); + } + gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset); + GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); + pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, worker, + deadline, now); + GPR_TIMER_END("maybe_work_and_unlock", 0); + locked = 0; + gpr_tls_set(&g_current_thread_poller, 0); + } else { + pollset->kicked_without_pollers = 0; + } + /* Finished execution - start cleaning up. + Note that we may arrive here from outside the enclosing while() loop. + In that case we won't loop though as we haven't added worker to the + worker list, which means nobody could ask us to re-evaluate polling). */ + done: + if (!locked) { + queued_work |= grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + locked = 1; + } + /* If we're forced to re-evaluate polling (via grpc_pollset_kick with + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force + a loop */ + if (worker->reevaluate_polling_on_wakeup) { + worker->reevaluate_polling_on_wakeup = 0; + pollset->kicked_without_pollers = 0; + if (queued_work || worker->kicked_specifically) { + /* If there's queued work on the list, then set the deadline to be + immediate so we get back out of the polling loop quickly */ + deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); + } + keep_polling = 1; + } } -done: - grpc_wakeup_fd_destroy(&worker->wakeup_fd); if (added_worker) { remove_worker(pollset, worker); + gpr_tls_set(&g_current_thread_worker, 0); } + grpc_wakeup_fd_destroy(&worker->wakeup_fd); if (pollset->shutting_down) { if (grpc_pollset_has_workers(pollset)) { grpc_pollset_kick(pollset, NULL); } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); - finish_shutdown(pollset); + finish_shutdown(exec_ctx, pollset); + grpc_exec_ctx_flush(exec_ctx); /* Continuing to access pollset here is safe -- it is the caller's * responsibility to not destroy when it has outstanding calls to * grpc_pollset_work. * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ gpr_mu_lock(&pollset->mu); + } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); } } + GPR_TIMER_END("grpc_pollset_work", 0); } -void grpc_pollset_shutdown(grpc_pollset *pollset, - void (*shutdown_done)(void *arg), - void *shutdown_done_arg) { +void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { int call_shutdown = 0; gpr_mu_lock(&pollset->mu); GPR_ASSERT(!pollset->shutting_down); @@ -229,13 +358,15 @@ void grpc_pollset_shutdown(grpc_pollset *pollset, pollset->called_shutdown = 1; call_shutdown = 1; } - pollset->shutdown_done_cb = shutdown_done; - pollset->shutdown_done_arg = shutdown_done_arg; + if (!grpc_pollset_has_workers(pollset)) { + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); + } + pollset->shutdown_done = closure; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); gpr_mu_unlock(&pollset->mu); if (call_shutdown) { - finish_shutdown(pollset); + finish_shutdown(exec_ctx, pollset); } } @@ -261,7 +392,7 @@ int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, } timeout = gpr_time_sub(deadline, now); return gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_SEC - 1, GPR_TIMESPAN))); + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); } /* @@ -273,15 +404,14 @@ typedef struct grpc_unary_promote_args { const grpc_pollset_vtable *original_vtable; grpc_pollset *pollset; grpc_fd *fd; - grpc_iomgr_closure promotion_closure; + grpc_closure promotion_closure; } grpc_unary_promote_args; -static void basic_do_promote(void *args, int success) { +static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, int success) { grpc_unary_promote_args *up_args = args; const grpc_pollset_vtable *original_vtable = up_args->original_vtable; grpc_pollset *pollset = up_args->pollset; grpc_fd *fd = up_args->fd; - int do_shutdown_cb = 0; /* * This is quite tricky. There are a number of cases to keep in mind here: @@ -294,12 +424,7 @@ static void basic_do_promote(void *args, int success) { gpr_mu_lock(&pollset->mu); /* First we need to ensure that nobody is polling concurrently */ - if (grpc_pollset_has_workers(pollset)) { - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - grpc_iomgr_add_callback(&up_args->promotion_closure); - gpr_mu_unlock(&pollset->mu); - return; - } + GPR_ASSERT(!grpc_pollset_has_workers(pollset)); gpr_free(up_args); /* At this point the pollset may no longer be a unary poller. In that case @@ -311,21 +436,21 @@ static void basic_do_promote(void *args, int success) { if (pollset->shutting_down) { /* We don't care about this pollset anymore. */ if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) { - GPR_ASSERT(!grpc_pollset_has_workers(pollset)); pollset->called_shutdown = 1; - do_shutdown_cb = 1; + finish_shutdown(exec_ctx, pollset); } } else if (grpc_fd_is_orphaned(fd)) { /* Don't try to add it to anything, we'll drop our ref on it below */ } else if (pollset->vtable != original_vtable) { - pollset->vtable->add_fd(pollset, fd, 0); + pollset->vtable->add_fd(exec_ctx, pollset, fd, 0); } else if (fd != pollset->data.ptr) { grpc_fd *fds[2]; fds[0] = pollset->data.ptr; fds[1] = fd; if (fds[0] && !grpc_fd_is_orphaned(fds[0])) { - grpc_platform_become_multipoller(pollset, fds, GPR_ARRAY_SIZE(fds)); + grpc_platform_become_multipoller(exec_ctx, pollset, fds, + GPR_ARRAY_SIZE(fds)); GRPC_FD_UNREF(fds[0], "basicpoll"); } else { /* old fd is orphaned and we haven't cleaned it up until now, so remain a @@ -340,16 +465,12 @@ static void basic_do_promote(void *args, int success) { gpr_mu_unlock(&pollset->mu); - if (do_shutdown_cb) { - pollset->shutdown_done_cb(pollset->shutdown_done_arg); - } - /* Matching ref in basic_pollset_add_fd */ GRPC_FD_UNREF(fd, "basicpoll_add"); } -static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd, - int and_unlock_pollset) { +static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd, int and_unlock_pollset) { grpc_unary_promote_args *up_args; GPR_ASSERT(fd); if (fd == pollset->data.ptr) goto exit; @@ -366,7 +487,8 @@ static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd, pollset->data.ptr = fd; GRPC_FD_REF(fd, "basicpoll"); } else if (!grpc_fd_is_orphaned(fds[0])) { - grpc_platform_become_multipoller(pollset, fds, GPR_ARRAY_SIZE(fds)); + grpc_platform_become_multipoller(exec_ctx, pollset, fds, + GPR_ARRAY_SIZE(fds)); GRPC_FD_UNREF(fds[0], "basicpoll"); } else { /* old fd is orphaned and we haven't cleaned it up until now, so remain a @@ -383,13 +505,13 @@ static void basic_pollset_add_fd(grpc_pollset *pollset, grpc_fd *fd, GRPC_FD_REF(fd, "basicpoll_add"); pollset->in_flight_cbs++; up_args = gpr_malloc(sizeof(*up_args)); - up_args->pollset = pollset; up_args->fd = fd; up_args->original_vtable = pollset->vtable; + up_args->pollset = pollset; up_args->promotion_closure.cb = basic_do_promote; up_args->promotion_closure.cb_arg = up_args; - grpc_iomgr_add_callback(&up_args->promotion_closure); + grpc_closure_list_add(&pollset->idle_jobs, &up_args->promotion_closure, 1); grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); exit: @@ -398,8 +520,8 @@ exit: } } -static void basic_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd, - int and_unlock_pollset) { +static void basic_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd, int and_unlock_pollset) { GPR_ASSERT(fd); if (fd == pollset->data.ptr) { GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); @@ -411,77 +533,85 @@ static void basic_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd, } } -static void basic_pollset_maybe_work(grpc_pollset *pollset, - grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now, - int allow_synchronous_callback) { - struct pollfd pfd[2]; +static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec deadline, + gpr_timespec now) { +#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) +#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) + + struct pollfd pfd[3]; grpc_fd *fd; grpc_fd_watcher fd_watcher; int timeout; int r; - int nfds; + nfds_t nfds; - if (pollset->in_flight_cbs) { - /* Give do_promote priority so we don't starve it out */ - gpr_mu_unlock(&pollset->mu); - gpr_mu_lock(&pollset->mu); - return; - } fd = pollset->data.ptr; if (fd && grpc_fd_is_orphaned(fd)) { GRPC_FD_UNREF(fd, "basicpoll"); fd = pollset->data.ptr = NULL; } timeout = grpc_poll_deadline_to_millis_timeout(deadline, now); - pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfd[0].events = POLLIN; pfd[0].revents = 0; - nfds = 1; + pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfd[1].events = POLLIN; + pfd[1].revents = 0; + nfds = 2; if (fd) { - pfd[1].fd = fd->fd; - pfd[1].revents = 0; + pfd[2].fd = fd->fd; + pfd[2].revents = 0; + GRPC_FD_REF(fd, "basicpoll_begin"); gpr_mu_unlock(&pollset->mu); - pfd[1].events = - grpc_fd_begin_poll(fd, pollset, POLLIN, POLLOUT, &fd_watcher); - if (pfd[1].events != 0) { + pfd[2].events = (short)grpc_fd_begin_poll(fd, pollset, worker, POLLIN, + POLLOUT, &fd_watcher); + if (pfd[2].events != 0) { nfds++; } } else { gpr_mu_unlock(&pollset->mu); } + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ /* poll fd count (argument 2) is shortened by one if we have no events to poll on - such that it only includes the kicker */ + GPR_TIMER_BEGIN("poll", 0); + GRPC_SCHEDULING_START_BLOCKING_REGION; r = grpc_poll_function(pfd, nfds, timeout); - GRPC_TIMER_MARK(GRPC_PTAG_POLL_FINISHED, r); - - if (fd) { - grpc_fd_end_poll(&fd_watcher, pfd[1].revents & POLLIN, - pfd[1].revents & POLLOUT); - } + GRPC_SCHEDULING_END_BLOCKING_REGION; + GPR_TIMER_END("poll", 0); if (r < 0) { - if (errno != EINTR) { - gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno)); + gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno)); + if (fd) { + grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); } } else if (r == 0) { - /* do nothing */ + if (fd) { + grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); + } } else { - if (pfd[0].revents & POLLIN) { + if (pfd[0].revents & POLLIN_CHECK) { + grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); + } + if (pfd[1].revents & POLLIN_CHECK) { grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); } - if (nfds > 1) { - if (pfd[1].revents & (POLLIN | POLLHUP | POLLERR)) { - grpc_fd_become_readable(fd, allow_synchronous_callback); - } - if (pfd[1].revents & (POLLOUT | POLLHUP | POLLERR)) { - grpc_fd_become_writable(fd, allow_synchronous_callback); - } + if (nfds > 2) { + grpc_fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, + pfd[2].revents & POLLOUT_CHECK); + } else if (fd) { + grpc_fd_end_poll(exec_ctx, &fd_watcher, 0, 0); } } - gpr_mu_lock(&pollset->mu); + if (fd) { + GRPC_FD_UNREF(fd, "basicpoll_begin"); + } } static void basic_pollset_destroy(grpc_pollset *pollset) { @@ -492,8 +622,9 @@ static void basic_pollset_destroy(grpc_pollset *pollset) { } static const grpc_pollset_vtable basic_pollset = { - basic_pollset_add_fd, basic_pollset_del_fd, basic_pollset_maybe_work, - basic_pollset_destroy, basic_pollset_destroy}; + basic_pollset_add_fd, basic_pollset_del_fd, + basic_pollset_maybe_work_and_unlock, basic_pollset_destroy, + basic_pollset_destroy}; static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) { pollset->vtable = &basic_pollset; diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index 69bd9cca8c..95ebeab1c2 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -37,6 +37,8 @@ #include <poll.h> #include <grpc/support/sync.h> +#include "src/core/iomgr/exec_ctx.h" +#include "src/core/iomgr/iomgr.h" #include "src/core/iomgr/wakeup_fd_posix.h" typedef struct grpc_pollset_vtable grpc_pollset_vtable; @@ -48,6 +50,8 @@ struct grpc_fd; typedef struct grpc_pollset_worker { grpc_wakeup_fd wakeup_fd; + int reevaluate_polling_on_wakeup; + int kicked_specifically; struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; } grpc_pollset_worker; @@ -64,8 +68,8 @@ typedef struct grpc_pollset { int shutting_down; int called_shutdown; int kicked_without_pollers; - void (*shutdown_done_cb)(void *arg); - void *shutdown_done_arg; + grpc_closure *shutdown_done; + grpc_closure_list idle_jobs; union { int fd; void *ptr; @@ -73,13 +77,13 @@ typedef struct grpc_pollset { } grpc_pollset; struct grpc_pollset_vtable { - void (*add_fd)(grpc_pollset *pollset, struct grpc_fd *fd, - int and_unlock_pollset); - void (*del_fd)(grpc_pollset *pollset, struct grpc_fd *fd, - int and_unlock_pollset); - void (*maybe_work)(grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now, - int allow_synchronous_callback); + void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd, int and_unlock_pollset); + void (*del_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd, int and_unlock_pollset); + void (*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now); void (*finish_shutdown)(grpc_pollset *pollset); void (*destroy)(grpc_pollset *pollset); }; @@ -87,10 +91,12 @@ struct grpc_pollset_vtable { #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) /* Add an fd to a pollset */ -void grpc_pollset_add_fd(grpc_pollset *pollset, struct grpc_fd *fd); +void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd); /* Force remove an fd from a pollset (normally they are removed on the next poll after an fd is orphaned) */ -void grpc_pollset_del_fd(grpc_pollset *pollset, struct grpc_fd *fd); +void grpc_pollset_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd); /* Returns the fd to listen on for kicks */ int grpc_kick_read_fd(grpc_pollset *p); @@ -107,13 +113,25 @@ void grpc_kick_drain(grpc_pollset *p); int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now); +/* Allow kick to wakeup the currently polling worker */ +#define GRPC_POLLSET_CAN_KICK_SELF 1 +/* Force the wakee to repoll when awoken */ +#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 +/* As per grpc_pollset_kick, with an extended set of flags (defined above) + -- mostly for fd_posix's use. */ +void grpc_pollset_kick_ext(grpc_pollset *p, + grpc_pollset_worker *specific_worker, + gpr_uint32 flags); + /* turn a pollset into a multipoller: platform specific */ -typedef void (*grpc_platform_become_multipoller_type)(grpc_pollset *pollset, +typedef void (*grpc_platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, struct grpc_fd **fds, size_t fd_count); extern grpc_platform_become_multipoller_type grpc_platform_become_multipoller; -void grpc_poll_become_multipoller(grpc_pollset *pollset, struct grpc_fd **fds, +void grpc_poll_become_multipoller(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, struct grpc_fd **fds, size_t fd_count); /* Return 1 if the pollset has active threads in grpc_pollset_work (pollset must @@ -123,5 +141,6 @@ int grpc_pollset_has_workers(grpc_pollset *pollset); /* override to allow tests to hook poll() usage */ typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int); extern grpc_poll_function_type grpc_poll_function; +extern grpc_wakeup_fd grpc_global_wakeup_fd; #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_POSIX_H */ diff --git a/src/core/iomgr/pollset_set.h b/src/core/iomgr/pollset_set.h index 6d73951c70..0fdcba01a4 100644 --- a/src/core/iomgr/pollset_set.h +++ b/src/core/iomgr/pollset_set.h @@ -49,11 +49,13 @@ #include "src/core/iomgr/pollset_set_windows.h" #endif -void grpc_pollset_set_init(grpc_pollset_set *pollset_set); -void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set); -void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, - grpc_pollset *pollset); -void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, - grpc_pollset *pollset); +void grpc_pollset_set_init(grpc_pollset_set* pollset_set); +void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set); +void grpc_pollset_set_add_pollset(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set, + grpc_pollset* pollset); +void grpc_pollset_set_del_pollset(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set, + grpc_pollset* pollset); #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_H */ diff --git a/src/core/iomgr/pollset_set_posix.c b/src/core/iomgr/pollset_set_posix.c index 2076ac70ef..c86ed3d5da 100644 --- a/src/core/iomgr/pollset_set_posix.c +++ b/src/core/iomgr/pollset_set_posix.c @@ -58,7 +58,8 @@ void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) { gpr_free(pollset_set->fds); } -void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, +void grpc_pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_pollset *pollset) { size_t i, j; gpr_mu_lock(&pollset_set->mu); @@ -74,7 +75,7 @@ void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, if (grpc_fd_is_orphaned(pollset_set->fds[i])) { GRPC_FD_UNREF(pollset_set->fds[i], "pollset"); } else { - grpc_pollset_add_fd(pollset, pollset_set->fds[i]); + grpc_pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]); pollset_set->fds[j++] = pollset_set->fds[i]; } } @@ -82,7 +83,8 @@ void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, gpr_mu_unlock(&pollset_set->mu); } -void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, +void grpc_pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_pollset *pollset) { size_t i; gpr_mu_lock(&pollset_set->mu); @@ -97,7 +99,8 @@ void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, gpr_mu_unlock(&pollset_set->mu); } -void grpc_pollset_set_add_fd(grpc_pollset_set *pollset_set, grpc_fd *fd) { +void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd) { size_t i; gpr_mu_lock(&pollset_set->mu); if (pollset_set->fd_count == pollset_set->fd_capacity) { @@ -108,12 +111,13 @@ void grpc_pollset_set_add_fd(grpc_pollset_set *pollset_set, grpc_fd *fd) { GRPC_FD_REF(fd, "pollset_set"); pollset_set->fds[pollset_set->fd_count++] = fd; for (i = 0; i < pollset_set->pollset_count; i++) { - grpc_pollset_add_fd(pollset_set->pollsets[i], fd); + grpc_pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd); } gpr_mu_unlock(&pollset_set->mu); } -void grpc_pollset_set_del_fd(grpc_pollset_set *pollset_set, grpc_fd *fd) { +void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd) { size_t i; gpr_mu_lock(&pollset_set->mu); for (i = 0; i < pollset_set->fd_count; i++) { diff --git a/src/core/iomgr/pollset_set_posix.h b/src/core/iomgr/pollset_set_posix.h index e88740bde1..05234fb642 100644 --- a/src/core/iomgr/pollset_set_posix.h +++ b/src/core/iomgr/pollset_set_posix.h @@ -49,7 +49,9 @@ typedef struct grpc_pollset_set { grpc_fd **fds; } grpc_pollset_set; -void grpc_pollset_set_add_fd(grpc_pollset_set *pollset_set, grpc_fd *fd); -void grpc_pollset_set_del_fd(grpc_pollset_set *pollset_set, grpc_fd *fd); +void grpc_pollset_set_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd); +void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd); #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/pollset_set_windows.c b/src/core/iomgr/pollset_set_windows.c index b9c209cd2c..53d5d3dcd4 100644 --- a/src/core/iomgr/pollset_set_windows.c +++ b/src/core/iomgr/pollset_set_windows.c @@ -37,14 +37,16 @@ #include "src/core/iomgr/pollset_set.h" -void grpc_pollset_set_init(grpc_pollset_set *pollset_set) {} +void grpc_pollset_set_init(grpc_pollset_set* pollset_set) {} -void grpc_pollset_set_destroy(grpc_pollset_set *pollset_set) {} +void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} -void grpc_pollset_set_add_pollset(grpc_pollset_set *pollset_set, - grpc_pollset *pollset) {} +void grpc_pollset_set_add_pollset(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set, + grpc_pollset* pollset) {} -void grpc_pollset_set_del_pollset(grpc_pollset_set *pollset_set, - grpc_pollset *pollset) {} +void grpc_pollset_set_del_pollset(grpc_exec_ctx* exec_ctx, + grpc_pollset_set* pollset_set, + grpc_pollset* pollset) {} #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 07522c8a0c..9f74580273 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -37,40 +37,65 @@ #include <grpc/support/thd.h> -#include "src/core/iomgr/alarm_internal.h" +#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_windows.h" -static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; +gpr_mu grpc_polling_mu; +static grpc_pollset_worker *g_active_poller; +static grpc_pollset_worker g_global_root_worker; + +void grpc_pollset_global_init() { + gpr_mu_init(&grpc_polling_mu); + g_active_poller = NULL; + g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = + g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = + &g_global_root_worker; +} + +void grpc_pollset_global_shutdown() { gpr_mu_destroy(&grpc_polling_mu); } + +static void remove_worker(grpc_pollset_worker *worker, + grpc_pollset_worker_link_type type) { + worker->links[type].prev->links[type].next = worker->links[type].next; + worker->links[type].next->links[type].prev = worker->links[type].prev; + worker->links[type].next = worker->links[type].prev = worker; } -static int has_workers(grpc_pollset *p) { - return p->root_worker.next != &p->root_worker; +static int has_workers(grpc_pollset_worker *root, + grpc_pollset_worker_link_type type) { + return root->links[type].next != root; } -static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { - if (has_workers(p)) { - grpc_pollset_worker *w = p->root_worker.next; - remove_worker(p, w); +static grpc_pollset_worker *pop_front_worker( + grpc_pollset_worker *root, grpc_pollset_worker_link_type type) { + if (has_workers(root, type)) { + grpc_pollset_worker *w = root->links[type].next; + remove_worker(w, type); return w; } else { return NULL; } } -static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->next = &p->root_worker; - worker->prev = worker->next->prev; - worker->prev->next = worker->next->prev = worker; +static void push_back_worker(grpc_pollset_worker *root, + grpc_pollset_worker_link_type type, + grpc_pollset_worker *worker) { + worker->links[type].next = root; + worker->links[type].prev = worker->links[type].next->links[type].prev; + worker->links[type].prev->links[type].next = + worker->links[type].next->links[type].prev = worker; } -static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev = &p->root_worker; - worker->next = worker->prev->next; - worker->prev->next = worker->next->prev = worker; +static void push_front_worker(grpc_pollset_worker *root, + grpc_pollset_worker_link_type type, + grpc_pollset_worker *worker) { + worker->links[type].prev = root; + worker->links[type].next = worker->links[type].prev->links[type].next; + worker->links[type].prev->links[type].next = + worker->links[type].next->links[type].prev = worker; } /* There isn't really any such thing as a pollset under Windows, due to the @@ -80,71 +105,135 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { void grpc_pollset_init(grpc_pollset *pollset) { memset(pollset, 0, sizeof(*pollset)); - gpr_mu_init(&pollset->mu); - pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; - pollset->kicked_without_pollers = 0; + pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = + pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + &pollset->root_worker; } -void grpc_pollset_shutdown(grpc_pollset *pollset, - void (*shutdown_done)(void *arg), - void *shutdown_done_arg) { - gpr_mu_lock(&pollset->mu); +void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + gpr_mu_lock(&grpc_polling_mu); pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - gpr_mu_unlock(&pollset->mu); - shutdown_done(shutdown_done_arg); + if (!pollset->is_iocp_worker) { + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); + } else { + pollset->on_shutdown = closure; + } + gpr_mu_unlock(&grpc_polling_mu); } -void grpc_pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); -} +void grpc_pollset_destroy(grpc_pollset *pollset) {} -void grpc_pollset_work(grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline) { +void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, gpr_timespec now, + gpr_timespec deadline) { int added_worker = 0; - worker->next = worker->prev = NULL; + worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = + worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = + worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL; + worker->kicked = 0; + worker->pollset = pollset; gpr_cv_init(&worker->cv); - if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1 /* GPR_TRUE */)) { - goto done; - } - if (grpc_alarm_check(&pollset->mu, now, &deadline)) { + if (grpc_timer_check(exec_ctx, now, &deadline)) { goto done; } if (!pollset->kicked_without_pollers && !pollset->shutting_down) { - push_front_worker(pollset, worker); + if (g_active_poller == NULL) { + grpc_pollset_worker *next_worker; + /* become poller */ + pollset->is_iocp_worker = 1; + g_active_poller = worker; + gpr_mu_unlock(&grpc_polling_mu); + grpc_iocp_work(exec_ctx, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&grpc_polling_mu); + pollset->is_iocp_worker = 0; + g_active_poller = NULL; + /* try to get a worker from this pollsets worker list */ + next_worker = pop_front_worker(&pollset->root_worker, + GRPC_POLLSET_WORKER_LINK_POLLSET); + if (next_worker == NULL) { + /* try to get a worker from the global list */ + next_worker = pop_front_worker(&g_global_root_worker, + GRPC_POLLSET_WORKER_LINK_GLOBAL); + } + if (next_worker != NULL) { + next_worker->kicked = 1; + gpr_cv_signal(&next_worker->cv); + } + + if (pollset->shutting_down && pollset->on_shutdown != NULL) { + grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, 1); + pollset->on_shutdown = NULL; + } + goto done; + } + push_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL, + worker); + push_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET, + worker); added_worker = 1; - gpr_cv_wait(&worker->cv, &pollset->mu, deadline); + while (!worker->kicked) { + if (gpr_cv_wait(&worker->cv, &grpc_polling_mu, deadline)) { + break; + } + } } else { pollset->kicked_without_pollers = 0; } done: - gpr_cv_destroy(&worker->cv); + if (!grpc_closure_list_empty(exec_ctx->closure_list)) { + gpr_mu_unlock(&grpc_polling_mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&grpc_polling_mu); + } if (added_worker) { - remove_worker(pollset, worker); + remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); + remove_worker(worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } + gpr_cv_destroy(&worker->cv); } void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { - for (specific_worker = p->root_worker.next; + for (specific_worker = + p->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next; specific_worker != &p->root_worker; - specific_worker = specific_worker->next) { + specific_worker = + specific_worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next) { + specific_worker->kicked = 1; gpr_cv_signal(&specific_worker->cv); } p->kicked_without_pollers = 1; + if (p->is_iocp_worker) { + grpc_iocp_kick(); + } } else { - gpr_cv_signal(&specific_worker->cv); + if (p->is_iocp_worker) { + if (g_active_poller == specific_worker) { + grpc_iocp_kick(); + } + } else { + specific_worker->kicked = 1; + gpr_cv_signal(&specific_worker->cv); + } } } else { - specific_worker = pop_front_worker(p); + specific_worker = + pop_front_worker(&p->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); if (specific_worker != NULL) { - push_back_worker(p, specific_worker); - gpr_cv_signal(&specific_worker->cv); + grpc_pollset_kick(p, specific_worker); + } else if (p->is_iocp_worker) { + grpc_iocp_kick(); } else { p->kicked_without_pollers = 1; } } } +void grpc_kick_poller(void) { grpc_iocp_kick(); } + #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index 4efa5a1717..65ba80619b 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -43,19 +43,37 @@ used to synchronize with the IOCP, and workers are condition variables used to block threads until work is ready. */ -typedef struct grpc_pollset_worker { - gpr_cv cv; +typedef enum { + GRPC_POLLSET_WORKER_LINK_POLLSET = 0, + GRPC_POLLSET_WORKER_LINK_GLOBAL, + GRPC_POLLSET_WORKER_LINK_TYPES +} grpc_pollset_worker_link_type; + +typedef struct grpc_pollset_worker_link { struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; +} grpc_pollset_worker_link; + +struct grpc_pollset; +typedef struct grpc_pollset grpc_pollset; + +typedef struct grpc_pollset_worker { + gpr_cv cv; + int kicked; + struct grpc_pollset *pollset; + grpc_pollset_worker_link links[GRPC_POLLSET_WORKER_LINK_TYPES]; } grpc_pollset_worker; -typedef struct grpc_pollset { - gpr_mu mu; +struct grpc_pollset { int shutting_down; int kicked_without_pollers; + int is_iocp_worker; grpc_pollset_worker root_worker; -} grpc_pollset; + grpc_closure *on_shutdown; +}; + +extern gpr_mu grpc_polling_mu; -#define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) +#define GRPC_POLLSET_MU(pollset) (&grpc_polling_mu) #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ diff --git a/src/core/iomgr/resolve_address.h b/src/core/iomgr/resolve_address.h index cc1bd428b0..01eedffa88 100644 --- a/src/core/iomgr/resolve_address.h +++ b/src/core/iomgr/resolve_address.h @@ -35,12 +35,14 @@ #define GRPC_INTERNAL_CORE_IOMGR_RESOLVE_ADDRESS_H #include <stddef.h> +#include "src/core/iomgr/exec_ctx.h" +#include "src/core/iomgr/iomgr.h" #define GRPC_MAX_SOCKADDR_SIZE 128 typedef struct { char addr[GRPC_MAX_SOCKADDR_SIZE]; - int len; + size_t len; } grpc_resolved_address; typedef struct { @@ -52,7 +54,8 @@ typedef struct { On success: addresses is the result, and the callee must call grpc_resolved_addresses_destroy when it's done with them On failure: addresses is NULL */ -typedef void (*grpc_resolve_cb)(void *arg, grpc_resolved_addresses *addresses); +typedef void (*grpc_resolve_cb)(grpc_exec_ctx *exec_ctx, void *arg, + grpc_resolved_addresses *addresses); /* Asynchronously resolve addr. Use default_port if a port isn't designated in addr, otherwise use the port in addr. */ /* TODO(ctiller): add a timeout here */ diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c index ce6972b797..555c74ce7e 100644 --- a/src/core/iomgr/resolve_address_posix.c +++ b/src/core/iomgr/resolve_address_posix.c @@ -41,8 +41,10 @@ #include <sys/un.h> #include <string.h> +#include "src/core/iomgr/executor.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" #include "src/core/support/string.h" #include <grpc/support/alloc.h> #include <grpc/support/host_port.h> @@ -50,13 +52,14 @@ #include <grpc/support/string_util.h> #include <grpc/support/thd.h> #include <grpc/support/time.h> +#include <grpc/support/useful.h> typedef struct { char *name; char *default_port; grpc_resolve_cb cb; + grpc_closure request_closure; void *arg; - grpc_iomgr_object iomgr_object; } request; grpc_resolved_addresses *grpc_blocking_resolve_address( @@ -102,14 +105,18 @@ grpc_resolved_addresses *grpc_blocking_resolve_address( hints.ai_socktype = SOCK_STREAM; /* stream socket */ hints.ai_flags = AI_PASSIVE; /* for wildcard IP address */ + GRPC_SCHEDULING_START_BLOCKING_REGION; s = getaddrinfo(host, port, &hints, &result); + GRPC_SCHEDULING_END_BLOCKING_REGION; + if (s != 0) { /* Retry if well-known service name is recognized */ char *svc[][2] = {{"http", "80"}, {"https", "443"}}; - int i; - for (i = 0; i < (int)(sizeof(svc) / sizeof(svc[0])); i++) { + for (i = 0; i < GPR_ARRAY_SIZE(svc); i++) { if (strcmp(port, svc[i][0]) == 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; s = getaddrinfo(host, svc[i][1], &hints, &result); + GRPC_SCHEDULING_END_BLOCKING_REGION; break; } } @@ -143,8 +150,9 @@ done: return addrs; } -/* Thread function to asynch-ify grpc_blocking_resolve_address */ -static void do_request(void *rp) { +/* Callback to be passed to grpc_executor to asynch-ify + * grpc_blocking_resolve_address */ +static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) { request *r = rp; grpc_resolved_addresses *resolved = grpc_blocking_resolve_address(r->name, r->default_port); @@ -152,8 +160,7 @@ static void do_request(void *rp) { grpc_resolve_cb cb = r->cb; gpr_free(r->name); gpr_free(r->default_port); - cb(arg, resolved); - grpc_iomgr_unregister_object(&r->iomgr_object); + cb(exec_ctx, arg, resolved); gpr_free(r); } @@ -165,17 +172,12 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) { void grpc_resolve_address(const char *name, const char *default_port, grpc_resolve_cb cb, void *arg) { request *r = gpr_malloc(sizeof(request)); - gpr_thd_id id; - char *tmp; - gpr_asprintf(&tmp, "resolve_address:name='%s':default_port='%s'", name, - default_port); - grpc_iomgr_register_object(&r->iomgr_object, tmp); - gpr_free(tmp); + grpc_closure_init(&r->request_closure, do_request_thread, r); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->cb = cb; r->arg = arg; - gpr_thd_new(&id, do_request, r, NULL); + grpc_executor_enqueue(&r->request_closure, 1); } #endif diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/iomgr/resolve_address_windows.c index fb5fd0d4f6..007c855d10 100644 --- a/src/core/iomgr/resolve_address_windows.c +++ b/src/core/iomgr/resolve_address_windows.c @@ -40,12 +40,15 @@ #include <sys/types.h> #include <string.h> +#include "src/core/iomgr/executor.h" #include "src/core/iomgr/iomgr_internal.h" #include "src/core/iomgr/sockaddr_utils.h" +#include "src/core/support/block_annotate.h" #include "src/core/support/string.h" #include <grpc/support/alloc.h> #include <grpc/support/host_port.h> #include <grpc/support/log.h> +#include <grpc/support/log_win32.h> #include <grpc/support/string_util.h> #include <grpc/support/thd.h> #include <grpc/support/time.h> @@ -54,8 +57,8 @@ typedef struct { char *name; char *default_port; grpc_resolve_cb cb; + grpc_closure request_closure; void *arg; - grpc_iomgr_object iomgr_object; } request; grpc_resolved_addresses *grpc_blocking_resolve_address( @@ -88,9 +91,13 @@ grpc_resolved_addresses *grpc_blocking_resolve_address( hints.ai_socktype = SOCK_STREAM; /* stream socket */ hints.ai_flags = AI_PASSIVE; /* for wildcard IP address */ + GRPC_SCHEDULING_START_BLOCKING_REGION; s = getaddrinfo(host, port, &hints, &result); + GRPC_SCHEDULING_END_BLOCKING_REGION; if (s != 0) { - gpr_log(GPR_ERROR, "getaddrinfo: %s", gai_strerror(s)); + char *error_message = gpr_format_message(s); + gpr_log(GPR_ERROR, "getaddrinfo: %s", error_message); + gpr_free(error_message); goto done; } @@ -126,8 +133,9 @@ done: return addrs; } -/* Thread function to asynch-ify grpc_blocking_resolve_address */ -static void do_request(void *rp) { +/* Callback to be passed to grpc_executor to asynch-ify + * grpc_blocking_resolve_address */ +static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, int success) { request *r = rp; grpc_resolved_addresses *resolved = grpc_blocking_resolve_address(r->name, r->default_port); @@ -135,9 +143,8 @@ static void do_request(void *rp) { grpc_resolve_cb cb = r->cb; gpr_free(r->name); gpr_free(r->default_port); - grpc_iomgr_unregister_object(&r->iomgr_object); + cb(exec_ctx, arg, resolved); gpr_free(r); - cb(arg, resolved); } void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) { @@ -148,16 +155,12 @@ void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) { void grpc_resolve_address(const char *name, const char *default_port, grpc_resolve_cb cb, void *arg) { request *r = gpr_malloc(sizeof(request)); - gpr_thd_id id; - const char *label; - gpr_asprintf(&label, "resolve:%s", name); - grpc_iomgr_register_object(&r->iomgr_object, label); - gpr_free(label); + grpc_closure_init(&r->request_closure, do_request_thread, r); r->name = gpr_strdup(name); r->default_port = gpr_strdup(default_port); r->cb = cb; r->arg = arg; - gpr_thd_new(&id, do_request, r, NULL); + grpc_executor_enqueue(&r->request_closure, 1); } #endif diff --git a/src/core/iomgr/sockaddr_utils.c b/src/core/iomgr/sockaddr_utils.c index efdc480365..511a5d5c59 100644 --- a/src/core/iomgr/sockaddr_utils.c +++ b/src/core/iomgr/sockaddr_utils.c @@ -123,15 +123,17 @@ void grpc_sockaddr_make_wildcards(int port, struct sockaddr_in *wild4_out, } void grpc_sockaddr_make_wildcard4(int port, struct sockaddr_in *wild_out) { + GPR_ASSERT(port >= 0 && port < 65536); memset(wild_out, 0, sizeof(*wild_out)); wild_out->sin_family = AF_INET; - wild_out->sin_port = htons(port); + wild_out->sin_port = htons((gpr_uint16)port); } void grpc_sockaddr_make_wildcard6(int port, struct sockaddr_in6 *wild_out) { + GPR_ASSERT(port >= 0 && port < 65536); memset(wild_out, 0, sizeof(*wild_out)); wild_out->sin6_family = AF_INET6; - wild_out->sin6_port = htons(port); + wild_out->sin6_port = htons((gpr_uint16)port); } int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr, @@ -156,8 +158,10 @@ int grpc_sockaddr_to_string(char **out, const struct sockaddr *addr, ip = &addr6->sin6_addr; port = ntohs(addr6->sin6_port); } + /* Windows inet_ntop wants a mutable ip pointer */ if (ip != NULL && - inet_ntop(addr->sa_family, ip, ntop_buf, sizeof(ntop_buf)) != NULL) { + inet_ntop(addr->sa_family, (void *)ip, ntop_buf, sizeof(ntop_buf)) != + NULL) { ret = gpr_join_host_port(out, ntop_buf, port); } else { ret = gpr_asprintf(out, "(sockaddr family=%d)", addr->sa_family); @@ -215,10 +219,12 @@ int grpc_sockaddr_get_port(const struct sockaddr *addr) { int grpc_sockaddr_set_port(const struct sockaddr *addr, int port) { switch (addr->sa_family) { case AF_INET: - ((struct sockaddr_in *)addr)->sin_port = htons(port); + GPR_ASSERT(port >= 0 && port < 65536); + ((struct sockaddr_in *)addr)->sin_port = htons((gpr_uint16)port); return 1; case AF_INET6: - ((struct sockaddr_in6 *)addr)->sin6_port = htons(port); + GPR_ASSERT(port >= 0 && port < 65536); + ((struct sockaddr_in6 *)addr)->sin6_port = htons((gpr_uint16)port); return 1; default: gpr_log(GPR_ERROR, "Unknown socket family %d in grpc_sockaddr_set_port", diff --git a/src/core/iomgr/socket_windows.c b/src/core/iomgr/socket_windows.c index 7d8421376b..fafb7b6622 100644 --- a/src/core/iomgr/socket_windows.c +++ b/src/core/iomgr/socket_windows.c @@ -35,8 +35,12 @@ #ifdef GPR_WINSOCK_SOCKET +#include <winsock2.h> +#include <mswsock.h> + #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/log_win32.h> #include <grpc/support/string_util.h> #include "src/core/iomgr/iocp_windows.h" @@ -62,46 +66,31 @@ grpc_winsocket *grpc_winsocket_create(SOCKET socket, const char *name) { operations to abort them. We need to do that this way because of the various callsites of that function, which happens to be in various mutex hold states, and that'd be unsafe to call them directly. */ -int grpc_winsocket_shutdown(grpc_winsocket *winsocket) { - int callbacks_set = 0; - SOCKET socket; - gpr_mu_lock(&winsocket->state_mu); - socket = winsocket->socket; - if (winsocket->read_info.cb) { - callbacks_set++; - grpc_iomgr_closure_init(&winsocket->shutdown_closure, - winsocket->read_info.cb, - winsocket->read_info.opaque); - grpc_iomgr_add_delayed_callback(&winsocket->shutdown_closure, 0); - } - if (winsocket->write_info.cb) { - callbacks_set++; - grpc_iomgr_closure_init(&winsocket->shutdown_closure, - winsocket->write_info.cb, - winsocket->write_info.opaque); - grpc_iomgr_add_delayed_callback(&winsocket->shutdown_closure, 0); - } - gpr_mu_unlock(&winsocket->state_mu); - closesocket(socket); - return callbacks_set; -} +void grpc_winsocket_shutdown(grpc_winsocket *winsocket) { + /* Grab the function pointer for DisconnectEx for that specific socket. + It may change depending on the interface. */ + int status; + GUID guid = WSAID_DISCONNECTEX; + LPFN_DISCONNECTEX DisconnectEx; + DWORD ioctl_num_bytes; -/* Abandons a socket. Either we're going to queue it up for garbage collecting - from the IO Completion Port thread, or destroy it immediately. Note that this - mechanisms assumes that we're either always waiting for an operation, or we - explicitly know that we don't. If there is a future case where we can have - an "idle" socket which is neither trying to read or write, we'd start leaking - both memory and sockets. */ -void grpc_winsocket_orphan(grpc_winsocket *winsocket) { - grpc_iomgr_unregister_object(&winsocket->iomgr_object); - if (winsocket->read_info.outstanding || winsocket->write_info.outstanding) { - grpc_iocp_socket_orphan(winsocket); + status = WSAIoctl(winsocket->socket, SIO_GET_EXTENSION_FUNCTION_POINTER, + &guid, sizeof(guid), &DisconnectEx, sizeof(DisconnectEx), + &ioctl_num_bytes, NULL, NULL); + + if (status == 0) { + DisconnectEx(winsocket->socket, NULL, 0, 0); } else { - grpc_winsocket_destroy(winsocket); + char *utf8_message = gpr_format_message(WSAGetLastError()); + gpr_log(GPR_ERROR, "Unable to retrieve DisconnectEx pointer : %s", + utf8_message); + gpr_free(utf8_message); } + closesocket(winsocket->socket); } void grpc_winsocket_destroy(grpc_winsocket *winsocket) { + grpc_iomgr_unregister_object(&winsocket->iomgr_object); gpr_mu_destroy(&winsocket->state_mu); gpr_free(winsocket); } diff --git a/src/core/iomgr/socket_windows.h b/src/core/iomgr/socket_windows.h index ecf2530173..dfbfabe1f9 100644 --- a/src/core/iomgr/socket_windows.h +++ b/src/core/iomgr/socket_windows.h @@ -41,6 +41,7 @@ #include <grpc/support/atm.h> #include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/exec_ctx.h" /* This holds the data for an outstanding read or write on a socket. The mutex to protect the concurrent access to that data is the one @@ -54,8 +55,7 @@ typedef struct grpc_winsocket_callback_info { OVERLAPPED overlapped; /* The callback information for the pending operation. May be empty if the caller hasn't registered a callback yet. */ - void (*cb)(void *opaque, int success); - void *opaque; + grpc_closure *closure; /* A boolean to describe if the IO Completion Port got a notification for that operation. This will happen if the operation completed before the called had time to register a callback. We could avoid that behavior @@ -68,8 +68,6 @@ typedef struct grpc_winsocket_callback_info { /* The results of the overlapped operation. */ DWORD bytes_transfered; int wsa_error; - /* A boolean indicating that we started an operation. */ - int outstanding; } grpc_winsocket_callback_info; /* This is a wrapper to a Windows socket. A socket can have one outstanding @@ -92,12 +90,8 @@ typedef struct grpc_winsocket { /* You can't add the same socket twice to the same IO Completion Port. This prevents that. */ int added_to_iocp; - /* A boolean to indicate that the caller has abandoned that socket, but - there is a pending operation that the IO Completion Port will have to - wait for. The socket will be collected at that time. */ - int orphan; - grpc_iomgr_closure shutdown_closure; + grpc_closure shutdown_closure; /* A label for iomgr to track outstanding objects */ grpc_iomgr_object iomgr_object; @@ -108,14 +102,10 @@ typedef struct grpc_winsocket { grpc_winsocket *grpc_winsocket_create(SOCKET socket, const char *name); /* Initiate an asynchronous shutdown of the socket. Will call off any pending - operation to cancel them. Returns the number of callbacks that got setup. */ -int grpc_winsocket_shutdown(grpc_winsocket *socket); + operation to cancel them. */ +void grpc_winsocket_shutdown(grpc_winsocket *socket); -/* Abandon a socket. */ -void grpc_winsocket_orphan(grpc_winsocket *socket); - -/* Destroy a socket. Should only be called by the IO Completion Port thread, - or by grpc_winsocket_orphan if there's no pending operation. */ +/* Destroy a socket. Should only be called if there's no pending operation. */ void grpc_winsocket_destroy(grpc_winsocket *socket); #endif /* GRPC_INTERNAL_CORE_IOMGR_SOCKET_WINDOWS_H */ diff --git a/src/core/iomgr/tcp_client.h b/src/core/iomgr/tcp_client.h index 8ad9b818e1..5e18e71ca2 100644 --- a/src/core/iomgr/tcp_client.h +++ b/src/core/iomgr/tcp_client.h @@ -44,9 +44,10 @@ NULL on failure). interested_parties points to a set of pollsets that would be interested in this connection being established (in order to continue their work) */ -void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), - void *arg, grpc_pollset_set *interested_parties, - const struct sockaddr *addr, int addr_len, +void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_connect, + grpc_endpoint **endpoint, + grpc_pollset_set *interested_parties, + const struct sockaddr *addr, size_t addr_len, gpr_timespec deadline); #endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_CLIENT_H */ diff --git a/src/core/iomgr/tcp_client_posix.c b/src/core/iomgr/tcp_client_posix.c index 66027f87a0..abd6315ca1 100644 --- a/src/core/iomgr/tcp_client_posix.c +++ b/src/core/iomgr/tcp_client_posix.c @@ -42,7 +42,7 @@ #include <string.h> #include <unistd.h> -#include "src/core/iomgr/alarm.h" +#include "src/core/iomgr/timer.h" #include "src/core/iomgr/iomgr_posix.h" #include "src/core/iomgr/pollset_posix.h" #include "src/core/iomgr/sockaddr_utils.h" @@ -54,17 +54,19 @@ #include <grpc/support/string_util.h> #include <grpc/support/time.h> +extern int grpc_tcp_trace; + typedef struct { - void (*cb)(void *arg, grpc_endpoint *tcp); - void *cb_arg; gpr_mu mu; grpc_fd *fd; gpr_timespec deadline; - grpc_alarm alarm; + grpc_timer alarm; int refs; - grpc_iomgr_closure write_closure; + grpc_closure write_closure; grpc_pollset_set *interested_parties; char *addr_str; + grpc_endpoint **ep; + grpc_closure *closure; } async_connect; static int prepare_socket(const struct sockaddr *addr, int fd) { @@ -89,12 +91,16 @@ error: return 0; } -static void tc_on_alarm(void *acp, int success) { +static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int success) { int done; async_connect *ac = acp; + if (grpc_tcp_trace) { + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: success=%d", ac->addr_str, + success); + } gpr_mu_lock(&ac->mu); if (ac->fd != NULL) { - grpc_fd_shutdown(ac->fd); + grpc_fd_shutdown(exec_ctx, ac->fd); } done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); @@ -105,24 +111,28 @@ static void tc_on_alarm(void *acp, int success) { } } -static void on_writable(void *acp, int success) { +static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, int success) { async_connect *ac = acp; int so_error = 0; socklen_t so_error_size; int err; int done; - grpc_endpoint *ep = NULL; - void (*cb)(void *arg, grpc_endpoint *tcp) = ac->cb; - void *cb_arg = ac->cb_arg; + grpc_endpoint **ep = ac->ep; + grpc_closure *closure = ac->closure; grpc_fd *fd; + if (grpc_tcp_trace) { + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: success=%d", + ac->addr_str, success); + } + gpr_mu_lock(&ac->mu); GPR_ASSERT(ac->fd); fd = ac->fd; ac->fd = NULL; gpr_mu_unlock(&ac->mu); - grpc_alarm_cancel(&ac->alarm); + grpc_timer_cancel(exec_ctx, &ac->alarm); gpr_mu_lock(&ac->mu); if (success) { @@ -131,7 +141,8 @@ static void on_writable(void *acp, int success) { err = getsockopt(fd->fd, SOL_SOCKET, SO_ERROR, &so_error, &so_error_size); } while (err < 0 && errno == EINTR); if (err < 0) { - gpr_log(GPR_ERROR, "getsockopt(ERROR): %s", strerror(errno)); + gpr_log(GPR_ERROR, "failed to connect to '%s': getsockopt(ERROR): %s", + ac->addr_str, strerror(errno)); goto finish; } else if (so_error != 0) { if (so_error == ENOBUFS) { @@ -151,36 +162,41 @@ static void on_writable(void *acp, int success) { don't do that! */ gpr_log(GPR_ERROR, "kernel out of buffers"); gpr_mu_unlock(&ac->mu); - grpc_fd_notify_on_write(fd, &ac->write_closure); + grpc_fd_notify_on_write(exec_ctx, fd, &ac->write_closure); return; } else { switch (so_error) { case ECONNREFUSED: - gpr_log(GPR_ERROR, "socket error: connection refused"); + gpr_log( + GPR_ERROR, + "failed to connect to '%s': socket error: connection refused", + ac->addr_str); break; default: - gpr_log(GPR_ERROR, "socket error: %d", so_error); + gpr_log(GPR_ERROR, "failed to connect to '%s': socket error: %d", + ac->addr_str, so_error); break; } goto finish; } } else { - grpc_pollset_set_del_fd(ac->interested_parties, fd); - ep = grpc_tcp_create(fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, ac->addr_str); + grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd); + *ep = grpc_tcp_create(fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, ac->addr_str); fd = NULL; goto finish; } } else { - gpr_log(GPR_ERROR, "on_writable failed during connect"); + gpr_log(GPR_ERROR, "failed to connect to '%s': timeout occurred", + ac->addr_str); goto finish; } - abort(); + GPR_UNREACHABLE_CODE(return ); finish: if (fd != NULL) { - grpc_pollset_set_del_fd(ac->interested_parties, fd); - grpc_fd_orphan(fd, NULL, "tcp_client_orphan"); + grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd); + grpc_fd_orphan(exec_ctx, fd, NULL, "tcp_client_orphan"); fd = NULL; } done = (--ac->refs == 0); @@ -190,12 +206,13 @@ finish: gpr_free(ac->addr_str); gpr_free(ac); } - cb(cb_arg, ep); + grpc_exec_ctx_enqueue(exec_ctx, closure, *ep != NULL); } -void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), - void *arg, grpc_pollset_set *interested_parties, - const struct sockaddr *addr, int addr_len, +void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_endpoint **ep, + grpc_pollset_set *interested_parties, + const struct sockaddr *addr, size_t addr_len, gpr_timespec deadline) { int fd; grpc_dualstack_mode dsmode; @@ -207,6 +224,8 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), char *name; char *addr_str; + *ep = NULL; + /* Use dualstack sockets where available. */ if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = (const struct sockaddr *)&addr6_v4mapped; @@ -224,12 +243,13 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), addr_len = sizeof(addr4_copy); } if (!prepare_socket(addr, fd)) { - cb(arg, NULL); + grpc_exec_ctx_enqueue(exec_ctx, closure, 0); return; } do { - err = connect(fd, addr, addr_len); + GPR_ASSERT(addr_len < ~(socklen_t)0); + err = connect(fd, addr, (socklen_t)addr_len); } while (err < 0 && errno == EINTR); addr_str = grpc_sockaddr_to_uri(addr); @@ -238,22 +258,23 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), fdobj = grpc_fd_create(fd, name); if (err >= 0) { - cb(arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str)); + *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str); + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); goto done; } if (errno != EWOULDBLOCK && errno != EINPROGRESS) { gpr_log(GPR_ERROR, "connect error to '%s': %s", addr_str, strerror(errno)); - grpc_fd_orphan(fdobj, NULL, "tcp_client_connect_error"); - cb(arg, NULL); + grpc_fd_orphan(exec_ctx, fdobj, NULL, "tcp_client_connect_error"); + grpc_exec_ctx_enqueue(exec_ctx, closure, 0); goto done; } - grpc_pollset_set_add_fd(interested_parties, fdobj); + grpc_pollset_set_add_fd(exec_ctx, interested_parties, fdobj); ac = gpr_malloc(sizeof(async_connect)); - ac->cb = cb; - ac->cb_arg = arg; + ac->closure = closure; + ac->ep = ep; ac->fd = fdobj; ac->interested_parties = interested_parties; ac->addr_str = addr_str; @@ -263,11 +284,16 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *ep), ac->write_closure.cb = on_writable; ac->write_closure.cb_arg = ac; + if (grpc_tcp_trace) { + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", + ac->addr_str); + } + gpr_mu_lock(&ac->mu); - grpc_alarm_init(&ac->alarm, + grpc_timer_init(exec_ctx, &ac->alarm, gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), tc_on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC)); - grpc_fd_notify_on_write(ac->fd, &ac->write_closure); + grpc_fd_notify_on_write(exec_ctx, ac->fd, &ac->write_closure); gpr_mu_unlock(&ac->mu); done: diff --git a/src/core/iomgr/tcp_client_windows.c b/src/core/iomgr/tcp_client_windows.c index 79a58fe2af..e5691b7e12 100644 --- a/src/core/iomgr/tcp_client_windows.c +++ b/src/core/iomgr/tcp_client_windows.c @@ -43,7 +43,7 @@ #include <grpc/support/slice_buffer.h> #include <grpc/support/useful.h> -#include "src/core/iomgr/alarm.h" +#include "src/core/iomgr/timer.h" #include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/tcp_client.h" #include "src/core/iomgr/tcp_windows.h" @@ -52,96 +52,77 @@ #include "src/core/iomgr/socket_windows.h" typedef struct { - void (*cb)(void *arg, grpc_endpoint *tcp); - void *cb_arg; + grpc_closure *on_done; gpr_mu mu; grpc_winsocket *socket; gpr_timespec deadline; - grpc_alarm alarm; + grpc_timer alarm; char *addr_name; int refs; - int aborted; + grpc_closure on_connect; + grpc_endpoint **endpoint; } async_connect; -static void async_connect_cleanup(async_connect *ac) { +static void async_connect_unlock_and_cleanup(async_connect *ac) { int done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { + if (ac->socket != NULL) grpc_winsocket_destroy(ac->socket); gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_name); gpr_free(ac); } } -static void on_alarm(void *acp, int occured) { +static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, int occured) { async_connect *ac = acp; gpr_mu_lock(&ac->mu); /* If the alarm didn't occur, it got cancelled. */ if (ac->socket != NULL && occured) { grpc_winsocket_shutdown(ac->socket); } - async_connect_cleanup(ac); + async_connect_unlock_and_cleanup(ac); } -static void on_connect(void *acp, int from_iocp) { +static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, int from_iocp) { async_connect *ac = acp; SOCKET sock = ac->socket->socket; - grpc_endpoint *ep = NULL; + grpc_endpoint **ep = ac->endpoint; grpc_winsocket_callback_info *info = &ac->socket->write_info; - void (*cb)(void *arg, grpc_endpoint *tcp) = ac->cb; - void *cb_arg = ac->cb_arg; - int aborted; + grpc_closure *on_done = ac->on_done; - grpc_alarm_cancel(&ac->alarm); + grpc_timer_cancel(exec_ctx, &ac->alarm); gpr_mu_lock(&ac->mu); - aborted = ac->aborted; if (from_iocp) { DWORD transfered_bytes = 0; DWORD flags; BOOL wsa_success = WSAGetOverlappedResult(sock, &info->overlapped, &transfered_bytes, FALSE, &flags); - info->outstanding = 0; GPR_ASSERT(transfered_bytes == 0); if (!wsa_success) { char *utf8_message = gpr_format_message(WSAGetLastError()); gpr_log(GPR_ERROR, "on_connect error: %s", utf8_message); gpr_free(utf8_message); - } else if (!aborted) { - ep = grpc_tcp_create(ac->socket, ac->addr_name); + } else { + *ep = grpc_tcp_create(ac->socket, ac->addr_name); + ac->socket = NULL; } - } else { - gpr_log(GPR_ERROR, "on_connect is shutting down"); - /* If the connection timeouts, we will still get a notification from - the IOCP whatever happens. So we're just going to flag that connection - as being in the process of being aborted, and wait for the IOCP. We - can't just orphan the socket now, because the IOCP might already have - gotten a successful connection, which is our worst-case scenario. - We need to call our callback now to respect the deadline. */ - ac->aborted = 1; - gpr_mu_unlock(&ac->mu); - cb(cb_arg, NULL); - return; } - ac->socket->write_info.outstanding = 0; - - /* If we don't have an endpoint, it means the connection failed, - so it doesn't matter if it aborted or failed. We need to orphan - that socket. */ - if (!ep || aborted) grpc_winsocket_orphan(ac->socket); - async_connect_cleanup(ac); + async_connect_unlock_and_cleanup(ac); /* If the connection was aborted, the callback was already called when the deadline was met. */ - if (!aborted) cb(cb_arg, ep); + on_done->cb(exec_ctx, on_done->cb_arg, *ep != NULL); } /* Tries to issue one async connection, then schedules both an IOCP notification request for the connection, and one timeout alert. */ -void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), - void *arg, grpc_pollset_set *interested_parties, - const struct sockaddr *addr, int addr_len, +void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, + grpc_endpoint **endpoint, + grpc_pollset_set *interested_parties, + const struct sockaddr *addr, size_t addr_len, gpr_timespec deadline) { SOCKET sock = INVALID_SOCKET; BOOL success; @@ -157,6 +138,8 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), char *utf8_message; grpc_winsocket_callback_info *info; + *endpoint = NULL; + /* Use dualstack sockets where available. */ if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = (const struct sockaddr *)&addr6_v4mapped; @@ -196,8 +179,8 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), socket = grpc_winsocket_create(sock, "client"); info = &socket->write_info; - info->outstanding = 1; - success = ConnectEx(sock, addr, addr_len, NULL, 0, NULL, &info->overlapped); + success = + ConnectEx(sock, addr, (int)addr_len, NULL, 0, NULL, &info->overlapped); /* It wouldn't be unusual to get a success immediately. But we'll still get an IOCP notification, so let's ignore it. */ @@ -210,30 +193,29 @@ void grpc_tcp_client_connect(void (*cb)(void *arg, grpc_endpoint *tcp), } ac = gpr_malloc(sizeof(async_connect)); - ac->cb = cb; - ac->cb_arg = arg; + ac->on_done = on_done; ac->socket = socket; gpr_mu_init(&ac->mu); ac->refs = 2; ac->addr_name = grpc_sockaddr_to_uri(addr); - ac->aborted = 0; + ac->endpoint = endpoint; + grpc_closure_init(&ac->on_connect, on_connect, ac); - grpc_alarm_init(&ac->alarm, deadline, on_alarm, ac, + grpc_timer_init(exec_ctx, &ac->alarm, deadline, on_alarm, ac, gpr_now(GPR_CLOCK_MONOTONIC)); - socket->write_info.outstanding = 1; - grpc_socket_notify_on_write(socket, on_connect, ac); + grpc_socket_notify_on_write(exec_ctx, socket, &ac->on_connect); return; failure: utf8_message = gpr_format_message(WSAGetLastError()); gpr_log(GPR_ERROR, message, utf8_message); gpr_free(utf8_message); - if (socket) { - grpc_winsocket_orphan(socket); + if (socket != NULL) { + grpc_winsocket_destroy(socket); } else if (sock != INVALID_SOCKET) { closesocket(sock); } - cb(arg, NULL); + grpc_exec_ctx_enqueue(exec_ctx, on_done, 0); } #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index 360e6ebd8c..915553d509 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -61,292 +61,135 @@ #define SENDMSG_FLAGS 0 #endif -/* Holds a slice array and associated state. */ -typedef struct grpc_tcp_slice_state { - gpr_slice *slices; /* Array of slices */ - size_t nslices; /* Size of slices array. */ - ssize_t first_slice; /* First valid slice in array */ - ssize_t last_slice; /* Last valid slice in array */ - gpr_slice working_slice; /* pointer to original final slice */ - int working_slice_valid; /* True if there is a working slice */ - int memory_owned; /* True if slices array is owned */ -} grpc_tcp_slice_state; +#ifdef GPR_MSG_IOVLEN_TYPE +typedef GPR_MSG_IOVLEN_TYPE msg_iovlen_type; +#else +typedef size_t msg_iovlen_type; +#endif int grpc_tcp_trace = 0; -static void slice_state_init(grpc_tcp_slice_state *state, gpr_slice *slices, - size_t nslices, size_t valid_slices) { - state->slices = slices; - state->nslices = nslices; - if (valid_slices == 0) { - state->first_slice = -1; - } else { - state->first_slice = 0; - } - state->last_slice = valid_slices - 1; - state->working_slice_valid = 0; - state->memory_owned = 0; -} - -/* Returns true if there is still available data */ -static int slice_state_has_available(grpc_tcp_slice_state *state) { - return state->first_slice != -1 && state->last_slice >= state->first_slice; -} - -static ssize_t slice_state_slices_allocated(grpc_tcp_slice_state *state) { - if (state->first_slice == -1) { - return 0; - } else { - return state->last_slice - state->first_slice + 1; - } -} - -static void slice_state_realloc(grpc_tcp_slice_state *state, size_t new_size) { - /* TODO(klempner): use realloc instead when first_slice is 0 */ - /* TODO(klempner): Avoid a realloc in cases where it is unnecessary */ - gpr_slice *slices = state->slices; - size_t original_size = slice_state_slices_allocated(state); - size_t i; - gpr_slice *new_slices = gpr_malloc(sizeof(gpr_slice) * new_size); - - for (i = 0; i < original_size; ++i) { - new_slices[i] = slices[i + state->first_slice]; - } - - state->slices = new_slices; - state->last_slice = original_size - 1; - if (original_size > 0) { - state->first_slice = 0; - } else { - state->first_slice = -1; - } - state->nslices = new_size; - - if (state->memory_owned) { - gpr_free(slices); - } - state->memory_owned = 1; -} - -static void slice_state_remove_prefix(grpc_tcp_slice_state *state, - size_t prefix_bytes) { - gpr_slice *current_slice = &state->slices[state->first_slice]; - size_t current_slice_size; - - while (slice_state_has_available(state)) { - current_slice_size = GPR_SLICE_LENGTH(*current_slice); - if (current_slice_size > prefix_bytes) { - /* TODO(klempner): Get rid of the extra refcount created here by adding a - native "trim the first N bytes" operation to splice */ - /* TODO(klempner): This really shouldn't be modifying the current slice - unless we own the slices array. */ - gpr_slice tail; - tail = gpr_slice_split_tail(current_slice, prefix_bytes); - gpr_slice_unref(*current_slice); - *current_slice = tail; - return; - } else { - gpr_slice_unref(*current_slice); - ++state->first_slice; - ++current_slice; - prefix_bytes -= current_slice_size; - } - } -} - -static void slice_state_destroy(grpc_tcp_slice_state *state) { - while (slice_state_has_available(state)) { - gpr_slice_unref(state->slices[state->first_slice]); - ++state->first_slice; - } - - if (state->memory_owned) { - gpr_free(state->slices); - state->memory_owned = 0; - } -} - -void slice_state_transfer_ownership(grpc_tcp_slice_state *state, - gpr_slice **slices, size_t *nslices) { - *slices = state->slices + state->first_slice; - *nslices = state->last_slice - state->first_slice + 1; - - state->first_slice = -1; - state->last_slice = -1; -} - -/* Fills iov with the first min(iov_size, available) slices, returns number - filled */ -static size_t slice_state_to_iovec(grpc_tcp_slice_state *state, - struct iovec *iov, size_t iov_size) { - size_t nslices = state->last_slice - state->first_slice + 1; - gpr_slice *slices = state->slices + state->first_slice; - size_t i; - if (nslices < iov_size) { - iov_size = nslices; - } - - for (i = 0; i < iov_size; ++i) { - iov[i].iov_base = GPR_SLICE_START_PTR(slices[i]); - iov[i].iov_len = GPR_SLICE_LENGTH(slices[i]); - } - return iov_size; -} - -/* Makes n blocks available at the end of state, writes them into iov, and - returns the number of bytes allocated */ -static size_t slice_state_append_blocks_into_iovec(grpc_tcp_slice_state *state, - struct iovec *iov, size_t n, - size_t slice_size) { - size_t target_size; - size_t i; - size_t allocated_bytes; - ssize_t allocated_slices = slice_state_slices_allocated(state); - - if (n - state->working_slice_valid >= state->nslices - state->last_slice) { - /* Need to grow the slice array */ - target_size = state->nslices; - do { - target_size = target_size * 2; - } while (target_size < allocated_slices + n - state->working_slice_valid); - /* TODO(klempner): If this ever needs to support both prefix removal and - append, we should be smarter about the growth logic here */ - slice_state_realloc(state, target_size); - } - - i = 0; - allocated_bytes = 0; - - if (state->working_slice_valid) { - iov[0].iov_base = GPR_SLICE_END_PTR(state->slices[state->last_slice]); - iov[0].iov_len = GPR_SLICE_LENGTH(state->working_slice) - - GPR_SLICE_LENGTH(state->slices[state->last_slice]); - allocated_bytes += iov[0].iov_len; - ++i; - state->slices[state->last_slice] = state->working_slice; - state->working_slice_valid = 0; - } - - for (; i < n; ++i) { - ++state->last_slice; - state->slices[state->last_slice] = gpr_slice_malloc(slice_size); - iov[i].iov_base = GPR_SLICE_START_PTR(state->slices[state->last_slice]); - iov[i].iov_len = slice_size; - allocated_bytes += slice_size; - } - if (state->first_slice == -1) { - state->first_slice = 0; - } - return allocated_bytes; -} - -/* Remove the last n bytes from state */ -/* TODO(klempner): Consider having this defer actual deletion until later */ -static void slice_state_remove_last(grpc_tcp_slice_state *state, size_t bytes) { - while (bytes > 0 && slice_state_has_available(state)) { - if (GPR_SLICE_LENGTH(state->slices[state->last_slice]) > bytes) { - state->working_slice = state->slices[state->last_slice]; - state->working_slice_valid = 1; - /* TODO(klempner): Combine these into a single operation that doesn't need - to refcount */ - gpr_slice_unref(gpr_slice_split_tail( - &state->slices[state->last_slice], - GPR_SLICE_LENGTH(state->slices[state->last_slice]) - bytes)); - bytes = 0; - } else { - bytes -= GPR_SLICE_LENGTH(state->slices[state->last_slice]); - gpr_slice_unref(state->slices[state->last_slice]); - --state->last_slice; - if (state->last_slice == -1) { - state->first_slice = -1; - } - } - } -} - typedef struct { grpc_endpoint base; grpc_fd *em_fd; int fd; - int iov_size; /* Number of slices to allocate per read attempt */ int finished_edge; + msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */ size_t slice_size; gpr_refcount refcount; - grpc_endpoint_read_cb read_cb; - void *read_user_data; - grpc_endpoint_write_cb write_cb; - void *write_user_data; + /* garbage after the last read */ + gpr_slice_buffer last_read_buffer; - grpc_tcp_slice_state write_state; + gpr_slice_buffer *incoming_buffer; + gpr_slice_buffer *outgoing_buffer; + /** slice within outgoing_buffer to write next */ + size_t outgoing_slice_idx; + /** byte within outgoing_buffer->slices[outgoing_slice_idx] to write next */ + size_t outgoing_byte_idx; - grpc_iomgr_closure read_closure; - grpc_iomgr_closure write_closure; + grpc_closure *read_cb; + grpc_closure *write_cb; - grpc_iomgr_closure handle_read_closure; + grpc_closure read_closure; + grpc_closure write_closure; char *peer_string; } grpc_tcp; -static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success); -static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success); +static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + int success); +static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + int success); -static void grpc_tcp_shutdown(grpc_endpoint *ep) { +static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - grpc_fd_shutdown(tcp->em_fd); + grpc_fd_shutdown(exec_ctx, tcp->em_fd); } -static void grpc_tcp_unref(grpc_tcp *tcp) { - int refcount_zero = gpr_unref(&tcp->refcount); - if (refcount_zero) { - grpc_fd_orphan(tcp->em_fd, NULL, "tcp_unref_orphan"); - gpr_free(tcp->peer_string); - gpr_free(tcp); +static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { + grpc_fd_orphan(exec_ctx, tcp->em_fd, NULL, "tcp_unref_orphan"); + gpr_slice_buffer_destroy(&tcp->last_read_buffer); + gpr_free(tcp->peer_string); + gpr_free(tcp); +} + +/*#define GRPC_TCP_REFCOUNT_DEBUG*/ +#ifdef GRPC_TCP_REFCOUNT_DEBUG +#define TCP_UNREF(cl, tcp, reason) \ + tcp_unref((cl), (tcp), (reason), __FILE__, __LINE__) +#define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__) +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, + const char *reason, const char *file, int line) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP unref %p : %s %d -> %d", tcp, + reason, tcp->refcount.count, tcp->refcount.count - 1); + if (gpr_unref(&tcp->refcount)) { + tcp_free(exec_ctx, tcp); + } +} + +static void tcp_ref(grpc_tcp *tcp, const char *reason, const char *file, + int line) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP ref %p : %s %d -> %d", tcp, + reason, tcp->refcount.count, tcp->refcount.count + 1); + gpr_ref(&tcp->refcount); +} +#else +#define TCP_UNREF(cl, tcp, reason) tcp_unref((cl), (tcp)) +#define TCP_REF(tcp, reason) tcp_ref((tcp)) +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { + if (gpr_unref(&tcp->refcount)) { + tcp_free(exec_ctx, tcp); } } -static void grpc_tcp_destroy(grpc_endpoint *ep) { +static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } +#endif + +static void tcp_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - grpc_tcp_unref(tcp); + TCP_UNREF(exec_ctx, tcp, "destroy"); } -static void call_read_cb(grpc_tcp *tcp, gpr_slice *slices, size_t nslices, - grpc_endpoint_cb_status status) { - grpc_endpoint_read_cb cb = tcp->read_cb; +static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, int success) { + grpc_closure *cb = tcp->read_cb; if (grpc_tcp_trace) { size_t i; - gpr_log(GPR_DEBUG, "read: status=%d", status); - for (i = 0; i < nslices; i++) { - char *dump = gpr_dump_slice(slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); + gpr_log(GPR_DEBUG, "read: success=%d", success); + for (i = 0; i < tcp->incoming_buffer->count; i++) { + char *dump = gpr_dump_slice(tcp->incoming_buffer->slices[i], + GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "READ %p: %s", tcp, dump); gpr_free(dump); } } tcp->read_cb = NULL; - cb(tcp->read_user_data, slices, nslices, status); + tcp->incoming_buffer = NULL; + cb->cb(exec_ctx, cb->cb_arg, success); } -#define INLINE_SLICE_BUFFER_SIZE 8 #define MAX_READ_IOVEC 4 -static void grpc_tcp_continue_read(grpc_tcp *tcp) { - gpr_slice static_read_slices[INLINE_SLICE_BUFFER_SIZE]; +static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; ssize_t read_bytes; - ssize_t allocated_bytes; - struct grpc_tcp_slice_state read_state; - gpr_slice *final_slices; - size_t final_nslices; + size_t i; GPR_ASSERT(!tcp->finished_edge); - GRPC_TIMER_BEGIN(GRPC_PTAG_HANDLE_READ, 0); - slice_state_init(&read_state, static_read_slices, INLINE_SLICE_BUFFER_SIZE, - 0); + GPR_ASSERT(tcp->iov_size <= MAX_READ_IOVEC); + GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC); + GPR_TIMER_BEGIN("tcp_continue_read", 0); - allocated_bytes = slice_state_append_blocks_into_iovec( - &read_state, iov, tcp->iov_size, tcp->slice_size); + while (tcp->incoming_buffer->count < (size_t)tcp->iov_size) { + gpr_slice_buffer_add_indexed(tcp->incoming_buffer, + gpr_slice_malloc(tcp->slice_size)); + } + for (i = 0; i < tcp->incoming_buffer->count; i++) { + iov[i].iov_base = GPR_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); + iov[i].iov_len = GPR_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); + } msg.msg_name = NULL; msg.msg_namelen = 0; @@ -356,112 +199,114 @@ static void grpc_tcp_continue_read(grpc_tcp *tcp) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_BEGIN(GRPC_PTAG_RECVMSG, 0); + GPR_TIMER_BEGIN("recvmsg", 1); do { read_bytes = recvmsg(tcp->fd, &msg, 0); } while (read_bytes < 0 && errno == EINTR); - GRPC_TIMER_END(GRPC_PTAG_RECVMSG, 0); - - if (read_bytes < allocated_bytes) { - /* TODO(klempner): Consider a second read first, in hopes of getting a - * quick EAGAIN and saving a bunch of allocations. */ - slice_state_remove_last(&read_state, read_bytes < 0 - ? allocated_bytes - : allocated_bytes - read_bytes); - } + GPR_TIMER_END("recvmsg", 0); if (read_bytes < 0) { - /* NB: After calling the user_cb a parallel call of the read handler may + /* NB: After calling call_read_cb a parallel call of the read handler may * be running. */ if (errno == EAGAIN) { if (tcp->iov_size > 1) { tcp->iov_size /= 2; } - if (slice_state_has_available(&read_state)) { - /* TODO(klempner): We should probably do the call into the application - without all this junk on the stack */ - /* FIXME(klempner): Refcount properly */ - slice_state_transfer_ownership(&read_state, &final_slices, - &final_nslices); - tcp->finished_edge = 1; - call_read_cb(tcp, final_slices, final_nslices, GRPC_ENDPOINT_CB_OK); - slice_state_destroy(&read_state); - grpc_tcp_unref(tcp); - } else { - /* We've consumed the edge, request a new one */ - slice_state_destroy(&read_state); - grpc_fd_notify_on_read(tcp->em_fd, &tcp->read_closure); - } + /* We've consumed the edge, request a new one */ + grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { /* TODO(klempner): Log interesting errors */ - call_read_cb(tcp, NULL, 0, GRPC_ENDPOINT_CB_ERROR); - slice_state_destroy(&read_state); - grpc_tcp_unref(tcp); + gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + call_read_cb(exec_ctx, tcp, 0); + TCP_UNREF(exec_ctx, tcp, "read"); } } else if (read_bytes == 0) { /* 0 read size ==> end of stream */ - if (slice_state_has_available(&read_state)) { - /* there were bytes already read: pass them up to the application */ - slice_state_transfer_ownership(&read_state, &final_slices, - &final_nslices); - call_read_cb(tcp, final_slices, final_nslices, GRPC_ENDPOINT_CB_EOF); - } else { - call_read_cb(tcp, NULL, 0, GRPC_ENDPOINT_CB_EOF); - } - slice_state_destroy(&read_state); - grpc_tcp_unref(tcp); + gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + call_read_cb(exec_ctx, tcp, 0); + TCP_UNREF(exec_ctx, tcp, "read"); } else { - if (tcp->iov_size < MAX_READ_IOVEC) { + GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); + if ((size_t)read_bytes < tcp->incoming_buffer->length) { + gpr_slice_buffer_trim_end( + tcp->incoming_buffer, + tcp->incoming_buffer->length - (size_t)read_bytes, + &tcp->last_read_buffer); + } else if (tcp->iov_size < MAX_READ_IOVEC) { ++tcp->iov_size; } - GPR_ASSERT(slice_state_has_available(&read_state)); - slice_state_transfer_ownership(&read_state, &final_slices, &final_nslices); - call_read_cb(tcp, final_slices, final_nslices, GRPC_ENDPOINT_CB_OK); - slice_state_destroy(&read_state); - grpc_tcp_unref(tcp); + GPR_ASSERT((size_t)read_bytes == tcp->incoming_buffer->length); + call_read_cb(exec_ctx, tcp, 1); + TCP_UNREF(exec_ctx, tcp, "read"); } - GRPC_TIMER_END(GRPC_PTAG_HANDLE_READ, 0); + GPR_TIMER_END("tcp_continue_read", 0); } -static void grpc_tcp_handle_read(void *arg /* grpc_tcp */, int success) { +static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + int success) { grpc_tcp *tcp = (grpc_tcp *)arg; GPR_ASSERT(!tcp->finished_edge); if (!success) { - call_read_cb(tcp, NULL, 0, GRPC_ENDPOINT_CB_SHUTDOWN); - grpc_tcp_unref(tcp); + gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + call_read_cb(exec_ctx, tcp, 0); + TCP_UNREF(exec_ctx, tcp, "read"); } else { - grpc_tcp_continue_read(tcp); + tcp_continue_read(exec_ctx, tcp); } } -static void grpc_tcp_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, - void *user_data) { +static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *incoming_buffer, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; GPR_ASSERT(tcp->read_cb == NULL); tcp->read_cb = cb; - tcp->read_user_data = user_data; - gpr_ref(&tcp->refcount); + tcp->incoming_buffer = incoming_buffer; + gpr_slice_buffer_reset_and_unref(incoming_buffer); + gpr_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer); + TCP_REF(tcp, "read"); if (tcp->finished_edge) { tcp->finished_edge = 0; - grpc_fd_notify_on_read(tcp->em_fd, &tcp->read_closure); + grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { - tcp->handle_read_closure.cb_arg = tcp; - grpc_iomgr_add_delayed_callback(&tcp->handle_read_closure, 1); + grpc_exec_ctx_enqueue(exec_ctx, &tcp->read_closure, 1); } } +typedef enum { FLUSH_DONE, FLUSH_PENDING, FLUSH_ERROR } flush_result; + #define MAX_WRITE_IOVEC 16 -static grpc_endpoint_write_status grpc_tcp_flush(grpc_tcp *tcp) { +static flush_result tcp_flush(grpc_tcp *tcp) { struct msghdr msg; struct iovec iov[MAX_WRITE_IOVEC]; - int iov_size; + msg_iovlen_type iov_size; ssize_t sent_length; - grpc_tcp_slice_state *state = &tcp->write_state; + size_t sending_length; + size_t trailing; + size_t unwind_slice_idx; + size_t unwind_byte_idx; for (;;) { - iov_size = slice_state_to_iovec(state, iov, MAX_WRITE_IOVEC); + sending_length = 0; + unwind_slice_idx = tcp->outgoing_slice_idx; + unwind_byte_idx = tcp->outgoing_byte_idx; + for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count && + iov_size != MAX_WRITE_IOVEC; + iov_size++) { + iov[iov_size].iov_base = + GPR_SLICE_START_PTR( + tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) + + tcp->outgoing_byte_idx; + iov[iov_size].iov_len = + GPR_SLICE_LENGTH( + tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) - + tcp->outgoing_byte_idx; + sending_length += iov[iov_size].iov_len; + tcp->outgoing_slice_idx++; + tcp->outgoing_byte_idx = 0; + } + GPR_ASSERT(iov_size > 0); msg.msg_name = NULL; msg.msg_namelen = 0; @@ -471,123 +316,133 @@ static grpc_endpoint_write_status grpc_tcp_flush(grpc_tcp *tcp) { msg.msg_controllen = 0; msg.msg_flags = 0; - GRPC_TIMER_BEGIN(GRPC_PTAG_SENDMSG, 0); + GPR_TIMER_BEGIN("sendmsg", 1); do { /* TODO(klempner): Cork if this is a partial write */ sent_length = sendmsg(tcp->fd, &msg, SENDMSG_FLAGS); } while (sent_length < 0 && errno == EINTR); - GRPC_TIMER_END(GRPC_PTAG_SENDMSG, 0); + GPR_TIMER_END("sendmsg", 0); if (sent_length < 0) { if (errno == EAGAIN) { - return GRPC_ENDPOINT_WRITE_PENDING; + tcp->outgoing_slice_idx = unwind_slice_idx; + tcp->outgoing_byte_idx = unwind_byte_idx; + return FLUSH_PENDING; } else { /* TODO(klempner): Log some of these */ - slice_state_destroy(state); - return GRPC_ENDPOINT_WRITE_ERROR; + return FLUSH_ERROR; } } - /* TODO(klempner): Probably better to batch this after we finish flushing */ - slice_state_remove_prefix(state, sent_length); + GPR_ASSERT(tcp->outgoing_byte_idx == 0); + trailing = sending_length - (size_t)sent_length; + while (trailing > 0) { + size_t slice_length; + + tcp->outgoing_slice_idx--; + slice_length = GPR_SLICE_LENGTH( + tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]); + if (slice_length > trailing) { + tcp->outgoing_byte_idx = slice_length - trailing; + break; + } else { + trailing -= slice_length; + } + } - if (!slice_state_has_available(state)) { - return GRPC_ENDPOINT_WRITE_DONE; + if (tcp->outgoing_slice_idx == tcp->outgoing_buffer->count) { + return FLUSH_DONE; } }; } -static void grpc_tcp_handle_write(void *arg /* grpc_tcp */, int success) { +static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, + int success) { grpc_tcp *tcp = (grpc_tcp *)arg; - grpc_endpoint_write_status write_status; - grpc_endpoint_cb_status cb_status; - grpc_endpoint_write_cb cb; + flush_result status; + grpc_closure *cb; if (!success) { - slice_state_destroy(&tcp->write_state); cb = tcp->write_cb; tcp->write_cb = NULL; - cb(tcp->write_user_data, GRPC_ENDPOINT_CB_SHUTDOWN); - grpc_tcp_unref(tcp); + cb->cb(exec_ctx, cb->cb_arg, 0); + TCP_UNREF(exec_ctx, tcp, "write"); return; } - GRPC_TIMER_BEGIN(GRPC_PTAG_TCP_CB_WRITE, 0); - write_status = grpc_tcp_flush(tcp); - if (write_status == GRPC_ENDPOINT_WRITE_PENDING) { - grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); + status = tcp_flush(tcp); + if (status == FLUSH_PENDING) { + grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { - slice_state_destroy(&tcp->write_state); - if (write_status == GRPC_ENDPOINT_WRITE_DONE) { - cb_status = GRPC_ENDPOINT_CB_OK; - } else { - cb_status = GRPC_ENDPOINT_CB_ERROR; - } cb = tcp->write_cb; tcp->write_cb = NULL; - cb(tcp->write_user_data, cb_status); - grpc_tcp_unref(tcp); + GPR_TIMER_BEGIN("tcp_handle_write.cb", 0); + cb->cb(exec_ctx, cb->cb_arg, status == FLUSH_DONE); + GPR_TIMER_END("tcp_handle_write.cb", 0); + TCP_UNREF(exec_ctx, tcp, "write"); } - GRPC_TIMER_END(GRPC_PTAG_TCP_CB_WRITE, 0); } -static grpc_endpoint_write_status grpc_tcp_write(grpc_endpoint *ep, - gpr_slice *slices, - size_t nslices, - grpc_endpoint_write_cb cb, - void *user_data) { +static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *buf, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; - grpc_endpoint_write_status status; + flush_result status; if (grpc_tcp_trace) { size_t i; - for (i = 0; i < nslices; i++) { - char *data = gpr_dump_slice(slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); + for (i = 0; i < buf->count; i++) { + char *data = + gpr_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "WRITE %p: %s", tcp, data); gpr_free(data); } } - GRPC_TIMER_BEGIN(GRPC_PTAG_TCP_WRITE, 0); + GPR_TIMER_BEGIN("tcp_write", 0); GPR_ASSERT(tcp->write_cb == NULL); - slice_state_init(&tcp->write_state, slices, nslices, nslices); - status = grpc_tcp_flush(tcp); - if (status == GRPC_ENDPOINT_WRITE_PENDING) { - /* TODO(klempner): Consider inlining rather than malloc for small nslices */ - slice_state_realloc(&tcp->write_state, nslices); - gpr_ref(&tcp->refcount); + if (buf->length == 0) { + GPR_TIMER_END("tcp_write", 0); + grpc_exec_ctx_enqueue(exec_ctx, cb, 1); + return; + } + tcp->outgoing_buffer = buf; + tcp->outgoing_slice_idx = 0; + tcp->outgoing_byte_idx = 0; + + status = tcp_flush(tcp); + if (status == FLUSH_PENDING) { + TCP_REF(tcp, "write"); tcp->write_cb = cb; - tcp->write_user_data = user_data; - grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_closure); + grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); + } else { + grpc_exec_ctx_enqueue(exec_ctx, cb, status == FLUSH_DONE); } - GRPC_TIMER_END(GRPC_PTAG_TCP_WRITE, 0); - return status; + GPR_TIMER_END("tcp_write", 0); } -static void grpc_tcp_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) { +static void tcp_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset *pollset) { grpc_tcp *tcp = (grpc_tcp *)ep; - grpc_pollset_add_fd(pollset, tcp->em_fd); + grpc_pollset_add_fd(exec_ctx, pollset, tcp->em_fd); } -static void grpc_tcp_add_to_pollset_set(grpc_endpoint *ep, - grpc_pollset_set *pollset_set) { +static void tcp_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset_set *pollset_set) { grpc_tcp *tcp = (grpc_tcp *)ep; - grpc_pollset_set_add_fd(pollset_set, tcp->em_fd); + grpc_pollset_set_add_fd(exec_ctx, pollset_set, tcp->em_fd); } -static char *grpc_tcp_get_peer(grpc_endpoint *ep) { +static char *tcp_get_peer(grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; return gpr_strdup(tcp->peer_string); } static const grpc_endpoint_vtable vtable = { - grpc_tcp_notify_on_read, grpc_tcp_write, - grpc_tcp_add_to_pollset, grpc_tcp_add_to_pollset_set, - grpc_tcp_shutdown, grpc_tcp_destroy, - grpc_tcp_get_peer}; + tcp_read, tcp_write, tcp_add_to_pollset, tcp_add_to_pollset_set, + tcp_shutdown, tcp_destroy, tcp_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, const char *peer_string) { @@ -597,21 +452,19 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, tcp->fd = em_fd->fd; tcp->read_cb = NULL; tcp->write_cb = NULL; - tcp->read_user_data = NULL; - tcp->write_user_data = NULL; + tcp->incoming_buffer = NULL; tcp->slice_size = slice_size; tcp->iov_size = 1; tcp->finished_edge = 1; - slice_state_init(&tcp->write_state, NULL, 0, 0); /* paired with unref in grpc_tcp_destroy */ gpr_ref_init(&tcp->refcount, 1); tcp->em_fd = em_fd; - tcp->read_closure.cb = grpc_tcp_handle_read; + tcp->read_closure.cb = tcp_handle_read; tcp->read_closure.cb_arg = tcp; - tcp->write_closure.cb = grpc_tcp_handle_write; + tcp->write_closure.cb = tcp_handle_write; tcp->write_closure.cb_arg = tcp; + gpr_slice_buffer_init(&tcp->last_read_buffer); - tcp->handle_read_closure.cb = grpc_tcp_handle_read; return &tcp->base; } diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index 66bb3ef701..882635f638 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -39,16 +39,17 @@ /* Forward decl of grpc_tcp_server */ typedef struct grpc_tcp_server grpc_tcp_server; -/* New server callback: tcp is the newly connected tcp connection */ -typedef void (*grpc_tcp_server_cb)(void *arg, grpc_endpoint *ep); +/* Called for newly connected TCP connections. */ +typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg, + grpc_endpoint *ep); /* Create a server, initially not bound to any ports */ grpc_tcp_server *grpc_tcp_server_create(void); /* Start listening to bound ports */ -void grpc_tcp_server_start(grpc_tcp_server *server, grpc_pollset **pollsets, - size_t pollset_count, grpc_tcp_server_cb cb, - void *cb_arg); +void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, + grpc_pollset **pollsets, size_t pollset_count, + grpc_tcp_server_cb on_accept_cb, void *cb_arg); /* Add a port to the server, returning port number on success, or negative on failure. @@ -62,7 +63,7 @@ void grpc_tcp_server_start(grpc_tcp_server *server, grpc_pollset **pollsets, /* TODO(ctiller): deprecate this, and make grpc_tcp_server_add_ports to handle all of the multiple socket port matching logic in one place */ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - int addr_len); + size_t addr_len); /* Returns the file descriptor of the Nth listening socket on this server, or -1 if the index is out of bounds. @@ -71,8 +72,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, up when grpc_tcp_server_destroy is called. */ int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index); -void grpc_tcp_server_destroy(grpc_tcp_server *server, - void (*shutdown_done)(void *shutdown_done_arg), - void *shutdown_done_arg); +void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, + grpc_closure *closure); #endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */ diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index 6399aaadb9..99c76dcbe9 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -83,9 +83,9 @@ typedef struct { struct sockaddr sockaddr; struct sockaddr_un un; } addr; - int addr_len; - grpc_iomgr_closure read_closure; - grpc_iomgr_closure destroyed_closure; + size_t addr_len; + grpc_closure read_closure; + grpc_closure destroyed_closure; } server_port; static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { @@ -98,8 +98,9 @@ static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { /* the overall server */ struct grpc_tcp_server { - grpc_tcp_server_cb cb; - void *cb_arg; + /* Called whenever accept() succeeds on a server port. */ + grpc_tcp_server_cb on_accept_cb; + void *on_accept_cb_arg; gpr_mu mu; @@ -117,8 +118,7 @@ struct grpc_tcp_server { size_t port_capacity; /* shutdown callback */ - void (*shutdown_complete)(void *); - void *shutdown_complete_arg; + grpc_closure *shutdown_complete; /* all pollsets interested in new connections */ grpc_pollset **pollsets; @@ -132,17 +132,16 @@ grpc_tcp_server *grpc_tcp_server_create(void) { s->active_ports = 0; s->destroyed_ports = 0; s->shutdown = 0; - s->cb = NULL; - s->cb_arg = NULL; + s->on_accept_cb = NULL; + s->on_accept_cb_arg = NULL; s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); s->nports = 0; s->port_capacity = INIT_PORT_CAP; return s; } -static void finish_shutdown(grpc_tcp_server *s) { - s->shutdown_complete(s->shutdown_complete_arg); - s->shutdown_complete = NULL; +static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); gpr_mu_destroy(&s->mu); @@ -150,25 +149,23 @@ static void finish_shutdown(grpc_tcp_server *s) { gpr_free(s); } -static void destroyed_port(void *server, int success) { +static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) { grpc_tcp_server *s = server; gpr_mu_lock(&s->mu); s->destroyed_ports++; if (s->destroyed_ports == s->nports) { gpr_mu_unlock(&s->mu); - finish_shutdown(s); + finish_shutdown(exec_ctx, s); } else { GPR_ASSERT(s->destroyed_ports < s->nports); gpr_mu_unlock(&s->mu); } } -static void dont_care_about_shutdown_completion(void *ignored) {} - /* called when all listening endpoints have been shutdown, so no further events will be received on them - at this point it's safe to destroy things */ -static void deactivated_all_ports(grpc_tcp_server *s) { +static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { size_t i; /* delete ALL the things */ @@ -187,38 +184,35 @@ static void deactivated_all_ports(grpc_tcp_server *s) { } sp->destroyed_closure.cb = destroyed_port; sp->destroyed_closure.cb_arg = s; - grpc_fd_orphan(sp->emfd, &sp->destroyed_closure, "tcp_listener_shutdown"); + grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, + "tcp_listener_shutdown"); } gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); - finish_shutdown(s); + finish_shutdown(exec_ctx, s); } } -void grpc_tcp_server_destroy( - grpc_tcp_server *s, void (*shutdown_complete)(void *shutdown_complete_arg), - void *shutdown_complete_arg) { +void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, + grpc_closure *closure) { size_t i; gpr_mu_lock(&s->mu); GPR_ASSERT(!s->shutdown); s->shutdown = 1; - s->shutdown_complete = shutdown_complete - ? shutdown_complete - : dont_care_about_shutdown_completion; - s->shutdown_complete_arg = shutdown_complete_arg; + s->shutdown_complete = closure; /* shutdown all fd's */ if (s->active_ports) { for (i = 0; i < s->nports; i++) { - grpc_fd_shutdown(s->ports[i].emfd); + grpc_fd_shutdown(exec_ctx, s->ports[i].emfd); } gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); - deactivated_all_ports(s); + deactivated_all_ports(exec_ctx, s); } } @@ -236,7 +230,7 @@ static void init_max_accept_queue_size(void) { char *end; long i = strtol(buf, &end, 10); if (i > 0 && i <= INT_MAX && end && *end == 0) { - n = i; + n = (int)i; } } fclose(fp); @@ -256,7 +250,8 @@ static int get_max_accept_queue_size(void) { } /* Prepare a recently-created socket for listening. */ -static int prepare_socket(int fd, const struct sockaddr *addr, int addr_len) { +static int prepare_socket(int fd, const struct sockaddr *addr, + size_t addr_len) { struct sockaddr_storage sockname_temp; socklen_t sockname_len; @@ -273,7 +268,8 @@ static int prepare_socket(int fd, const struct sockaddr *addr, int addr_len) { goto error; } - if (bind(fd, addr, addr_len) < 0) { + GPR_ASSERT(addr_len < ~(socklen_t)0); + if (bind(fd, addr, (socklen_t)addr_len) < 0) { char *addr_str; grpc_sockaddr_to_string(&addr_str, addr, 0); gpr_log(GPR_ERROR, "bind addr=%s: %s", addr_str, strerror(errno)); @@ -301,7 +297,7 @@ error: } /* event manager callback when reads are ready */ -static void on_read(void *arg, int success) { +static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { server_port *sp = arg; grpc_fd *fdobj; size_t i; @@ -317,14 +313,14 @@ static void on_read(void *arg, int success) { char *addr_str; char *name; /* Note: If we ever decide to return this address to the user, remember to - strip off the ::ffff:0.0.0.0/96 prefix first. */ + strip off the ::ffff:0.0.0.0/96 prefix first. */ int fd = grpc_accept4(sp->fd, (struct sockaddr *)&addr, &addrlen, 1, 1); if (fd < 0) { switch (errno) { case EINTR: continue; case EAGAIN: - grpc_fd_notify_on_read(sp->emfd, &sp->read_closure); + grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); return; default: gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno)); @@ -337,35 +333,39 @@ static void on_read(void *arg, int success) { addr_str = grpc_sockaddr_to_uri((struct sockaddr *)&addr); gpr_asprintf(&name, "tcp-server-connection:%s", addr_str); + if (grpc_tcp_trace) { + gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str); + } + fdobj = grpc_fd_create(fd, name); /* TODO(ctiller): revise this when we have server-side sharding of channels -- we certainly should not be automatically adding every incoming channel to every pollset owned by the server */ for (i = 0; i < sp->server->pollset_count; i++) { - grpc_pollset_add_fd(sp->server->pollsets[i], fdobj); + grpc_pollset_add_fd(exec_ctx, sp->server->pollsets[i], fdobj); } - sp->server->cb( - sp->server->cb_arg, + sp->server->on_accept_cb( + exec_ctx, sp->server->on_accept_cb_arg, grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str)); gpr_free(name); gpr_free(addr_str); } - abort(); + GPR_UNREACHABLE_CODE(return ); error: gpr_mu_lock(&sp->server->mu); if (0 == --sp->server->active_ports) { gpr_mu_unlock(&sp->server->mu); - deactivated_all_ports(sp->server); + deactivated_all_ports(exec_ctx, sp->server); } else { gpr_mu_unlock(&sp->server->mu); } } static int add_socket_to_server(grpc_tcp_server *s, int fd, - const struct sockaddr *addr, int addr_len) { + const struct sockaddr *addr, size_t addr_len) { server_port *sp; int port; char *addr_str; @@ -376,7 +376,7 @@ static int add_socket_to_server(grpc_tcp_server *s, int fd, grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1); gpr_asprintf(&name, "tcp-server-listener:%s", addr_str); gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb && "must add ports before starting server"); + GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); /* append it to the list under a lock */ if (s->nports == s->port_capacity) { s->port_capacity *= 2; @@ -398,7 +398,7 @@ static int add_socket_to_server(grpc_tcp_server *s, int fd, } int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - int addr_len) { + size_t addr_len) { int allocated_port1 = -1; int allocated_port2 = -1; unsigned i; @@ -478,29 +478,31 @@ done: return allocated_port1 >= 0 ? allocated_port1 : allocated_port2; } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index) { - return (index < s->nports) ? s->ports[index].fd : -1; +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { + return (port_index < s->nports) ? s->ports[port_index].fd : -1; } -void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset **pollsets, - size_t pollset_count, grpc_tcp_server_cb cb, - void *cb_arg) { +void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, + grpc_pollset **pollsets, size_t pollset_count, + grpc_tcp_server_cb on_accept_cb, + void *on_accept_cb_arg) { size_t i, j; - GPR_ASSERT(cb); + GPR_ASSERT(on_accept_cb); gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb); + GPR_ASSERT(!s->on_accept_cb); GPR_ASSERT(s->active_ports == 0); - s->cb = cb; - s->cb_arg = cb_arg; + s->on_accept_cb = on_accept_cb; + s->on_accept_cb_arg = on_accept_cb_arg; s->pollsets = pollsets; s->pollset_count = pollset_count; for (i = 0; i < s->nports; i++) { for (j = 0; j < pollset_count; j++) { - grpc_pollset_add_fd(pollsets[j], s->ports[i].emfd); + grpc_pollset_add_fd(exec_ctx, pollsets[j], s->ports[i].emfd); } s->ports[i].read_closure.cb = on_read; s->ports[i].read_closure.cb_arg = &s->ports[i]; - grpc_fd_notify_on_read(s->ports[i].emfd, &s->ports[i].read_closure); + grpc_fd_notify_on_read(exec_ctx, s->ports[i].emfd, + &s->ports[i].read_closure); s->active_ports++; } gpr_mu_unlock(&s->mu); diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index d0478d3604..3fea8b5b35 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -67,26 +67,28 @@ typedef struct server_port { /* The cached AcceptEx for that port. */ LPFN_ACCEPTEX AcceptEx; int shutting_down; + /* closure for socket notification of accept being ready */ + grpc_closure on_accept; } server_port; /* the overall server */ struct grpc_tcp_server { - grpc_tcp_server_cb cb; - void *cb_arg; + /* Called whenever accept() succeeds on a server port. */ + grpc_tcp_server_cb on_accept_cb; + void *on_accept_cb_arg; gpr_mu mu; - gpr_cv cv; /* active port count: how many ports are actually still listening */ int active_ports; - /* number of iomgr callbacks that have been explicitly scheduled during - * shutdown */ - int iomgr_callbacks_pending; /* all listening ports */ server_port *ports; size_t nports; size_t port_capacity; + + /* shutdown callback */ + grpc_closure *shutdown_complete; }; /* Public function. Allocates the proper data structures to hold a @@ -94,54 +96,63 @@ struct grpc_tcp_server { grpc_tcp_server *grpc_tcp_server_create(void) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); gpr_mu_init(&s->mu); - gpr_cv_init(&s->cv); s->active_ports = 0; - s->iomgr_callbacks_pending = 0; - s->cb = NULL; - s->cb_arg = NULL; + s->on_accept_cb = NULL; + s->on_accept_cb_arg = NULL; s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); s->nports = 0; s->port_capacity = INIT_PORT_CAP; + s->shutdown_complete = NULL; return s; } -/* Public function. Stops and destroys a grpc_tcp_server. */ -void grpc_tcp_server_destroy(grpc_tcp_server *s, - void (*shutdown_done)(void *shutdown_done_arg), - void *shutdown_done_arg) { +static void dont_care_about_shutdown_completion(void *arg) {} + +static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { size_t i; - gpr_mu_lock(&s->mu); - /* First, shutdown all fd's. This will queue abortion calls for all - of the pending accepts due to the normal operation mechanism. */ - for (i = 0; i < s->nports; i++) { - server_port *sp = &s->ports[i]; - sp->shutting_down = 1; - s->iomgr_callbacks_pending += grpc_winsocket_shutdown(sp->socket); - } - /* This happens asynchronously. Wait while that happens. */ - while (s->active_ports || s->iomgr_callbacks_pending) { - gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); - } - gpr_mu_unlock(&s->mu); + + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); /* Now that the accepts have been aborted, we can destroy the sockets. The IOCP won't get notified on these, so we can flag them as already closed by the system. */ for (i = 0; i < s->nports; i++) { server_port *sp = &s->ports[i]; - grpc_winsocket_orphan(sp->socket); + grpc_winsocket_destroy(sp->socket); } gpr_free(s->ports); gpr_free(s); +} - if (shutdown_done) { - shutdown_done(shutdown_done_arg); +/* Public function. Stops and destroys a grpc_tcp_server. */ +void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, + grpc_closure *shutdown_complete) { + size_t i; + int immediately_done = 0; + gpr_mu_lock(&s->mu); + + s->shutdown_complete = shutdown_complete; + + /* First, shutdown all fd's. This will queue abortion calls for all + of the pending accepts due to the normal operation mechanism. */ + if (s->active_ports == 0) { + immediately_done = 1; + } + for (i = 0; i < s->nports; i++) { + server_port *sp = &s->ports[i]; + sp->shutting_down = 1; + grpc_winsocket_shutdown(sp->socket); + } + gpr_mu_unlock(&s->mu); + + if (immediately_done) { + finish_shutdown(exec_ctx, s); } } /* Prepare (bind) a recently-created socket for listening. */ static int prepare_socket(SOCKET sock, const struct sockaddr *addr, - int addr_len) { + size_t addr_len) { struct sockaddr_storage sockname_temp; socklen_t sockname_len; @@ -154,7 +165,7 @@ static int prepare_socket(SOCKET sock, const struct sockaddr *addr, goto error; } - if (bind(sock, addr, addr_len) == SOCKET_ERROR) { + if (bind(sock, addr, (int)addr_len) == SOCKET_ERROR) { char *addr_str; char *utf8_message = gpr_format_message(WSAGetLastError()); grpc_sockaddr_to_string(&addr_str, addr, 0); @@ -187,23 +198,25 @@ error: return -1; } -static void decrement_active_ports_and_notify(server_port *sp) { +static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx, + server_port *sp) { + int notify = 0; sp->shutting_down = 0; - sp->socket->read_info.outstanding = 0; gpr_mu_lock(&sp->server->mu); GPR_ASSERT(sp->server->active_ports > 0); - if (0 == --sp->server->active_ports) { - gpr_cv_broadcast(&sp->server->cv); + if (0 == --sp->server->active_ports && + sp->server->shutdown_complete != NULL) { + notify = 1; } gpr_mu_unlock(&sp->server->mu); + if (notify) { + finish_shutdown(exec_ctx, sp->server); + } } -/* start_accept will reference that for the IOCP notification request. */ -static void on_accept(void *arg, int from_iocp); - /* In order to do an async accept, we need to create a socket first which will be the one assigned to the new incoming connection. */ -static void start_accept(server_port *port) { +static void start_accept(grpc_exec_ctx *exec_ctx, server_port *port) { SOCKET sock = INVALID_SOCKET; char *message; char *utf8_message; @@ -242,7 +255,7 @@ static void start_accept(server_port *port) { /* We're ready to do the accept. Calling grpc_socket_notify_on_read may immediately process an accept that happened in the meantime. */ port->new_socket = sock; - grpc_socket_notify_on_read(port->socket, on_accept, port); + grpc_socket_notify_on_read(exec_ctx, port->socket, &port->on_accept); return; failure: @@ -252,7 +265,7 @@ failure: change is not seen by on_accept and we proceed to trying new accept, but we fail there because the listening port has been closed in the meantime. */ - decrement_active_ports_and_notify(port); + decrement_active_ports_and_notify(exec_ctx, port); return; } utf8_message = gpr_format_message(WSAGetLastError()); @@ -262,7 +275,7 @@ failure: } /* Event manager callback when reads are ready. */ -static void on_accept(void *arg, int from_iocp) { +static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { server_port *sp = arg; SOCKET sock = sp->new_socket; grpc_winsocket_callback_info *info = &sp->socket->read_info; @@ -280,17 +293,11 @@ static void on_accept(void *arg, int from_iocp) { this is necessary in the read/write case, it's useless for the accept case. We only need to adjust the pending callback count */ if (!from_iocp) { - gpr_mu_lock(&sp->server->mu); - GPR_ASSERT(sp->server->iomgr_callbacks_pending > 0); - if (0 == --sp->server->iomgr_callbacks_pending) { - gpr_cv_broadcast(&sp->server->cv); - } - gpr_mu_unlock(&sp->server->mu); return; } /* The IOCP notified us of a completed operation. Let's grab the results, - and act accordingly. */ + and act accordingly. */ transfered_bytes = 0; wsa_success = WSAGetOverlappedResult(sock, &info->overlapped, &transfered_bytes, FALSE, &flags); @@ -298,7 +305,7 @@ static void on_accept(void *arg, int from_iocp) { if (sp->shutting_down) { /* During the shutdown case, we ARE expecting an error. So that's well, and we can wake up the shutdown thread. */ - decrement_active_ports_and_notify(sp); + decrement_active_ports_and_notify(exec_ctx, sp); return; } else { char *utf8_message = gpr_format_message(WSAGetLastError()); @@ -329,21 +336,23 @@ static void on_accept(void *arg, int from_iocp) { peer_name_string); gpr_free(fd_name); gpr_free(peer_name_string); + } else { + closesocket(sock); } } /* The only time we should call our callback, is where we successfully managed to accept a connection, and created an endpoint. */ - if (ep) sp->server->cb(sp->server->cb_arg, ep); + if (ep) sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep); /* As we were notified from the IOCP of one and exactly one accept, the former socked we created has now either been destroy or assigned to the new connection. We need to create a new one for the next connection. */ - start_accept(sp); + start_accept(exec_ctx, sp); } static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, - const struct sockaddr *addr, int addr_len) { + const struct sockaddr *addr, size_t addr_len) { server_port *sp; int port; int status; @@ -370,11 +379,12 @@ static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, port = prepare_socket(sock, addr, addr_len); if (port >= 0) { gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb && "must add ports before starting server"); + GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); /* append it to the list under a lock */ if (s->nports == s->port_capacity) { - s->port_capacity *= 2; - s->ports = gpr_realloc(s->ports, sizeof(server_port) * s->port_capacity); + /* too many ports, and we need to store their address in a closure */ + /* TODO(ctiller): make server_port a linked list */ + abort(); } sp = &s->ports[s->nports++]; sp->server = s; @@ -382,6 +392,7 @@ static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, sp->shutting_down = 0; sp->AcceptEx = AcceptEx; sp->new_socket = INVALID_SOCKET; + grpc_closure_init(&sp->on_accept, on_accept, sp); GPR_ASSERT(sp->socket); gpr_mu_unlock(&s->mu); } @@ -390,7 +401,7 @@ static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, } int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - int addr_len) { + size_t addr_len) { int allocated_port = -1; unsigned i; SOCKET sock; @@ -447,23 +458,24 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, return allocated_port; } -SOCKET grpc_tcp_server_get_socket(grpc_tcp_server *s, unsigned index) { +SOCKET +grpc_tcp_server_get_socket(grpc_tcp_server *s, unsigned index) { return (index < s->nports) ? s->ports[index].socket->socket : INVALID_SOCKET; } -void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset **pollset, - size_t pollset_count, grpc_tcp_server_cb cb, - void *cb_arg) { +void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, + grpc_pollset **pollset, size_t pollset_count, + grpc_tcp_server_cb on_accept_cb, + void *on_accept_cb_arg) { size_t i; - GPR_ASSERT(cb); + GPR_ASSERT(on_accept_cb); gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb); + GPR_ASSERT(!s->on_accept_cb); GPR_ASSERT(s->active_ports == 0); - s->cb = cb; - s->cb_arg = cb_arg; + s->on_accept_cb = on_accept_cb; + s->on_accept_cb_arg = on_accept_cb_arg; for (i = 0; i < s->nports; i++) { - s->ports[i].socket->read_info.outstanding = 1; - start_accept(s->ports + i); + start_accept(exec_ctx, s->ports + i); s->active_ports++; } gpr_mu_unlock(&s->mu); diff --git a/src/core/iomgr/tcp_windows.c b/src/core/iomgr/tcp_windows.c index 901793ec43..5ff78231bd 100644 --- a/src/core/iomgr/tcp_windows.c +++ b/src/core/iomgr/tcp_windows.c @@ -44,7 +44,7 @@ #include <grpc/support/string_util.h> #include <grpc/support/useful.h> -#include "src/core/iomgr/alarm.h" +#include "src/core/iomgr/timer.h" #include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/sockaddr.h" #include "src/core/iomgr/sockaddr_utils.h" @@ -82,13 +82,14 @@ typedef struct grpc_tcp { /* Refcounting how many operations are in progress. */ gpr_refcount refcount; - grpc_endpoint_read_cb read_cb; - void *read_user_data; - gpr_slice read_slice; + grpc_closure on_read; + grpc_closure on_write; - grpc_endpoint_write_cb write_cb; - void *write_user_data; - gpr_slice_buffer write_slices; + grpc_closure *read_cb; + grpc_closure *write_cb; + gpr_slice read_slice; + gpr_slice_buffer *write_slices; + gpr_slice_buffer *read_slices; /* The IO Completion Port runs from another thread. We need some mechanism to protect ourselves when requesting a shutdown. */ @@ -98,82 +99,85 @@ typedef struct grpc_tcp { char *peer_string; } grpc_tcp; -static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } +static void tcp_free(grpc_tcp *tcp) { + grpc_winsocket_destroy(tcp->socket); + gpr_mu_destroy(&tcp->mu); + gpr_free(tcp->peer_string); + gpr_free(tcp); +} +/*#define GRPC_TCP_REFCOUNT_DEBUG*/ +#ifdef GRPC_TCP_REFCOUNT_DEBUG +#define TCP_UNREF(tcp, reason) tcp_unref((tcp), (reason), __FILE__, __LINE__) +#define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__) +static void tcp_unref(grpc_tcp *tcp, const char *reason, const char *file, + int line) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP unref %p : %s %d -> %d", tcp, + reason, tcp->refcount.count, tcp->refcount.count - 1); + if (gpr_unref(&tcp->refcount)) { + tcp_free(tcp); + } +} + +static void tcp_ref(grpc_tcp *tcp, const char *reason, const char *file, + int line) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP ref %p : %s %d -> %d", tcp, + reason, tcp->refcount.count, tcp->refcount.count + 1); + gpr_ref(&tcp->refcount); +} +#else +#define TCP_UNREF(tcp, reason) tcp_unref((tcp)) +#define TCP_REF(tcp, reason) tcp_ref((tcp)) static void tcp_unref(grpc_tcp *tcp) { if (gpr_unref(&tcp->refcount)) { - gpr_slice_buffer_destroy(&tcp->write_slices); - grpc_winsocket_orphan(tcp->socket); - gpr_mu_destroy(&tcp->mu); - gpr_free(tcp->peer_string); - gpr_free(tcp); + tcp_free(tcp); } } +static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } +#endif + /* Asynchronous callback from the IOCP, or the background thread. */ -static void on_read(void *tcpp, int from_iocp) { - grpc_tcp *tcp = (grpc_tcp *)tcpp; +static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, int success) { + grpc_tcp *tcp = tcpp; + grpc_closure *cb = tcp->read_cb; grpc_winsocket *socket = tcp->socket; gpr_slice sub; gpr_slice *slice = NULL; size_t nslices = 0; - grpc_endpoint_cb_status status; - grpc_endpoint_read_cb cb; grpc_winsocket_callback_info *info = &socket->read_info; - void *opaque = tcp->read_user_data; int do_abort = 0; - gpr_mu_lock(&tcp->mu); - cb = tcp->read_cb; - tcp->read_cb = NULL; - if (!from_iocp || tcp->shutting_down) { - /* If we are here with from_iocp set to true, it means we got raced to - shutting down the endpoint. No actual abort callback will happen - though, so we're going to do it from here. */ - do_abort = 1; - } - gpr_mu_unlock(&tcp->mu); - - if (do_abort) { - if (from_iocp) { - tcp->socket->read_info.outstanding = 0; + if (success) { + if (socket->read_info.wsa_error != 0 && !tcp->shutting_down) { + if (socket->read_info.wsa_error != WSAECONNRESET) { + char *utf8_message = gpr_format_message(info->wsa_error); + gpr_log(GPR_ERROR, "ReadFile overlapped error: %s", utf8_message); + gpr_free(utf8_message); + } + success = 0; gpr_slice_unref(tcp->read_slice); - } - tcp_unref(tcp); - if (cb) cb(opaque, NULL, 0, GRPC_ENDPOINT_CB_SHUTDOWN); - return; - } - - GPR_ASSERT(tcp->socket->read_info.outstanding); - - if (socket->read_info.wsa_error != 0) { - if (socket->read_info.wsa_error != WSAECONNRESET) { - char *utf8_message = gpr_format_message(info->wsa_error); - gpr_log(GPR_ERROR, "ReadFile overlapped error: %s", utf8_message); - gpr_free(utf8_message); - } - gpr_slice_unref(tcp->read_slice); - status = GRPC_ENDPOINT_CB_ERROR; - } else { - if (info->bytes_transfered != 0) { - sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered); - status = GRPC_ENDPOINT_CB_OK; - slice = ⊂ - nslices = 1; } else { - gpr_slice_unref(tcp->read_slice); - status = GRPC_ENDPOINT_CB_EOF; + if (info->bytes_transfered != 0 && !tcp->shutting_down) { + sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered); + gpr_slice_buffer_add(tcp->read_slices, sub); + success = 1; + } else { + gpr_slice_unref(tcp->read_slice); + success = 0; + } } } - tcp->socket->read_info.outstanding = 0; - - tcp_unref(tcp); - cb(opaque, slice, nslices, status); + tcp->read_cb = NULL; + TCP_UNREF(tcp, "read"); + if (cb) { + cb->cb(exec_ctx, cb->cb_arg, success); + } } -static void win_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, - void *arg) { +static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *read_slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; grpc_winsocket *handle = tcp->socket; grpc_winsocket_callback_info *info = &handle->read_info; @@ -182,21 +186,22 @@ static void win_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, DWORD flags = 0; WSABUF buffer; - GPR_ASSERT(!tcp->socket->read_info.outstanding); if (tcp->shutting_down) { - cb(arg, NULL, 0, GRPC_ENDPOINT_CB_SHUTDOWN); + grpc_exec_ctx_enqueue(exec_ctx, cb, 0); return; } - tcp_ref(tcp); - tcp->socket->read_info.outstanding = 1; + tcp->read_cb = cb; - tcp->read_user_data = arg; + tcp->read_slices = read_slices; + gpr_slice_buffer_reset_and_unref(read_slices); tcp->read_slice = gpr_slice_malloc(8192); buffer.len = GPR_SLICE_LENGTH(tcp->read_slice); buffer.buf = (char *)GPR_SLICE_START_PTR(tcp->read_slice); + TCP_REF(tcp, "read"); + /* First let's try a synchronous, non-blocking read. */ status = WSARecv(tcp->socket->socket, &buffer, 1, &bytes_read, &flags, NULL, NULL); @@ -205,8 +210,7 @@ static void win_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, /* Did we get data immediately ? Yay. */ if (info->wsa_error != WSAEWOULDBLOCK) { info->bytes_transfered = bytes_read; - /* This might heavily recurse. */ - on_read(tcp, 1); + grpc_exec_ctx_enqueue(exec_ctx, &tcp->on_read, 1); return; } @@ -219,70 +223,47 @@ static void win_notify_on_read(grpc_endpoint *ep, grpc_endpoint_read_cb cb, int wsa_error = WSAGetLastError(); if (wsa_error != WSA_IO_PENDING) { info->wsa_error = wsa_error; - on_read(tcp, 1); + grpc_exec_ctx_enqueue(exec_ctx, &tcp->on_read, 0); return; } } - grpc_socket_notify_on_read(tcp->socket, on_read, tcp); + grpc_socket_notify_on_read(exec_ctx, tcp->socket, &tcp->on_read); } /* Asynchronous callback from the IOCP, or the background thread. */ -static void on_write(void *tcpp, int from_iocp) { +static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, int success) { grpc_tcp *tcp = (grpc_tcp *)tcpp; grpc_winsocket *handle = tcp->socket; grpc_winsocket_callback_info *info = &handle->write_info; - grpc_endpoint_cb_status status = GRPC_ENDPOINT_CB_OK; - grpc_endpoint_write_cb cb; - void *opaque = tcp->write_user_data; + grpc_closure *cb; int do_abort = 0; gpr_mu_lock(&tcp->mu); cb = tcp->write_cb; tcp->write_cb = NULL; - if (!from_iocp || tcp->shutting_down) { - /* If we are here with from_iocp set to true, it means we got raced to - shutting down the endpoint. No actual abort callback will happen - though, so we're going to do it from here. */ - do_abort = 1; - } gpr_mu_unlock(&tcp->mu); - if (do_abort) { - if (from_iocp) { - tcp->socket->write_info.outstanding = 0; - gpr_slice_buffer_reset_and_unref(&tcp->write_slices); - } - tcp_unref(tcp); - if (cb) cb(opaque, GRPC_ENDPOINT_CB_SHUTDOWN); - return; - } - - GPR_ASSERT(tcp->socket->write_info.outstanding); - - if (info->wsa_error != 0) { - if (info->wsa_error != WSAECONNRESET) { - char *utf8_message = gpr_format_message(info->wsa_error); - gpr_log(GPR_ERROR, "WSASend overlapped error: %s", utf8_message); - gpr_free(utf8_message); + if (success) { + if (info->wsa_error != 0) { + if (info->wsa_error != WSAECONNRESET) { + char *utf8_message = gpr_format_message(info->wsa_error); + gpr_log(GPR_ERROR, "WSASend overlapped error: %s", utf8_message); + gpr_free(utf8_message); + } + success = 0; + } else { + GPR_ASSERT(info->bytes_transfered == tcp->write_slices->length); } - status = GRPC_ENDPOINT_CB_ERROR; - } else { - GPR_ASSERT(info->bytes_transfered == tcp->write_slices.length); } - gpr_slice_buffer_reset_and_unref(&tcp->write_slices); - tcp->socket->write_info.outstanding = 0; - - tcp_unref(tcp); - cb(opaque, status); + TCP_UNREF(tcp, "write"); + cb->cb(exec_ctx, cb->cb_arg, success); } /* Initiates a write. */ -static grpc_endpoint_write_status win_write(grpc_endpoint *ep, - gpr_slice *slices, size_t nslices, - grpc_endpoint_write_cb cb, - void *arg) { +static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + gpr_slice_buffer *slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; grpc_winsocket *socket = tcp->socket; grpc_winsocket_callback_info *info = &socket->write_info; @@ -293,30 +274,26 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep, WSABUF *allocated = NULL; WSABUF *buffers = local_buffers; - GPR_ASSERT(!tcp->socket->write_info.outstanding); if (tcp->shutting_down) { - return GRPC_ENDPOINT_WRITE_ERROR; + grpc_exec_ctx_enqueue(exec_ctx, cb, 0); + return; } - tcp_ref(tcp); - tcp->socket->write_info.outstanding = 1; tcp->write_cb = cb; - tcp->write_user_data = arg; - - gpr_slice_buffer_addn(&tcp->write_slices, slices, nslices); + tcp->write_slices = slices; - if (tcp->write_slices.count > GPR_ARRAY_SIZE(local_buffers)) { - buffers = (WSABUF *)gpr_malloc(sizeof(WSABUF) * tcp->write_slices.count); + if (tcp->write_slices->count > GPR_ARRAY_SIZE(local_buffers)) { + buffers = (WSABUF *)gpr_malloc(sizeof(WSABUF) * tcp->write_slices->count); allocated = buffers; } - for (i = 0; i < tcp->write_slices.count; i++) { - buffers[i].len = GPR_SLICE_LENGTH(tcp->write_slices.slices[i]); - buffers[i].buf = (char *)GPR_SLICE_START_PTR(tcp->write_slices.slices[i]); + for (i = 0; i < tcp->write_slices->count; i++) { + buffers[i].len = GPR_SLICE_LENGTH(tcp->write_slices->slices[i]); + buffers[i].buf = (char *)GPR_SLICE_START_PTR(tcp->write_slices->slices[i]); } /* First, let's try a synchronous, non-blocking write. */ - status = WSASend(socket->socket, buffers, tcp->write_slices.count, + status = WSASend(socket->socket, buffers, tcp->write_slices->count, &bytes_sent, 0, NULL, NULL); info->wsa_error = status == 0 ? 0 : WSAGetLastError(); @@ -324,10 +301,10 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep, connection that has its send queue filled up. But if we don't, then we can avoid doing an async write operation at all. */ if (info->wsa_error != WSAEWOULDBLOCK) { - grpc_endpoint_write_status ret = GRPC_ENDPOINT_WRITE_ERROR; + int ok = 0; if (status == 0) { - ret = GRPC_ENDPOINT_WRITE_DONE; - GPR_ASSERT(bytes_sent == tcp->write_slices.length); + ok = 1; + GPR_ASSERT(bytes_sent == tcp->write_slices->length); } else { if (socket->read_info.wsa_error != WSAECONNRESET) { char *utf8_message = gpr_format_message(info->wsa_error); @@ -336,43 +313,43 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep, } } if (allocated) gpr_free(allocated); - gpr_slice_buffer_reset_and_unref(&tcp->write_slices); - tcp->socket->write_info.outstanding = 0; - tcp_unref(tcp); - return ret; + grpc_exec_ctx_enqueue(exec_ctx, cb, ok); + return; } + TCP_REF(tcp, "write"); + /* If we got a WSAEWOULDBLOCK earlier, then we need to re-do the same operation, this time asynchronously. */ memset(&socket->write_info.overlapped, 0, sizeof(OVERLAPPED)); - status = WSASend(socket->socket, buffers, tcp->write_slices.count, + status = WSASend(socket->socket, buffers, tcp->write_slices->count, &bytes_sent, 0, &socket->write_info.overlapped, NULL); if (allocated) gpr_free(allocated); if (status != 0) { int wsa_error = WSAGetLastError(); if (wsa_error != WSA_IO_PENDING) { - gpr_slice_buffer_reset_and_unref(&tcp->write_slices); - tcp->socket->write_info.outstanding = 0; - tcp_unref(tcp); - return GRPC_ENDPOINT_WRITE_ERROR; + TCP_UNREF(tcp, "write"); + grpc_exec_ctx_enqueue(exec_ctx, cb, 0); + return; } } /* As all is now setup, we can now ask for the IOCP notification. It may trigger the callback immediately however, but no matter. */ - grpc_socket_notify_on_write(socket, on_write, tcp); - return GRPC_ENDPOINT_WRITE_PENDING; + grpc_socket_notify_on_write(exec_ctx, socket, &tcp->on_write); } -static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *ps) { +static void win_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset *ps) { grpc_tcp *tcp; (void)ps; tcp = (grpc_tcp *)ep; grpc_iocp_add_socket(tcp->socket); } -static void win_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pss) { +static void win_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, + grpc_pollset_set *pss) { grpc_tcp *tcp; (void)pss; tcp = (grpc_tcp *)ep; @@ -385,21 +362,19 @@ static void win_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pss) { we're not going to protect against these. However the IO Completion Port callback will happen from another thread, so we need to protect against concurrent access of the data structure in that regard. */ -static void win_shutdown(grpc_endpoint *ep) { +static void win_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - int extra_refs = 0; gpr_mu_lock(&tcp->mu); /* At that point, what may happen is that we're already inside the IOCP callback. See the comments in on_read and on_write. */ tcp->shutting_down = 1; - extra_refs = grpc_winsocket_shutdown(tcp->socket); - while (extra_refs--) tcp_ref(tcp); + grpc_winsocket_shutdown(tcp->socket); gpr_mu_unlock(&tcp->mu); } -static void win_destroy(grpc_endpoint *ep) { +static void win_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - tcp_unref(tcp); + TCP_UNREF(tcp, "destroy"); } static char *win_get_peer(grpc_endpoint *ep) { @@ -407,9 +382,9 @@ static char *win_get_peer(grpc_endpoint *ep) { return gpr_strdup(tcp->peer_string); } -static grpc_endpoint_vtable vtable = { - win_notify_on_read, win_write, win_add_to_pollset, win_add_to_pollset_set, - win_shutdown, win_destroy, win_get_peer}; +static grpc_endpoint_vtable vtable = {win_read, win_write, win_add_to_pollset, + win_add_to_pollset_set, win_shutdown, + win_destroy, win_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); @@ -417,8 +392,9 @@ grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) { tcp->base.vtable = &vtable; tcp->socket = socket; gpr_mu_init(&tcp->mu); - gpr_slice_buffer_init(&tcp->write_slices); gpr_ref_init(&tcp->refcount, 1); + grpc_closure_init(&tcp->on_read, on_read, tcp); + grpc_closure_init(&tcp->on_write, on_write, tcp); tcp->peer_string = gpr_strdup(peer_string); return &tcp->base; } diff --git a/src/core/iomgr/time_averaged_stats.c b/src/core/iomgr/time_averaged_stats.c index f881dde9fc..e075db4373 100644 --- a/src/core/iomgr/time_averaged_stats.c +++ b/src/core/iomgr/time_averaged_stats.c @@ -33,7 +33,7 @@ #include "src/core/iomgr/time_averaged_stats.h" -void grpc_time_averaged_stats_init(grpc_time_averaged_stats *stats, +void grpc_time_averaged_stats_init(grpc_time_averaged_stats* stats, double init_avg, double regress_weight, double persistence_factor) { stats->init_avg = init_avg; @@ -45,14 +45,14 @@ void grpc_time_averaged_stats_init(grpc_time_averaged_stats *stats, stats->aggregate_weighted_avg = init_avg; } -void grpc_time_averaged_stats_add_sample(grpc_time_averaged_stats *stats, +void grpc_time_averaged_stats_add_sample(grpc_time_averaged_stats* stats, double value) { stats->batch_total_value += value; ++stats->batch_num_samples; } double grpc_time_averaged_stats_update_average( - grpc_time_averaged_stats *stats) { + grpc_time_averaged_stats* stats) { /* Start with the current batch: */ double weighted_sum = stats->batch_total_value; double total_weight = stats->batch_num_samples; diff --git a/src/core/iomgr/time_averaged_stats.h b/src/core/iomgr/time_averaged_stats.h index e6dec1b4cd..4e9e3956c2 100644 --- a/src/core/iomgr/time_averaged_stats.h +++ b/src/core/iomgr/time_averaged_stats.h @@ -75,14 +75,14 @@ typedef struct { /* See the comments on the members above for an explanation of init_avg, regress_weight, and persistence_factor. */ -void grpc_time_averaged_stats_init(grpc_time_averaged_stats *stats, +void grpc_time_averaged_stats_init(grpc_time_averaged_stats* stats, double init_avg, double regress_weight, double persistence_factor); /* Add a sample to the current batch. */ -void grpc_time_averaged_stats_add_sample(grpc_time_averaged_stats *stats, +void grpc_time_averaged_stats_add_sample(grpc_time_averaged_stats* stats, double value); /* Complete a batch and compute the new estimate of the average sample value. */ -double grpc_time_averaged_stats_update_average(grpc_time_averaged_stats *stats); +double grpc_time_averaged_stats_update_average(grpc_time_averaged_stats* stats); #endif /* GRPC_INTERNAL_CORE_IOMGR_TIME_AVERAGED_STATS_H */ diff --git a/src/core/iomgr/alarm.c b/src/core/iomgr/timer.c index ddb30dc4bb..66fafe75ad 100644 --- a/src/core/iomgr/alarm.c +++ b/src/core/iomgr/timer.c @@ -31,10 +31,10 @@ * */ -#include "src/core/iomgr/alarm.h" +#include "src/core/iomgr/timer.h" -#include "src/core/iomgr/alarm_heap.h" -#include "src/core/iomgr/alarm_internal.h" +#include "src/core/iomgr/timer_heap.h" +#include "src/core/iomgr/timer_internal.h" #include "src/core/iomgr/time_averaged_stats.h" #include <grpc/support/log.h> #include <grpc/support/sync.h> @@ -44,7 +44,6 @@ #define LOG2_NUM_SHARDS 5 #define NUM_SHARDS (1 << LOG2_NUM_SHARDS) -#define MAX_ALARMS_PER_CHECK 128 #define ADD_DEADLINE_SCALE 0.33 #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 @@ -52,38 +51,38 @@ typedef struct { gpr_mu mu; grpc_time_averaged_stats stats; - /* All and only alarms with deadlines <= this will be in the heap. */ + /* All and only timers with deadlines <= this will be in the heap. */ gpr_timespec queue_deadline_cap; gpr_timespec min_deadline; /* Index in the g_shard_queue */ gpr_uint32 shard_queue_index; - /* This holds all alarms with deadlines < queue_deadline_cap. Alarms in this + /* This holds all timers with deadlines < queue_deadline_cap. Timers in this list have the top bit of their deadline set to 0. */ - grpc_alarm_heap heap; - /* This holds alarms whose deadline is >= queue_deadline_cap. */ - grpc_alarm list; + grpc_timer_heap heap; + /* This holds timers whose deadline is >= queue_deadline_cap. */ + grpc_timer list; } shard_type; /* Protects g_shard_queue */ static gpr_mu g_mu; -/* Allow only one run_some_expired_alarms at once */ +/* Allow only one run_some_expired_timers at once */ static gpr_mu g_checker_mu; static gpr_clock_type g_clock_type; static shard_type g_shards[NUM_SHARDS]; /* Protected by g_mu */ static shard_type *g_shard_queue[NUM_SHARDS]; -static int run_some_expired_alarms(gpr_mu *drop_mu, gpr_timespec now, +static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec *next, int success); static gpr_timespec compute_min_deadline(shard_type *shard) { - return grpc_alarm_heap_is_empty(&shard->heap) + return grpc_timer_heap_is_empty(&shard->heap) ? shard->queue_deadline_cap - : grpc_alarm_heap_top(&shard->heap)->deadline; + : grpc_timer_heap_top(&shard->heap)->deadline; } -void grpc_alarm_list_init(gpr_timespec now) { - int i; +void grpc_timer_list_init(gpr_timespec now) { + gpr_uint32 i; gpr_mu_init(&g_mu); gpr_mu_init(&g_checker_mu); @@ -96,56 +95,55 @@ void grpc_alarm_list_init(gpr_timespec now) { 0.5); shard->queue_deadline_cap = now; shard->shard_queue_index = i; - grpc_alarm_heap_init(&shard->heap); + grpc_timer_heap_init(&shard->heap); shard->list.next = shard->list.prev = &shard->list; shard->min_deadline = compute_min_deadline(shard); g_shard_queue[i] = shard; } } -void grpc_alarm_list_shutdown(void) { +void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) { int i; - while (run_some_expired_alarms(NULL, gpr_inf_future(g_clock_type), NULL, 0)) - ; + run_some_expired_timers(exec_ctx, gpr_inf_future(g_clock_type), NULL, 0); for (i = 0; i < NUM_SHARDS; i++) { shard_type *shard = &g_shards[i]; gpr_mu_destroy(&shard->mu); - grpc_alarm_heap_destroy(&shard->heap); + grpc_timer_heap_destroy(&shard->heap); } gpr_mu_destroy(&g_mu); gpr_mu_destroy(&g_checker_mu); } /* This is a cheap, but good enough, pointer hash for sharding the tasks: */ -static size_t shard_idx(const grpc_alarm *info) { +static size_t shard_idx(const grpc_timer *info) { size_t x = (size_t)info; return ((x >> 4) ^ (x >> 9) ^ (x >> 14)) & (NUM_SHARDS - 1); } static double ts_to_dbl(gpr_timespec ts) { - return ts.tv_sec + 1e-9 * ts.tv_nsec; + return (double)ts.tv_sec + 1e-9 * ts.tv_nsec; } static gpr_timespec dbl_to_ts(double d) { gpr_timespec ts; - ts.tv_sec = d; - ts.tv_nsec = 1e9 * (d - ts.tv_sec); + ts.tv_sec = (time_t)d; + ts.tv_nsec = (int)(1e9 * (d - (double)ts.tv_sec)); ts.clock_type = GPR_TIMESPAN; return ts; } -static void list_join(grpc_alarm *head, grpc_alarm *alarm) { - alarm->next = head; - alarm->prev = head->prev; - alarm->next->prev = alarm->prev->next = alarm; +static void list_join(grpc_timer *head, grpc_timer *timer) { + timer->next = head; + timer->prev = head->prev; + timer->next->prev = timer->prev->next = timer; } -static void list_remove(grpc_alarm *alarm) { - alarm->next->prev = alarm->prev; - alarm->prev->next = alarm->next; +static void list_remove(grpc_timer *timer) { + timer->next->prev = timer->prev; + timer->prev->next = timer->next; } -static void swap_adjacent_shards_in_queue(size_t first_shard_queue_index) { +static void swap_adjacent_shards_in_queue(gpr_uint32 first_shard_queue_index) { shard_type *temp; temp = g_shard_queue[first_shard_queue_index]; g_shard_queue[first_shard_queue_index] = @@ -172,17 +170,16 @@ static void note_deadline_change(shard_type *shard) { } } -void grpc_alarm_init(grpc_alarm *alarm, gpr_timespec deadline, - grpc_iomgr_cb_func alarm_cb, void *alarm_cb_arg, - gpr_timespec now) { - int is_first_alarm = 0; - shard_type *shard = &g_shards[shard_idx(alarm)]; +void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, + gpr_timespec deadline, grpc_iomgr_cb_func timer_cb, + void *timer_cb_arg, gpr_timespec now) { + int is_first_timer = 0; + shard_type *shard = &g_shards[shard_idx(timer)]; GPR_ASSERT(deadline.clock_type == g_clock_type); GPR_ASSERT(now.clock_type == g_clock_type); - alarm->cb = alarm_cb; - alarm->cb_arg = alarm_cb_arg; - alarm->deadline = deadline; - alarm->triggered = 0; + grpc_closure_init(&timer->closure, timer_cb, timer_cb_arg); + timer->deadline = deadline; + timer->triggered = 0; /* TODO(ctiller): check deadline expired */ @@ -190,25 +187,25 @@ void grpc_alarm_init(grpc_alarm *alarm, gpr_timespec deadline, grpc_time_averaged_stats_add_sample(&shard->stats, ts_to_dbl(gpr_time_sub(deadline, now))); if (gpr_time_cmp(deadline, shard->queue_deadline_cap) < 0) { - is_first_alarm = grpc_alarm_heap_add(&shard->heap, alarm); + is_first_timer = grpc_timer_heap_add(&shard->heap, timer); } else { - alarm->heap_index = INVALID_HEAP_INDEX; - list_join(&shard->list, alarm); + timer->heap_index = INVALID_HEAP_INDEX; + list_join(&shard->list, timer); } gpr_mu_unlock(&shard->mu); /* Deadline may have decreased, we need to adjust the master queue. Note that there is a potential racy unlocked region here. There could be a - reordering of multiple grpc_alarm_init calls, at this point, but the < test + reordering of multiple grpc_timer_init calls, at this point, but the < test below should ensure that we err on the side of caution. There could - also be a race with grpc_alarm_check, which might beat us to the lock. In - that case, it is possible that the alarm that we added will have already + also be a race with grpc_timer_check, which might beat us to the lock. In + that case, it is possible that the timer that we added will have already run by the time we hold the lock, but that too is a safe error. - Finally, it's possible that the grpc_alarm_check that intervened failed to - trigger the new alarm because the min_deadline hadn't yet been reduced. - In that case, the alarm will simply have to wait for the next - grpc_alarm_check. */ - if (is_first_alarm) { + Finally, it's possible that the grpc_timer_check that intervened failed to + trigger the new timer because the min_deadline hadn't yet been reduced. + In that case, the timer will simply have to wait for the next + grpc_timer_check. */ + if (is_first_timer) { gpr_mu_lock(&g_mu); if (gpr_time_cmp(deadline, shard->min_deadline) < 0) { gpr_timespec old_min_deadline = g_shard_queue[0]->min_deadline; @@ -223,29 +220,24 @@ void grpc_alarm_init(grpc_alarm *alarm, gpr_timespec deadline, } } -void grpc_alarm_cancel(grpc_alarm *alarm) { - shard_type *shard = &g_shards[shard_idx(alarm)]; - int triggered = 0; +void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { + shard_type *shard = &g_shards[shard_idx(timer)]; gpr_mu_lock(&shard->mu); - if (!alarm->triggered) { - triggered = 1; - alarm->triggered = 1; - if (alarm->heap_index == INVALID_HEAP_INDEX) { - list_remove(alarm); + if (!timer->triggered) { + grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, 0); + timer->triggered = 1; + if (timer->heap_index == INVALID_HEAP_INDEX) { + list_remove(timer); } else { - grpc_alarm_heap_remove(&shard->heap, alarm); + grpc_timer_heap_remove(&shard->heap, timer); } } gpr_mu_unlock(&shard->mu); - - if (triggered) { - alarm->cb(alarm->cb_arg, 0); - } } /* This is called when the queue is empty and "now" has reached the queue_deadline_cap. We compute a new queue deadline and then scan the map - for alarms that fall at or under it. Returns true if the queue is no + for timers that fall at or under it. Returns true if the queue is no longer empty. REQUIRES: shard->mu locked */ static int refill_queue(shard_type *shard, gpr_timespec now) { @@ -256,80 +248,78 @@ static int refill_queue(shard_type *shard, gpr_timespec now) { double deadline_delta = GPR_CLAMP(computed_deadline_delta, MIN_QUEUE_WINDOW_DURATION, MAX_QUEUE_WINDOW_DURATION); - grpc_alarm *alarm, *next; + grpc_timer *timer, *next; - /* Compute the new cap and put all alarms under it into the queue: */ + /* Compute the new cap and put all timers under it into the queue: */ shard->queue_deadline_cap = gpr_time_add( gpr_time_max(now, shard->queue_deadline_cap), dbl_to_ts(deadline_delta)); - for (alarm = shard->list.next; alarm != &shard->list; alarm = next) { - next = alarm->next; + for (timer = shard->list.next; timer != &shard->list; timer = next) { + next = timer->next; - if (gpr_time_cmp(alarm->deadline, shard->queue_deadline_cap) < 0) { - list_remove(alarm); - grpc_alarm_heap_add(&shard->heap, alarm); + if (gpr_time_cmp(timer->deadline, shard->queue_deadline_cap) < 0) { + list_remove(timer); + grpc_timer_heap_add(&shard->heap, timer); } } - return !grpc_alarm_heap_is_empty(&shard->heap); + return !grpc_timer_heap_is_empty(&shard->heap); } -/* This pops the next non-cancelled alarm with deadline <= now from the queue, +/* This pops the next non-cancelled timer with deadline <= now from the queue, or returns NULL if there isn't one. REQUIRES: shard->mu locked */ -static grpc_alarm *pop_one(shard_type *shard, gpr_timespec now) { - grpc_alarm *alarm; +static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) { + grpc_timer *timer; for (;;) { - if (grpc_alarm_heap_is_empty(&shard->heap)) { + if (grpc_timer_heap_is_empty(&shard->heap)) { if (gpr_time_cmp(now, shard->queue_deadline_cap) < 0) return NULL; if (!refill_queue(shard, now)) return NULL; } - alarm = grpc_alarm_heap_top(&shard->heap); - if (gpr_time_cmp(alarm->deadline, now) > 0) return NULL; - alarm->triggered = 1; - grpc_alarm_heap_pop(&shard->heap); - return alarm; + timer = grpc_timer_heap_top(&shard->heap); + if (gpr_time_cmp(timer->deadline, now) > 0) return NULL; + timer->triggered = 1; + grpc_timer_heap_pop(&shard->heap); + return timer; } } /* REQUIRES: shard->mu unlocked */ -static size_t pop_alarms(shard_type *shard, gpr_timespec now, - grpc_alarm **alarms, size_t max_alarms, - gpr_timespec *new_min_deadline) { +static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard, + gpr_timespec now, gpr_timespec *new_min_deadline, + int success) { size_t n = 0; - grpc_alarm *alarm; + grpc_timer *timer; gpr_mu_lock(&shard->mu); - while (n < max_alarms && (alarm = pop_one(shard, now))) { - alarms[n++] = alarm; + while ((timer = pop_one(shard, now))) { + grpc_exec_ctx_enqueue(exec_ctx, &timer->closure, success); + n++; } *new_min_deadline = compute_min_deadline(shard); gpr_mu_unlock(&shard->mu); return n; } -static int run_some_expired_alarms(gpr_mu *drop_mu, gpr_timespec now, +static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec *next, int success) { size_t n = 0; - size_t i; - grpc_alarm *alarms[MAX_ALARMS_PER_CHECK]; - /* TODO(ctiller): verify that there are any alarms (atomically) here */ + /* TODO(ctiller): verify that there are any timers (atomically) here */ if (gpr_mu_trylock(&g_checker_mu)) { gpr_mu_lock(&g_mu); - while (n < MAX_ALARMS_PER_CHECK && - gpr_time_cmp(g_shard_queue[0]->min_deadline, now) < 0) { + while (gpr_time_cmp(g_shard_queue[0]->min_deadline, now) < 0) { gpr_timespec new_min_deadline; - /* For efficiency, we pop as many available alarms as we can from the - shard. This may violate perfect alarm deadline ordering, but that + /* For efficiency, we pop as many available timers as we can from the + shard. This may violate perfect timer deadline ordering, but that shouldn't be a big deal because we don't make ordering guarantees. */ - n += pop_alarms(g_shard_queue[0], now, alarms + n, - MAX_ALARMS_PER_CHECK - n, &new_min_deadline); + n += pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, + success); - /* An grpc_alarm_init() on the shard could intervene here, adding a new - alarm that is earlier than new_min_deadline. However, - grpc_alarm_init() will block on the master_lock before it can call - set_min_deadline, so this one will complete first and then the AddAlarm + /* An grpc_timer_init() on the shard could intervene here, adding a new + timer that is earlier than new_min_deadline. However, + grpc_timer_init() will block on the master_lock before it can call + set_min_deadline, so this one will complete first and then the Addtimer will reduce the min_deadline (perhaps unnecessarily). */ g_shard_queue[0]->min_deadline = new_min_deadline; note_deadline_change(g_shard_queue[0]); @@ -343,29 +333,18 @@ static int run_some_expired_alarms(gpr_mu *drop_mu, gpr_timespec now, gpr_mu_unlock(&g_checker_mu); } - if (n && drop_mu) { - gpr_mu_unlock(drop_mu); - } - - for (i = 0; i < n; i++) { - alarms[i]->cb(alarms[i]->cb_arg, success); - } - - if (n && drop_mu) { - gpr_mu_lock(drop_mu); - } - - return n; + return (int)n; } -int grpc_alarm_check(gpr_mu *drop_mu, gpr_timespec now, gpr_timespec *next) { +int grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, + gpr_timespec *next) { GPR_ASSERT(now.clock_type == g_clock_type); - return run_some_expired_alarms( - drop_mu, now, next, + return run_some_expired_timers( + exec_ctx, now, next, gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0); } -gpr_timespec grpc_alarm_list_next_timeout(void) { +gpr_timespec grpc_timer_list_next_timeout(void) { gpr_timespec out; gpr_mu_lock(&g_mu); out = g_shard_queue[0]->min_deadline; diff --git a/src/core/iomgr/alarm.h b/src/core/iomgr/timer.h index 4a13527e64..9abe58133d 100644 --- a/src/core/iomgr/alarm.h +++ b/src/core/iomgr/timer.h @@ -31,59 +31,59 @@ * */ -#ifndef GRPC_INTERNAL_CORE_IOMGR_ALARM_H -#define GRPC_INTERNAL_CORE_IOMGR_ALARM_H +#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_H +#define GRPC_INTERNAL_CORE_IOMGR_TIMER_H #include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/exec_ctx.h" #include <grpc/support/port_platform.h> #include <grpc/support/time.h> -typedef struct grpc_alarm { +typedef struct grpc_timer { gpr_timespec deadline; gpr_uint32 heap_index; /* INVALID_HEAP_INDEX if not in heap */ int triggered; - struct grpc_alarm *next; - struct grpc_alarm *prev; - grpc_iomgr_cb_func cb; - void *cb_arg; -} grpc_alarm; + struct grpc_timer *next; + struct grpc_timer *prev; + grpc_closure closure; +} grpc_timer; -/* Initialize *alarm. When expired or canceled, alarm_cb will be called with - *alarm_cb_arg and status to indicate if it expired (SUCCESS) or was - canceled (CANCELLED). alarm_cb is guaranteed to be called exactly once, +/* Initialize *timer. When expired or canceled, timer_cb will be called with + *timer_cb_arg and status to indicate if it expired (SUCCESS) or was + canceled (CANCELLED). timer_cb is guaranteed to be called exactly once, and application code should check the status to determine how it was invoked. The application callback is also responsible for maintaining information about when to free up any user-level state. */ -void grpc_alarm_init(grpc_alarm *alarm, gpr_timespec deadline, - grpc_iomgr_cb_func alarm_cb, void *alarm_cb_arg, - gpr_timespec now); +void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, + gpr_timespec deadline, grpc_iomgr_cb_func timer_cb, + void *timer_cb_arg, gpr_timespec now); -/* Note that there is no alarm destroy function. This is because the - alarm is a one-time occurrence with a guarantee that the callback will +/* Note that there is no timer destroy function. This is because the + timer is a one-time occurrence with a guarantee that the callback will be called exactly once, either at expiration or cancellation. Thus, all - the internal alarm event management state is destroyed just before + the internal timer event management state is destroyed just before that callback is invoked. If the user has additional state associated with - the alarm, the user is responsible for determining when it is safe to + the timer, the user is responsible for determining when it is safe to destroy that state. */ -/* Cancel an *alarm. +/* Cancel an *timer. There are three cases: - 1. We normally cancel the alarm - 2. The alarm has already run - 3. We can't cancel the alarm because it is "in flight". + 1. We normally cancel the timer + 2. The timer has already run + 3. We can't cancel the timer because it is "in flight". In all of these cases, the cancellation is still considered successful. - They are essentially distinguished in that the alarm_cb will be run + They are essentially distinguished in that the timer_cb will be run exactly once from either the cancellation (with status CANCELLED) or from the activation (with status SUCCESS) Note carefully that the callback function MAY occur in the same callstack - as grpc_alarm_cancel. It's expected that most alarms will be cancelled (their + as grpc_timer_cancel. It's expected that most timers will be cancelled (their primary use is to implement deadlines), and so this code is optimized such that cancellation costs as little as possible. Making callbacks run inline matches this aim. - Requires: cancel() must happen after add() on a given alarm */ -void grpc_alarm_cancel(grpc_alarm *alarm); + Requires: cancel() must happen after add() on a given timer */ +void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer); -#endif /* GRPC_INTERNAL_CORE_IOMGR_ALARM_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_H */ diff --git a/src/core/iomgr/alarm_heap.c b/src/core/iomgr/timer_heap.c index daed251982..31d41d6750 100644 --- a/src/core/iomgr/alarm_heap.c +++ b/src/core/iomgr/timer_heap.c @@ -31,7 +31,7 @@ * */ -#include "src/core/iomgr/alarm_heap.h" +#include "src/core/iomgr/timer_heap.h" #include <string.h> @@ -43,9 +43,9 @@ position. This functor is called each time immediately after modifying a value in the underlying container, with the offset of the modified element as its argument. */ -static void adjust_upwards(grpc_alarm **first, int i, grpc_alarm *t) { +static void adjust_upwards(grpc_timer **first, gpr_uint32 i, grpc_timer *t) { while (i > 0) { - int parent = (i - 1) / 2; + gpr_uint32 parent = (gpr_uint32)(((int)i - 1) / 2); if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break; first[i] = first[parent]; first[i]->heap_index = i; @@ -58,12 +58,12 @@ static void adjust_upwards(grpc_alarm **first, int i, grpc_alarm *t) { /* Adjusts a heap so as to move a hole at position i farther away from the root, until a suitable position is found for element t. Then, copies t into that position. */ -static void adjust_downwards(grpc_alarm **first, int i, int length, - grpc_alarm *t) { +static void adjust_downwards(grpc_timer **first, gpr_uint32 i, + gpr_uint32 length, grpc_timer *t) { for (;;) { - int left_child = 1 + 2 * i; - int right_child; - int next_i; + gpr_uint32 left_child = 1u + 2u * i; + gpr_uint32 right_child; + gpr_uint32 next_i; if (left_child >= length) break; right_child = left_child + 1; next_i = right_child < length && @@ -83,66 +83,66 @@ static void adjust_downwards(grpc_alarm **first, int i, int length, #define SHRINK_MIN_ELEMS 8 #define SHRINK_FULLNESS_FACTOR 2 -static void maybe_shrink(grpc_alarm_heap *heap) { - if (heap->alarm_count >= 8 && - heap->alarm_count <= heap->alarm_capacity / SHRINK_FULLNESS_FACTOR / 2) { - heap->alarm_capacity = heap->alarm_count * SHRINK_FULLNESS_FACTOR; - heap->alarms = - gpr_realloc(heap->alarms, heap->alarm_capacity * sizeof(grpc_alarm *)); +static void maybe_shrink(grpc_timer_heap *heap) { + if (heap->timer_count >= 8 && + heap->timer_count <= heap->timer_capacity / SHRINK_FULLNESS_FACTOR / 2) { + heap->timer_capacity = heap->timer_count * SHRINK_FULLNESS_FACTOR; + heap->timers = + gpr_realloc(heap->timers, heap->timer_capacity * sizeof(grpc_timer *)); } } -static void note_changed_priority(grpc_alarm_heap *heap, grpc_alarm *alarm) { - int i = alarm->heap_index; - int parent = (i - 1) / 2; - if (gpr_time_cmp(heap->alarms[parent]->deadline, alarm->deadline) < 0) { - adjust_upwards(heap->alarms, i, alarm); +static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) { + gpr_uint32 i = timer->heap_index; + gpr_uint32 parent = (gpr_uint32)(((int)i - 1) / 2); + if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) { + adjust_upwards(heap->timers, i, timer); } else { - adjust_downwards(heap->alarms, i, heap->alarm_count, alarm); + adjust_downwards(heap->timers, i, heap->timer_count, timer); } } -void grpc_alarm_heap_init(grpc_alarm_heap *heap) { +void grpc_timer_heap_init(grpc_timer_heap *heap) { memset(heap, 0, sizeof(*heap)); } -void grpc_alarm_heap_destroy(grpc_alarm_heap *heap) { gpr_free(heap->alarms); } +void grpc_timer_heap_destroy(grpc_timer_heap *heap) { gpr_free(heap->timers); } -int grpc_alarm_heap_add(grpc_alarm_heap *heap, grpc_alarm *alarm) { - if (heap->alarm_count == heap->alarm_capacity) { - heap->alarm_capacity = - GPR_MAX(heap->alarm_capacity + 1, heap->alarm_capacity * 3 / 2); - heap->alarms = - gpr_realloc(heap->alarms, heap->alarm_capacity * sizeof(grpc_alarm *)); +int grpc_timer_heap_add(grpc_timer_heap *heap, grpc_timer *timer) { + if (heap->timer_count == heap->timer_capacity) { + heap->timer_capacity = + GPR_MAX(heap->timer_capacity + 1, heap->timer_capacity * 3 / 2); + heap->timers = + gpr_realloc(heap->timers, heap->timer_capacity * sizeof(grpc_timer *)); } - alarm->heap_index = heap->alarm_count; - adjust_upwards(heap->alarms, heap->alarm_count, alarm); - heap->alarm_count++; - return alarm->heap_index == 0; + timer->heap_index = heap->timer_count; + adjust_upwards(heap->timers, heap->timer_count, timer); + heap->timer_count++; + return timer->heap_index == 0; } -void grpc_alarm_heap_remove(grpc_alarm_heap *heap, grpc_alarm *alarm) { - int i = alarm->heap_index; - if (i == heap->alarm_count - 1) { - heap->alarm_count--; +void grpc_timer_heap_remove(grpc_timer_heap *heap, grpc_timer *timer) { + gpr_uint32 i = timer->heap_index; + if (i == heap->timer_count - 1) { + heap->timer_count--; maybe_shrink(heap); return; } - heap->alarms[i] = heap->alarms[heap->alarm_count - 1]; - heap->alarms[i]->heap_index = i; - heap->alarm_count--; + heap->timers[i] = heap->timers[heap->timer_count - 1]; + heap->timers[i]->heap_index = i; + heap->timer_count--; maybe_shrink(heap); - note_changed_priority(heap, heap->alarms[i]); + note_changed_priority(heap, heap->timers[i]); } -int grpc_alarm_heap_is_empty(grpc_alarm_heap *heap) { - return heap->alarm_count == 0; +int grpc_timer_heap_is_empty(grpc_timer_heap *heap) { + return heap->timer_count == 0; } -grpc_alarm *grpc_alarm_heap_top(grpc_alarm_heap *heap) { - return heap->alarms[0]; +grpc_timer *grpc_timer_heap_top(grpc_timer_heap *heap) { + return heap->timers[0]; } -void grpc_alarm_heap_pop(grpc_alarm_heap *heap) { - grpc_alarm_heap_remove(heap, grpc_alarm_heap_top(heap)); +void grpc_timer_heap_pop(grpc_timer_heap *heap) { + grpc_timer_heap_remove(heap, grpc_timer_heap_top(heap)); } diff --git a/src/core/iomgr/alarm_heap.h b/src/core/iomgr/timer_heap.h index 60db6c991b..cd5258f93e 100644 --- a/src/core/iomgr/alarm_heap.h +++ b/src/core/iomgr/timer_heap.h @@ -31,27 +31,27 @@ * */ -#ifndef GRPC_INTERNAL_CORE_IOMGR_ALARM_HEAP_H -#define GRPC_INTERNAL_CORE_IOMGR_ALARM_HEAP_H +#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H +#define GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H -#include "src/core/iomgr/alarm.h" +#include "src/core/iomgr/timer.h" typedef struct { - grpc_alarm **alarms; - int alarm_count; - int alarm_capacity; -} grpc_alarm_heap; + grpc_timer **timers; + gpr_uint32 timer_count; + gpr_uint32 timer_capacity; +} grpc_timer_heap; -/* return 1 if the new alarm is the first alarm in the heap */ -int grpc_alarm_heap_add(grpc_alarm_heap *heap, grpc_alarm *alarm); +/* return 1 if the new timer is the first timer in the heap */ +int grpc_timer_heap_add(grpc_timer_heap *heap, grpc_timer *timer); -void grpc_alarm_heap_init(grpc_alarm_heap *heap); -void grpc_alarm_heap_destroy(grpc_alarm_heap *heap); +void grpc_timer_heap_init(grpc_timer_heap *heap); +void grpc_timer_heap_destroy(grpc_timer_heap *heap); -void grpc_alarm_heap_remove(grpc_alarm_heap *heap, grpc_alarm *alarm); -grpc_alarm *grpc_alarm_heap_top(grpc_alarm_heap *heap); -void grpc_alarm_heap_pop(grpc_alarm_heap *heap); +void grpc_timer_heap_remove(grpc_timer_heap *heap, grpc_timer *timer); +grpc_timer *grpc_timer_heap_top(grpc_timer_heap *heap); +void grpc_timer_heap_pop(grpc_timer_heap *heap); -int grpc_alarm_heap_is_empty(grpc_alarm_heap *heap); +int grpc_timer_heap_is_empty(grpc_timer_heap *heap); -#endif /* GRPC_INTERNAL_CORE_IOMGR_ALARM_HEAP_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_HEAP_H */ diff --git a/src/core/iomgr/alarm_internal.h b/src/core/iomgr/timer_internal.h index e9f98a3444..f180eca36e 100644 --- a/src/core/iomgr/alarm_internal.h +++ b/src/core/iomgr/timer_internal.h @@ -31,32 +31,33 @@ * */ -#ifndef GRPC_INTERNAL_CORE_IOMGR_ALARM_INTERNAL_H -#define GRPC_INTERNAL_CORE_IOMGR_ALARM_INTERNAL_H +#ifndef GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H +#define GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H +#include "src/core/iomgr/exec_ctx.h" #include <grpc/support/sync.h> #include <grpc/support/time.h> -/* iomgr internal api for dealing with alarms */ +/* iomgr internal api for dealing with timers */ -/* Check for alarms to be run, and run them. - Return non zero if alarm callbacks were executed. +/* Check for timers to be run, and run them. + Return non zero if timer callbacks were executed. Drops drop_mu if it is non-null before executing callbacks. - If next is non-null, TRY to update *next with the next running alarm - IF that alarm occurs before *next current value. + If next is non-null, TRY to update *next with the next running timer + IF that timer occurs before *next current value. *next is never guaranteed to be updated on any given execution; however, with high probability at least one thread in the system will see an update at any time slice. */ -int grpc_alarm_check(gpr_mu *drop_mu, gpr_timespec now, gpr_timespec *next); +int grpc_timer_check(grpc_exec_ctx* exec_ctx, gpr_timespec now, + gpr_timespec* next); +void grpc_timer_list_init(gpr_timespec now); +void grpc_timer_list_shutdown(grpc_exec_ctx* exec_ctx); -void grpc_alarm_list_init(gpr_timespec now); -void grpc_alarm_list_shutdown(void); - -gpr_timespec grpc_alarm_list_next_timeout(void); +gpr_timespec grpc_timer_list_next_timeout(void); /* the following must be implemented by each iomgr implementation */ void grpc_kick_poller(void); -#endif /* GRPC_INTERNAL_CORE_IOMGR_ALARM_INTERNAL_H */ +#endif /* GRPC_INTERNAL_CORE_IOMGR_TIMER_INTERNAL_H */ diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c index 6429c38b28..9903e970e6 100644 --- a/src/core/iomgr/udp_server.c +++ b/src/core/iomgr/udp_server.c @@ -78,9 +78,9 @@ typedef struct { struct sockaddr sockaddr; struct sockaddr_un un; } addr; - int addr_len; - grpc_iomgr_closure read_closure; - grpc_iomgr_closure destroyed_closure; + size_t addr_len; + grpc_closure read_closure; + grpc_closure destroyed_closure; grpc_udp_server_read_cb read_cb; } server_port; @@ -94,9 +94,6 @@ static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { /* the overall server */ struct grpc_udp_server { - grpc_udp_server_cb cb; - void *cb_arg; - gpr_mu mu; gpr_cv cv; @@ -114,13 +111,14 @@ struct grpc_udp_server { size_t port_capacity; /* shutdown callback */ - void (*shutdown_complete)(void *); - void *shutdown_complete_arg; + grpc_closure *shutdown_complete; /* all pollsets interested in new connections */ grpc_pollset **pollsets; /* number of pollsets in the pollsets array */ size_t pollset_count; + /* The parent grpc server */ + grpc_server *grpc_server; }; grpc_udp_server *grpc_udp_server_create(void) { @@ -130,8 +128,6 @@ grpc_udp_server *grpc_udp_server_create(void) { s->active_ports = 0; s->destroyed_ports = 0; s->shutdown = 0; - s->cb = NULL; - s->cb_arg = NULL; s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); s->nports = 0; s->port_capacity = INIT_PORT_CAP; @@ -139,8 +135,8 @@ grpc_udp_server *grpc_udp_server_create(void) { return s; } -static void finish_shutdown(grpc_udp_server *s) { - s->shutdown_complete(s->shutdown_complete_arg); +static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); gpr_mu_destroy(&s->mu); gpr_cv_destroy(&s->cv); @@ -149,24 +145,22 @@ static void finish_shutdown(grpc_udp_server *s) { gpr_free(s); } -static void destroyed_port(void *server, int success) { +static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) { grpc_udp_server *s = server; gpr_mu_lock(&s->mu); s->destroyed_ports++; if (s->destroyed_ports == s->nports) { gpr_mu_unlock(&s->mu); - finish_shutdown(s); + finish_shutdown(exec_ctx, s); } else { gpr_mu_unlock(&s->mu); } } -static void dont_care_about_shutdown_completion(void *ignored) {} - /* called when all listening endpoints have been shutdown, so no further events will be received on them - at this point it's safe to destroy things */ -static void deactivated_all_ports(grpc_udp_server *s) { +static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { size_t i; /* delete ALL the things */ @@ -185,43 +179,41 @@ static void deactivated_all_ports(grpc_udp_server *s) { } sp->destroyed_closure.cb = destroyed_port; sp->destroyed_closure.cb_arg = s; - grpc_fd_orphan(sp->emfd, &sp->destroyed_closure, "udp_listener_shutdown"); + grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, + "udp_listener_shutdown"); } gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); - finish_shutdown(s); + finish_shutdown(exec_ctx, s); } } -void grpc_udp_server_destroy( - grpc_udp_server *s, void (*shutdown_complete)(void *shutdown_complete_arg), - void *shutdown_complete_arg) { +void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, + grpc_closure *on_done) { size_t i; gpr_mu_lock(&s->mu); GPR_ASSERT(!s->shutdown); s->shutdown = 1; - s->shutdown_complete = shutdown_complete - ? shutdown_complete - : dont_care_about_shutdown_completion; - s->shutdown_complete_arg = shutdown_complete_arg; + s->shutdown_complete = on_done; /* shutdown all fd's */ if (s->active_ports) { for (i = 0; i < s->nports; i++) { - grpc_fd_shutdown(s->ports[i].emfd); + grpc_fd_shutdown(exec_ctx, s->ports[i].emfd); } gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); - deactivated_all_ports(s); + deactivated_all_ports(exec_ctx, s); } } /* Prepare a recently-created socket for listening. */ -static int prepare_socket(int fd, const struct sockaddr *addr, int addr_len) { +static int prepare_socket(int fd, const struct sockaddr *addr, + size_t addr_len) { struct sockaddr_storage sockname_temp; socklen_t sockname_len; int get_local_ip; @@ -231,17 +223,23 @@ static int prepare_socket(int fd, const struct sockaddr *addr, int addr_len) { goto error; } + if (!grpc_set_socket_nonblocking(fd, 1) || !grpc_set_socket_cloexec(fd, 1)) { + gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd, + strerror(errno)); + } + get_local_ip = 1; rc = setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &get_local_ip, sizeof(get_local_ip)); if (rc == 0 && addr->sa_family == AF_INET6) { -#if !TARGET_OS_IPHONE +#if !defined(__APPLE__) rc = setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &get_local_ip, sizeof(get_local_ip)); #endif } - if (bind(fd, addr, addr_len) < 0) { + GPR_ASSERT(addr_len < ~(socklen_t)0); + if (bind(fd, addr, (socklen_t)addr_len) < 0) { char *addr_str; grpc_sockaddr_to_string(&addr_str, addr, 0); gpr_log(GPR_ERROR, "bind addr=%s: %s", addr_str, strerror(errno)); @@ -264,14 +262,14 @@ error: } /* event manager callback when reads are ready */ -static void on_read(void *arg, int success) { +static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { server_port *sp = arg; if (success == 0) { gpr_mu_lock(&sp->server->mu); if (0 == --sp->server->active_ports) { gpr_mu_unlock(&sp->server->mu); - deactivated_all_ports(sp->server); + deactivated_all_ports(exec_ctx, sp->server); } else { gpr_mu_unlock(&sp->server->mu); } @@ -280,14 +278,14 @@ static void on_read(void *arg, int success) { /* Tell the registered callback that data is available to read. */ GPR_ASSERT(sp->read_cb); - sp->read_cb(sp->fd, sp->server->cb, sp->server->cb_arg); + sp->read_cb(exec_ctx, sp->emfd, sp->server->grpc_server); /* Re-arm the notification event so we get another chance to read. */ - grpc_fd_notify_on_read(sp->emfd, &sp->read_closure); + grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); } static int add_socket_to_server(grpc_udp_server *s, int fd, - const struct sockaddr *addr, int addr_len, + const struct sockaddr *addr, size_t addr_len, grpc_udp_server_read_cb read_cb) { server_port *sp; int port; @@ -298,8 +296,8 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, if (port >= 0) { grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1); gpr_asprintf(&name, "udp-server-listener:%s", addr_str); + gpr_free(addr_str); gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb && "must add ports before starting server"); /* append it to the list under a lock */ if (s->nports == s->port_capacity) { s->port_capacity *= 2; @@ -314,13 +312,14 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, sp->read_cb = read_cb; GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); + gpr_free(name); } return port; } -int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr, int addr_len, - grpc_udp_server_read_cb read_cb) { +int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr, + size_t addr_len, grpc_udp_server_read_cb read_cb) { int allocated_port1 = -1; int allocated_port2 = -1; unsigned i; @@ -400,28 +399,26 @@ done: return allocated_port1 >= 0 ? allocated_port1 : allocated_port2; } -int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index) { - return (index < s->nports) ? s->ports[index].fd : -1; +int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index) { + return (port_index < s->nports) ? s->ports[port_index].fd : -1; } -void grpc_udp_server_start(grpc_udp_server *s, grpc_pollset **pollsets, - size_t pollset_count, - grpc_udp_server_cb new_transport_cb, void *cb_arg) { +void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, + grpc_pollset **pollsets, size_t pollset_count, + grpc_server *server) { size_t i, j; - GPR_ASSERT(new_transport_cb); gpr_mu_lock(&s->mu); - GPR_ASSERT(!s->cb); GPR_ASSERT(s->active_ports == 0); - s->cb = new_transport_cb; - s->cb_arg = cb_arg; s->pollsets = pollsets; + s->grpc_server = server; for (i = 0; i < s->nports; i++) { for (j = 0; j < pollset_count; j++) { - grpc_pollset_add_fd(pollsets[j], s->ports[i].emfd); + grpc_pollset_add_fd(exec_ctx, pollsets[j], s->ports[i].emfd); } s->ports[i].read_closure.cb = on_read; s->ports[i].read_closure.cb_arg = &s->ports[i]; - grpc_fd_notify_on_read(s->ports[i].emfd, &s->ports[i].read_closure); + grpc_fd_notify_on_read(exec_ctx, s->ports[i].emfd, + &s->ports[i].read_closure); s->active_ports++; } gpr_mu_unlock(&s->mu); @@ -430,7 +427,7 @@ void grpc_udp_server_start(grpc_udp_server *s, grpc_pollset **pollsets, /* TODO(rjshade): Add a test for this method. */ void grpc_udp_server_write(server_port *sp, const char *buffer, size_t buf_len, const struct sockaddr *peer_address) { - int rc; + ssize_t rc; rc = sendto(sp->fd, buffer, buf_len, 0, peer_address, sizeof(peer_address)); if (rc < 0) { gpr_log(GPR_ERROR, "Unable to send data: %s", strerror(errno)); diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h index fcc4ba6e97..de5736c426 100644 --- a/src/core/iomgr/udp_server.h +++ b/src/core/iomgr/udp_server.h @@ -36,24 +36,23 @@ #include "src/core/iomgr/endpoint.h" +/* Forward decl of grpc_server */ +typedef struct grpc_server grpc_server; + /* Forward decl of grpc_udp_server */ typedef struct grpc_udp_server grpc_udp_server; -/* New server callback: ep is the newly connected connection */ -typedef void (*grpc_udp_server_cb)(void *arg, grpc_endpoint *ep); - /* Called when data is available to read from the socket. */ -typedef void (*grpc_udp_server_read_cb)(int fd, - grpc_udp_server_cb new_transport_cb, - void *cb_arg); +typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, + grpc_server *server); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); /* Start listening to bound ports */ -void grpc_udp_server_start(grpc_udp_server *server, grpc_pollset **pollsets, - size_t pollset_count, grpc_udp_server_cb cb, - void *cb_arg); +void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *udp_server, + grpc_pollset **pollsets, size_t pollset_count, + grpc_server *server); int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index); @@ -67,12 +66,11 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index); /* TODO(ctiller): deprecate this, and make grpc_udp_server_add_ports to handle all of the multiple socket port matching logic in one place */ -int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr, int addr_len, - grpc_udp_server_read_cb read_cb); +int grpc_udp_server_add_port(grpc_udp_server *s, const void *addr, + size_t addr_len, grpc_udp_server_read_cb read_cb); -void grpc_udp_server_destroy(grpc_udp_server *server, - void (*shutdown_done)(void *shutdown_done_arg), - void *shutdown_done_arg); +void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *server, + grpc_closure *on_done); /* Write the contents of buffer to the underlying UDP socket. */ /* diff --git a/src/core/iomgr/wakeup_fd_eventfd.c b/src/core/iomgr/wakeup_fd_eventfd.c index 08fdc74f17..f67379e4fc 100644 --- a/src/core/iomgr/wakeup_fd_eventfd.c +++ b/src/core/iomgr/wakeup_fd_eventfd.c @@ -39,10 +39,12 @@ #include <sys/eventfd.h> #include <unistd.h> -#include "src/core/iomgr/wakeup_fd_posix.h" #include <grpc/support/log.h> -static void eventfd_create(grpc_wakeup_fd *fd_info) { +#include "src/core/iomgr/wakeup_fd_posix.h" +#include "src/core/profiling/timers.h" + +static void eventfd_create(grpc_wakeup_fd* fd_info) { int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); /* TODO(klempner): Handle failure more gracefully */ GPR_ASSERT(efd >= 0); @@ -50,7 +52,7 @@ static void eventfd_create(grpc_wakeup_fd *fd_info) { fd_info->write_fd = -1; } -static void eventfd_consume(grpc_wakeup_fd *fd_info) { +static void eventfd_consume(grpc_wakeup_fd* fd_info) { eventfd_t value; int err; do { @@ -58,15 +60,17 @@ static void eventfd_consume(grpc_wakeup_fd *fd_info) { } while (err < 0 && errno == EINTR); } -static void eventfd_wakeup(grpc_wakeup_fd *fd_info) { +static void eventfd_wakeup(grpc_wakeup_fd* fd_info) { int err; + GPR_TIMER_BEGIN("eventfd_wakeup", 0); do { err = eventfd_write(fd_info->read_fd, 1); } while (err < 0 && errno == EINTR); + GPR_TIMER_END("eventfd_wakeup", 0); } -static void eventfd_destroy(grpc_wakeup_fd *fd_info) { - close(fd_info->read_fd); +static void eventfd_destroy(grpc_wakeup_fd* fd_info) { + if (fd_info->read_fd != 0) close(fd_info->read_fd); } static int eventfd_check_availability(void) { diff --git a/src/core/iomgr/wakeup_fd_pipe.c b/src/core/iomgr/wakeup_fd_pipe.c index bd643e8061..80de181d9d 100644 --- a/src/core/iomgr/wakeup_fd_pipe.c +++ b/src/core/iomgr/wakeup_fd_pipe.c @@ -44,7 +44,7 @@ #include "src/core/iomgr/socket_utils_posix.h" #include <grpc/support/log.h> -static void pipe_init(grpc_wakeup_fd *fd_info) { +static void pipe_init(grpc_wakeup_fd* fd_info) { int pipefd[2]; /* TODO(klempner): Make this nonfatal */ GPR_ASSERT(0 == pipe(pipefd)); @@ -54,9 +54,9 @@ static void pipe_init(grpc_wakeup_fd *fd_info) { fd_info->write_fd = pipefd[1]; } -static void pipe_consume(grpc_wakeup_fd *fd_info) { +static void pipe_consume(grpc_wakeup_fd* fd_info) { char buf[128]; - int r; + ssize_t r; for (;;) { r = read(fd_info->read_fd, buf, sizeof(buf)); @@ -74,15 +74,15 @@ static void pipe_consume(grpc_wakeup_fd *fd_info) { } } -static void pipe_wakeup(grpc_wakeup_fd *fd_info) { +static void pipe_wakeup(grpc_wakeup_fd* fd_info) { char c = 0; while (write(fd_info->write_fd, &c, 1) != 1 && errno == EINTR) ; } -static void pipe_destroy(grpc_wakeup_fd *fd_info) { - close(fd_info->read_fd); - close(fd_info->write_fd); +static void pipe_destroy(grpc_wakeup_fd* fd_info) { + if (fd_info->read_fd != 0) close(fd_info->read_fd); + if (fd_info->write_fd != 0) close(fd_info->write_fd); } static int pipe_check_availability(void) { diff --git a/src/core/iomgr/wakeup_fd_posix.h b/src/core/iomgr/wakeup_fd_posix.h index b6c086900d..fe71b5abe9 100644 --- a/src/core/iomgr/wakeup_fd_posix.h +++ b/src/core/iomgr/wakeup_fd_posix.h @@ -72,10 +72,10 @@ void grpc_wakeup_fd_global_init_force_fallback(void); typedef struct grpc_wakeup_fd grpc_wakeup_fd; typedef struct grpc_wakeup_fd_vtable { - void (*init)(grpc_wakeup_fd *fd_info); - void (*consume)(grpc_wakeup_fd *fd_info); - void (*wakeup)(grpc_wakeup_fd *fd_info); - void (*destroy)(grpc_wakeup_fd *fd_info); + void (*init)(grpc_wakeup_fd* fd_info); + void (*consume)(grpc_wakeup_fd* fd_info); + void (*wakeup)(grpc_wakeup_fd* fd_info); + void (*destroy)(grpc_wakeup_fd* fd_info); /* Must be called before calling any other functions */ int (*check_availability)(void); } grpc_wakeup_fd_vtable; @@ -87,10 +87,10 @@ struct grpc_wakeup_fd { #define GRPC_WAKEUP_FD_GET_READ_FD(fd_info) ((fd_info)->read_fd) -void grpc_wakeup_fd_init(grpc_wakeup_fd *fd_info); -void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd *fd_info); -void grpc_wakeup_fd_wakeup(grpc_wakeup_fd *fd_info); -void grpc_wakeup_fd_destroy(grpc_wakeup_fd *fd_info); +void grpc_wakeup_fd_init(grpc_wakeup_fd* fd_info); +void grpc_wakeup_fd_consume_wakeup(grpc_wakeup_fd* fd_info); +void grpc_wakeup_fd_wakeup(grpc_wakeup_fd* fd_info); +void grpc_wakeup_fd_destroy(grpc_wakeup_fd* fd_info); /* Defined in some specialized implementation's .c file, or by * wakeup_fd_nospecial.c if no such implementation exists. */ diff --git a/src/core/iomgr/workqueue.h b/src/core/iomgr/workqueue.h new file mode 100644 index 0000000000..714536233c --- /dev/null +++ b/src/core/iomgr/workqueue.h @@ -0,0 +1,85 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_H +#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_H + +#include "src/core/iomgr/iomgr.h" +#include "src/core/iomgr/pollset.h" +#include "src/core/iomgr/closure.h" +#include "src/core/iomgr/exec_ctx.h" + +#ifdef GPR_POSIX_SOCKET +#include "src/core/iomgr/workqueue_posix.h" +#endif + +#ifdef GPR_WIN32 +#include "src/core/iomgr/workqueue_windows.h" +#endif + +/** A workqueue represents a list of work to be executed asynchronously. */ +struct grpc_workqueue; +typedef struct grpc_workqueue grpc_workqueue; + +/** Create a work queue */ +grpc_workqueue *grpc_workqueue_create(grpc_exec_ctx *exec_ctx); + +void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue); + +#define GRPC_WORKQUEUE_REFCOUNT_DEBUG +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +#define GRPC_WORKQUEUE_REF(p, r) \ + grpc_workqueue_ref((p), __FILE__, __LINE__, (r)) +#define GRPC_WORKQUEUE_UNREF(cl, p, r) \ + grpc_workqueue_unref((cl), (p), __FILE__, __LINE__, (r)) +void grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, + const char *reason); +void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason); +#else +#define GRPC_WORKQUEUE_REF(p, r) grpc_workqueue_ref((p)) +#define GRPC_WORKQUEUE_UNREF(cl, p, r) grpc_workqueue_unref((cl), (p)) +void grpc_workqueue_ref(grpc_workqueue *workqueue); +void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue); +#endif + +/** Bind this workqueue to a pollset */ +void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue, + grpc_pollset *pollset); + +/** Add a work item to a workqueue */ +void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure, + int success); + +#endif diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c new file mode 100644 index 0000000000..0a0f3c364e --- /dev/null +++ b/src/core/iomgr/workqueue_posix.c @@ -0,0 +1,142 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/port_platform.h> + +#ifdef GPR_POSIX_SOCKET + +#include "src/core/iomgr/workqueue.h" + +#include <stdio.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/useful.h> + +#include "src/core/iomgr/fd_posix.h" + +static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, int success); + +grpc_workqueue *grpc_workqueue_create(grpc_exec_ctx *exec_ctx) { + char name[32]; + grpc_workqueue *workqueue = gpr_malloc(sizeof(grpc_workqueue)); + gpr_ref_init(&workqueue->refs, 1); + gpr_mu_init(&workqueue->mu); + workqueue->closure_list.head = workqueue->closure_list.tail = NULL; + grpc_wakeup_fd_init(&workqueue->wakeup_fd); + sprintf(name, "workqueue:%p", (void *)workqueue); + workqueue->wakeup_read_fd = + grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&workqueue->wakeup_fd), name); + grpc_closure_init(&workqueue->read_closure, on_readable, workqueue); + grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd, + &workqueue->read_closure); + return workqueue; +} + +static void workqueue_destroy(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + GPR_ASSERT(grpc_closure_list_empty(workqueue->closure_list)); + grpc_fd_shutdown(exec_ctx, workqueue->wakeup_read_fd); +} + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +void grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, + const char *reason) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "WORKQUEUE:%p ref %d -> %d %s", + workqueue, (int)workqueue->refs.count, (int)workqueue->refs.count + 1, + reason); +#else +void grpc_workqueue_ref(grpc_workqueue *workqueue) { +#endif + gpr_ref(&workqueue->refs); +} + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "WORKQUEUE:%p unref %d -> %d %s", + workqueue, (int)workqueue->refs.count, (int)workqueue->refs.count - 1, + reason); +#else +void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { +#endif + if (gpr_unref(&workqueue->refs)) { + workqueue_destroy(exec_ctx, workqueue); + } +} + +void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue, + grpc_pollset *pollset) { + grpc_pollset_add_fd(exec_ctx, pollset, workqueue->wakeup_read_fd); +} + +void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { + gpr_mu_lock(&workqueue->mu); + grpc_closure_list_move(&exec_ctx->closure_list, &workqueue->closure_list); + gpr_mu_unlock(&workqueue->mu); +} + +static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, int success) { + grpc_workqueue *workqueue = arg; + + if (!success) { + gpr_mu_destroy(&workqueue->mu); + /* HACK: let wakeup_fd code know that we stole the fd */ + workqueue->wakeup_fd.read_fd = 0; + grpc_wakeup_fd_destroy(&workqueue->wakeup_fd); + grpc_fd_orphan(exec_ctx, workqueue->wakeup_read_fd, NULL, "destroy"); + gpr_free(workqueue); + } else { + gpr_mu_lock(&workqueue->mu); + grpc_closure_list_move(&workqueue->closure_list, &exec_ctx->closure_list); + grpc_wakeup_fd_consume_wakeup(&workqueue->wakeup_fd); + gpr_mu_unlock(&workqueue->mu); + grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd, + &workqueue->read_closure); + } +} + +void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure, + int success) { + closure->success = success; + closure->next = NULL; + gpr_mu_lock(&workqueue->mu); + if (grpc_closure_list_empty(workqueue->closure_list)) { + grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd); + } + grpc_closure_list_add(&workqueue->closure_list, closure, success); + gpr_mu_unlock(&workqueue->mu); +} + +#endif /* GPR_POSIX_SOCKET */ diff --git a/src/core/iomgr/workqueue_posix.h b/src/core/iomgr/workqueue_posix.h new file mode 100644 index 0000000000..589034fe1b --- /dev/null +++ b/src/core/iomgr/workqueue_posix.h @@ -0,0 +1,51 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H +#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H + +struct grpc_fd; + +struct grpc_workqueue { + gpr_refcount refs; + + gpr_mu mu; + grpc_closure_list closure_list; + + grpc_wakeup_fd wakeup_fd; + struct grpc_fd *wakeup_read_fd; + + grpc_closure read_closure; +}; + +#endif /* GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_POSIX_H */ diff --git a/src/core/iomgr/workqueue_windows.c b/src/core/iomgr/workqueue_windows.c new file mode 100644 index 0000000000..f9ca57557b --- /dev/null +++ b/src/core/iomgr/workqueue_windows.c @@ -0,0 +1,40 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/port_platform.h> + +#ifdef GPR_WIN32 + +#include "src/core/iomgr/workqueue.h" + +#endif /* GPR_WIN32 */ diff --git a/src/core/iomgr/workqueue_windows.h b/src/core/iomgr/workqueue_windows.h new file mode 100644 index 0000000000..941f195f51 --- /dev/null +++ b/src/core/iomgr/workqueue_windows.h @@ -0,0 +1,37 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H +#define GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H + +#endif /* GRPC_INTERNAL_CORE_IOMGR_WORKQUEUE_WINDOWS_H */ |