diff options
Diffstat (limited to 'src/core/lib')
112 files changed, 3459 insertions, 673 deletions
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 569be4dc28..79ceeb66b3 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -35,6 +35,7 @@ #include <grpc/grpc.h> #include "src/core/lib/support/string.h" +#include <grpc/compression.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> @@ -148,6 +149,7 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) { void grpc_channel_args_destroy(grpc_channel_args *a) { size_t i; + if (!a) return; for (i = 0; i < a->num_args; i++) { switch (a->args[i].type) { case GRPC_ARG_STRING: @@ -181,6 +183,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( grpc_channel_args *grpc_channel_args_set_compression_algorithm( grpc_channel_args *a, grpc_compression_algorithm algorithm) { + GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; @@ -200,7 +203,8 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a, !strcmp(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET, a->args[i].key)) { *states_arg = &a->args[i].value.integer; - return 1; /* GPR_TRUE */ + **states_arg |= 0x1; /* forcefully enable support for no compression */ + return 1; } } } @@ -209,15 +213,23 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a, grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( grpc_channel_args **a, grpc_compression_algorithm algorithm, int state) { - int *states_arg; + int *states_arg = NULL; grpc_channel_args *result = *a; const int states_arg_found = find_compression_algorithm_states_bitset(*a, &states_arg); - if (states_arg_found) { + if (grpc_channel_args_get_compression_algorithm(*a) == algorithm && + state == 0) { + char *algo_name = NULL; + GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0); + gpr_log(GPR_ERROR, + "Tried to disable default compression algorithm '%s'. The " + "operation has been ignored.", + algo_name); + } else if (states_arg_found) { if (state != 0) { GPR_BITSET((unsigned *)states_arg, algorithm); - } else { + } else if (algorithm != GRPC_COMPRESS_NONE) { GPR_BITCLEAR((unsigned *)states_arg, algorithm); } } else { @@ -229,7 +241,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { GPR_BITSET((unsigned *)&tmp.value.integer, algorithm); - } else { + } else if (algorithm != GRPC_COMPRESS_NONE) { GPR_BITCLEAR((unsigned *)&tmp.value.integer, algorithm); } result = grpc_channel_args_copy_and_add(*a, &tmp, 1); @@ -239,11 +251,11 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( return result; } -int grpc_channel_args_compression_algorithm_get_states( +uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args *a) { int *states_arg; if (find_compression_algorithm_states_bitset(a, &states_arg)) { - return *states_arg; + return (uint32_t)*states_arg; } else { return (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; /* All algs. enabled */ } diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 23c7b7b897..653d04f427 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -81,7 +81,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( * * The i-th bit of the returned bitset corresponds to the i-th entry in the * grpc_compression_algorithm enum. */ -int grpc_channel_args_compression_algorithm_get_states( +uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args *a); int grpc_channel_args_compare(const grpc_channel_args *a, diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index ad182d1f69..bbba85d80b 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -106,6 +106,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs, const grpc_channel_filter **filters, size_t filter_count, const grpc_channel_args *channel_args, + grpc_transport *optional_transport, const char *name, grpc_channel_stack *stack) { size_t call_size = ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) + @@ -127,6 +128,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs, for (i = 0; i < filter_count; i++) { args.channel_stack = stack; args.channel_args = channel_args; + args.optional_transport = optional_transport; args.is_first = i == 0; args.is_last = i == (filter_count - 1); elems[i].filter = filters[i]; @@ -189,9 +191,9 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx, } } -void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_call_stack *call_stack, - grpc_pollset *pollset) { +void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_call_stack *call_stack, + grpc_polling_entity *pollent) { size_t count = call_stack->count; grpc_call_element *call_elems; char *user_data; @@ -203,17 +205,19 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, /* init per-filter data */ for (i = 0; i < count; i++) { - call_elems[i].filter->set_pollset(exec_ctx, &call_elems[i], pollset); + call_elems[i].filter->set_pollset_or_pollset_set(exec_ctx, &call_elems[i], + pollent); user_data += ROUND_UP_TO_ALIGNMENT_SIZE(call_elems[i].filter->sizeof_call_data); } } -void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_pollset *pollset) {} +void grpc_call_stack_ignore_set_pollset_or_pollset_set( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_polling_entity *pollent) {} void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, + const grpc_call_stats *call_stats, void *and_free_memory) { grpc_call_element *elems = CALL_ELEMS_FROM_STACK(stack); size_t count = stack->count; @@ -221,7 +225,7 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, /* destroy per-filter data */ for (i = 0; i < count; i++) { - elems[i].filter->destroy_call_elem(exec_ctx, &elems[i], + elems[i].filter->destroy_call_elem(exec_ctx, &elems[i], call_stats, i == count - 1 ? and_free_memory : NULL); } } diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 36c17cb467..41dd4a0d8a 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -45,7 +45,10 @@ #include <grpc/grpc.h> #include <grpc/support/log.h> +#include <grpc/support/time.h> + #include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/transport/transport.h" typedef struct grpc_channel_element grpc_channel_element; @@ -57,6 +60,8 @@ typedef struct grpc_call_stack grpc_call_stack; typedef struct { grpc_channel_stack *channel_stack; const grpc_channel_args *channel_args; + /** Transport, iff it is known */ + grpc_transport *optional_transport; int is_first; int is_last; } grpc_channel_element_args; @@ -67,6 +72,12 @@ typedef struct { grpc_call_context_element *context; } grpc_call_element_args; +typedef struct { + grpc_transport_stream_stats transport_stream_stats; + gpr_timespec latency; /* From call creating to enqueing of received status */ + grpc_status_code final_status; +} grpc_call_stats; + /* Channel filters specify: 1. the amount of memory needed in the channel & call (via the sizeof_XXX members) @@ -101,14 +112,16 @@ typedef struct { argument. */ void (*init_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_call_element_args *args); - void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_pollset *pollset); + void (*set_pollset_or_pollset_set)(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_polling_entity *pollent); /* Destroy per call data. The filter does not need to do any chaining. The bottom filter of a stack will be passed a non-NULL pointer to \a and_free_memory that should be passed to gpr_free when destruction is complete. */ void (*destroy_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + const grpc_call_stats *stats, void *and_free_memory); /* sizeof(per channel data) */ @@ -187,6 +200,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs, grpc_iomgr_cb_func destroy, void *destroy_arg, const grpc_channel_filter **filters, size_t filter_count, const grpc_channel_args *args, + grpc_transport *optional_transport, const char *name, grpc_channel_stack *stack); /* Destroy a channel stack */ void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx, @@ -201,10 +215,11 @@ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx, grpc_call_context_element *context, const void *transport_server_data, grpc_call_stack *call_stack); -/* Set a pollset for a call stack: must occur before the first op is started */ -void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_call_stack *call_stack, - grpc_pollset *pollset); +/* Set a pollset or a pollset_set for a call stack: must occur before the first + * op is started */ +void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_call_stack *call_stack, + grpc_polling_entity *pollent); #ifdef GRPC_STREAM_REFCOUNT_DEBUG #define GRPC_CALL_STACK_REF(call_stack, reason) \ @@ -228,13 +243,14 @@ void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, /* Destroy a call stack */ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, + const grpc_call_stats *call_stats, void *and_free_memory); -/* Ignore set pollset - used by filters to implement the set_pollset method - if they don't care about pollsets at all. Does nothing. */ -void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_pollset *pollset); +/* Ignore set pollset{_set} - used by filters if they don't care about pollsets + * at all. Does nothing. */ +void grpc_call_stack_ignore_set_pollset_or_pollset_set( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_polling_entity *pollent); /* Call the next operation in a call stack */ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op); diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index a8646c9565..eda4968f48 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -257,8 +257,8 @@ void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx, // and initialize it grpc_channel_stack_init(exec_ctx, initial_refs, destroy, destroy_arg == NULL ? result : destroy_arg, filters, - num_filters, builder->args, builder->name, - channel_stack); + num_filters, builder->args, builder->transport, + builder->name, channel_stack); // run post-initialization functions i = 0; diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index 27a87db0c2..32ebe53ee6 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -73,8 +73,8 @@ typedef struct call_data { typedef struct channel_data { /** The default, channel-level, compression algorithm */ grpc_compression_algorithm default_compression_algorithm; - /** Compression options for the channel */ - grpc_compression_options compression_options; + /** Bitset of enabled algorithms */ + uint32_t enabled_algorithms_bitset; /** Supported compression algorithms */ uint32_t supported_compression_algorithms; } channel_data; @@ -96,9 +96,8 @@ static grpc_mdelem *compression_md_filter(void *user_data, grpc_mdelem *md) { md_c_str); calld->compression_algorithm = GRPC_COMPRESS_NONE; } - if (grpc_compression_options_is_algorithm_enabled( - &channeld->compression_options, calld->compression_algorithm) == - 0) { + if (!GPR_BITGET(channeld->enabled_algorithms_bitset, + calld->compression_algorithm)) { gpr_log(GPR_ERROR, "Invalid compression algorithm: '%s' (previously disabled). " "Ignoring.", @@ -178,8 +177,8 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, const float savings_ratio = 1.0f - (float)after_size / (float)before_size; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); - gpr_log(GPR_DEBUG, - "Compressed[%s] %d bytes vs. %d bytes (%.2f%% savings)", + gpr_log(GPR_DEBUG, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR + " bytes (%.2f%% savings)", algo_name, before_size, after_size, 100 * savings_ratio); } gpr_slice_buffer_swap(&calld->slices, &tmp); @@ -189,10 +188,10 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); - gpr_log( - GPR_DEBUG, - "Algorithm '%s' enabled but decided not to compress. Input size: %d", - algo_name, calld->slices.length); + gpr_log(GPR_DEBUG, + "Algorithm '%s' enabled but decided not to compress. Input size: " + "%" PRIuPTR, + algo_name, calld->slices.length); } } @@ -271,7 +270,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) { + const grpc_call_stats *stats, void *ignored) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; gpr_slice_buffer_destroy(&calld->slices); @@ -282,32 +281,26 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_channel_element_args *args) { channel_data *channeld = elem->channel_data; - grpc_compression_algorithm algo_idx; - grpc_compression_options_init(&channeld->compression_options); - channeld->compression_options.enabled_algorithms_bitset = - (uint32_t)grpc_channel_args_compression_algorithm_get_states( - args->channel_args); + channeld->enabled_algorithms_bitset = + grpc_channel_args_compression_algorithm_get_states(args->channel_args); channeld->default_compression_algorithm = grpc_channel_args_get_compression_algorithm(args->channel_args); /* Make sure the default isn't disabled. */ - if (!grpc_compression_options_is_algorithm_enabled( - &channeld->compression_options, - channeld->default_compression_algorithm)) { + if (!GPR_BITGET(channeld->enabled_algorithms_bitset, + channeld->default_compression_algorithm)) { gpr_log(GPR_DEBUG, "compression algorithm %d not enabled: switching to none", channeld->default_compression_algorithm); channeld->default_compression_algorithm = GRPC_COMPRESS_NONE; } - channeld->compression_options.default_compression_algorithm = - channeld->default_compression_algorithm; - channeld->supported_compression_algorithms = 0; - for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { + channeld->supported_compression_algorithms = 1; /* always support identity */ + for (grpc_compression_algorithm algo_idx = 1; + algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { /* skip disabled algorithms */ - if (grpc_compression_options_is_algorithm_enabled( - &channeld->compression_options, algo_idx) == 0) { + if (!GPR_BITGET(channeld->enabled_algorithms_bitset, algo_idx)) { continue; } channeld->supported_compression_algorithms |= 1u << algo_idx; @@ -325,7 +318,7 @@ const grpc_channel_filter grpc_compress_filter = { grpc_channel_next_op, sizeof(call_data), init_call_elem, - grpc_call_stack_ignore_set_pollset, + grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, diff --git a/src/core/lib/channel/compress_filter.h b/src/core/lib/channel/compress_filter.h index 0ce5d08837..e4a2a829d5 100644 --- a/src/core/lib/channel/compress_filter.h +++ b/src/core/lib/channel/compress_filter.h @@ -34,9 +34,9 @@ #ifndef GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H #define GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H -#include "src/core/lib/channel/channel_stack.h" +#include <grpc/impl/codegen/compression_types.h> -#define GRPC_COMPRESS_REQUEST_ALGORITHM_KEY "grpc-internal-encoding-request" +#include "src/core/lib/channel/channel_stack.h" extern int grpc_compression_trace; @@ -48,7 +48,7 @@ extern int grpc_compression_trace; * - Channel configuration, as established at channel creation time. * - The metadata accompanying the outgoing data to be compressed. This is * taken as a request only. We may choose not to honor it. The metadata key - * is given by \a GRPC_COMPRESS_REQUEST_ALGORITHM_KEY. + * is given by \a GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY. * * Compression can be disabled for concrete messages (for instance in order to * prevent CRIME/BEAST type attacks) by having the GRPC_WRITE_NO_COMPRESS set in diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index 68a3a7d6fd..0a7d27a1dc 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -93,16 +93,18 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, GPR_ASSERT(r == 0); } -static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_pollset *pollset) { +static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_polling_entity *pollent) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - grpc_transport_set_pollset(exec_ctx, chand->transport, - TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollset); + grpc_transport_set_pops(exec_ctx, chand->transport, + TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollent); } /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + const grpc_call_stats *stats, void *and_free_memory) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; @@ -137,7 +139,7 @@ static const grpc_channel_filter connected_channel_filter = { con_start_transport_op, sizeof(call_data), init_call_elem, - set_pollset, + set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index ced3d75cd2..ab6c6c9ef0 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -38,6 +38,10 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" +#include "src/core/lib/transport/transport_impl.h" + +#define EXPECTED_CONTENT_TYPE "application/grpc" +#define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 typedef struct call_data { grpc_linked_mdelem method; @@ -74,7 +78,24 @@ static grpc_mdelem *client_recv_filter(void *user_data, grpc_mdelem *md) { } else if (md->key == GRPC_MDSTR_STATUS) { grpc_call_element_send_cancel(a->exec_ctx, a->elem); return NULL; + } else if (md == GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC) { + return NULL; } else if (md->key == GRPC_MDSTR_CONTENT_TYPE) { + const char *value_str = grpc_mdstr_as_c_string(md->value); + if (strncmp(value_str, EXPECTED_CONTENT_TYPE, + EXPECTED_CONTENT_TYPE_LENGTH) == 0 && + (value_str[EXPECTED_CONTENT_TYPE_LENGTH] == '+' || + value_str[EXPECTED_CONTENT_TYPE_LENGTH] == ';')) { + /* Although the C implementation doesn't (currently) generate them, + any custom +-suffix is explicitly valid. */ + /* TODO(klempner): We should consider preallocating common values such + as +proto or +json, or at least stashing them if we see them. */ + /* TODO(klempner): Should we be surfacing this to application code? */ + } else { + /* TODO(klempner): We're currently allowing this, but we shouldn't + see it without a proxy so log for now. */ + gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str); + } return NULL; } return md; @@ -157,7 +178,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) {} + const grpc_call_stats *stats, void *ignored) {} static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) { unsigned i; @@ -180,7 +201,8 @@ static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) { return GRPC_MDELEM_SCHEME_HTTP; } -static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) { +static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args, + const char *transport_name) { gpr_strvec v; size_t i; int is_first = 1; @@ -202,8 +224,8 @@ static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args) { } } - gpr_asprintf(&tmp, "%sgrpc-c/%s (%s)", is_first ? "" : " ", - grpc_version_string(), GPR_PLATFORM_STRING); + gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s)", is_first ? "" : " ", + grpc_version_string(), GPR_PLATFORM_STRING, transport_name); is_first = 0; gpr_strvec_add(&v, tmp); @@ -234,9 +256,12 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; GPR_ASSERT(!args->is_last); + GPR_ASSERT(args->optional_transport != NULL); chand->static_scheme = scheme_from_args(args->channel_args); chand->user_agent = grpc_mdelem_from_metadata_strings( - GRPC_MDSTR_USER_AGENT, user_agent_from_args(args->channel_args)); + GRPC_MDSTR_USER_AGENT, + user_agent_from_args(args->channel_args, + args->optional_transport->vtable->name)); } /* Destructor for channel data */ @@ -251,7 +276,7 @@ const grpc_channel_filter grpc_http_client_filter = { grpc_channel_next_op, sizeof(call_data), init_call_elem, - grpc_call_stack_ignore_set_pollset, + grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, diff --git a/src/core/lib/channel/http_client_filter.h b/src/core/lib/channel/http_client_filter.h index a884b36318..47081175ea 100644 --- a/src/core/lib/channel/http_client_filter.h +++ b/src/core/lib/channel/http_client_filter.h @@ -1,5 +1,4 @@ /* - * * Copyright 2015, Google Inc. * All rights reserved. * @@ -39,6 +38,7 @@ /* Processes metadata on the client side for HTTP2 transports */ extern const grpc_channel_filter grpc_http_client_filter; +/* Channel arg to override the http2 :scheme header */ #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme" #endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */ diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index 876eaddc1e..d0beebd817 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -108,7 +108,7 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) { } else { /* TODO(klempner): We're currently allowing this, but we shouldn't see it without a proxy so log for now. */ - gpr_log(GPR_INFO, "Unexpected content-type %s", value_str); + gpr_log(GPR_INFO, "Unexpected content-type '%s'", value_str); } return NULL; } else if (md->key == GRPC_MDSTR_TE || md->key == GRPC_MDSTR_METHOD || @@ -235,7 +235,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) {} + const grpc_call_stats *stats, void *ignored) {} /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, @@ -253,7 +253,7 @@ const grpc_channel_filter grpc_http_server_filter = { grpc_channel_next_op, sizeof(call_data), init_call_elem, - grpc_call_stack_ignore_set_pollset, + grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression.c index 820871d579..54efb5e855 100644 --- a/src/core/lib/compression/compression_algorithm.c +++ b/src/core/lib/compression/compression.c @@ -125,6 +125,28 @@ grpc_mdelem *grpc_compression_encoding_mdelem( return NULL; } +void grpc_compression_options_init(grpc_compression_options *opts) { + memset(opts, 0, sizeof(*opts)); + /* all enabled by default */ + opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; +} + +void grpc_compression_options_enable_algorithm( + grpc_compression_options *opts, grpc_compression_algorithm algorithm) { + GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm); +} + +void grpc_compression_options_disable_algorithm( + grpc_compression_options *opts, grpc_compression_algorithm algorithm) { + GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm); +} + +int grpc_compression_options_is_algorithm_enabled( + const grpc_compression_options *opts, + grpc_compression_algorithm algorithm) { + return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm); +} + /* TODO(dgq): Add the ability to specify parameters to the individual * compression algorithms */ grpc_compression_algorithm grpc_compression_algorithm_for_level( @@ -180,25 +202,3 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level( abort(); }; } - -void grpc_compression_options_init(grpc_compression_options *opts) { - opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; - opts->default_compression_algorithm = GRPC_COMPRESS_NONE; -} - -void grpc_compression_options_enable_algorithm( - grpc_compression_options *opts, grpc_compression_algorithm algorithm) { - GPR_BITSET(&opts->enabled_algorithms_bitset, algorithm); -} - -void grpc_compression_options_disable_algorithm( - grpc_compression_options *opts, grpc_compression_algorithm algorithm) { - GPR_BITCLEAR(&opts->enabled_algorithms_bitset, algorithm); -} - -int grpc_compression_options_is_algorithm_enabled( - const grpc_compression_options *opts, - grpc_compression_algorithm algorithm) { - if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0; - return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm); -} diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 4bda734bfa..18135bcb58 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -63,7 +63,7 @@ typedef struct { const grpc_httpcli_handshaker *handshaker; grpc_closure *on_done; grpc_httpcli_context *context; - grpc_pollset *pollset; + grpc_polling_entity *pollent; grpc_iomgr_object iomgr_obj; gpr_slice_buffer incoming; gpr_slice_buffer outgoing; @@ -78,6 +78,7 @@ static grpc_httpcli_post_override g_post_override = NULL; static void plaintext_handshake(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint, const char *host, + gpr_timespec deadline, void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint)) { @@ -100,9 +101,9 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, grpc_error *error) { - grpc_pollset_set_del_pollset(exec_ctx, req->context->pollset_set, - req->pollset); - grpc_exec_ctx_push(exec_ctx, req->on_done, error, NULL); + grpc_polling_entity_del_from_pollset_set(exec_ctx, req->pollent, + req->context->pollset_set); + grpc_exec_ctx_sched(exec_ctx, req->on_done, error, NULL); grpc_http_parser_destroy(&req->parser); if (req->addresses != NULL) { grpc_resolved_addresses_destroy(req->addresses); @@ -206,7 +207,7 @@ static void on_connected(grpc_exec_ctx *exec_ctx, void *arg, req->handshaker->handshake( exec_ctx, req, req->ep, req->ssl_host_override ? req->ssl_host_override : req->host, - on_handshake_done); + req->deadline, on_handshake_done); } static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, @@ -240,7 +241,7 @@ static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { static void internal_request_begin(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_done, grpc_httpcli_response *response, @@ -254,7 +255,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, req->handshaker = request->handshaker ? request->handshaker : &grpc_httpcli_plaintext; req->context = context; - req->pollset = pollset; + req->pollent = pollent; req->overall_error = GRPC_ERROR_NONE; grpc_closure_init(&req->on_read, on_read, req); grpc_closure_init(&req->done_write, done_write, req); @@ -264,14 +265,15 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, req->host = gpr_strdup(request->host); req->ssl_host_override = gpr_strdup(request->ssl_host_override); - grpc_pollset_set_add_pollset(exec_ctx, req->context->pollset_set, - req->pollset); + GPR_ASSERT(pollent); + grpc_polling_entity_add_to_pollset_set(exec_ctx, req->pollent, + req->context->pollset_set); grpc_resolve_address(exec_ctx, request->host, req->handshaker->default_port, grpc_closure_create(on_resolved, req), &req->addresses); } void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_done, grpc_httpcli_response *response) { @@ -281,14 +283,14 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, return; } gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path); - internal_request_begin(exec_ctx, context, pollset, request, deadline, on_done, + internal_request_begin(exec_ctx, context, pollent, request, deadline, on_done, response, name, grpc_httpcli_format_get_request(request)); gpr_free(name); } void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, const grpc_httpcli_request *request, const char *body_bytes, size_t body_size, gpr_timespec deadline, grpc_closure *on_done, @@ -301,7 +303,7 @@ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, } gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path); internal_request_begin( - exec_ctx, context, pollset, request, deadline, on_done, response, name, + exec_ctx, context, pollent, request, deadline, on_done, response, name, grpc_httpcli_format_post_request(request, body_bytes, body_size)); gpr_free(name); } diff --git a/src/core/lib/http/httpcli.h b/src/core/lib/http/httpcli.h index 2d57864a1f..662e176f4c 100644 --- a/src/core/lib/http/httpcli.h +++ b/src/core/lib/http/httpcli.h @@ -41,6 +41,7 @@ #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/pollset_set.h" /* User agent this library reports */ @@ -56,7 +57,7 @@ typedef struct grpc_httpcli_context { typedef struct { const char *default_port; void (*handshake)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint, - const char *host, + const char *host, gpr_timespec deadline, void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint)); } grpc_httpcli_handshaker; @@ -95,7 +96,7 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context); 'on_response' is a callback to report results to (and 'user_data' is a user supplied pointer to pass to said call) */ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_complete, grpc_httpcli_response *response); @@ -116,7 +117,7 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, supplied pointer to pass to said call) Does not support ?var1=val1&var2=val2 in the path. */ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, const grpc_httpcli_request *request, const char *body_bytes, size_t body_size, gpr_timespec deadline, grpc_closure *on_complete, diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 5590928968..a57d93bb7b 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -61,6 +61,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) { static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_httpcli_ssl_channel_security_connector *c = @@ -79,7 +80,7 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, - nonsecure_endpoint, cb, user_data); + nonsecure_endpoint, deadline, cb, user_data); } } @@ -163,6 +164,7 @@ static void on_secure_transport_setup_done(grpc_exec_ctx *exec_ctx, void *rp, static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, const char *host, + gpr_timespec deadline, void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint)) { grpc_channel_security_connector *sc = NULL; @@ -181,7 +183,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, pem_root_certs, pem_root_certs_size, host, &sc) == GRPC_SECURITY_OK); grpc_channel_security_connector_do_handshake( - exec_ctx, sc, tcp, on_secure_transport_setup_done, c); + exec_ctx, sc, tcp, deadline, on_secure_transport_setup_done, c); GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli"); } diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 344b7d3c3e..08e59a168e 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -53,8 +53,8 @@ typedef struct grpc_closure_list { /** gRPC Callback definition. * * \param arg Arbitrary input. - * \param success An indication on the state of the iomgr. On false, cleanup - * actions should be taken (eg, shutdown). */ + * \param error GRPC_ERROR_NONE if no error occurred, otherwise some grpc_error + * describing what went wrong */ typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); @@ -66,8 +66,11 @@ struct grpc_closure { /** Arguments to be passed to "cb". */ void *cb_arg; + /** Once queued, the result of the closure. Before then: scratch space */ grpc_error *error; + /** Once queued, next indicates the next queued closure; before then, scratch + * space */ union { grpc_closure *next; uintptr_t scratch; @@ -84,8 +87,8 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg); #define GRPC_CLOSURE_LIST_INIT \ { NULL, NULL } -/** add \a closure to the end of \a list and set \a closure's success to \a - * success */ +/** add \a closure to the end of \a list + and set \a closure's result to \a error */ void grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure, grpc_error *error); diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h index 3877ceb1e2..f9808bbda1 100644 --- a/src/core/lib/iomgr/endpoint.h +++ b/src/core/lib/iomgr/endpoint.h @@ -82,7 +82,7 @@ char *grpc_endpoint_get_peer(grpc_endpoint *ep); void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, gpr_slice_buffer *slices, grpc_closure *cb); -/* Causes any pending read/write callbacks to run immediately with +/* Causes any pending and future read/write callbacks to run immediately with success==0 */ void grpc_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); void grpc_endpoint_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 6da0b00582..540fb4fa7e 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,6 +33,7 @@ #include "src/core/lib/iomgr/error.h" +#include <inttypes.h> #include <stdbool.h> #include <string.h> @@ -42,6 +43,10 @@ #include <grpc/support/string_util.h> #include <grpc/support/useful.h> +#ifdef GPR_WINDOWS +#include <grpc/support/log_windows.h> +#endif + static void destroy_integer(void *key) {} static void *copy_integer(void *key) { return key; } @@ -84,14 +89,10 @@ static const gpr_avl_vtable avl_vtable_errs = { static const char *error_int_name(grpc_error_ints key) { switch (key) { - case GRPC_ERROR_INT_STATUS_CODE: - return "status_code"; case GRPC_ERROR_INT_ERRNO: return "errno"; case GRPC_ERROR_INT_FILE_LINE: return "file_line"; - case GRPC_ERROR_INT_WARNING: - return "warning"; case GRPC_ERROR_INT_STREAM_ID: return "stream_id"; case GRPC_ERROR_INT_GRPC_STATUS: @@ -112,6 +113,8 @@ static const char *error_int_name(grpc_error_ints key) { return "fd"; case GRPC_ERROR_INT_WSA_ERROR: return "wsa_error"; + case GRPC_ERROR_INT_HTTP_STATUS: + return "http_status"; } GPR_UNREACHABLE_CODE(return "unknown"); } @@ -336,7 +339,7 @@ static char *key_time(void *p) { static char *fmt_int(void *p) { char *s; - gpr_asprintf(&s, "%lld", (intptr_t)p); + gpr_asprintf(&s, "%" PRIdPTR, (intptr_t)p); return s; } @@ -418,7 +421,7 @@ static char *fmt_time(void *p) { pfx = ""; break; } - gpr_asprintf(&out, "\"%s%d.%09d\"", pfx, tm.tv_sec, tm.tv_nsec); + gpr_asprintf(&out, "\"%s%" PRId64 ".%09d\"", pfx, tm.tv_sec, tm.tv_nsec); return out; } @@ -506,7 +509,7 @@ grpc_error *grpc_os_error(const char *file, int line, int err, GRPC_ERROR_STR_SYSCALL, call_name); } -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS grpc_error *grpc_wsa_error(const char *file, int line, int err, const char *call_name) { char *utf8_message = gpr_format_message(err); diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 695724c0be..69cdf3028e 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,49 +39,84 @@ #include <grpc/support/time.h> -// Opaque representation of an error. -// Errors are refcounted objects that represent the result of an operation. -// Ownership laws: -// if a grpc_error is returned by a function, the caller owns a ref to that -// instance -// if a grpc_error is passed to a grpc_closure callback function (functions -// with the signature: -// void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error)) -// then those functions do not automatically own a ref to error -// if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes -// ownership of the error +/// Opaque representation of an error. +/// Errors are refcounted objects that represent the result of an operation. +/// Ownership laws: +/// if a grpc_error is returned by a function, the caller owns a ref to that +/// instance +/// if a grpc_error is passed to a grpc_closure callback function (functions +/// with the signature: +/// void (*f)(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error)) +/// then those functions do not automatically own a ref to error +/// if a grpc_error is passed to *ANY OTHER FUNCTION* then that function takes +/// ownership of the error +/// Errors have: +/// a set of ints, strings, and timestamps that describe the error +/// always present are: +/// GRPC_ERROR_STR_FILE, GRPC_ERROR_INT_FILE_LINE - source location the error +/// was generated +/// GRPC_ERROR_STR_DESCRIPTION - a human readable description of the error +/// GRPC_ERROR_TIME_CREATED - a timestamp indicating when the error happened +/// an error can also have children; these are other errors that are believed +/// to have contributed to this one. By accumulating children, we can begin +/// to root cause high level failures from low level failures, without having +/// to derive execution paths from log lines typedef struct grpc_error grpc_error; typedef enum { + /// 'errno' from the operating system GRPC_ERROR_INT_ERRNO, + /// __LINE__ from the call site creating the error GRPC_ERROR_INT_FILE_LINE, - GRPC_ERROR_INT_STATUS_CODE, - GRPC_ERROR_INT_WARNING, + /// stream identifier: for errors that are associated with an individual + /// wire stream GRPC_ERROR_INT_STREAM_ID, + /// grpc status code representing this error GRPC_ERROR_INT_GRPC_STATUS, + /// offset into some binary blob (usually represented by + /// GRPC_ERROR_STR_RAW_BYTES) where the error occurred GRPC_ERROR_INT_OFFSET, + /// context sensitive index associated with the error GRPC_ERROR_INT_INDEX, + /// context sensitive size associated with the error GRPC_ERROR_INT_SIZE, + /// http2 error code associated with the error (see the HTTP2 RFC) GRPC_ERROR_INT_HTTP2_ERROR, + /// TSI status code associated with the error GRPC_ERROR_INT_TSI_CODE, + /// grpc_security_status associated with the error GRPC_ERROR_INT_SECURITY_STATUS, + /// WSAGetLastError() reported when this error occurred GRPC_ERROR_INT_WSA_ERROR, + /// File descriptor associated with this error GRPC_ERROR_INT_FD, + /// HTTP status (i.e. 404) + GRPC_ERROR_INT_HTTP_STATUS, } grpc_error_ints; typedef enum { + /// top-level textual description of this error GRPC_ERROR_STR_DESCRIPTION, + /// source file in which this error occurred GRPC_ERROR_STR_FILE, + /// operating system description of this error GRPC_ERROR_STR_OS_ERROR, + /// syscall that generated this error GRPC_ERROR_STR_SYSCALL, + /// peer that we were trying to communicate when this error occurred GRPC_ERROR_STR_TARGET_ADDRESS, + /// grpc status message associated with this error GRPC_ERROR_STR_GRPC_MESSAGE, + /// hex dump (or similar) with the data that generated this error GRPC_ERROR_STR_RAW_BYTES, + /// tsi error string associated with this error GRPC_ERROR_STR_TSI_ERROR, + /// filename that we were trying to read/write when this error occurred GRPC_ERROR_STR_FILENAME, } grpc_error_strs; typedef enum { + /// timestamp of error creation GRPC_ERROR_TIME_CREATED, } grpc_error_times; @@ -92,8 +127,17 @@ typedef enum { const char *grpc_error_string(grpc_error *error); void grpc_error_free_string(const char *str); +/// Create an error - but use GRPC_ERROR_CREATE instead grpc_error *grpc_error_create(const char *file, int line, const char *desc, grpc_error **referencing, size_t num_referencing); +/// Create an error (this is the preferred way of generating an error that is +/// not due to a system call - for system calls, use GRPC_OS_ERROR or +/// GRPC_WSA_ERROR as appropriate) +/// \a referencing is an array of num_referencing elements indicating one or +/// more errors that are believed to have contributed to this one +/// err = grpc_error_create(x, y, z, r, nr) is equivalent to: +/// err = grpc_error_create(x, y, z, NULL, 0); +/// for (i=0; i<nr; i++) err = grpc_error_add_child(err, r[i]); #define GRPC_ERROR_CREATE(desc) \ grpc_error_create(__FILE__, __LINE__, desc, NULL, 0) @@ -125,13 +169,18 @@ grpc_error *grpc_error_set_time(grpc_error *src, grpc_error_times which, gpr_timespec value); grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, const char *value); +/// Add a child error: an error that is believed to have contributed to this +/// error occurring. Allows root causing high level errors from lower level +/// errors that contributed to them. grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child); grpc_error *grpc_os_error(const char *file, int line, int err, const char *call_name); +/// create an error associated with errno!=0 (an 'operating system' error) #define GRPC_OS_ERROR(err, call_name) \ grpc_os_error(__FILE__, __LINE__, err, call_name) grpc_error *grpc_wsa_error(const char *file, int line, int err, const char *call_name); +/// windows only: create an error associated with WSAGetLastError()!=0 #define GRPC_WSA_ERROR(err, call_name) \ grpc_wsa_error(__FILE__, __LINE__, err, call_name) diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c new file mode 100644 index 0000000000..9e306af5fa --- /dev/null +++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c @@ -0,0 +1,2043 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* This file will be removed shortly: it's here to keep refactoring + * steps simple and auditable. + * It's the combination of the old files: + * - fd_posix.{h,c} + * - pollset_posix.{h,c} + * - pullset_multipoller_with_{poll,epoll}.{h,c} + * The new version will be split into: + * - ev_poll_posix.{h,c} + * - ev_epoll_posix.{h,c} + */ + +#include <grpc/support/port_platform.h> + +#ifdef GPR_POSIX_SOCKET + +#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h" + +#include <assert.h> +#include <errno.h> +#include <poll.h> +#include <string.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/tls.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/******************************************************************************* + * FD declarations + */ + +typedef struct grpc_fd_watcher { + struct grpc_fd_watcher *next; + struct grpc_fd_watcher *prev; + grpc_pollset *pollset; + grpc_pollset_worker *worker; + grpc_fd *fd; +} grpc_fd_watcher; + +struct grpc_fd { + int fd; + /* refst format: + bit0: 1=active/0=orphaned + bit1-n: refcount + meaning that mostly we ref by two to avoid altering the orphaned bit, + and just unref by 1 when we're ready to flag the object as orphaned */ + gpr_atm refst; + + gpr_mu mu; + int shutdown; + int closed; + int released; + + /* The watcher list. + + The following watcher related fields are protected by watcher_mu. + + An fd_watcher is an ephemeral object created when an fd wants to + begin polling, and destroyed after the poll. + + It denotes the fd's interest in whether to read poll or write poll + or both or neither on this fd. + + If a watcher is asked to poll for reads or writes, the read_watcher + or write_watcher fields are set respectively. A watcher may be asked + to poll for both, in which case both fields will be set. + + read_watcher and write_watcher may be NULL if no watcher has been + asked to poll for reads or writes. + + If an fd_watcher is not asked to poll for reads or writes, it's added + to a linked list of inactive watchers, rooted at inactive_watcher_root. + If at a later time there becomes need of a poller to poll, one of + the inactive pollers may be kicked out of their poll loops to take + that responsibility. */ + grpc_fd_watcher inactive_watcher_root; + grpc_fd_watcher *read_watcher; + grpc_fd_watcher *write_watcher; + + grpc_closure *read_closure; + grpc_closure *write_closure; + + struct grpc_fd *freelist_next; + + grpc_closure *on_done_closure; + + grpc_iomgr_object iomgr_object; + + /* The pollset that last noticed and notified that the fd is readable */ + grpc_pollset *read_notifier_pollset; +}; + +/* Begin polling on an fd. + Registers that the given pollset is interested in this fd - so that if read + or writability interest changes, the pollset can be kicked to pick up that + new interest. + Return value is: + (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0) + i.e. a combination of read_mask and write_mask determined by the fd's current + interest in said events. + Polling strategies that do not need to alter their behavior depending on the + fd's current interest (such as epoll) do not need to call this function. + MUST NOT be called with a pollset lock taken */ +static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, + grpc_pollset_worker *worker, uint32_t read_mask, + uint32_t write_mask, grpc_fd_watcher *rec); +/* Complete polling previously started with fd_begin_poll + MUST NOT be called with a pollset lock taken + if got_read or got_write are 1, also does the become_{readable,writable} as + appropriate. */ +static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec, + int got_read, int got_write, + grpc_pollset *read_notifier_pollset); + +/* Return 1 if this fd is orphaned, 0 otherwise */ +static bool fd_is_orphaned(grpc_fd *fd); + +/* Reference counting for fds */ +/*#define GRPC_FD_REF_COUNT_DEBUG*/ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +#define CLOSURE_NOT_READY ((grpc_closure *)0) +#define CLOSURE_READY ((grpc_closure *)1) + +/******************************************************************************* + * pollset declarations + */ + +typedef struct grpc_pollset_vtable grpc_pollset_vtable; + +typedef struct grpc_cached_wakeup_fd { + grpc_wakeup_fd fd; + struct grpc_cached_wakeup_fd *next; +} grpc_cached_wakeup_fd; + +struct grpc_pollset_worker { + grpc_cached_wakeup_fd *wakeup_fd; + int reevaluate_polling_on_wakeup; + int kicked_specifically; + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset { + /* pollsets under posix can mutate representation as fds are added and + removed. + For example, we may choose a poll() based implementation on linux for + few fds, and an epoll() based implementation for many fds */ + const grpc_pollset_vtable *vtable; + gpr_mu mu; + grpc_pollset_worker root_worker; + int in_flight_cbs; + int shutting_down; + int called_shutdown; + int kicked_without_pollers; + grpc_closure *shutdown_done; + grpc_closure_list idle_jobs; + union { + int fd; + void *ptr; + } data; + /* Local cache of eventfds for workers */ + grpc_cached_wakeup_fd *local_wakeup_cache; +}; + +struct grpc_pollset_vtable { + void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd, int and_unlock_pollset); + grpc_error *(*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now); + void (*finish_shutdown)(grpc_pollset *pollset); + void (*destroy)(grpc_pollset *pollset); +}; + +/* Add an fd to a pollset */ +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + struct grpc_fd *fd); + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd); + +/* Convert a timespec to milliseconds: + - very small or negative poll times are clamped to zero to do a + non-blocking poll (which becomes spin polling) + - other small values are rounded up to one millisecond + - longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now); + +/* Allow kick to wakeup the currently polling worker */ +#define GRPC_POLLSET_CAN_KICK_SELF 1 +/* Force the wakee to repoll when awoken */ +#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 +/* As per pollset_kick, with an extended set of flags (defined above) + -- mostly for fd_posix's use. */ +static grpc_error *pollset_kick_ext(grpc_pollset *p, + grpc_pollset_worker *specific_worker, + uint32_t flags) GRPC_MUST_USE_RESULT; + +/* turn a pollset into a multipoller: platform specific */ +typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + struct grpc_fd **fds, + size_t fd_count); +static platform_become_multipoller_type platform_become_multipoller; + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *pollset); + +static void remove_fd_from_all_epoll_sets(int fd); + +/******************************************************************************* + * pollset_set definitions + */ + +struct grpc_pollset_set { + gpr_mu mu; + + size_t pollset_count; + size_t pollset_capacity; + grpc_pollset **pollsets; + + size_t pollset_set_count; + size_t pollset_set_capacity; + struct grpc_pollset_set **pollset_sets; + + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; +}; + +/******************************************************************************* + * fd_posix.c + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ +/* TODO(klempner): We could use some form of polling generation count to know + * when these are safe to free. */ +/* TODO(klempner): Consider disabling freelisting if we don't have multiple + * threads in poll on the same fd */ +/* TODO(klempner): Batch these allocations to reduce fragmentation */ +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +static void freelist_fd(grpc_fd *fd) { + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + gpr_mu_unlock(&fd_freelist_mu); +} + +static grpc_fd *alloc_fd(int fd) { + grpc_fd *r = NULL; + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + r = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + if (r == NULL) { + r = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&r->mu); + } + + gpr_mu_lock(&r->mu); + gpr_atm_rel_store(&r->refst, 1); + r->shutdown = 0; + r->read_closure = CLOSURE_NOT_READY; + r->write_closure = CLOSURE_NOT_READY; + r->fd = fd; + r->inactive_watcher_root.next = r->inactive_watcher_root.prev = + &r->inactive_watcher_root; + r->freelist_next = NULL; + r->read_watcher = r->write_watcher = NULL; + r->on_done_closure = NULL; + r->closed = 0; + r->released = 0; + r->read_notifier_pollset = NULL; + gpr_mu_unlock(&r->mu); + return r; +} + +static void destroy(grpc_fd *fd) { + gpr_mu_destroy(&fd->mu); + gpr_free(fd); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, + gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, + gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + freelist_fd(fd); + } else { + GPR_ASSERT(old > n); + } +} + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + destroy(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *r = alloc_fd(fd); + char *name2; + gpr_asprintf(&name2, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&r->iomgr_object, name2); + gpr_free(name2); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name); +#endif + return r; +} + +static bool fd_is_orphaned(grpc_fd *fd) { + return (gpr_atm_acq_load(&fd->refst) & 1) == 0; +} + +static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { + gpr_mu_lock(&watcher->pollset->mu); + GPR_ASSERT(watcher->worker); + grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + gpr_mu_unlock(&watcher->pollset->mu); + return err; +} + +static void maybe_wake_one_watcher_locked(grpc_fd *fd) { + if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { + pollset_kick_locked(fd->inactive_watcher_root.next); + } else if (fd->read_watcher) { + pollset_kick_locked(fd->read_watcher); + } else if (fd->write_watcher) { + pollset_kick_locked(fd->write_watcher); + } +} + +static void wake_all_watchers_locked(grpc_fd *fd) { + grpc_fd_watcher *watcher; + for (watcher = fd->inactive_watcher_root.next; + watcher != &fd->inactive_watcher_root; watcher = watcher->next) { + pollset_kick_locked(watcher); + } + if (fd->read_watcher) { + pollset_kick_locked(fd->read_watcher); + } + if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { + pollset_kick_locked(fd->write_watcher); + } +} + +static int has_watchers(grpc_fd *fd) { + return fd->read_watcher != NULL || fd->write_watcher != NULL || + fd->inactive_watcher_root.next != &fd->inactive_watcher_root; +} + +static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + fd->closed = 1; + if (!fd->released) { + close(fd->fd); + } else { + remove_fd_from_all_epoll_sets(fd->fd); + } + grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL); +} + +static int fd_wrapped_fd(grpc_fd *fd) { + if (fd->released || fd->closed) { + return -1; + } else { + return fd->fd; + } +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + fd->on_done_closure = on_done; + fd->released = release_fd != NULL; + if (!fd->released) { + shutdown(fd->fd, SHUT_RDWR); + } else { + *release_fd = fd->fd; + } + gpr_mu_lock(&fd->mu); + REF_BY(fd, 1, reason); /* remove active status, but keep referenced */ + if (!has_watchers(fd)) { + close_fd_locked(exec_ctx, fd); + } else { + wake_all_watchers_locked(fd); + } + gpr_mu_unlock(&fd->mu); + UNREF_BY(fd, 2, reason); /* drop the reference */ +} + +/* increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } + +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static grpc_error *fd_shutdown_error(bool shutdown) { + if (!shutdown) { + return GRPC_ERROR_NONE; + } else { + return GRPC_ERROR_CREATE("FD shutdown"); + } +} + +static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure **st, grpc_closure *closure) { + if (fd->shutdown) { + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"), + NULL); + } else if (*st == CLOSURE_NOT_READY) { + /* not ready ==> switch to a waiting state by setting the closure */ + *st = closure; + } else if (*st == CLOSURE_READY) { + /* already ready ==> queue the closure to run immediately */ + *st = CLOSURE_NOT_READY; + grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown), + NULL); + maybe_wake_one_watcher_locked(fd); + } else { + /* upcallptr was set to a different closure. This is an error! */ + gpr_log(GPR_ERROR, + "User called a notify_on function with a previous callback still " + "pending"); + abort(); + } +} + +/* returns 1 if state becomes not ready */ +static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure **st) { + if (*st == CLOSURE_READY) { + /* duplicate ready ==> ignore */ + return 0; + } else if (*st == CLOSURE_NOT_READY) { + /* not ready, and not waiting ==> flag ready */ + *st = CLOSURE_READY; + return 0; + } else { + /* waiting ==> queue closure */ + grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL); + *st = CLOSURE_NOT_READY; + return 1; + } +} + +static void set_read_notifier_pollset_locked( + grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) { + fd->read_notifier_pollset = read_notifier_pollset; +} + +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + gpr_mu_lock(&fd->mu); + /* only shutdown once */ + if (!fd->shutdown) { + fd->shutdown = 1; + /* signal read/write closed to OS so that future operations fail */ + shutdown(fd->fd, SHUT_RDWR); + set_ready_locked(exec_ctx, fd, &fd->read_closure); + set_ready_locked(exec_ctx, fd, &fd->write_closure); + } + gpr_mu_unlock(&fd->mu); +} + +static bool fd_is_shutdown(grpc_fd *fd) { + gpr_mu_lock(&fd->mu); + bool r = fd->shutdown; + gpr_mu_unlock(&fd->mu); + return r; +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + gpr_mu_lock(&fd->mu); + notify_on_locked(exec_ctx, fd, &fd->read_closure, closure); + gpr_mu_unlock(&fd->mu); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + gpr_mu_lock(&fd->mu); + notify_on_locked(exec_ctx, fd, &fd->write_closure, closure); + gpr_mu_unlock(&fd->mu); +} + +/* Return the read-notifier pollset */ +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + grpc_pollset *notifier = NULL; + + gpr_mu_lock(&fd->mu); + notifier = fd->read_notifier_pollset; + gpr_mu_unlock(&fd->mu); + + return notifier; +} + +static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, + grpc_pollset_worker *worker, uint32_t read_mask, + uint32_t write_mask, grpc_fd_watcher *watcher) { + uint32_t mask = 0; + grpc_closure *cur; + int requested; + /* keep track of pollers that have requested our events, in case they change + */ + GRPC_FD_REF(fd, "poll"); + + gpr_mu_lock(&fd->mu); + + /* if we are shutdown, then don't add to the watcher set */ + if (fd->shutdown) { + watcher->fd = NULL; + watcher->pollset = NULL; + watcher->worker = NULL; + gpr_mu_unlock(&fd->mu); + GRPC_FD_UNREF(fd, "poll"); + return 0; + } + + /* if there is nobody polling for read, but we need to, then start doing so */ + cur = fd->read_closure; + requested = cur != CLOSURE_READY; + if (read_mask && fd->read_watcher == NULL && requested) { + fd->read_watcher = watcher; + mask |= read_mask; + } + /* if there is nobody polling for write, but we need to, then start doing so + */ + cur = fd->write_closure; + requested = cur != CLOSURE_READY; + if (write_mask && fd->write_watcher == NULL && requested) { + fd->write_watcher = watcher; + mask |= write_mask; + } + /* if not polling, remember this watcher in case we need someone to later */ + if (mask == 0 && worker != NULL) { + watcher->next = &fd->inactive_watcher_root; + watcher->prev = watcher->next->prev; + watcher->next->prev = watcher->prev->next = watcher; + } + watcher->pollset = pollset; + watcher->worker = worker; + watcher->fd = fd; + gpr_mu_unlock(&fd->mu); + + return mask; +} + +static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, + int got_read, int got_write, + grpc_pollset *read_notifier_pollset) { + int was_polling = 0; + int kick = 0; + grpc_fd *fd = watcher->fd; + + if (fd == NULL) { + return; + } + + gpr_mu_lock(&fd->mu); + + if (watcher == fd->read_watcher) { + /* remove read watcher, kick if we still need a read */ + was_polling = 1; + if (!got_read) { + kick = 1; + } + fd->read_watcher = NULL; + } + if (watcher == fd->write_watcher) { + /* remove write watcher, kick if we still need a write */ + was_polling = 1; + if (!got_write) { + kick = 1; + } + fd->write_watcher = NULL; + } + if (!was_polling && watcher->worker != NULL) { + /* remove from inactive list */ + watcher->next->prev = watcher->prev; + watcher->prev->next = watcher->next; + } + if (got_read) { + if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) { + kick = 1; + } + + if (read_notifier_pollset != NULL) { + set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset); + } + } + if (got_write) { + if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) { + kick = 1; + } + } + if (kick) { + maybe_wake_one_watcher_locked(fd); + } + if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { + close_fd_locked(exec_ctx, fd); + } + gpr_mu_unlock(&fd->mu); + + GRPC_FD_UNREF(fd, "poll"); +} + +/******************************************************************************* + * pollset_posix.c + */ + +GPR_TLS_DECL(g_current_thread_poller); +GPR_TLS_DECL(g_current_thread_worker); + +/** The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). + * This wakeup_fd gives us something to alert on when such a case occurs. */ +grpc_wakeup_fd grpc_global_wakeup_fd; + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +static void kick_append_error(grpc_error **composite, grpc_error *error) { + if (error == GRPC_ERROR_NONE) return; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE("Kick Failure"); + } + *composite = grpc_error_add_child(*composite, error); +} + +static grpc_error *pollset_kick_ext(grpc_pollset *p, + grpc_pollset_worker *specific_worker, + uint32_t flags) { + GPR_TIMER_BEGIN("pollset_kick_ext", 0); + grpc_error *error = GRPC_ERROR_NONE; + + /* pollset->mu already held */ + if (specific_worker != NULL) { + if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { + GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0); + GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); + for (specific_worker = p->root_worker.next; + specific_worker != &p->root_worker; + specific_worker = specific_worker->next) { + kick_append_error( + &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); + } + p->kicked_without_pollers = true; + GPR_TIMER_END("pollset_kick_ext.broadcast", 0); + } else if (gpr_tls_get(&g_current_thread_worker) != + (intptr_t)specific_worker) { + GPR_TIMER_MARK("different_thread_worker", 0); + if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { + specific_worker->reevaluate_polling_on_wakeup = true; + } + specific_worker->kicked_specifically = true; + kick_append_error(&error, + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); + } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { + GPR_TIMER_MARK("kick_yoself", 0); + if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { + specific_worker->reevaluate_polling_on_wakeup = true; + } + specific_worker->kicked_specifically = true; + kick_append_error(&error, + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); + } + } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) { + GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); + GPR_TIMER_MARK("kick_anonymous", 0); + specific_worker = pop_front_worker(p); + if (specific_worker != NULL) { + if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + GPR_TIMER_MARK("kick_anonymous_not_self", 0); + push_back_worker(p, specific_worker); + specific_worker = pop_front_worker(p); + if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 && + gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + push_back_worker(p, specific_worker); + specific_worker = NULL; + } + } + if (specific_worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, specific_worker); + kick_append_error( + &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); + } + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick_ext", 0); + return error; +} + +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + return pollset_kick_ext(p, specific_worker, 0); +} + +/* global state management */ + +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_poller); + gpr_tls_init(&g_current_thread_worker); + return grpc_wakeup_fd_init(&grpc_global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_poller); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); +} + +/* main interface */ + +static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->in_flight_cbs = 0; + pollset->shutting_down = 0; + pollset->called_shutdown = 0; + pollset->kicked_without_pollers = 0; + pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; + pollset->local_wakeup_cache = NULL; + pollset->kicked_without_pollers = 0; + become_basic_pollset(pollset, NULL); +} + +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + while (pollset->local_wakeup_cache) { + grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; + grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); + gpr_free(pollset->local_wakeup_cache); + pollset->local_wakeup_cache = next; + } + gpr_mu_destroy(&pollset->mu); +} + +static void pollset_reset(grpc_pollset *pollset) { + GPR_ASSERT(pollset->shutting_down); + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + pollset->shutting_down = 0; + pollset->called_shutdown = 0; + pollset->kicked_without_pollers = 0; + become_basic_pollset(pollset, NULL); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + gpr_mu_lock(&pollset->mu); + pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); +/* the following (enabled only in debug) will reacquire and then release + our lock - meaning that if the unlocking flag passed to add_fd above is + not respected, the code will deadlock (in a way that we have a chance of + debugging) */ +#ifndef NDEBUG + gpr_mu_lock(&pollset->mu); + gpr_mu_unlock(&pollset->mu); +#endif +} + +static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs)); + pollset->vtable->finish_shutdown(pollset); + grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL); +} + +static void work_combine_error(grpc_error **composite, grpc_error *error) { + if (error == GRPC_ERROR_NONE) return; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE("pollset_work"); + } + *composite = grpc_error_add_child(*composite, error); +} + +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + *worker_hdl = &worker; + grpc_error *error = GRPC_ERROR_NONE; + + /* pollset->mu already held */ + int added_worker = 0; + int locked = 1; + int queued_work = 0; + int keep_polling = 0; + GPR_TIMER_BEGIN("pollset_work", 0); + /* this must happen before we (potentially) drop pollset->mu */ + worker.next = worker.prev = NULL; + worker.reevaluate_polling_on_wakeup = 0; + if (pollset->local_wakeup_cache != NULL) { + worker.wakeup_fd = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker.wakeup_fd->next; + } else { + worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd)); + error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd); + if (error != GRPC_ERROR_NONE) { + return error; + } + } + worker.kicked_specifically = 0; + /* If there's work waiting for the pollset to be idle, and the + pollset is idle, then do that work */ + if (!pollset_has_workers(pollset) && + !grpc_closure_list_empty(pollset->idle_jobs)) { + GPR_TIMER_MARK("pollset_work.idle_jobs", 0); + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); + goto done; + } + /* If we're shutting down then we don't execute any extended work */ + if (pollset->shutting_down) { + GPR_TIMER_MARK("pollset_work.shutting_down", 0); + goto done; + } + /* Give do_promote priority so we don't starve it out */ + if (pollset->in_flight_cbs) { + GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0); + gpr_mu_unlock(&pollset->mu); + locked = 0; + goto done; + } + /* Start polling, and keep doing so while we're being asked to + re-evaluate our pollers (this allows poll() based pollers to + ensure they don't miss wakeups) */ + keep_polling = 1; + while (keep_polling) { + keep_polling = 0; + if (!pollset->kicked_without_pollers) { + if (!added_worker) { + push_front_worker(pollset, &worker); + added_worker = 1; + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + } + gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset); + GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); + work_combine_error(&error, + pollset->vtable->maybe_work_and_unlock( + exec_ctx, pollset, &worker, deadline, now)); + GPR_TIMER_END("maybe_work_and_unlock", 0); + locked = 0; + gpr_tls_set(&g_current_thread_poller, 0); + } else { + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } + /* Finished execution - start cleaning up. + Note that we may arrive here from outside the enclosing while() loop. + In that case we won't loop though as we haven't added worker to the + worker list, which means nobody could ask us to re-evaluate polling). */ + done: + if (!locked) { + queued_work |= grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + locked = 1; + } + /* If we're forced to re-evaluate polling (via pollset_kick with + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force + a loop */ + if (worker.reevaluate_polling_on_wakeup) { + worker.reevaluate_polling_on_wakeup = 0; + pollset->kicked_without_pollers = 0; + if (queued_work || worker.kicked_specifically) { + /* If there's queued work on the list, then set the deadline to be + immediate so we get back out of the polling loop quickly */ + deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); + } + keep_polling = 1; + } + } + if (added_worker) { + remove_worker(pollset, &worker); + gpr_tls_set(&g_current_thread_worker, 0); + } + /* release wakeup fd to the local pool */ + worker.wakeup_fd->next = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker.wakeup_fd; + /* check shutdown conditions */ + if (pollset->shutting_down) { + if (pollset_has_workers(pollset)) { + pollset_kick(pollset, NULL); + } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { + pollset->called_shutdown = 1; + gpr_mu_unlock(&pollset->mu); + finish_shutdown(exec_ctx, pollset); + grpc_exec_ctx_flush(exec_ctx); + /* Continuing to access pollset here is safe -- it is the caller's + * responsibility to not destroy when it has outstanding calls to + * pollset_work. + * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ + gpr_mu_lock(&pollset->mu); + } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + } + *worker_hdl = NULL; + GPR_TIMER_END("pollset_work", 0); + return error; +} + +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = 1; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + if (!pollset_has_workers(pollset)) { + grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); + } + if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && + !pollset_has_workers(pollset)) { + pollset->called_shutdown = 1; + finish_shutdown(exec_ctx, pollset); + } +} + +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + return gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); +} + +/* + * basic_pollset - a vtable that provides polling for zero or one file + * descriptor via poll() + */ + +typedef struct grpc_unary_promote_args { + const grpc_pollset_vtable *original_vtable; + grpc_pollset *pollset; + grpc_fd *fd; + grpc_closure promotion_closure; +} grpc_unary_promote_args; + +static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, + grpc_error *error) { + grpc_unary_promote_args *up_args = args; + const grpc_pollset_vtable *original_vtable = up_args->original_vtable; + grpc_pollset *pollset = up_args->pollset; + grpc_fd *fd = up_args->fd; + + /* + * This is quite tricky. There are a number of cases to keep in mind here: + * 1. fd may have been orphaned + * 2. The pollset may no longer be a unary poller (and we can't let case #1 + * leak to other pollset types!) + * 3. pollset's fd (which may have changed) may have been orphaned + * 4. The pollset may be shutting down. + */ + + gpr_mu_lock(&pollset->mu); + /* First we need to ensure that nobody is polling concurrently */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + gpr_free(up_args); + /* At this point the pollset may no longer be a unary poller. In that case + * we should just call the right add function and be done. */ + /* TODO(klempner): If we're not careful this could cause infinite recursion. + * That's not a problem for now because empty_pollset has a trivial poller + * and we don't have any mechanism to unbecome multipoller. */ + pollset->in_flight_cbs--; + if (pollset->shutting_down) { + /* We don't care about this pollset anymore. */ + if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) { + pollset->called_shutdown = 1; + finish_shutdown(exec_ctx, pollset); + } + } else if (fd_is_orphaned(fd)) { + /* Don't try to add it to anything, we'll drop our ref on it below */ + } else if (pollset->vtable != original_vtable) { + pollset->vtable->add_fd(exec_ctx, pollset, fd, 0); + } else if (fd != pollset->data.ptr) { + grpc_fd *fds[2]; + fds[0] = pollset->data.ptr; + fds[1] = fd; + + if (fds[0] && !fd_is_orphaned(fds[0])) { + platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds)); + GRPC_FD_UNREF(fds[0], "basicpoll"); + } else { + /* old fd is orphaned and we haven't cleaned it up until now, so remain a + * unary poller */ + /* Note that it is possible that fds[1] is also orphaned at this point. + * That's okay, we'll correct it at the next add or poll. */ + if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll"); + pollset->data.ptr = fd; + GRPC_FD_REF(fd, "basicpoll"); + } + } + + gpr_mu_unlock(&pollset->mu); + + /* Matching ref in basic_pollset_add_fd */ + GRPC_FD_UNREF(fd, "basicpoll_add"); +} + +static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd, int and_unlock_pollset) { + grpc_unary_promote_args *up_args; + GPR_ASSERT(fd); + if (fd == pollset->data.ptr) goto exit; + + if (!pollset_has_workers(pollset)) { + /* Fast path -- no in flight cbs */ + /* TODO(klempner): Comment this out and fix any test failures or establish + * they are due to timing issues */ + grpc_fd *fds[2]; + fds[0] = pollset->data.ptr; + fds[1] = fd; + + if (fds[0] == NULL) { + pollset->data.ptr = fd; + GRPC_FD_REF(fd, "basicpoll"); + } else if (!fd_is_orphaned(fds[0])) { + platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds)); + GRPC_FD_UNREF(fds[0], "basicpoll"); + } else { + /* old fd is orphaned and we haven't cleaned it up until now, so remain a + * unary poller */ + GRPC_FD_UNREF(fds[0], "basicpoll"); + pollset->data.ptr = fd; + GRPC_FD_REF(fd, "basicpoll"); + } + goto exit; + } + + /* Now we need to promote. This needs to happen when we're not polling. Since + * this may be called from poll, the wait needs to happen asynchronously. */ + GRPC_FD_REF(fd, "basicpoll_add"); + pollset->in_flight_cbs++; + up_args = gpr_malloc(sizeof(*up_args)); + up_args->fd = fd; + up_args->original_vtable = pollset->vtable; + up_args->pollset = pollset; + up_args->promotion_closure.cb = basic_do_promote; + up_args->promotion_closure.cb_arg = up_args; + + grpc_closure_list_append(&pollset->idle_jobs, &up_args->promotion_closure, + GRPC_ERROR_NONE); + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + +exit: + if (and_unlock_pollset) { + gpr_mu_unlock(&pollset->mu); + } +} + +static grpc_error *basic_pollset_maybe_work_and_unlock( + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now) { +#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) +#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) + + struct pollfd pfd[3]; + grpc_fd *fd; + grpc_fd_watcher fd_watcher; + int timeout; + int r; + nfds_t nfds; + grpc_error *error = GRPC_ERROR_NONE; + + fd = pollset->data.ptr; + if (fd && fd_is_orphaned(fd)) { + GRPC_FD_UNREF(fd, "basicpoll"); + fd = pollset->data.ptr = NULL; + } + timeout = poll_deadline_to_millis_timeout(deadline, now); + pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); + pfd[0].events = POLLIN; + pfd[0].revents = 0; + pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); + pfd[1].events = POLLIN; + pfd[1].revents = 0; + nfds = 2; + if (fd) { + pfd[2].fd = fd->fd; + pfd[2].revents = 0; + GRPC_FD_REF(fd, "basicpoll_begin"); + gpr_mu_unlock(&pollset->mu); + pfd[2].events = + (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher); + if (pfd[2].events != 0) { + nfds++; + } + } else { + gpr_mu_unlock(&pollset->mu); + } + + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ + /* poll fd count (argument 2) is shortened by one if we have no events + to poll on - such that it only includes the kicker */ + GPR_TIMER_BEGIN("poll", 0); + GRPC_SCHEDULING_START_BLOCKING_REGION; + r = grpc_poll_function(pfd, nfds, timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; + GPR_TIMER_END("poll", 0); + + if (r < 0) { + if (errno != EINTR) { + work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); + } + if (fd) { + fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); + } + } else if (r == 0) { + if (fd) { + fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); + } + } else { + if (pfd[0].revents & POLLIN_CHECK) { + work_combine_error(&error, + grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd)); + } + if (pfd[1].revents & POLLIN_CHECK) { + work_combine_error(&error, + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); + } + if (nfds > 2) { + fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, + pfd[2].revents & POLLOUT_CHECK, pollset); + } else if (fd) { + fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); + } + } + + if (fd) { + GRPC_FD_UNREF(fd, "basicpoll_begin"); + } + + return error; +} + +static void basic_pollset_destroy(grpc_pollset *pollset) { + if (pollset->data.ptr != NULL) { + GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); + pollset->data.ptr = NULL; + } +} + +static const grpc_pollset_vtable basic_pollset = { + basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock, + basic_pollset_destroy, basic_pollset_destroy}; + +static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) { + pollset->vtable = &basic_pollset; + pollset->data.ptr = fd_or_null; + if (fd_or_null != NULL) { + GRPC_FD_REF(fd_or_null, "basicpoll"); + } +} + +/******************************************************************************* + * pollset_multipoller_with_poll_posix.c + */ + +#ifndef GPR_LINUX_MULTIPOLL_WITH_EPOLL + +typedef struct { + /* all polled fds */ + size_t fd_count; + size_t fd_capacity; + grpc_fd **fds; + /* fds that have been removed from the pollset explicitly */ + size_t del_count; + size_t del_capacity; + grpc_fd **dels; +} poll_hdr; + +static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_fd *fd, + int and_unlock_pollset) { + size_t i; + poll_hdr *h = pollset->data.ptr; + /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */ + for (i = 0; i < h->fd_count; i++) { + if (h->fds[i] == fd) goto exit; + } + if (h->fd_count == h->fd_capacity) { + h->fd_capacity = GPR_MAX(h->fd_capacity + 8, h->fd_count * 3 / 2); + h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity); + } + h->fds[h->fd_count++] = fd; + GRPC_FD_REF(fd, "multipoller"); +exit: + if (and_unlock_pollset) { + gpr_mu_unlock(&pollset->mu); + } +} + +static grpc_error *multipoll_with_poll_pollset_maybe_work_and_unlock( + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now) { +#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) +#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) + + int timeout; + int r; + size_t i, j, fd_count; + nfds_t pfd_count; + poll_hdr *h; + /* TODO(ctiller): inline some elements to avoid an allocation */ + grpc_fd_watcher *watchers; + struct pollfd *pfds; + grpc_error *error = GRPC_ERROR_NONE; + + h = pollset->data.ptr; + timeout = poll_deadline_to_millis_timeout(deadline, now); + /* TODO(ctiller): perform just one malloc here if we exceed the inline case */ + pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2)); + watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2)); + fd_count = 0; + pfd_count = 2; + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); + pfds[0].events = POLLIN; + pfds[0].revents = 0; + pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); + pfds[1].events = POLLIN; + pfds[1].revents = 0; + for (i = 0; i < h->fd_count; i++) { + int remove = fd_is_orphaned(h->fds[i]); + for (j = 0; !remove && j < h->del_count; j++) { + if (h->fds[i] == h->dels[j]) remove = 1; + } + if (remove) { + GRPC_FD_UNREF(h->fds[i], "multipoller"); + } else { + h->fds[fd_count++] = h->fds[i]; + watchers[pfd_count].fd = h->fds[i]; + GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start"); + pfds[pfd_count].fd = h->fds[i]->fd; + pfds[pfd_count].revents = 0; + pfd_count++; + } + } + for (j = 0; j < h->del_count; j++) { + GRPC_FD_UNREF(h->dels[j], "multipoller_del"); + } + h->del_count = 0; + h->fd_count = fd_count; + gpr_mu_unlock(&pollset->mu); + + for (i = 2; i < pfd_count; i++) { + grpc_fd *fd = watchers[i].fd; + pfds[i].events = (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, + &watchers[i]); + GRPC_FD_UNREF(fd, "multipoller_start"); + } + + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ + GRPC_SCHEDULING_START_BLOCKING_REGION; + r = grpc_poll_function(pfds, pfd_count, timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + if (r < 0) { + if (errno != EINTR) { + work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); + } + for (i = 2; i < pfd_count; i++) { + fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); + } + } else if (r == 0) { + for (i = 2; i < pfd_count; i++) { + fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); + } + } else { + if (pfds[0].revents & POLLIN_CHECK) { + work_combine_error(&error, + grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd)); + } + if (pfds[1].revents & POLLIN_CHECK) { + work_combine_error(&error, + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); + } + for (i = 2; i < pfd_count; i++) { + if (watchers[i].fd == NULL) { + fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); + continue; + } + fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK, + pfds[i].revents & POLLOUT_CHECK, pollset); + } + } + + gpr_free(pfds); + gpr_free(watchers); + + return error; +} + +static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) { + size_t i; + poll_hdr *h = pollset->data.ptr; + for (i = 0; i < h->fd_count; i++) { + GRPC_FD_UNREF(h->fds[i], "multipoller"); + } + for (i = 0; i < h->del_count; i++) { + GRPC_FD_UNREF(h->dels[i], "multipoller_del"); + } + h->fd_count = 0; + h->del_count = 0; +} + +static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { + poll_hdr *h = pollset->data.ptr; + multipoll_with_poll_pollset_finish_shutdown(pollset); + gpr_free(h->fds); + gpr_free(h->dels); + gpr_free(h); +} + +static const grpc_pollset_vtable multipoll_with_poll_pollset = { + multipoll_with_poll_pollset_add_fd, + multipoll_with_poll_pollset_maybe_work_and_unlock, + multipoll_with_poll_pollset_finish_shutdown, + multipoll_with_poll_pollset_destroy}; + +static void poll_become_multipoller(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd **fds, + size_t nfds) { + size_t i; + poll_hdr *h = gpr_malloc(sizeof(poll_hdr)); + pollset->vtable = &multipoll_with_poll_pollset; + pollset->data.ptr = h; + h->fd_count = nfds; + h->fd_capacity = nfds; + h->fds = gpr_malloc(nfds * sizeof(grpc_fd *)); + h->del_count = 0; + h->del_capacity = 0; + h->dels = NULL; + for (i = 0; i < nfds; i++) { + h->fds[i] = fds[i]; + GRPC_FD_REF(fds[i], "multipoller"); + } +} + +#endif /* !GPR_LINUX_MULTIPOLL_WITH_EPOLL */ + +/******************************************************************************* + * pollset_multipoller_with_epoll_posix.c + */ + +#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL + +#include <errno.h> +#include <poll.h> +#include <string.h> +#include <sys/epoll.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st, + grpc_pollset *read_notifier_pollset) { + /* only one set_ready can be active at once (but there may be a racing + notify_on) */ + gpr_mu_lock(&fd->mu); + set_ready_locked(exec_ctx, fd, st); + + /* A non-NULL read_notifier_pollset means that the fd is readable. */ + if (read_notifier_pollset != NULL) { + /* Note: Since the fd might be a part of multiple pollsets, this might be + * called multiple times (for each time the fd becomes readable) and it is + * okay to set the fd's read-notifier pollset to anyone of these pollsets */ + set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset); + } + + gpr_mu_unlock(&fd->mu); +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier_pollset) { + set_ready(exec_ctx, fd, &fd->read_closure, notifier_pollset); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + set_ready(exec_ctx, fd, &fd->write_closure, NULL); +} + +struct epoll_fd_list { + int *epoll_fds; + size_t count; + size_t capacity; +}; + +static struct epoll_fd_list epoll_fd_global_list; +static gpr_once init_epoll_fd_list_mu = GPR_ONCE_INIT; +static gpr_mu epoll_fd_list_mu; + +static void init_mu(void) { gpr_mu_init(&epoll_fd_list_mu); } + +static void add_epoll_fd_to_global_list(int epoll_fd) { + gpr_once_init(&init_epoll_fd_list_mu, init_mu); + + gpr_mu_lock(&epoll_fd_list_mu); + if (epoll_fd_global_list.count == epoll_fd_global_list.capacity) { + epoll_fd_global_list.capacity = + GPR_MAX((size_t)8, epoll_fd_global_list.capacity * 2); + epoll_fd_global_list.epoll_fds = + gpr_realloc(epoll_fd_global_list.epoll_fds, + epoll_fd_global_list.capacity * sizeof(int)); + } + epoll_fd_global_list.epoll_fds[epoll_fd_global_list.count++] = epoll_fd; + gpr_mu_unlock(&epoll_fd_list_mu); +} + +static void remove_epoll_fd_from_global_list(int epoll_fd) { + gpr_mu_lock(&epoll_fd_list_mu); + GPR_ASSERT(epoll_fd_global_list.count > 0); + for (size_t i = 0; i < epoll_fd_global_list.count; i++) { + if (epoll_fd == epoll_fd_global_list.epoll_fds[i]) { + epoll_fd_global_list.epoll_fds[i] = + epoll_fd_global_list.epoll_fds[--(epoll_fd_global_list.count)]; + break; + } + } + gpr_mu_unlock(&epoll_fd_list_mu); +} + +static void remove_fd_from_all_epoll_sets(int fd) { + int err; + gpr_once_init(&init_epoll_fd_list_mu, init_mu); + gpr_mu_lock(&epoll_fd_list_mu); + if (epoll_fd_global_list.count == 0) { + gpr_mu_unlock(&epoll_fd_list_mu); + return; + } + for (size_t i = 0; i < epoll_fd_global_list.count; i++) { + err = epoll_ctl(epoll_fd_global_list.epoll_fds[i], EPOLL_CTL_DEL, fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd, + strerror(errno)); + } + } + gpr_mu_unlock(&epoll_fd_list_mu); +} + +typedef struct { + grpc_pollset *pollset; + grpc_fd *fd; + grpc_closure closure; +} delayed_add; + +typedef struct { int epoll_fd; } epoll_hdr; + +static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + epoll_hdr *h = pollset->data.ptr; + struct epoll_event ev; + int err; + grpc_fd_watcher watcher; + + /* We pretend to be polling whilst adding an fd to keep the fd from being + closed during the add. This may result in a spurious wakeup being assigned + to this pollset whilst adding, but that should be benign. */ + GPR_ASSERT(fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0); + if (watcher.fd != NULL) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fd; + err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0) { + /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ + if (errno != EEXIST) { + gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, + strerror(errno)); + } + } + } + fd_end_poll(exec_ctx, &watcher, 0, 0, NULL); +} + +static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + delayed_add *da = arg; + + if (!fd_is_orphaned(da->fd)) { + finally_add_fd(exec_ctx, da->pollset, da->fd); + } + + gpr_mu_lock(&da->pollset->mu); + da->pollset->in_flight_cbs--; + if (da->pollset->shutting_down) { + /* We don't care about this pollset anymore. */ + if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) { + da->pollset->called_shutdown = 1; + grpc_exec_ctx_sched(exec_ctx, da->pollset->shutdown_done, GRPC_ERROR_NONE, + NULL); + } + } + gpr_mu_unlock(&da->pollset->mu); + + GRPC_FD_UNREF(da->fd, "delayed_add"); + + gpr_free(da); +} + +static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_fd *fd, + int and_unlock_pollset) { + if (and_unlock_pollset) { + gpr_mu_unlock(&pollset->mu); + finally_add_fd(exec_ctx, pollset, fd); + } else { + delayed_add *da = gpr_malloc(sizeof(*da)); + da->pollset = pollset; + da->fd = fd; + GRPC_FD_REF(fd, "delayed_add"); + grpc_closure_init(&da->closure, perform_delayed_add, da); + pollset->in_flight_cbs++; + grpc_exec_ctx_sched(exec_ctx, &da->closure, GRPC_ERROR_NONE, NULL); + } +} + +/* TODO(klempner): We probably want to turn this down a bit */ +#define GRPC_EPOLL_MAX_EVENTS 1000 + +static grpc_error *multipoll_with_epoll_pollset_maybe_work_and_unlock( + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, + gpr_timespec deadline, gpr_timespec now) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + int poll_rv; + epoll_hdr *h = pollset->data.ptr; + int timeout_ms; + struct pollfd pfds[2]; + grpc_error *error = GRPC_ERROR_NONE; + + /* If you want to ignore epoll's ability to sanely handle parallel pollers, + * for a more apples-to-apples performance comparison with poll, add a + * if (pollset->counter != 0) { return 0; } + * here. + */ + + gpr_mu_unlock(&pollset->mu); + + timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); + pfds[0].events = POLLIN; + pfds[0].revents = 0; + pfds[1].fd = h->epoll_fd; + pfds[1].events = POLLIN; + pfds[1].revents = 0; + + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ + GPR_TIMER_BEGIN("poll", 0); + GRPC_SCHEDULING_START_BLOCKING_REGION; + poll_rv = grpc_poll_function(pfds, 2, timeout_ms); + GRPC_SCHEDULING_END_BLOCKING_REGION; + GPR_TIMER_END("poll", 0); + + if (poll_rv < 0) { + if (errno != EINTR) { + work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); + } + } else if (poll_rv == 0) { + /* do nothing */ + } else { + if (pfds[0].revents) { + work_combine_error(&error, + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); + } + if (pfds[1].revents) { + do { + /* The following epoll_wait never blocks; it has a timeout of 0 */ + ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + if (ep_rv < 0) { + if (errno != EINTR) { + work_combine_error(&error, GRPC_OS_ERROR(errno, "epoll_wait")); + } + } else { + int i; + for (i = 0; i < ep_rv; ++i) { + grpc_fd *fd = ep_ev[i].data.ptr; + /* TODO(klempner): We might want to consider making err and pri + * separate events */ + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (fd == NULL) { + work_combine_error(&error, grpc_wakeup_fd_consume_wakeup( + &grpc_global_wakeup_fd)); + } else { + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + } + } while (ep_rv == GRPC_EPOLL_MAX_EVENTS); + } + } + return error; +} + +static void multipoll_with_epoll_pollset_finish_shutdown( + grpc_pollset *pollset) {} + +static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) { + epoll_hdr *h = pollset->data.ptr; + close(h->epoll_fd); + remove_epoll_fd_from_global_list(h->epoll_fd); + gpr_free(h); +} + +static const grpc_pollset_vtable multipoll_with_epoll_pollset = { + multipoll_with_epoll_pollset_add_fd, + multipoll_with_epoll_pollset_maybe_work_and_unlock, + multipoll_with_epoll_pollset_finish_shutdown, + multipoll_with_epoll_pollset_destroy}; + +static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd **fds, + size_t nfds) { + size_t i; + epoll_hdr *h = gpr_malloc(sizeof(epoll_hdr)); + struct epoll_event ev; + int err; + + pollset->vtable = &multipoll_with_epoll_pollset; + pollset->data.ptr = h; + h->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if (h->epoll_fd < 0) { + /* TODO(klempner): Fall back to poll here, especially on ENOSYS */ + gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno)); + abort(); + } + add_epoll_fd_to_global_list(h->epoll_fd); + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = NULL; + err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev); + if (err < 0) { + gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", + GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), + strerror(errno)); + } + + for (i = 0; i < nfds; i++) { + multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0); + } +} + +#else /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */ + +static void remove_fd_from_all_epoll_sets(int fd) {} + +#endif /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */ + +/******************************************************************************* + * pollset_set_posix.c + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set)); + memset(pollset_set, 0, sizeof(*pollset_set)); + gpr_mu_init(&pollset_set->mu); + return pollset_set; +} + +static void pollset_set_destroy(grpc_pollset_set *pollset_set) { + size_t i; + gpr_mu_destroy(&pollset_set->mu); + for (i = 0; i < pollset_set->fd_count; i++) { + GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set"); + } + gpr_free(pollset_set->pollsets); + gpr_free(pollset_set->pollset_sets); + gpr_free(pollset_set->fds); + gpr_free(pollset_set); +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, + grpc_pollset *pollset) { + size_t i, j; + gpr_mu_lock(&pollset_set->mu); + if (pollset_set->pollset_count == pollset_set->pollset_capacity) { + pollset_set->pollset_capacity = + GPR_MAX(8, 2 * pollset_set->pollset_capacity); + pollset_set->pollsets = + gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity * + sizeof(*pollset_set->pollsets)); + } + pollset_set->pollsets[pollset_set->pollset_count++] = pollset; + for (i = 0, j = 0; i < pollset_set->fd_count; i++) { + if (fd_is_orphaned(pollset_set->fds[i])) { + GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set"); + } else { + pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]); + pollset_set->fds[j++] = pollset_set->fds[i]; + } + } + pollset_set->fd_count = j; + gpr_mu_unlock(&pollset_set->mu); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, + grpc_pollset *pollset) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + for (i = 0; i < pollset_set->pollset_count; i++) { + if (pollset_set->pollsets[i] == pollset) { + pollset_set->pollset_count--; + GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i], + pollset_set->pollsets[pollset_set->pollset_count]); + break; + } + } + gpr_mu_unlock(&pollset_set->mu); +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + size_t i, j; + gpr_mu_lock(&bag->mu); + if (bag->pollset_set_count == bag->pollset_set_capacity) { + bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity); + bag->pollset_sets = + gpr_realloc(bag->pollset_sets, + bag->pollset_set_capacity * sizeof(*bag->pollset_sets)); + } + bag->pollset_sets[bag->pollset_set_count++] = item; + for (i = 0, j = 0; i < bag->fd_count; i++) { + if (fd_is_orphaned(bag->fds[i])) { + GRPC_FD_UNREF(bag->fds[i], "pollset_set"); + } else { + pollset_set_add_fd(exec_ctx, item, bag->fds[i]); + bag->fds[j++] = bag->fds[i]; + } + } + bag->fd_count = j; + gpr_mu_unlock(&bag->mu); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + size_t i; + gpr_mu_lock(&bag->mu); + for (i = 0; i < bag->pollset_set_count; i++) { + if (bag->pollset_sets[i] == item) { + bag->pollset_set_count--; + GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i], + bag->pollset_sets[bag->pollset_set_count]); + break; + } + } + gpr_mu_unlock(&bag->mu); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + if (pollset_set->fd_count == pollset_set->fd_capacity) { + pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity); + pollset_set->fds = gpr_realloc( + pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds)); + } + GRPC_FD_REF(fd, "pollset_set"); + pollset_set->fds[pollset_set->fd_count++] = fd; + for (i = 0; i < pollset_set->pollset_count; i++) { + pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd); + } + for (i = 0; i < pollset_set->pollset_set_count; i++) { + pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd); + } + gpr_mu_unlock(&pollset_set->mu); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pollset_set, grpc_fd *fd) { + size_t i; + gpr_mu_lock(&pollset_set->mu); + for (i = 0; i < pollset_set->fd_count; i++) { + if (pollset_set->fds[i] == fd) { + pollset_set->fd_count--; + GPR_SWAP(grpc_fd *, pollset_set->fds[i], + pollset_set->fds[pollset_set->fd_count]); + GRPC_FD_UNREF(fd, "pollset_set"); + break; + } + } + for (i = 0; i < pollset_set->pollset_set_count; i++) { + pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd); + } + gpr_mu_unlock(&pollset_set->mu); +} + +/******************************************************************************* + * event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_reset = pollset_reset, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .shutdown_engine = shutdown_engine, +}; + +const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void) { +#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL + platform_become_multipoller = epoll_become_multipoller; +#else + platform_become_multipoller = poll_become_multipoller; +#endif + fd_global_init(); + pollset_global_init(); + return &vtable; +} + +#endif diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h b/src/core/lib/iomgr/ev_poll_and_epoll_posix.h new file mode 100644 index 0000000000..06d6dbf29d --- /dev/null +++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.h @@ -0,0 +1,41 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H +#define GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H + +#include "src/core/lib/iomgr/ev_posix.h" + +const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H */ diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 52efb0d6b2..45c0a5e954 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -59,8 +59,6 @@ * FD declarations */ -grpc_wakeup_fd grpc_global_wakeup_fd; - typedef struct grpc_fd_watcher { struct grpc_fd_watcher *next; struct grpc_fd_watcher *prev; @@ -373,7 +371,7 @@ static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { if (!fd->released) { close(fd->fd); } - grpc_exec_ctx_push(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL); } static int fd_wrapped_fd(grpc_fd *fd) { @@ -432,14 +430,17 @@ static grpc_error *fd_shutdown_error(bool shutdown) { static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st, grpc_closure *closure) { - if (*st == CLOSURE_NOT_READY) { + if (fd->shutdown) { + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"), + NULL); + } else if (*st == CLOSURE_NOT_READY) { /* not ready ==> switch to a waiting state by setting the closure */ *st = closure; } else if (*st == CLOSURE_READY) { /* already ready ==> queue the closure to run immediately */ *st = CLOSURE_NOT_READY; - grpc_exec_ctx_push(exec_ctx, closure, fd_shutdown_error(fd->shutdown), - NULL); + grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown), + NULL); maybe_wake_one_watcher_locked(fd); } else { /* upcallptr was set to a different closure. This is an error! */ @@ -462,7 +463,7 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, return 0; } else { /* waiting ==> queue closure */ - grpc_exec_ctx_push(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL); + grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL); *st = CLOSURE_NOT_READY; return 1; } @@ -475,11 +476,22 @@ static void set_read_notifier_pollset_locked( static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { gpr_mu_lock(&fd->mu); - GPR_ASSERT(!fd->shutdown); - fd->shutdown = 1; - set_ready_locked(exec_ctx, fd, &fd->read_closure); - set_ready_locked(exec_ctx, fd, &fd->write_closure); + /* only shutdown once */ + if (!fd->shutdown) { + fd->shutdown = 1; + /* signal read/write closed to OS so that future operations fail */ + shutdown(fd->fd, SHUT_RDWR); + set_ready_locked(exec_ctx, fd, &fd->read_closure); + set_ready_locked(exec_ctx, fd, &fd->write_closure); + } + gpr_mu_unlock(&fd->mu); +} + +static bool fd_is_shutdown(grpc_fd *fd) { + gpr_mu_lock(&fd->mu); + bool r = fd->shutdown; gpr_mu_unlock(&fd->mu); + return r; } static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, @@ -717,6 +729,7 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, } GPR_TIMER_END("pollset_kick_ext", 0); + GRPC_LOG_IF_ERROR("pollset_kick_ext", GRPC_ERROR_REF(error)); return error; } @@ -811,7 +824,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_FD_UNREF(pollset->fds[i], "multipoller"); } pollset->fd_count = 0; - grpc_exec_ctx_push(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL); } static void work_combine_error(grpc_error **composite, grpc_error *error) { @@ -845,6 +858,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd)); error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd); if (error != GRPC_ERROR_NONE) { + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; } } @@ -1024,6 +1038,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } *worker_hdl = NULL; GPR_TIMER_END("pollset_work", 0); + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; } @@ -1215,6 +1230,7 @@ static const grpc_event_engine_vtable vtable = { .fd_wrapped_fd = fd_wrapped_fd, .fd_orphan = fd_orphan, .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, .fd_notify_on_read = fd_notify_on_read, .fd_notify_on_write = fd_notify_on_write, .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, @@ -1242,7 +1258,9 @@ static const grpc_event_engine_vtable vtable = { }; const grpc_event_engine_vtable *grpc_init_poll_posix(void) { - pollset_global_init(); + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } return &vtable; } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index ad8eaab454..4c360f13a3 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,6 +44,7 @@ #include <grpc/support/string_util.h> #include <grpc/support/useful.h> +#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -61,7 +62,7 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { - {"poll", grpc_init_poll_posix}, + {"poll", grpc_init_poll_posix}, {"legacy", grpc_init_poll_and_epoll_posix}, }; static void add(const char *beg, const char *end, char ***ss, size_t *ns) { @@ -152,6 +153,10 @@ void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { g_event_engine->fd_shutdown(exec_ctx, fd); } +bool grpc_fd_is_shutdown(grpc_fd *fd) { + return g_event_engine->fd_is_shutdown(fd); +} + void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { g_event_engine->fd_notify_on_read(exec_ctx, fd, closure); diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 3058f22c91..87e9bc4d46 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -55,6 +55,7 @@ typedef struct grpc_event_engine_vtable { grpc_closure *closure); void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure); + bool (*fd_is_shutdown)(grpc_fd *fd); grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx, grpc_fd *fd); @@ -116,7 +117,10 @@ int grpc_fd_wrapped_fd(grpc_fd *fd); void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, const char *reason); -/* Cause any current callbacks to error out with GRPC_CALLBACK_CANCELLED. */ +/* Has grpc_fd_shutdown been called on an fd? */ +bool grpc_fd_is_shutdown(grpc_fd *fd); + +/* Cause any current and future callbacks to fail. */ void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd); /* Register read interest, causing read_cb to be called once when fd becomes diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c index fcf39ffd54..c44aafcddf 100644 --- a/src/core/lib/iomgr/exec_ctx.c +++ b/src/core/lib/iomgr/exec_ctx.c @@ -82,9 +82,9 @@ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) { grpc_exec_ctx_flush(exec_ctx); } -void grpc_exec_ctx_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error, - grpc_workqueue *offload_target_or_null) { +void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error, + grpc_workqueue *offload_target_or_null) { GPR_ASSERT(offload_target_or_null == NULL); grpc_closure_list_append(&exec_ctx->closure_list, closure, error); } diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h index 617f48f1b8..38f27d9b13 100644 --- a/src/core/lib/iomgr/exec_ctx.h +++ b/src/core/lib/iomgr/exec_ctx.h @@ -94,9 +94,9 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx); * the instance is destroyed, or work may be lost. */ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx); /** Add a closure to be executed at the next flush/finish point */ -void grpc_exec_ctx_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error, - grpc_workqueue *offload_target_or_null); +void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error, + grpc_workqueue *offload_target_or_null); /** Returns true if we'd like to leave this execution context as soon as possible: useful for deciding whether to do something more or not depending on outside context */ diff --git a/src/core/lib/iomgr/iocp_windows.c b/src/core/lib/iomgr/iocp_windows.c index 1faa8eeedf..2532e52e48 100644 --- a/src/core/lib/iomgr/iocp_windows.c +++ b/src/core/lib/iomgr/iocp_windows.c @@ -39,7 +39,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/thd.h> #include "src/core/lib/iomgr/iocp_windows.h" diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 60cef8ba77..89292a153e 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -96,7 +96,8 @@ void grpc_iomgr_shutdown(void) { gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), last_warning_time), gpr_time_from_seconds(1, GPR_TIMESPAN)) >= 0) { if (g_root_object.next != &g_root_object) { - gpr_log(GPR_DEBUG, "Waiting for %d iomgr objects to be destroyed", + gpr_log(GPR_DEBUG, + "Waiting for %" PRIuPTR " iomgr objects to be destroyed", count_objects()); } last_warning_time = gpr_now(GPR_CLOCK_REALTIME); @@ -114,9 +115,9 @@ void grpc_iomgr_shutdown(void) { if (gpr_cv_wait(&g_rcv, &g_mu, short_deadline)) { if (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), shutdown_deadline) > 0) { if (g_root_object.next != &g_root_object) { - gpr_log(GPR_DEBUG, - "Failed to free %d iomgr objects before shutdown deadline: " - "memory leaks are likely", + gpr_log(GPR_DEBUG, "Failed to free %" PRIuPTR + " iomgr objects before shutdown deadline: " + "memory leaks are likely", count_objects()); dump_objects("LEAKED"); if (grpc_iomgr_abort_on_leaks()) { diff --git a/src/core/lib/iomgr/iomgr_windows.c b/src/core/lib/iomgr/iomgr_windows.c index 398517fc75..7653f6e635 100644 --- a/src/core/lib/iomgr/iomgr_windows.c +++ b/src/core/lib/iomgr/iomgr_windows.c @@ -35,7 +35,7 @@ #ifdef GPR_WINSOCK_SOCKET -#include "src/core/lib/iomgr/sockaddr_win32.h" +#include "src/core/lib/iomgr/sockaddr_windows.h" #include <grpc/support/log.h> diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c new file mode 100644 index 0000000000..d1686aa12f --- /dev/null +++ b/src/core/lib/iomgr/polling_entity.c @@ -0,0 +1,104 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> + +#include "src/core/lib/iomgr/polling_entity.h" + +grpc_polling_entity grpc_polling_entity_create_from_pollset_set( + grpc_pollset_set *pollset_set) { + grpc_polling_entity pollent; + pollent.pollent.pollset_set = pollset_set; + pollent.tag = POPS_POLLSET_SET; + return pollent; +} + +grpc_polling_entity grpc_polling_entity_create_from_pollset( + grpc_pollset *pollset) { + grpc_polling_entity pollent; + pollent.pollent.pollset = pollset; + pollent.tag = POPS_POLLSET; + return pollent; +} + +grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { + if (pollent->tag == POPS_POLLSET) { + return pollent->pollent.pollset; + } + return NULL; +} + +grpc_pollset_set *grpc_polling_entity_pollset_set( + grpc_polling_entity *pollent) { + if (pollent->tag == POPS_POLLSET_SET) { + return pollent->pollent.pollset_set; + } + return NULL; +} + +bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) { + return pollent->tag == POPS_NONE; +} + +void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_polling_entity *pollent, + grpc_pollset_set *pss_dst) { + if (pollent->tag == POPS_POLLSET) { + GPR_ASSERT(pollent->pollent.pollset != NULL); + grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); + } else if (pollent->tag == POPS_POLLSET_SET) { + GPR_ASSERT(pollent->pollent.pollset_set != NULL); + grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst, + pollent->pollent.pollset_set); + } else { + gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag); + abort(); + } +} + +void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_polling_entity *pollent, + grpc_pollset_set *pss_dst) { + if (pollent->tag == POPS_POLLSET) { + GPR_ASSERT(pollent->pollent.pollset != NULL); + grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); + } else if (pollent->tag == POPS_POLLSET_SET) { + GPR_ASSERT(pollent->pollent.pollset_set != NULL); + grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst, + pollent->pollent.pollset_set); + } else { + gpr_log(GPR_ERROR, "Invalid grpc_polling_entity tag '%d'", pollent->tag); + abort(); + } +} diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h new file mode 100644 index 0000000000..e81531053c --- /dev/null +++ b/src/core/lib/iomgr/polling_entity.h @@ -0,0 +1,81 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H +#define GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H + +#include "src/core/lib/iomgr/pollset.h" +#include "src/core/lib/iomgr/pollset_set.h" + +/* A grpc_polling_entity is a pollset-or-pollset_set container. It allows + * functions that + * accept a pollset XOR a pollset_set to do so through an abstract interface. + * No ownership is taken. */ + +typedef struct grpc_polling_entity { + union { + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + } pollent; + enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag; +} grpc_polling_entity; + +grpc_polling_entity grpc_polling_entity_create_from_pollset_set( + grpc_pollset_set *pollset_set); +grpc_polling_entity grpc_polling_entity_create_from_pollset( + grpc_pollset *pollset); + +/** If \a pollent contains a pollset, return it. Otherwise, return NULL */ +grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent); + +/** If \a pollent contains a pollset_set, return it. Otherwise, return NULL */ +grpc_pollset_set *grpc_polling_entity_pollset_set(grpc_polling_entity *pollent); + +bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent); + +/** Add the pollset or pollset_set in \a pollent to the destination pollset_set + * \a + * pss_dst */ +void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_polling_entity *pollent, + grpc_pollset_set *pss_dst); + +/** Delete the pollset or pollset_set in \a pollent from the destination + * pollset_set \a + * pss_dst */ +void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_polling_entity *pollent, + grpc_pollset_set *pss_dst); +/* pollset_set specific */ + +#endif /* GRPC_CORE_LIB_IOMGR_POLLING_ENTITY_H */ diff --git a/src/core/lib/iomgr/pollset_set_windows.c b/src/core/lib/iomgr/pollset_set_windows.c index 89f60b92fb..a35a9766fc 100644 --- a/src/core/lib/iomgr/pollset_set_windows.c +++ b/src/core/lib/iomgr/pollset_set_windows.c @@ -32,12 +32,15 @@ */ #include <grpc/support/port_platform.h> +#include <stdint.h> #ifdef GPR_WINSOCK_SOCKET #include "src/core/lib/iomgr/pollset_set_windows.h" -grpc_pollset_set* grpc_pollset_set_create(void) { return NULL; } +grpc_pollset_set* grpc_pollset_set_create(void) { + return (grpc_pollset_set*)((intptr_t)0xdeafbeef); +} void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) {} diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 5882d8d71d..626dd784b3 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -109,7 +109,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { - grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL); } else { pollset->on_shutdown = closure; } @@ -167,8 +167,8 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } if (pollset->shutting_down && pollset->on_shutdown != NULL) { - grpc_exec_ctx_push(exec_ctx, pollset->on_shutdown, GRPC_ERROR_NONE, - NULL); + grpc_exec_ctx_sched(exec_ctx, pollset->on_shutdown, GRPC_ERROR_NONE, + NULL); pollset->on_shutdown = NULL; } goto done; diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c index 9a320dc952..4e9f978584 100644 --- a/src/core/lib/iomgr/resolve_address_posix.c +++ b/src/core/lib/iomgr/resolve_address_posix.c @@ -163,7 +163,7 @@ typedef struct { static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, grpc_error *error) { request *r = rp; - grpc_exec_ctx_push( + grpc_exec_ctx_sched( exec_ctx, r->on_done, grpc_blocking_resolve_address(r->name, r->default_port, r->addrs_out), NULL); diff --git a/src/core/lib/iomgr/resolve_address_windows.c b/src/core/lib/iomgr/resolve_address_windows.c index 22a9feb72d..2af8af82dc 100644 --- a/src/core/lib/iomgr/resolve_address_windows.c +++ b/src/core/lib/iomgr/resolve_address_windows.c @@ -43,7 +43,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/host_port.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/string_util.h> #include <grpc/support/thd.h> #include <grpc/support/time.h> @@ -154,7 +154,7 @@ static void do_request_thread(grpc_exec_ctx *exec_ctx, void *rp, } else { GRPC_ERROR_REF(error); } - grpc_exec_ctx_push(exec_ctx, r->on_done, error, NULL); + grpc_exec_ctx_sched(exec_ctx, r->on_done, error, NULL); gpr_free(r->name); gpr_free(r->default_port); gpr_free(r); diff --git a/src/core/lib/iomgr/sockaddr.h b/src/core/lib/iomgr/sockaddr.h index 891a2f094f..5563d0b8a6 100644 --- a/src/core/lib/iomgr/sockaddr.h +++ b/src/core/lib/iomgr/sockaddr.h @@ -36,8 +36,8 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 -#include "src/core/lib/iomgr/sockaddr_win32.h" +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" #endif #ifdef GPR_POSIX_SOCKETADDR diff --git a/src/core/lib/iomgr/sockaddr_win32.h b/src/core/lib/iomgr/sockaddr_windows.h index 02aeae7619..971db5b32b 100644 --- a/src/core/lib/iomgr/sockaddr_win32.h +++ b/src/core/lib/iomgr/sockaddr_windows.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H -#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H +#ifndef GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H +#define GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H #include <winsock2.h> #include <ws2tcpip.h> @@ -40,4 +40,4 @@ // must be included after the above #include <mswsock.h> -#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WIN32_H */ +#endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_WINDOWS_H */ diff --git a/src/core/lib/iomgr/socket_utils_common_posix.c b/src/core/lib/iomgr/socket_utils_common_posix.c index bd8ef00c85..d2f6261e2a 100644 --- a/src/core/lib/iomgr/socket_utils_common_posix.c +++ b/src/core/lib/iomgr/socket_utils_common_posix.c @@ -49,6 +49,7 @@ #include <sys/types.h> #include <unistd.h> +#include <grpc/support/alloc.h> #include <grpc/support/host_port.h> #include <grpc/support/log.h> #include <grpc/support/port_platform.h> @@ -116,6 +117,20 @@ grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd) { return GRPC_ERROR_NONE; } +grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes) { + return 0 == setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffer_size_bytes, + sizeof(buffer_size_bytes)) + ? GRPC_ERROR_NONE + : GRPC_OS_ERROR(errno, "setsockopt(SO_SNDBUF)"); +} + +grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes) { + return 0 == setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &buffer_size_bytes, + sizeof(buffer_size_bytes)) + ? GRPC_ERROR_NONE + : GRPC_OS_ERROR(errno, "setsockopt(SO_RCVBUF)"); +} + /* set a socket to close on exec */ grpc_error *grpc_set_socket_cloexec(int fd, int close_on_exec) { int oldflags = fcntl(fd, F_GETFD, 0); @@ -238,6 +253,16 @@ static int set_socket_dualstack(int fd) { } } +static grpc_error *error_for_fd(int fd, const struct sockaddr *addr) { + if (fd >= 0) return GRPC_ERROR_NONE; + char *addr_str; + grpc_sockaddr_to_string(&addr_str, addr, 0); + grpc_error *err = grpc_error_set_str(GRPC_OS_ERROR(errno, "socket"), + GRPC_ERROR_STR_TARGET_ADDRESS, addr_str); + gpr_free(addr_str); + return err; +} + grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type, int protocol, grpc_dualstack_mode *dsmode, @@ -258,7 +283,7 @@ grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type, /* If this isn't an IPv4 address, then return whatever we've got. */ if (!grpc_sockaddr_is_v4mapped(addr, NULL)) { *dsmode = GRPC_DSMODE_IPV6; - return GRPC_ERROR_NONE; + return error_for_fd(*newfd, addr); } /* Fall back to AF_INET. */ if (*newfd >= 0) { @@ -268,10 +293,7 @@ grpc_error *grpc_create_dualstack_socket(const struct sockaddr *addr, int type, } *dsmode = family == AF_INET ? GRPC_DSMODE_IPV4 : GRPC_DSMODE_NONE; *newfd = socket(family, type, protocol); - if (*newfd == -1) { - return GRPC_OS_ERROR(errno, "socket"); - } - return GRPC_ERROR_NONE; + return error_for_fd(*newfd, addr); } #endif diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h index 607fbc7a79..7bcc2219ae 100644 --- a/src/core/lib/iomgr/socket_utils_posix.h +++ b/src/core/lib/iomgr/socket_utils_posix.h @@ -69,20 +69,23 @@ grpc_error *grpc_set_socket_reuse_port(int fd, int reuse); int grpc_ipv6_loopback_available(void); /* Tries to set SO_NOSIGPIPE if available on this platform. - Returns 1 on success, 0 on failure. If SO_NO_SIGPIPE is not available, returns 1. */ grpc_error *grpc_set_socket_no_sigpipe_if_possible(int fd); /* Tries to set IP_PKTINFO if available on this platform. - Returns 1 on success, 0 on failure. If IP_PKTINFO is not available, returns 1. */ grpc_error *grpc_set_socket_ip_pktinfo_if_possible(int fd); /* Tries to set IPV6_RECVPKTINFO if available on this platform. - Returns 1 on success, 0 on failure. If IPV6_RECVPKTINFO is not available, returns 1. */ grpc_error *grpc_set_socket_ipv6_recvpktinfo_if_possible(int fd); +/* Tries to set the socket's send buffer to given size. */ +grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes); + +/* Tries to set the socket's receive buffer to given size. */ +grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes); + /* An enum to keep track of IPv4/IPv6 socket modes. Currently, this information is only used when a socket is first created, but diff --git a/src/core/lib/iomgr/socket_windows.c b/src/core/lib/iomgr/socket_windows.c index 8919452751..d7d5f6f157 100644 --- a/src/core/lib/iomgr/socket_windows.c +++ b/src/core/lib/iomgr/socket_windows.c @@ -42,7 +42,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/string_util.h> #include "src/core/lib/iomgr/iocp_windows.h" @@ -123,7 +123,7 @@ static void socket_notify_on_iocp(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&socket->state_mu); if (info->has_pending_iocp) { info->has_pending_iocp = 0; - grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL); } else { info->closure = closure; } @@ -146,7 +146,7 @@ void grpc_socket_become_ready(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, GPR_ASSERT(!info->has_pending_iocp); gpr_mu_lock(&socket->state_mu); if (info->closure) { - grpc_exec_ctx_push(exec_ctx, info->closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, info->closure, GRPC_ERROR_NONE, NULL); info->closure = NULL; } else { info->has_pending_iocp = 1; diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index 9c4ca1617f..80c7a3f128 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -221,7 +221,7 @@ finish: gpr_free(ac->addr_str); gpr_free(ac); } - grpc_exec_ctx_push(exec_ctx, closure, error, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); } static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, @@ -250,7 +250,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, error = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); if (error != GRPC_ERROR_NONE) { - grpc_exec_ctx_push(exec_ctx, closure, error, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); return; } if (dsmode == GRPC_DSMODE_IPV4) { @@ -260,7 +260,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, addr_len = sizeof(addr4_copy); } if ((error = prepare_socket(addr, fd)) != GRPC_ERROR_NONE) { - grpc_exec_ctx_push(exec_ctx, closure, error, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); return; } @@ -276,14 +276,14 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, if (err >= 0) { *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str); - grpc_exec_ctx_push(exec_ctx, closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL); goto done; } if (errno != EWOULDBLOCK && errno != EINPROGRESS) { grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error"); - grpc_exec_ctx_push(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"), - NULL); + grpc_exec_ctx_sched(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"), + NULL); goto done; } diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index dbf116422f..562cb9c6bf 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -35,11 +35,11 @@ #ifdef GPR_WINSOCK_SOCKET -#include "src/core/lib/iomgr/sockaddr_win32.h" +#include "src/core/lib/iomgr/sockaddr_windows.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/slice_buffer.h> #include <grpc/support/useful.h> @@ -121,7 +121,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { async_connect_unlock_and_cleanup(ac, socket); /* If the connection was aborted, the callback was already called when the deadline was met. */ - grpc_exec_ctx_push(exec_ctx, on_done, error, NULL); + grpc_exec_ctx_sched(exec_ctx, on_done, error, NULL); } /* Tries to issue one async connection, then schedules both an IOCP @@ -225,7 +225,7 @@ failure: } else if (sock != INVALID_SOCKET) { closesocket(sock); } - grpc_exec_ctx_push(exec_ctx, on_done, final_error, NULL); + grpc_exec_ctx_sched(exec_ctx, on_done, final_error, NULL); } #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 48a19bed84..017f52c367 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -160,7 +160,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, grpc_error *error) { grpc_closure *cb = tcp->read_cb; - if (grpc_tcp_trace) { + if (false && grpc_tcp_trace) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -175,7 +175,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, tcp->read_cb = NULL; tcp->incoming_buffer = NULL; - grpc_exec_ctx_push(exec_ctx, cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } #define MAX_READ_IOVEC 4 @@ -277,7 +277,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->finished_edge = false; grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { - grpc_exec_ctx_push(exec_ctx, &tcp->read_closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, &tcp->read_closure, GRPC_ERROR_NONE, NULL); } } @@ -394,7 +394,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_tcp *tcp = (grpc_tcp *)ep; grpc_error *error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (false && grpc_tcp_trace) { size_t i; for (i = 0; i < buf->count; i++) { @@ -410,7 +410,10 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, if (buf->length == 0) { GPR_TIMER_END("tcp_write", 0); - grpc_exec_ctx_push(exec_ctx, cb, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, grpc_fd_is_shutdown(tcp->em_fd) + ? GRPC_ERROR_CREATE("EOF") + : GRPC_ERROR_NONE, + NULL); return; } tcp->outgoing_buffer = buf; @@ -422,7 +425,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->write_cb = cb; grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { - grpc_exec_ctx_push(exec_ctx, cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } GPR_TIMER_END("tcp_write", 0); diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index aac3beb8bd..a1a463550a 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -132,6 +132,9 @@ struct grpc_tcp_server { grpc_pollset **pollsets; /* number of pollsets in the pollsets array */ size_t pollset_count; + + /* next pollset to assign a channel to */ + size_t next_pollset_to_assign; }; static gpr_once check_init = GPR_ONCE_INIT; @@ -178,13 +181,14 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete, s->head = NULL; s->tail = NULL; s->nports = 0; + s->next_pollset_to_assign = 0; *server = s; return GRPC_ERROR_NONE; } static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (s->shutdown_complete != NULL) { - grpc_exec_ctx_push(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL); } gpr_mu_destroy(&s->mu); @@ -364,7 +368,9 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { goto error; } - read_notifier_pollset = grpc_fd_get_read_notifier_pollset(exec_ctx, sp->emfd); + read_notifier_pollset = + sp->server->pollsets[(sp->server->next_pollset_to_assign++) % + sp->server->pollset_count]; /* loop until accept4 returns EAGAIN, and then re-arm notification */ for (;;) { @@ -588,9 +594,9 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, if (port == 0 && sp != NULL) { grpc_sockaddr_set_port((struct sockaddr *)&wild4, sp->port); } - addr = (struct sockaddr *)&wild4; - addr_len = sizeof(wild4); } + addr = (struct sockaddr *)&wild4; + addr_len = sizeof(wild4); } errs[1] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index 481111c9f9..2a51671ec7 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -41,7 +41,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/string_util.h> #include <grpc/support/sync.h> #include <grpc/support/time.h> @@ -121,7 +121,7 @@ grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete, static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { if (s->shutdown_complete != NULL) { - grpc_exec_ctx_push(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL); } /* Now that the accepts have been aborted, we can destroy the sockets. diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index 88c9354f6e..b2af8030aa 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -37,11 +37,11 @@ #include <limits.h> -#include "src/core/lib/iomgr/sockaddr_win32.h" +#include "src/core/lib/iomgr/sockaddr_windows.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/slice_buffer.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -183,7 +183,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { tcp->read_cb = NULL; TCP_UNREF(tcp, "read"); - grpc_exec_ctx_push(exec_ctx, cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, @@ -197,8 +197,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, WSABUF buffer; if (tcp->shutting_down) { - grpc_exec_ctx_push(exec_ctx, cb, - GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL); + grpc_exec_ctx_sched(exec_ctx, cb, + GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL); return; } @@ -222,7 +222,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, /* Did we get data immediately ? Yay. */ if (info->wsa_error != WSAEWOULDBLOCK) { info->bytes_transfered = bytes_read; - grpc_exec_ctx_push(exec_ctx, &tcp->on_read, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, &tcp->on_read, GRPC_ERROR_NONE, NULL); return; } @@ -235,8 +235,8 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, int wsa_error = WSAGetLastError(); if (wsa_error != WSA_IO_PENDING) { info->wsa_error = wsa_error; - grpc_exec_ctx_push(exec_ctx, &tcp->on_read, - GRPC_WSA_ERROR(info->wsa_error, "WSARecv"), NULL); + grpc_exec_ctx_sched(exec_ctx, &tcp->on_read, + GRPC_WSA_ERROR(info->wsa_error, "WSARecv"), NULL); return; } } @@ -267,7 +267,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { } TCP_UNREF(tcp, "write"); - grpc_exec_ctx_push(exec_ctx, cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } /* Initiates a write. */ @@ -285,8 +285,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, size_t len; if (tcp->shutting_down) { - grpc_exec_ctx_push(exec_ctx, cb, - GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL); + grpc_exec_ctx_sched(exec_ctx, cb, + GRPC_ERROR_CREATE("TCP socket is shutting down"), NULL); return; } @@ -317,7 +317,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_error *error = status == 0 ? GRPC_ERROR_NONE : GRPC_WSA_ERROR(info->wsa_error, "WSASend"); - grpc_exec_ctx_push(exec_ctx, cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); return; } @@ -334,8 +334,8 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, int wsa_error = WSAGetLastError(); if (wsa_error != WSA_IO_PENDING) { TCP_UNREF(tcp, "write"); - grpc_exec_ctx_push(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"), - NULL); + grpc_exec_ctx_sched(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"), + NULL); return; } } diff --git a/src/core/lib/iomgr/timer.c b/src/core/lib/iomgr/timer.c index b46c6df287..9975fa1671 100644 --- a/src/core/lib/iomgr/timer.c +++ b/src/core/lib/iomgr/timer.c @@ -186,7 +186,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, if (!g_initialized) { timer->triggered = 1; - grpc_exec_ctx_push( + grpc_exec_ctx_sched( exec_ctx, &timer->closure, GRPC_ERROR_CREATE("Attempt to create timer before initialization"), NULL); @@ -195,7 +195,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, if (gpr_time_cmp(deadline, now) <= 0) { timer->triggered = 1; - grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_NONE, NULL); return; } @@ -247,7 +247,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { shard_type *shard = &g_shards[shard_idx(timer)]; gpr_mu_lock(&shard->mu); if (!timer->triggered) { - grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_CANCELLED, NULL); timer->triggered = 1; if (timer->heap_index == INVALID_HEAP_INDEX) { list_remove(timer); @@ -287,8 +287,8 @@ static int refill_queue(shard_type *shard, gpr_timespec now) { return !grpc_timer_heap_is_empty(&shard->heap); } -/* This pops the next non-cancelled timer with deadline <= now from the queue, - or returns NULL if there isn't one. +/* This pops the next non-cancelled timer with deadline <= now from the + queue, or returns NULL if there isn't one. REQUIRES: shard->mu locked */ static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) { grpc_timer *timer; @@ -313,7 +313,7 @@ static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard, grpc_timer *timer; gpr_mu_lock(&shard->mu); while ((timer = pop_one(shard, now))) { - grpc_exec_ctx_push(exec_ctx, &timer->closure, GRPC_ERROR_REF(error), NULL); + grpc_exec_ctx_sched(exec_ctx, &timer->closure, GRPC_ERROR_REF(error), NULL); n++; } *new_min_deadline = compute_min_deadline(shard); diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 98ffccd59b..16150687d3 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -210,6 +210,8 @@ static int prepare_socket(int fd, const struct sockaddr *addr, size_t addr_len) { struct sockaddr_storage sockname_temp; socklen_t sockname_len; + /* Set send/receive socket buffers to 1 MB */ + int buffer_size_bytes = 1024 * 1024; if (fd < 0) { goto error; @@ -239,6 +241,18 @@ static int prepare_socket(int fd, const struct sockaddr *addr, goto error; } + if (!grpc_set_socket_sndbuf(fd, buffer_size_bytes)) { + gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes", + buf_size_bytes); + goto error; + } + + if (!grpc_set_socket_rcvbuf(fd, buffer_size_bytes)) { + gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes", + buf_size_bytes); + goto error; + } + return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); error: diff --git a/src/core/lib/iomgr/workqueue.h b/src/core/lib/iomgr/workqueue.h index 6e8b1218be..5cc40eea50 100644 --- a/src/core/lib/iomgr/workqueue.h +++ b/src/core/lib/iomgr/workqueue.h @@ -43,7 +43,7 @@ #include "src/core/lib/iomgr/workqueue_posix.h" #endif -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include "src/core/lib/iomgr/workqueue_windows.h" #endif @@ -78,7 +78,7 @@ void grpc_workqueue_add_to_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); /** Add a work item to a workqueue */ -void grpc_workqueue_push(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - grpc_closure *closure, grpc_error *error); +void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + grpc_closure *closure, grpc_error *error); #endif /* GRPC_CORE_LIB_IOMGR_WORKQUEUE_H */ diff --git a/src/core/lib/iomgr/workqueue_posix.c b/src/core/lib/iomgr/workqueue_posix.c index 297e8d3102..45e0f6063b 100644 --- a/src/core/lib/iomgr/workqueue_posix.c +++ b/src/core/lib/iomgr/workqueue_posix.c @@ -137,8 +137,8 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } } -void grpc_workqueue_push(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - grpc_closure *closure, grpc_error *error) { +void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + grpc_closure *closure, grpc_error *error) { grpc_error *push_error = GRPC_ERROR_NONE; gpr_mu_lock(&workqueue->mu); if (grpc_closure_list_empty(workqueue->closure_list)) { diff --git a/src/core/lib/iomgr/workqueue_windows.c b/src/core/lib/iomgr/workqueue_windows.c index c3c0446a57..275f040b1c 100644 --- a/src/core/lib/iomgr/workqueue_windows.c +++ b/src/core/lib/iomgr/workqueue_windows.c @@ -33,8 +33,8 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include "src/core/lib/iomgr/workqueue.h" -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c index 18189a8fb8..07db8bfd75 100644 --- a/src/core/lib/security/credentials/composite/composite_credentials.c +++ b/src/core/lib/security/credentials/composite/composite_credentials.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,6 +35,7 @@ #include <string.h> +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/surface/api_trace.h" #include <grpc/support/alloc.h> @@ -49,7 +50,7 @@ typedef struct { grpc_credentials_md_store *md_elems; grpc_auth_metadata_context auth_md_context; void *user_data; - grpc_pollset *pollset; + grpc_polling_entity *pollent; grpc_credentials_metadata_cb cb; } grpc_composite_call_credentials_metadata_context; @@ -93,7 +94,7 @@ static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data, grpc_call_credentials *inner_creds = ctx->composite_creds->inner.creds_array[ctx->creds_index++]; grpc_call_credentials_get_request_metadata( - exec_ctx, inner_creds, ctx->pollset, ctx->auth_md_context, + exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context, composite_call_metadata_cb, ctx); return; } @@ -106,7 +107,7 @@ static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data, static void composite_call_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context auth_md_context, + grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context, grpc_credentials_metadata_cb cb, void *user_data) { grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds; grpc_composite_call_credentials_metadata_context *ctx; @@ -117,10 +118,10 @@ static void composite_call_get_request_metadata( ctx->user_data = user_data; ctx->cb = cb; ctx->composite_creds = c; - ctx->pollset = pollset; + ctx->pollent = pollent; ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds); grpc_call_credentials_get_request_metadata( - exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset, + exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent, auth_md_context, composite_call_metadata_cb, ctx); } diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h index 43974a9623..0d8966f464 100644 --- a/src/core/lib/security/credentials/composite/composite_credentials.h +++ b/src/core/lib/security/credentials/composite/composite_credentials.h @@ -68,4 +68,5 @@ typedef struct { grpc_call_credentials_array inner; } grpc_composite_call_credentials; -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \ + */ diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c index ce31919f74..0eadaec191 100644 --- a/src/core/lib/security/credentials/credentials.c +++ b/src/core/lib/security/credentials/credentials.c @@ -113,7 +113,7 @@ void grpc_call_credentials_release(grpc_call_credentials *creds) { void grpc_call_credentials_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context context, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { if (creds == NULL || creds->vtable->get_request_metadata == NULL) { if (cb != NULL) { @@ -121,7 +121,7 @@ void grpc_call_credentials_get_request_metadata( } return; } - creds->vtable->get_request_metadata(exec_ctx, creds, pollset, context, cb, + creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb, user_data); } diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 675e02b58d..ce235e3a1d 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -41,6 +41,7 @@ #include "src/core/lib/http/httpcli.h" #include "src/core/lib/http/parser.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/security/transport/security_connector.h" struct grpc_http_response; @@ -164,7 +165,8 @@ typedef void (*grpc_credentials_metadata_cb)(grpc_exec_ctx *exec_ctx, typedef struct { void (*destruct)(grpc_call_credentials *c); void (*get_request_metadata)(grpc_exec_ctx *exec_ctx, - grpc_call_credentials *c, grpc_pollset *pollset, + grpc_call_credentials *c, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data); @@ -180,7 +182,7 @@ grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds); void grpc_call_credentials_unref(grpc_call_credentials *creds); void grpc_call_credentials_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context context, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data); /* Metadata-only credentials with the specified key and value where diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c index 1ff7bd14a5..ee6d964de1 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.c +++ b/src/core/lib/security/credentials/fake/fake_credentials.c @@ -106,7 +106,7 @@ static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx, static void md_only_test_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context context, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds; diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index adeba94bdb..9cf38084a3 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -53,4 +53,4 @@ typedef struct { int is_async; } grpc_md_only_test_credentials; -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */ diff --git a/src/core/lib/security/credentials/google_default/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_windows.c index cd8b48080a..208b8fd9ad 100644 --- a/src/core/lib/security/credentials/google_default/credentials_win32.c +++ b/src/core/lib/security/credentials/google_default/credentials_windows.c @@ -33,7 +33,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include "src/core/lib/security/credentials/google_default/google_default_credentials.h" @@ -58,4 +58,4 @@ char *grpc_get_well_known_google_credentials_file_path_impl(void) { return result; } -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index 29f818b8c8..312a3d4f90 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -42,6 +42,7 @@ #include "src/core/lib/http/httpcli.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/security/credentials/jwt/jwt_credentials.h" #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h" #include "src/core/lib/support/env.h" @@ -63,7 +64,7 @@ static gpr_once g_once = GPR_ONCE_INIT; static void init_default_credentials(void) { gpr_mu_init(&g_state_mu); } typedef struct { - grpc_pollset *pollset; + grpc_polling_entity pollent; int is_done; int success; grpc_http_response response; @@ -89,7 +90,9 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, } gpr_mu_lock(g_polling_mu); detector->is_done = 1; - GRPC_LOG_IF_ERROR("Pollset kick", grpc_pollset_kick(detector->pollset, NULL)); + GRPC_LOG_IF_ERROR( + "Pollset kick", + grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL)); gpr_mu_unlock(g_polling_mu); } @@ -108,8 +111,9 @@ static int is_stack_running_on_compute_engine(void) { on compute engine. */ gpr_timespec max_detection_delay = gpr_time_from_seconds(1, GPR_TIMESPAN); - detector.pollset = gpr_malloc(grpc_pollset_size()); - grpc_pollset_init(detector.pollset, &g_polling_mu); + grpc_pollset *pollset = gpr_malloc(grpc_pollset_size()); + grpc_pollset_init(pollset, &g_polling_mu); + detector.pollent = grpc_polling_entity_create_from_pollset(pollset); detector.is_done = 0; detector.success = 0; @@ -121,7 +125,7 @@ static int is_stack_running_on_compute_engine(void) { grpc_httpcli_context_init(&context); grpc_httpcli_get( - &exec_ctx, &context, detector.pollset, &request, + &exec_ctx, &context, &detector.pollent, &request, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay), grpc_closure_create(on_compute_engine_detection_http_response, &detector), &detector.response); @@ -135,8 +139,9 @@ static int is_stack_running_on_compute_engine(void) { grpc_pollset_worker *worker = NULL; if (!GRPC_LOG_IF_ERROR( "pollset_work", - grpc_pollset_work(&exec_ctx, detector.pollset, &worker, - gpr_now(GPR_CLOCK_MONOTONIC), + grpc_pollset_work(&exec_ctx, + grpc_polling_entity_pollset(&detector.pollent), + &worker, gpr_now(GPR_CLOCK_MONOTONIC), gpr_inf_future(GPR_CLOCK_MONOTONIC)))) { detector.is_done = 1; detector.success = 0; @@ -145,12 +150,15 @@ static int is_stack_running_on_compute_engine(void) { gpr_mu_unlock(g_polling_mu); grpc_httpcli_context_destroy(&context); - grpc_closure_init(&destroy_closure, destroy_pollset, detector.pollset); - grpc_pollset_shutdown(&exec_ctx, detector.pollset, &destroy_closure); + grpc_closure_init(&destroy_closure, destroy_pollset, + grpc_polling_entity_pollset(&detector.pollent)); + grpc_pollset_shutdown(&exec_ctx, + grpc_polling_entity_pollset(&detector.pollent), + &destroy_closure); grpc_exec_ctx_finish(&exec_ctx); g_polling_mu = NULL; - gpr_free(detector.pollset); + gpr_free(grpc_polling_entity_pollset(&detector.pollent)); grpc_http_response_destroy(&detector.response); return detector.success; diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h index 0d14cbfdad..fac4377e2c 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.h +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h @@ -42,4 +42,5 @@ void grpc_flush_cached_google_default_credentials(void); -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \ + */ diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c index 89defa7c60..64d5871844 100644 --- a/src/core/lib/security/credentials/iam/iam_credentials.c +++ b/src/core/lib/security/credentials/iam/iam_credentials.c @@ -49,7 +49,7 @@ static void iam_destruct(grpc_call_credentials *creds) { static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h index e989bfc28e..af54faa586 100644 --- a/src/core/lib/security/credentials/iam/iam_credentials.h +++ b/src/core/lib/security/credentials/iam/iam_credentials.h @@ -41,4 +41,4 @@ typedef struct { grpc_credentials_md_store *iam_md; } grpc_google_iam_credentials; -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */ diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c index 8755a96af4..973fb75eaa 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.c +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c @@ -64,7 +64,7 @@ static void jwt_destruct(grpc_call_credentials *creds) { static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h index dfa4a1ebd9..d572606179 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.h +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h @@ -59,4 +59,4 @@ grpc_call_credentials * grpc_service_account_jwt_access_credentials_create_from_auth_json_key( grpc_auth_json_key key, gpr_timespec token_lifetime); -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */ diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index bdee6c85f7..73eb2e3258 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -37,6 +37,7 @@ #include <string.h> #include "src/core/lib/http/httpcli.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/security/util/b64.h" #include "src/core/lib/tsi/ssl_types.h" @@ -320,9 +321,15 @@ grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, /* --- verifier_cb_ctx object. --- */ +typedef enum { + HTTP_RESPONSE_OPENID = 0, + HTTP_RESPONSE_KEYS, + HTTP_RESPONSE_COUNT /* must be last */ +} http_response_index; + typedef struct { grpc_jwt_verifier *verifier; - grpc_pollset *pollset; + grpc_polling_entity pollent; jose_header *header; grpc_jwt_claims *claims; char *audience; @@ -330,7 +337,7 @@ typedef struct { gpr_slice signed_data; void *user_data; grpc_jwt_verification_done_cb user_cb; - grpc_http_response responses[2]; + grpc_http_response responses[HTTP_RESPONSE_COUNT]; } verifier_cb_ctx; /* Takes ownership of the header, claims and signature. */ @@ -339,10 +346,11 @@ static verifier_cb_ctx *verifier_cb_ctx_create( grpc_jwt_claims *claims, const char *audience, gpr_slice signature, const char *signed_jwt, size_t signed_jwt_len, void *user_data, grpc_jwt_verification_done_cb cb) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; verifier_cb_ctx *ctx = gpr_malloc(sizeof(verifier_cb_ctx)); memset(ctx, 0, sizeof(verifier_cb_ctx)); ctx->verifier = verifier; - ctx->pollset = pollset; + ctx->pollent = grpc_polling_entity_create_from_pollset(pollset); ctx->header = header; ctx->audience = gpr_strdup(audience); ctx->claims = claims; @@ -350,6 +358,7 @@ static verifier_cb_ctx *verifier_cb_ctx_create( ctx->signed_data = gpr_slice_from_copied_buffer(signed_jwt, signed_jwt_len); ctx->user_data = user_data; ctx->user_cb = cb; + grpc_exec_ctx_finish(&exec_ctx); return ctx; } @@ -359,7 +368,7 @@ void verifier_cb_ctx_destroy(verifier_cb_ctx *ctx) { gpr_slice_unref(ctx->signature); gpr_slice_unref(ctx->signed_data); jose_header_destroy(ctx->header); - for (size_t i = 0; i < GPR_ARRAY_SIZE(ctx->responses); i++) { + for (size_t i = 0; i < HTTP_RESPONSE_COUNT; i++) { grpc_http_response_destroy(&ctx->responses[i]); } /* TODO: see what to do with claims... */ @@ -578,7 +587,7 @@ end: static void on_keys_retrieved(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *error) { verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data; - grpc_json *json = json_from_http(&ctx->responses[1]); + grpc_json *json = json_from_http(&ctx->responses[HTTP_RESPONSE_KEYS]); EVP_PKEY *verification_key = NULL; grpc_jwt_verifier_status status = GRPC_JWT_VERIFIER_GENERIC_ERROR; grpc_jwt_claims *claims = NULL; @@ -620,7 +629,7 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *error) { const grpc_json *cur; verifier_cb_ctx *ctx = (verifier_cb_ctx *)user_data; - const grpc_http_response *response = &ctx->responses[0]; + const grpc_http_response *response = &ctx->responses[HTTP_RESPONSE_OPENID]; grpc_json *json = json_from_http(response); grpc_httpcli_request req; const char *jwks_uri; @@ -649,9 +658,10 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data, } grpc_httpcli_get( - exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req, + exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay), - grpc_closure_create(on_keys_retrieved, ctx), &ctx->responses[1]); + grpc_closure_create(on_keys_retrieved, ctx), + &ctx->responses[HTTP_RESPONSE_KEYS]); grpc_json_destroy(json); gpr_free(req.host); return; @@ -699,7 +709,7 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, grpc_httpcli_request req; memset(&req, 0, sizeof(grpc_httpcli_request)); req.handshaker = &grpc_httpcli_ssl; - int rsp_idx; + http_response_index rsp_idx; GPR_ASSERT(ctx != NULL && ctx->header != NULL && ctx->claims != NULL); iss = ctx->claims->iss; @@ -739,7 +749,7 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, gpr_asprintf(&req.http.path, "/%s/%s", path_prefix, iss); } http_cb = grpc_closure_create(on_keys_retrieved, ctx); - rsp_idx = 1; + rsp_idx = HTTP_RESPONSE_KEYS; } else { req.host = gpr_strdup(strstr(iss, "https://") == iss ? iss + 8 : iss); path_prefix = strchr(req.host, '/'); @@ -751,11 +761,11 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx, GRPC_OPENID_CONFIG_URL_SUFFIX); } http_cb = grpc_closure_create(on_openid_config_retrieved, ctx); - rsp_idx = 0; + rsp_idx = HTTP_RESPONSE_OPENID; } grpc_httpcli_get( - exec_ctx, &ctx->verifier->http_ctx, ctx->pollset, &req, + exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay), http_cb, &ctx->responses[rsp_idx]); gpr_free(req.host); diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index 671dea31e6..1102553dd3 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -226,6 +226,8 @@ static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx, gpr_timespec token_lifetime; grpc_credentials_status status; + GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error)); + gpr_mu_lock(&c->mu); status = grpc_oauth2_token_fetcher_credentials_parse_server_response( &r->response, &c->access_token_md, &token_lifetime); @@ -244,7 +246,7 @@ static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx, static void oauth2_token_fetcher_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context context, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { grpc_oauth2_token_fetcher_credentials *c = (grpc_oauth2_token_fetcher_credentials *)creds; @@ -270,7 +272,7 @@ static void oauth2_token_fetcher_get_request_metadata( c->fetch_func( exec_ctx, grpc_credentials_metadata_request_create(creds, cb, user_data), - &c->httpcli_context, pollset, on_oauth2_token_fetcher_http_response, + &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response, gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold)); } } @@ -295,7 +297,7 @@ static grpc_call_credentials_vtable compute_engine_vtable = { static void compute_engine_fetch_oauth2( grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req, - grpc_httpcli_context *httpcli_context, grpc_pollset *pollset, + grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent, grpc_iomgr_cb_func response_cb, gpr_timespec deadline) { grpc_http_header header = {"Metadata-Flavor", "Google"}; grpc_httpcli_request request; @@ -304,7 +306,7 @@ static void compute_engine_fetch_oauth2( request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH; request.http.hdr_count = 1; request.http.hdrs = &header; - grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline, + grpc_httpcli_get(exec_ctx, httpcli_context, pollent, &request, deadline, grpc_closure_create(response_cb, metadata_req), &metadata_req->response); } @@ -337,7 +339,7 @@ static grpc_call_credentials_vtable refresh_token_vtable = { static void refresh_token_fetch_oauth2( grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req, - grpc_httpcli_context *httpcli_context, grpc_pollset *pollset, + grpc_httpcli_context *httpcli_context, grpc_polling_entity *pollent, grpc_iomgr_cb_func response_cb, gpr_timespec deadline) { grpc_google_refresh_token_credentials *c = (grpc_google_refresh_token_credentials *)metadata_req->creds; @@ -354,7 +356,7 @@ static void refresh_token_fetch_oauth2( request.http.hdr_count = 1; request.http.hdrs = &header; request.handshaker = &grpc_httpcli_ssl; - grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body, + grpc_httpcli_post(exec_ctx, httpcli_context, pollent, &request, body, strlen(body), deadline, grpc_closure_create(response_cb, metadata_req), &metadata_req->response); @@ -399,7 +401,7 @@ static void access_token_destruct(grpc_call_credentials *creds) { static void access_token_get_request_metadata( grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, grpc_auth_metadata_context context, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds; cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK); diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h index be7bc81626..7f6f205c22 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h @@ -70,7 +70,7 @@ void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token); typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *req, grpc_httpcli_context *http_context, - grpc_pollset *pollset, + grpc_polling_entity *pollent, grpc_iomgr_cb_func cb, gpr_timespec deadline); typedef struct { @@ -106,4 +106,4 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response( const struct grpc_http_response *response, grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime); -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */ diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c index bae357321e..9fb55e8466 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.c +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c @@ -94,7 +94,7 @@ static void plugin_md_request_metadata_ready(void *request, static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, - grpc_pollset *pollset, + grpc_polling_entity *pollent, grpc_auth_metadata_context context, grpc_credentials_metadata_cb cb, void *user_data) { diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h index b5dd3ad5e2..89073cb3d1 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.h +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h @@ -42,4 +42,4 @@ typedef struct { grpc_credentials_md_store *plugin_md; } grpc_plugin_credentials; -#endif // GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H +#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */ diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index e3cbcb4433..76be2acd72 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -54,11 +54,11 @@ typedef struct { grpc_call_credentials *creds; grpc_mdstr *host; grpc_mdstr *method; - /* pollset bound to this call; if we need to make external - network requests, they should be done under this pollset - so that work can progress when this call wants work to - progress */ - grpc_pollset *pollset; + /* pollset{_set} bound to this call; if we need to make external + network requests, they should be done under a pollset added to this + pollset_set so that work can progress when this call wants work to progress + */ + grpc_polling_entity *pollent; grpc_transport_stream_op op; uint8_t security_context_set; grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT]; @@ -184,9 +184,9 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx, build_auth_metadata_context(&chand->security_connector->base, chand->auth_context, calld); calld->op = *op; /* Copy op (originates from the caller's stack). */ - GPR_ASSERT(calld->pollset); + GPR_ASSERT(calld->pollent != NULL); grpc_call_credentials_get_request_metadata( - exec_ctx, calld->creds, calld->pollset, calld->auth_md_context, + exec_ctx, calld->creds, calld->pollent, calld->auth_md_context, on_credentials_metadata, elem); } @@ -270,15 +270,16 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, memset(calld, 0, sizeof(*calld)); } -static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_pollset *pollset) { +static void set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_polling_entity *pollent) { call_data *calld = elem->call_data; - calld->pollset = pollset; + calld->pollent = pollent; } /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) { + const grpc_call_stats *stats, void *ignored) { call_data *calld = elem->call_data; grpc_call_credentials_unref(calld->creds); if (calld->host != NULL) { @@ -329,8 +330,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, GRPC_AUTH_CONTEXT_UNREF(chand->auth_context, "client_auth_filter"); } -const grpc_channel_filter grpc_client_auth_filter = { - auth_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, set_pollset, destroy_call_elem, - sizeof(channel_data), init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, "client-auth"}; +const grpc_channel_filter grpc_client_auth_filter = {auth_start_transport_op, + grpc_channel_next_op, + sizeof(call_data), + init_call_elem, + set_pollset_or_pollset_set, + destroy_call_elem, + sizeof(channel_data), + init_channel_elem, + destroy_channel_elem, + grpc_call_next_get_peer, + "client-auth"}; diff --git a/src/core/lib/security/transport/handshake.c b/src/core/lib/security/transport/handshake.c index c87262f124..b374ca7633 100644 --- a/src/core/lib/security/transport/handshake.c +++ b/src/core/lib/security/transport/handshake.c @@ -39,6 +39,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/slice_buffer.h> +#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/security/context/security_context.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/tsi_error.h" @@ -61,6 +62,8 @@ typedef struct { grpc_closure on_handshake_data_sent_to_peer; grpc_closure on_handshake_data_received_from_peer; grpc_auth_context *auth_context; + grpc_timer timer; + gpr_refcount refs; } grpc_security_handshake; static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, @@ -97,9 +100,23 @@ static void security_connector_remove_handshake(grpc_security_handshake *h) { gpr_mu_unlock(&sc->mu); } +static void unref_handshake(grpc_security_handshake *h) { + if (gpr_unref(&h->refs)) { + if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker); + if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer); + gpr_slice_buffer_destroy(&h->left_overs); + gpr_slice_buffer_destroy(&h->outgoing); + gpr_slice_buffer_destroy(&h->incoming); + GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake"); + GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake"); + gpr_free(h); + } +} + static void security_handshake_done(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h, grpc_error *error) { + grpc_timer_cancel(exec_ctx, &h->timer); if (!h->is_client_side) { security_connector_remove_handshake(h); } @@ -119,14 +136,7 @@ static void security_handshake_done(grpc_exec_ctx *exec_ctx, } h->cb(exec_ctx, h->user_data, GRPC_SECURITY_ERROR, NULL, NULL); } - if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker); - if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer); - gpr_slice_buffer_destroy(&h->left_overs); - gpr_slice_buffer_destroy(&h->outgoing); - gpr_slice_buffer_destroy(&h->incoming); - GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake"); - GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake"); - gpr_free(h); + unref_handshake(h); GRPC_ERROR_UNREF(error); } @@ -149,7 +159,7 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data, if (result != TSI_OK) { security_handshake_done( exec_ctx, h, - grpc_set_tsi_error_bits( + grpc_set_tsi_error_result( GRPC_ERROR_CREATE("Frame protector creation failed"), result)); return; } @@ -168,7 +178,7 @@ static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) { if (result != TSI_OK) { security_handshake_done( - exec_ctx, h, grpc_set_tsi_error_bits( + exec_ctx, h, grpc_set_tsi_error_result( GRPC_ERROR_CREATE("Peer extraction failed"), result)); return; } @@ -195,9 +205,9 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx, } while (result == TSI_INCOMPLETE_DATA); if (result != TSI_OK) { - security_handshake_done( - exec_ctx, h, - grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Handshake failed"), result)); + security_handshake_done(exec_ctx, h, + grpc_set_tsi_error_result( + GRPC_ERROR_CREATE("Handshake failed"), result)); return; } @@ -249,9 +259,9 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, } if (result != TSI_OK) { - security_handshake_done( - exec_ctx, h, - grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Handshake failed"), result)); + security_handshake_done(exec_ctx, h, + grpc_set_tsi_error_result( + GRPC_ERROR_CREATE("Handshake failed"), result)); return; } @@ -304,13 +314,19 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, } } -void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, - tsi_handshaker *handshaker, - grpc_security_connector *connector, - bool is_client_side, - grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, - void *user_data) { +static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_security_handshake *h = arg; + if (error == GRPC_ERROR_NONE) { + grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint); + } + unref_handshake(h); +} + +void grpc_do_security_handshake( + grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, + grpc_security_connector *connector, bool is_client_side, + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data) { grpc_security_connector_handshake_list *handshake_node; grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake)); memset(h, 0, sizeof(grpc_security_handshake)); @@ -322,6 +338,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, h->wrapped_endpoint = nonsecure_endpoint; h->user_data = user_data; h->cb = cb; + gpr_ref_init(&h->refs, 2); /* timer and handshake proper each get a ref */ grpc_closure_init(&h->on_handshake_data_sent_to_peer, on_handshake_data_sent_to_peer, h); grpc_closure_init(&h->on_handshake_data_received_from_peer, @@ -340,6 +357,8 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, gpr_mu_unlock(&server_connector->mu); } send_handshake_bytes_to_peer(exec_ctx, h); + grpc_timer_init(exec_ctx, &h->timer, deadline, on_timeout, h, + gpr_now(deadline.clock_type)); } void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/security/transport/handshake.h b/src/core/lib/security/transport/handshake.h index 6ed850b315..c0906dd6af 100644 --- a/src/core/lib/security/transport/handshake.h +++ b/src/core/lib/security/transport/handshake.h @@ -38,13 +38,11 @@ #include "src/core/lib/security/transport/security_connector.h" /* Calls the callback upon completion. Takes owership of handshaker. */ -void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx, - tsi_handshaker *handshaker, - grpc_security_connector *connector, - bool is_client_side, - grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, - void *user_data); +void grpc_do_security_handshake( + grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, + grpc_security_connector *connector, bool is_client_side, + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data); void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake); diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 23b51f33f6..7650d68e89 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -128,7 +128,7 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, grpc_error *error) { - if (grpc_trace_secure_endpoint) { + if (false && grpc_trace_secure_endpoint) { size_t i; for (i = 0; i < ep->read_buffer->count; i++) { char *data = gpr_dump_slice(ep->read_buffer->slices[i], @@ -138,7 +138,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, } } ep->read_buffer = NULL; - grpc_exec_ctx_push(exec_ctx, ep->read_cb, error, NULL); + grpc_exec_ctx_sched(exec_ctx, ep->read_cb, error, NULL); SECURE_ENDPOINT_UNREF(exec_ctx, ep, "read"); } @@ -211,7 +211,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, if (result != TSI_OK) { gpr_slice_buffer_reset_and_unref(ep->read_buffer); - call_read_cb(exec_ctx, ep, grpc_set_tsi_error_bits( + call_read_cb(exec_ctx, ep, grpc_set_tsi_error_result( GRPC_ERROR_CREATE("Unwrap failed"), result)); return; } @@ -256,7 +256,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, gpr_slice_buffer_reset_and_unref(&ep->output_buffer); - if (grpc_trace_secure_endpoint) { + if (false && grpc_trace_secure_endpoint) { for (i = 0; i < slices->count; i++) { char *data = gpr_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -319,9 +319,9 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, if (result != TSI_OK) { /* TODO(yangg) do different things according to the error type? */ gpr_slice_buffer_reset_and_unref(&ep->output_buffer); - grpc_exec_ctx_push( + grpc_exec_ctx_sched( exec_ctx, cb, - grpc_set_tsi_error_bits(GRPC_ERROR_CREATE("Wrap failed"), result), + grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Wrap failed"), result), NULL); return; } diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index 03b64c5121..f0ee6770e5 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -127,23 +127,25 @@ void grpc_server_security_connector_shutdown( void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, - void *user_data) { + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data); + sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, deadline, cb, user_data); } } void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, void *user_data) { + gpr_timespec deadline, grpc_security_handshake_done_cb cb, + void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL); } else { - sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, cb, user_data); + sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, deadline, cb, + user_data); } } @@ -310,20 +312,23 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base, - true, nonsecure_endpoint, cb, user_data); + true, nonsecure_endpoint, deadline, cb, user_data); } static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base, - false, nonsecure_endpoint, cb, user_data); + false, nonsecure_endpoint, deadline, cb, + user_data); } static grpc_security_connector_vtable fake_channel_vtable = { @@ -413,6 +418,7 @@ static grpc_security_status ssl_create_handshaker( static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_ssl_channel_security_connector *c = @@ -427,7 +433,7 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, cb(exec_ctx, user_data, status, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true, - nonsecure_endpoint, cb, user_data); + nonsecure_endpoint, deadline, cb, user_data); } } @@ -435,6 +441,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_ssl_server_security_connector *c = @@ -446,7 +453,7 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx, cb(exec_ctx, user_data, status, NULL, NULL); } else { grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false, - nonsecure_endpoint, cb, user_data); + nonsecure_endpoint, deadline, cb, user_data); } } diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index 84e586deaa..c2ddf5ee1e 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -143,7 +143,7 @@ struct grpc_channel_security_connector { grpc_security_call_host_check_cb cb, void *user_data); void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; @@ -156,8 +156,8 @@ void grpc_channel_security_connector_check_call_host( /* Handshake. */ void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector, - grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb, - void *user_data); + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, + grpc_security_handshake_done_cb cb, void *user_data); /* --- server_security_connector object. --- @@ -174,14 +174,14 @@ struct grpc_server_security_connector { void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, - grpc_endpoint *nonsecure_endpoint, + grpc_endpoint *nonsecure_endpoint, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - grpc_security_handshake_done_cb cb, void *user_data); + gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); void grpc_server_security_connector_shutdown( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector); diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index d45ec4020c..12e789bde9 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -128,7 +128,7 @@ static void on_md_processing_done( grpc_metadata_batch_filter(calld->recv_initial_metadata, remove_consumed_md, elem); grpc_metadata_array_destroy(&calld->md); - grpc_exec_ctx_push(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL); } else { gpr_slice message; grpc_transport_stream_op close_op; @@ -146,10 +146,10 @@ static void on_md_processing_done( calld->transport_op.send_trailing_metadata = NULL; grpc_transport_stream_op_add_close(&close_op, status, &message); grpc_call_next_op(&exec_ctx, elem, &close_op); - grpc_exec_ctx_push(&exec_ctx, calld->on_done_recv, - grpc_error_set_int(GRPC_ERROR_CREATE(error_details), - GRPC_ERROR_INT_GRPC_STATUS, status), - NULL); + grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, + grpc_error_set_int(GRPC_ERROR_CREATE(error_details), + GRPC_ERROR_INT_GRPC_STATUS, status), + NULL); } grpc_exec_ctx_finish(&exec_ctx); @@ -169,8 +169,8 @@ static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, return; } } - grpc_exec_ctx_push(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error), - NULL); + grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error), + NULL); } static void set_recv_ops_md_callbacks(grpc_call_element *elem, @@ -224,12 +224,9 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_server_security_context_destroy; } -static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_pollset *pollset) {} - /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) {} + const grpc_call_stats *stats, void *ignored) {} /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, @@ -262,7 +259,14 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } const grpc_channel_filter grpc_server_auth_filter = { - auth_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, set_pollset, destroy_call_elem, - sizeof(channel_data), init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, "server-auth"}; + auth_start_transport_op, + grpc_channel_next_op, + sizeof(call_data), + init_call_elem, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + destroy_call_elem, + sizeof(channel_data), + init_channel_elem, + destroy_channel_elem, + grpc_call_next_get_peer, + "server-auth"}; diff --git a/src/core/lib/security/transport/tsi_error.c b/src/core/lib/security/transport/tsi_error.c index b9fb814905..afc1733567 100644 --- a/src/core/lib/security/transport/tsi_error.c +++ b/src/core/lib/security/transport/tsi_error.c @@ -33,7 +33,7 @@ #include "src/core/lib/security/transport/tsi_error.h" -grpc_error *grpc_set_tsi_error_bits(grpc_error *error, tsi_result result) { +grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result) { return grpc_error_set_int(grpc_error_set_str(error, GRPC_ERROR_STR_TSI_ERROR, tsi_result_to_string(result)), GRPC_ERROR_INT_TSI_CODE, result); diff --git a/src/core/lib/security/transport/tsi_error.h b/src/core/lib/security/transport/tsi_error.h index 0b691a427a..636fbb89cf 100644 --- a/src/core/lib/security/transport/tsi_error.h +++ b/src/core/lib/security/transport/tsi_error.h @@ -37,6 +37,6 @@ #include "src/core/lib/iomgr/error.h" #include "src/core/lib/tsi/transport_security_interface.h" -grpc_error *grpc_set_tsi_error_bits(grpc_error *error, tsi_result result); +grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result); #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H */ diff --git a/src/core/lib/security/util/json_util.h b/src/core/lib/security/util/json_util.h index 5959626a5f..137900593f 100644 --- a/src/core/lib/security/util/json_util.h +++ b/src/core/lib/security/util/json_util.h @@ -52,4 +52,4 @@ const char *grpc_json_get_string_property(const grpc_json *json, bool grpc_copy_json_string_property(const grpc_json *json, const char *prop_name, char **copied_value); -#endif // GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H +#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */ diff --git a/src/core/lib/support/cpu_windows.c b/src/core/lib/support/cpu_windows.c index ce32eb0a9d..34d006bfc8 100644 --- a/src/core/lib/support/cpu_windows.c +++ b/src/core/lib/support/cpu_windows.c @@ -33,7 +33,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include <grpc/support/log.h> unsigned gpr_cpu_num_cores(void) { @@ -44,4 +44,4 @@ unsigned gpr_cpu_num_cores(void) { unsigned gpr_cpu_current_cpu(void) { return GetCurrentProcessorNumber(); } -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/support/env_win32.c b/src/core/lib/support/env_windows.c index e670e1e8d0..9116959442 100644 --- a/src/core/lib/support/env_win32.c +++ b/src/core/lib/support/env_windows.c @@ -33,13 +33,13 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32_ENV +#ifdef GPR_WINDOWS_ENV #include <windows.h> #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" -#include "src/core/lib/support/string_win32.h" +#include "src/core/lib/support/string_windows.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> @@ -76,4 +76,4 @@ void gpr_setenv(const char *name, const char *value) { GPR_ASSERT(res); } -#endif /* GPR_WIN32_ENV */ +#endif /* GPR_WINDOWS_ENV */ diff --git a/src/core/lib/support/log.c b/src/core/lib/support/log.c index 882abf673c..bae0957df7 100644 --- a/src/core/lib/support/log.c +++ b/src/core/lib/support/log.c @@ -96,4 +96,6 @@ void gpr_log_verbosity_init() { } } -void gpr_set_log_function(gpr_log_func f) { g_log_func = f; } +void gpr_set_log_function(gpr_log_func f) { + g_log_func = f ? f : gpr_default_log; +} diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c index ca04c022e3..508fae4eec 100644 --- a/src/core/lib/support/log_linux.c +++ b/src/core/lib/support/log_linux.c @@ -95,9 +95,9 @@ void gpr_default_log(gpr_log_func_args *args) { strcpy(time_buffer, "error:strftime"); } - gpr_asprintf(&prefix, "%s%s.%09d %7tu %s:%d]", + gpr_asprintf(&prefix, "%s%s.%09" PRId32 " %7ld %s:%d]", gpr_log_severity_string(args->severity), time_buffer, - (int)(now.tv_nsec), gettid(), display_file, args->line); + now.tv_nsec, gettid(), display_file, args->line); fprintf(stderr, "%-60s %s\n", prefix, args->message); gpr_free(prefix); diff --git a/src/core/lib/support/log_win32.c b/src/core/lib/support/log_windows.c index 29735bd18c..ea898c359d 100644 --- a/src/core/lib/support/log_win32.c +++ b/src/core/lib/support/log_windows.c @@ -33,19 +33,19 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32_LOG +#ifdef GPR_WINDOWS_LOG #include <stdarg.h> #include <stdio.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/log_win32.h> +#include <grpc/support/log_windows.h> #include <grpc/support/string_util.h> #include <grpc/support/time.h> #include "src/core/lib/support/string.h" -#include "src/core/lib/support/string_win32.h" +#include "src/core/lib/support/string_windows.h" void gpr_log(const char *file, int line, gpr_log_severity severity, const char *format, ...) { @@ -109,4 +109,4 @@ void gpr_default_log(gpr_log_func_args *args) { fflush(stderr); } -#endif /* GPR_WIN32_LOG */ +#endif /* GPR_WINDOWS_LOG */ diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index a2ab6c5f1f..30c1e67647 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -194,6 +194,16 @@ int int64_ttoa(int64_t value, char *string) { return i; } +char *gpr_leftpad(const char *str, char flag, size_t length) { + const size_t str_length = strlen(str); + const size_t out_length = str_length > length ? str_length : length; + char *out = gpr_malloc(out_length + 1); + memset(out, flag, out_length - str_length); + memcpy(out + out_length - str_length, str, str_length); + out[out_length] = 0; + return out; +} + char *gpr_strjoin(const char **strs, size_t nstrs, size_t *final_length) { return gpr_strjoin_sep(strs, nstrs, "", final_length); } diff --git a/src/core/lib/support/string.h b/src/core/lib/support/string.h index ea58610914..2b6bb3eec6 100644 --- a/src/core/lib/support/string.h +++ b/src/core/lib/support/string.h @@ -83,6 +83,10 @@ int int64_ttoa(int64_t value, char *output); /* Reverse a run of bytes */ void gpr_reverse_bytes(char *str, int len); +/* Pad a string with flag characters. The given length specifies the minimum + field width. The input string is never truncated. */ +char *gpr_leftpad(const char *str, char flag, size_t length); + /* Join a set of strings, returning the resulting string. Total combined length (excluding null terminator) is returned in total_length if it is non-null. */ diff --git a/src/core/lib/support/string_util_win32.c b/src/core/lib/support/string_util_windows.c index 0d7bcdb5aa..049c9a8c04 100644 --- a/src/core/lib/support/string_util_win32.c +++ b/src/core/lib/support/string_util_windows.c @@ -35,7 +35,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS /* Some platforms (namely msys) need wchar to be included BEFORE anything else, especially strsafe.h. */ @@ -91,4 +91,4 @@ char *gpr_format_message(int messageid) { return message; } -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/support/string_win32.c b/src/core/lib/support/string_windows.c index 6b92f79253..ecc2a3a4e5 100644 --- a/src/core/lib/support/string_win32.c +++ b/src/core/lib/support/string_windows.c @@ -35,7 +35,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32_STRING +#ifdef GPR_WINDOWS_STRING #include <stdarg.h> #include <stdio.h> @@ -80,4 +80,4 @@ int gpr_asprintf(char **strp, const char *format, ...) { return -1; } -#endif /* GPR_WIN32_STRING */ +#endif /* GPR_WINDOWS_STRING */ diff --git a/src/core/lib/support/string_win32.h b/src/core/lib/support/string_windows.h index ff4a694ca9..899563b72d 100644 --- a/src/core/lib/support/string_win32.h +++ b/src/core/lib/support/string_windows.h @@ -31,17 +31,17 @@ * */ -#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H -#define GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H +#ifndef GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H +#define GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS /* These allocate new strings using gpr_malloc to convert from and to utf-8. */ LPTSTR gpr_char_to_tchar(LPCSTR input); LPSTR gpr_tchar_to_char(LPCTSTR input); -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ -#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WIN32_H */ +#endif /* GRPC_CORE_LIB_SUPPORT_STRING_WINDOWS_H */ diff --git a/src/core/lib/support/subprocess_windows.c b/src/core/lib/support/subprocess_windows.c index 264306f1bd..dee8c44ac1 100644 --- a/src/core/lib/support/subprocess_windows.c +++ b/src/core/lib/support/subprocess_windows.c @@ -43,7 +43,7 @@ #include <grpc/support/log.h> #include <grpc/support/subprocess.h> #include "src/core/lib/support/string.h" -#include "src/core/lib/support/string_win32.h" +#include "src/core/lib/support/string_windows.h" struct gpr_subprocess { PROCESS_INFORMATION pi; diff --git a/src/core/lib/support/sync_win32.c b/src/core/lib/support/sync_windows.c index 470a9f9704..8f0e8ff69f 100644 --- a/src/core/lib/support/sync_win32.c +++ b/src/core/lib/support/sync_windows.c @@ -35,7 +35,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include <grpc/support/log.h> #include <grpc/support/sync.h> @@ -130,4 +130,4 @@ void gpr_once_init(gpr_once *once, void (*init_function)(void)) { InitOnceExecuteOnce(once, run_once_func, &arg, &dummy); } -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/support/thd_win32.c b/src/core/lib/support/thd_windows.c index 6deb3140eb..74d2250df4 100644 --- a/src/core/lib/support/thd_win32.c +++ b/src/core/lib/support/thd_windows.c @@ -35,7 +35,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32 +#ifdef GPR_WINDOWS #include <grpc/support/alloc.h> #include <grpc/support/log.h> @@ -114,4 +114,4 @@ void gpr_thd_join(gpr_thd_id t) { destroy_thread(info); } -#endif /* GPR_WIN32 */ +#endif /* GPR_WINDOWS */ diff --git a/src/core/lib/support/time_win32.c b/src/core/lib/support/time_windows.c index 9e924ab3f4..6459732879 100644 --- a/src/core/lib/support/time_win32.c +++ b/src/core/lib/support/time_windows.c @@ -35,7 +35,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32_TIME +#ifdef GPR_WINDOWS_TIME #include <grpc/support/log.h> #include <grpc/support/time.h> @@ -107,4 +107,4 @@ void gpr_sleep_until(gpr_timespec until) { } } -#endif /* GPR_WIN32_TIME */ +#endif /* GPR_WINDOWS_TIME */ diff --git a/src/core/lib/support/tmpfile_msys.c b/src/core/lib/support/tmpfile_msys.c index 2fdc89a64f..4f566c4c28 100644 --- a/src/core/lib/support/tmpfile_msys.c +++ b/src/core/lib/support/tmpfile_msys.c @@ -44,7 +44,7 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -#include "src/core/lib/support/string_win32.h" +#include "src/core/lib/support/string_windows.h" #include "src/core/lib/support/tmpfile.h" FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) { diff --git a/src/core/lib/support/tmpfile_win32.c b/src/core/lib/support/tmpfile_windows.c index 9ac73128c3..542f53e589 100644 --- a/src/core/lib/support/tmpfile_win32.c +++ b/src/core/lib/support/tmpfile_windows.c @@ -33,7 +33,7 @@ #include <grpc/support/port_platform.h> -#ifdef GPR_WIN32_TMPFILE +#ifdef GPR_WINDOWS_TMPFILE #include <io.h> #include <stdio.h> @@ -44,7 +44,7 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -#include "src/core/lib/support/string_win32.h" +#include "src/core/lib/support/string_windows.h" #include "src/core/lib/support/tmpfile.h" FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) { @@ -81,4 +81,4 @@ end: return result; } -#endif /* GPR_WIN32_TMPFILE */ +#endif /* GPR_WINDOWS_TMPFILE */ diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 3915ba57a2..04291b0ee0 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -40,6 +40,7 @@ #include <grpc/grpc.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/slice.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -52,7 +53,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" +#include "src/core/lib/transport/transport.h" /** The maximum number of concurrent batches possible. Based upon the maximum number of individually queueable ops in the batch @@ -65,12 +68,6 @@ - status/close recv (depending on client/server) */ #define MAX_CONCURRENT_BATCHES 6 -typedef struct { - grpc_ioreq_completion_func on_complete; - void *user_data; - int success; -} completed_request; - #define MAX_SEND_EXTRA_METADATA_COUNT 3 /* Status data for a request can come from several sources; this @@ -97,25 +94,6 @@ typedef struct { grpc_mdstr *details; } received_status; -/* How far through the GRPC stream have we read? */ -typedef enum { - /* We are still waiting for initial metadata to complete */ - READ_STATE_INITIAL = 0, - /* We have gotten initial metadata, and are reading either - messages or trailing metadata */ - READ_STATE_GOT_INITIAL_METADATA, - /* The stream is closed for reading */ - READ_STATE_READ_CLOSED, - /* The stream is closed for reading & writing */ - READ_STATE_STREAM_CLOSED -} read_state; - -typedef enum { - WRITE_STATE_INITIAL = 0, - WRITE_STATE_STARTED, - WRITE_STATE_WRITE_CLOSED -} write_state; - typedef struct batch_control { grpc_call *call; grpc_cq_completion cq_completion; @@ -135,6 +113,7 @@ typedef struct batch_control { struct grpc_call { grpc_completion_queue *cq; + grpc_polling_entity pollent; grpc_channel *channel; grpc_call *parent; grpc_call *first_child; @@ -176,10 +155,10 @@ struct grpc_call { received_status status[STATUS_SOURCE_COUNT]; /* Call stats: only valid after trailing metadata received */ - grpc_transport_stream_stats stats; + grpc_call_stats stats; - /* Compression algorithm for the call */ - grpc_compression_algorithm compression_algorithm; + /* Compression algorithm for *incoming* data */ + grpc_compression_algorithm incoming_compression_algorithm; /* Supported encodings (compression algorithms), a bitset */ uint32_t encodings_accepted_by_peer; @@ -238,18 +217,19 @@ static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, grpc_status_code status, const char *description); +static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + grpc_status_code status, + const char *description); static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack, grpc_error *error); static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error); -grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, - uint32_t propagation_mask, - grpc_completion_queue *cq, - const void *server_transport_data, - grpc_mdelem **add_initial_metadata, - size_t add_initial_metadata_count, - gpr_timespec send_deadline) { +grpc_call *grpc_call_create( + grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, + grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative, + const void *server_transport_data, grpc_mdelem **add_initial_metadata, + size_t add_initial_metadata_count, gpr_timespec send_deadline) { size_t i, j; grpc_channel_stack *channel_stack = grpc_channel_get_channel_stack(channel); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -286,9 +266,20 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, call->context, server_transport_data, CALL_STACK_FROM_CALL(call)); if (cq != NULL) { + GPR_ASSERT( + pollset_set_alternative == NULL && + "Only one of 'cq' and 'pollset_set_alternative' should be non-NULL."); GRPC_CQ_INTERNAL_REF(cq, "bind"); - grpc_call_stack_set_pollset(&exec_ctx, CALL_STACK_FROM_CALL(call), - grpc_cq_pollset(cq)); + call->pollent = + grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)); + } + if (pollset_set_alternative != NULL) { + call->pollent = + grpc_polling_entity_create_from_pollset_set(pollset_set_alternative); + } + if (!grpc_polling_entity_is_empty(&call->pollent)) { + grpc_call_stack_set_pollset_or_pollset_set( + &exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent); } if (parent_call != NULL) { GRPC_CALL_INTERNAL_REF(parent_call, "child"); @@ -343,10 +334,16 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_completion_queue *cq) { GPR_ASSERT(cq); + + if (grpc_polling_entity_pollset_set(&call->pollent) != NULL) { + gpr_log(GPR_ERROR, "A pollset_set is already registered for this call."); + abort(); + } call->cq = cq; GRPC_CQ_INTERNAL_REF(cq, "bind"); - grpc_call_stack_set_pollset(exec_ctx, CALL_STACK_FROM_CALL(call), - grpc_cq_pollset(cq)); + call->pollent = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)); + grpc_call_stack_set_pollset_or_pollset_set( + exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent); } #ifdef GRPC_STREAM_REFCOUNT_DEBUG @@ -394,7 +391,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); } grpc_channel *channel = c->channel; - grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), c); + grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->stats, c); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); GPR_TIMER_END("destroy_call", 0); } @@ -409,21 +406,27 @@ static void set_status_code(grpc_call *call, status_source source, /* TODO(ctiller): what to do about the flush that was previously here */ } -static void set_compression_algorithm(grpc_call *call, - grpc_compression_algorithm algo) { +static void set_incoming_compression_algorithm( + grpc_call *call, grpc_compression_algorithm algo) { GPR_ASSERT(algo < GRPC_COMPRESS_ALGORITHMS_COUNT); - call->compression_algorithm = algo; + call->incoming_compression_algorithm = algo; } grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( grpc_call *call) { grpc_compression_algorithm algorithm; gpr_mu_lock(&call->mu); - algorithm = call->compression_algorithm; + algorithm = call->incoming_compression_algorithm; gpr_mu_unlock(&call->mu); return algorithm; } +static grpc_compression_algorithm compression_algorithm_for_level_locked( + grpc_call *call, grpc_compression_level level) { + return grpc_compression_algorithm_for_level(level, + call->encodings_accepted_by_peer); +} + uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) { uint32_t flags; gpr_mu_lock(&call->mu); @@ -549,15 +552,28 @@ static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) { return (grpc_linked_mdelem *)&md->internal_data; } +static grpc_metadata *get_md_elem(grpc_metadata *metadata, + grpc_metadata *additional_metadata, int i, + int count) { + grpc_metadata *res = + i < count ? &metadata[i] : &additional_metadata[i - count]; + GPR_ASSERT(res); + return res; +} + static int prepare_application_metadata(grpc_call *call, int count, grpc_metadata *metadata, int is_trailing, - int prepend_extra_metadata) { + int prepend_extra_metadata, + grpc_metadata *additional_metadata, + int additional_metadata_count) { + int total_count = count + additional_metadata_count; int i; grpc_metadata_batch *batch = &call->metadata_batch[0 /* is_receiving */][is_trailing]; - for (i = 0; i < count; i++) { - grpc_metadata *md = &metadata[i]; + for (i = 0; i < total_count; i++) { + const grpc_metadata *md = + get_md_elem(metadata, additional_metadata, i, count); grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data; GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data)); l->md = grpc_mdelem_from_string_and_buffer( @@ -576,9 +592,10 @@ static int prepare_application_metadata(grpc_call *call, int count, break; } } - if (i != count) { + if (i != total_count) { for (int j = 0; j <= i; j++) { - grpc_metadata *md = &metadata[j]; + const grpc_metadata *md = + get_md_elem(metadata, additional_metadata, j, count); grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data; GRPC_MDELEM_UNREF(l->md); } @@ -599,24 +616,36 @@ static int prepare_application_metadata(grpc_call *call, int count, } } } - for (i = 1; i < count; i++) { - linked_from_md(&metadata[i])->prev = linked_from_md(&metadata[i - 1]); + for (i = 1; i < total_count; i++) { + grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count); + grpc_metadata *prev_md = + get_md_elem(metadata, additional_metadata, i - 1, count); + linked_from_md(md)->prev = linked_from_md(prev_md); } - for (i = 0; i < count - 1; i++) { - linked_from_md(&metadata[i])->next = linked_from_md(&metadata[i + 1]); + for (i = 0; i < total_count - 1; i++) { + grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count); + grpc_metadata *next_md = + get_md_elem(metadata, additional_metadata, i + 1, count); + linked_from_md(md)->next = linked_from_md(next_md); } - switch (prepend_extra_metadata * 2 + (count != 0)) { + + switch (prepend_extra_metadata * 2 + (total_count != 0)) { case 0: /* no prepend, no metadata => nothing to do */ batch->list.head = batch->list.tail = NULL; break; - case 1: + case 1: { /* metadata, but no prepend */ - batch->list.head = linked_from_md(&metadata[0]); - batch->list.tail = linked_from_md(&metadata[count - 1]); + grpc_metadata *first_md = + get_md_elem(metadata, additional_metadata, 0, count); + grpc_metadata *last_md = + get_md_elem(metadata, additional_metadata, total_count - 1, count); + batch->list.head = linked_from_md(first_md); + batch->list.tail = linked_from_md(last_md); batch->list.head->prev = NULL; batch->list.tail->next = NULL; break; + } case 2: /* prepend, but no md */ batch->list.head = &call->send_extra_metadata[0]; @@ -625,17 +654,22 @@ static int prepare_application_metadata(grpc_call *call, int count, batch->list.head->prev = NULL; batch->list.tail->next = NULL; break; - case 3: + case 3: { /* prepend AND md */ + grpc_metadata *first_md = + get_md_elem(metadata, additional_metadata, 0, count); + grpc_metadata *last_md = + get_md_elem(metadata, additional_metadata, total_count - 1, count); batch->list.head = &call->send_extra_metadata[0]; call->send_extra_metadata[call->send_extra_metadata_count - 1].next = - linked_from_md(&metadata[0]); - linked_from_md(&metadata[0])->prev = + linked_from_md(first_md); + linked_from_md(first_md)->prev = &call->send_extra_metadata[call->send_extra_metadata_count - 1]; - batch->list.tail = linked_from_md(&metadata[count - 1]); + batch->list.tail = linked_from_md(last_md); batch->list.head->prev = NULL; batch->list.tail->next = NULL; break; + } default: GPR_UNREACHABLE_CODE(return 0); } @@ -704,48 +738,103 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, return r; } -typedef struct cancel_closure { +typedef struct termination_closure { grpc_closure closure; grpc_call *call; grpc_status_code status; -} cancel_closure; + gpr_slice optional_message; + grpc_closure *op_closure; + enum { TC_CANCEL, TC_CLOSE } type; +} termination_closure; + +static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp, + grpc_error *error) { + termination_closure *tc = tcp; + switch (tc->type) { + case TC_CANCEL: + GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "cancel"); + break; + case TC_CLOSE: + GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "close"); + break; + } + gpr_slice_unref(tc->optional_message); + grpc_exec_ctx_sched(exec_ctx, tc->op_closure, GRPC_ERROR_NONE, NULL); + gpr_free(tc); +} -static void done_cancel(grpc_exec_ctx *exec_ctx, void *ccp, grpc_error *error) { - cancel_closure *cc = ccp; - GRPC_CALL_INTERNAL_UNREF(exec_ctx, cc->call, "cancel"); - gpr_free(cc); +static void send_cancel(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) { + grpc_transport_stream_op op; + termination_closure *tc = tcp; + memset(&op, 0, sizeof(op)); + op.cancel_with_status = tc->status; + /* reuse closure to catch completion */ + grpc_closure_init(&tc->closure, done_termination, tc); + op.on_complete = &tc->closure; + execute_op(exec_ctx, tc->call, &op); } -static void send_cancel(grpc_exec_ctx *exec_ctx, void *ccp, grpc_error *error) { +static void send_close(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) { grpc_transport_stream_op op; - cancel_closure *cc = ccp; + termination_closure *tc = tcp; memset(&op, 0, sizeof(op)); - op.cancel_with_status = cc->status; + tc->optional_message = gpr_slice_ref(tc->optional_message); + grpc_transport_stream_op_add_close(&op, tc->status, &tc->optional_message); /* reuse closure to catch completion */ - grpc_closure_init(&cc->closure, done_cancel, cc); - op.on_complete = &cc->closure; - execute_op(exec_ctx, cc->call, &op); + grpc_closure_init(&tc->closure, done_termination, tc); + tc->op_closure = op.on_complete; + op.on_complete = &tc->closure; + execute_op(exec_ctx, tc->call, &op); +} + +static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx, + termination_closure *tc) { + grpc_mdstr *details = NULL; + if (GPR_SLICE_LENGTH(tc->optional_message) > 0) { + tc->optional_message = gpr_slice_ref(tc->optional_message); + details = grpc_mdstr_from_slice(tc->optional_message); + } + + set_status_code(tc->call, STATUS_FROM_API_OVERRIDE, (uint32_t)tc->status); + set_status_details(tc->call, STATUS_FROM_API_OVERRIDE, details); + + if (tc->type == TC_CANCEL) { + grpc_closure_init(&tc->closure, send_cancel, tc); + GRPC_CALL_INTERNAL_REF(tc->call, "cancel"); + } else if (tc->type == TC_CLOSE) { + grpc_closure_init(&tc->closure, send_close, tc); + GRPC_CALL_INTERNAL_REF(tc->call, "close"); + } + grpc_exec_ctx_sched(exec_ctx, &tc->closure, GRPC_ERROR_NONE, NULL); + return GRPC_CALL_OK; } static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, grpc_status_code status, const char *description) { - grpc_mdstr *details = - description ? grpc_mdstr_from_string(description) : NULL; - cancel_closure *cc = gpr_malloc(sizeof(*cc)); - + termination_closure *tc = gpr_malloc(sizeof(*tc)); + memset(tc, 0, sizeof(termination_closure)); + tc->type = TC_CANCEL; + tc->call = c; + tc->optional_message = gpr_slice_from_copied_string(description); GPR_ASSERT(status != GRPC_STATUS_OK); + tc->status = status; - set_status_code(c, STATUS_FROM_API_OVERRIDE, (uint32_t)status); - set_status_details(c, STATUS_FROM_API_OVERRIDE, details); + return terminate_with_status(exec_ctx, tc); +} - grpc_closure_init(&cc->closure, send_cancel, cc); - cc->call = c; - cc->status = status; - GRPC_CALL_INTERNAL_REF(c, "cancel"); - grpc_exec_ctx_push(exec_ctx, &cc->closure, GRPC_ERROR_NONE, NULL); +static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + grpc_status_code status, + const char *description) { + termination_closure *tc = gpr_malloc(sizeof(*tc)); + memset(tc, 0, sizeof(termination_closure)); + tc->type = TC_CLOSE; + tc->call = c; + tc->optional_message = gpr_slice_from_copied_string(description); + GPR_ASSERT(status != GRPC_STATUS_OK); + tc->status = status; - return GRPC_CALL_OK; + return terminate_with_status(exec_ctx, tc); } static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, @@ -886,9 +975,9 @@ static grpc_mdelem *recv_initial_filter(void *callp, grpc_mdelem *elem) { if (elem == NULL) { return NULL; } else if (elem->key == GRPC_MDSTR_GRPC_ENCODING) { - GPR_TIMER_BEGIN("compression_algorithm", 0); - set_compression_algorithm(call, decode_compression(elem)); - GPR_TIMER_END("compression_algorithm", 0); + GPR_TIMER_BEGIN("incoming_compression_algorithm", 0); + set_incoming_compression_algorithm(call, decode_compression(elem)); + GPR_TIMER_END("incoming_compression_algorithm", 0); return NULL; } else if (elem->key == GRPC_MDSTR_GRPC_ACCEPT_ENCODING) { GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0); @@ -970,7 +1059,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, grpc_call *call = bctl->call; if (bctl->is_notify_tag_closure) { /* unrefs bctl->error */ - grpc_exec_ctx_push(exec_ctx, bctl->notify_tag, bctl->error, NULL); + grpc_exec_ctx_sched(exec_ctx, bctl->notify_tag, bctl->error, NULL); gpr_mu_lock(&call->mu); bctl->call->used_batches = (uint8_t)(bctl->call->used_batches & @@ -1020,6 +1109,9 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, call->receiving_slice); continue_receiving_slices(exec_ctx, bctl); } else { + if (grpc_trace_operation_failures) { + GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); + } grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); call->receiving_stream = NULL; grpc_byte_buffer_destroy(*call->receiving_buffer); @@ -1053,9 +1145,9 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl, } else { call->test_only_last_message_flags = call->receiving_stream->flags; if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) && - (call->compression_algorithm > GRPC_COMPRESS_NONE)) { + (call->incoming_compression_algorithm > GRPC_COMPRESS_NONE)) { *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create( - NULL, 0, call->compression_algorithm); + NULL, 0, call->incoming_compression_algorithm); } else { *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0); } @@ -1083,6 +1175,56 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } } +static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, + batch_control *bctl) { + grpc_call *call = bctl->call; + /* validate call->incoming_compression_algorithm */ + if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) { + const grpc_compression_algorithm algo = + call->incoming_compression_algorithm; + char *error_msg = NULL; + const grpc_compression_options compression_options = + grpc_channel_compression_options(call->channel); + /* check if algorithm is known */ + if (algo >= GRPC_COMPRESS_ALGORITHMS_COUNT) { + gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.", + algo); + gpr_log(GPR_ERROR, "%s", error_msg); + close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg); + } else if (grpc_compression_options_is_algorithm_enabled( + &compression_options, algo) == 0) { + /* check if algorithm is supported by current channel config */ + char *algo_name; + grpc_compression_algorithm_name(algo, &algo_name); + gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", + algo_name); + gpr_log(GPR_ERROR, "%s", error_msg); + close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg); + } else { + call->incoming_compression_algorithm = algo; + } + gpr_free(error_msg); + } + + /* make sure the received grpc-encoding is amongst the ones listed in + * grpc-accept-encoding */ + GPR_ASSERT(call->encodings_accepted_by_peer != 0); + if (!GPR_BITGET(call->encodings_accepted_by_peer, + call->incoming_compression_algorithm)) { + extern int grpc_compression_trace; + if (grpc_compression_trace) { + char *algo_name; + grpc_compression_algorithm_name(call->incoming_compression_algorithm, + &algo_name); + gpr_log(GPR_ERROR, + "Compression algorithm (grpc-encoding = '%s') not present in " + "the bitset of accepted encodings (grpc-accept-encodings: " + "'0x%x')", + algo_name, call->encodings_accepted_by_peer); + } + } +} + static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; @@ -1097,24 +1239,10 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; grpc_metadata_batch_filter(md, recv_initial_filter, call); - /* make sure the received grpc-encoding is amongst the ones listed in - * grpc-accept-encoding */ - - GPR_ASSERT(call->encodings_accepted_by_peer != 0); - if (!GPR_BITGET(call->encodings_accepted_by_peer, - call->compression_algorithm)) { - extern int grpc_compression_trace; - if (grpc_compression_trace) { - char *algo_name; - grpc_compression_algorithm_name(call->compression_algorithm, - &algo_name); - gpr_log(GPR_ERROR, - "Compression algorithm (grpc-encoding = '%s') not present in " - "the bitset of accepted encodings (grpc-accept-encodings: " - "'0x%x')", - algo_name, call->encodings_accepted_by_peer); - } - } + GPR_TIMER_BEGIN("validate_filtered_metadata", 0); + validate_filtered_metadata(exec_ctx, bctl); + GPR_TIMER_END("validate_filtered_metadata", 0); + if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) != 0 && !call->is_client) { @@ -1129,7 +1257,7 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, grpc_closure *saved_rsr_closure = grpc_closure_create( receiving_stream_ready, call->saved_receiving_stream_ready_bctlp); call->saved_receiving_stream_ready_bctlp = NULL; - grpc_exec_ctx_push(exec_ctx, saved_rsr_closure, error, NULL); + grpc_exec_ctx_sched(exec_ctx, saved_rsr_closure, error, NULL); } gpr_mu_unlock(&call->mu); @@ -1264,7 +1392,40 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - if (op->data.send_initial_metadata.count > INT_MAX) { + /* process compression level */ + grpc_metadata compression_md; + memset(&compression_md, 0, sizeof(grpc_metadata)); + size_t additional_metadata_count = 0; + grpc_compression_level effective_compression_level; + bool level_set = false; + if (op->data.send_initial_metadata.maybe_compression_level.is_set) { + effective_compression_level = + op->data.send_initial_metadata.maybe_compression_level.level; + level_set = true; + } else { + const grpc_compression_options copts = + grpc_channel_compression_options(call->channel); + level_set = copts.default_level.is_set; + if (level_set) { + effective_compression_level = copts.default_level.level; + } + } + if (level_set && !call->is_client) { + const grpc_compression_algorithm calgo = + compression_algorithm_for_level_locked( + call, effective_compression_level); + char *calgo_name; + grpc_compression_algorithm_name(calgo, &calgo_name); + // the following will be picked up by the compress filter and used as + // the call's compression algorithm. + compression_md.key = GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY; + compression_md.value = calgo_name; + compression_md.value_length = strlen(calgo_name); + additional_metadata_count++; + } + + if (op->data.send_initial_metadata.count + additional_metadata_count > + INT_MAX) { error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } @@ -1272,7 +1433,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, call->sent_initial_metadata = 1; if (!prepare_application_metadata( call, (int)op->data.send_initial_metadata.count, - op->data.send_initial_metadata.metadata, 0, call->is_client)) { + op->data.send_initial_metadata.metadata, 0, call->is_client, + &compression_md, (int)additional_metadata_count)) { error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } @@ -1360,7 +1522,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (!prepare_application_metadata( call, (int)op->data.send_status_from_server.trailing_metadata_count, - op->data.send_status_from_server.trailing_metadata, 1, 1)) { + op->data.send_status_from_server.trailing_metadata, 1, 1, NULL, + 0)) { error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } @@ -1432,7 +1595,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->recv_final_op = 1; stream_op.recv_trailing_metadata = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; - stream_op.collect_stats = &call->stats; + stream_op.collect_stats = &call->stats.transport_stream_stats; break; case GRPC_OP_RECV_CLOSE_ON_SERVER: /* Flag validation: currently allow no flags */ @@ -1454,7 +1617,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->recv_final_op = 1; stream_op.recv_trailing_metadata = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; - stream_op.collect_stats = &call->stats; + stream_op.collect_stats = &call->stats.transport_stream_stats; break; } } @@ -1549,9 +1712,10 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; } grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level) { gpr_mu_lock(&call->mu); - const uint32_t accepted_encodings = call->encodings_accepted_by_peer; + grpc_compression_algorithm algo = + compression_algorithm_for_level_locked(call, level); gpr_mu_unlock(&call->mu); - return grpc_compression_algorithm_for_level(level, accepted_encodings); + return algo; } const char *grpc_call_error_to_string(grpc_call_error error) { diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 2725e060b8..b640345c21 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -53,6 +53,8 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx, grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, grpc_completion_queue *cq, + /* if not NULL, it'll be used in lieu of \a cq */ + grpc_pollset_set *pollset_set_alternative, const void *server_transport_data, grpc_mdelem **add_initial_metadata, size_t add_initial_metadata_count, diff --git a/src/core/lib/surface/call_log_batch.c b/src/core/lib/surface/call_log_batch.c index a6d1d5149f..31c074f15d 100644 --- a/src/core/lib/surface/call_log_batch.c +++ b/src/core/lib/surface/call_log_batch.c @@ -112,7 +112,7 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity, size_t i; for (i = 0; i < nops; i++) { tmp = grpc_op_string(&ops[i]); - gpr_log(file, line, severity, "ops[%d]: %s", i, tmp); + gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i, tmp); gpr_free(tmp); } } diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 3c68dc0c4b..8936e7164f 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -36,16 +36,17 @@ #include <stdlib.h> #include <string.h> +#include <grpc/compression.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" -#include "src/core/lib/surface/init.h" #include "src/core/lib/transport/static_metadata.h" /** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS. @@ -64,10 +65,12 @@ typedef struct registered_call { struct grpc_channel { int is_client; uint32_t max_message_length; + grpc_compression_options compression_options; grpc_mdelem *default_authority; gpr_mu registered_call_mu; registered_call *registered_calls; + char *target; }; @@ -112,6 +115,7 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, channel->registered_calls = NULL; channel->max_message_length = DEFAULT_MAX_MESSAGE_LENGTH; + grpc_compression_options_init(&channel->compression_options); if (args) { for (size_t i = 0; i < args->num_args; i++) { if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_MESSAGE_LENGTH)) { @@ -152,6 +156,27 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, ":authority", args->args[i].value.string); } } + } else if (0 == strcmp(args->args[i].key, + GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) { + channel->compression_options.default_level.is_set = true; + GPR_ASSERT(args->args[i].value.integer >= 0 && + args->args[i].value.integer < GRPC_COMPRESS_LEVEL_COUNT); + channel->compression_options.default_level.level = + (grpc_compression_level)args->args[i].value.integer; + } else if (0 == strcmp(args->args[i].key, + GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { + channel->compression_options.default_algorithm.is_set = true; + GPR_ASSERT(args->args[i].value.integer >= 0 && + args->args[i].value.integer < + GRPC_COMPRESS_ALGORITHMS_COUNT); + channel->compression_options.default_algorithm.algorithm = + (grpc_compression_algorithm)args->args[i].value.integer; + } else if (0 == + strcmp(args->args[i].key, + GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { + channel->compression_options.enabled_algorithms_bitset = + (uint32_t)args->args[i].value.integer | + 0x1; /* always support no compression */ } } grpc_channel_args_destroy(args); @@ -167,12 +192,14 @@ char *grpc_channel_get_target(grpc_channel *channel) { static grpc_call *grpc_channel_create_call_internal( grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, - grpc_completion_queue *cq, grpc_mdelem *path_mdelem, - grpc_mdelem *authority_mdelem, gpr_timespec deadline) { + grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative, + grpc_mdelem *path_mdelem, grpc_mdelem *authority_mdelem, + gpr_timespec deadline) { grpc_mdelem *send_metadata[2]; size_t num_metadata = 0; GPR_ASSERT(channel->is_client); + GPR_ASSERT(!(cq != NULL && pollset_set_alternative != NULL)); send_metadata[num_metadata++] = path_mdelem; if (authority_mdelem != NULL) { @@ -181,8 +208,9 @@ static grpc_call *grpc_channel_create_call_internal( send_metadata[num_metadata++] = GRPC_MDELEM_REF(channel->default_authority); } - return grpc_call_create(channel, parent_call, propagation_mask, cq, NULL, - send_metadata, num_metadata, deadline); + return grpc_call_create(channel, parent_call, propagation_mask, cq, + pollset_set_alternative, NULL, send_metadata, + num_metadata, deadline); } grpc_call *grpc_channel_create_call(grpc_channel *channel, @@ -202,7 +230,22 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel, (int)deadline.clock_type, reserved)); GPR_ASSERT(!reserved); return grpc_channel_create_call_internal( - channel, parent_call, propagation_mask, cq, + channel, parent_call, propagation_mask, cq, NULL, + grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH, + grpc_mdstr_from_string(method)), + host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY, + grpc_mdstr_from_string(host)) + : NULL, + deadline); +} + +grpc_call *grpc_channel_create_pollset_set_call( + grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, + grpc_pollset_set *pollset_set, const char *method, const char *host, + gpr_timespec deadline, void *reserved) { + GPR_ASSERT(!reserved); + return grpc_channel_create_call_internal( + channel, parent_call, propagation_mask, NULL, pollset_set, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH, grpc_mdstr_from_string(method)), host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY, @@ -246,7 +289,7 @@ grpc_call *grpc_channel_create_registered_call( (int)deadline.tv_nsec, (int)deadline.clock_type, reserved)); GPR_ASSERT(!reserved); return grpc_channel_create_call_internal( - channel, parent_call, propagation_mask, completion_queue, + channel, parent_call, propagation_mask, completion_queue, NULL, GRPC_MDELEM_REF(rc->path), rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline); } @@ -307,6 +350,11 @@ grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel) { return CHANNEL_STACK_FROM_CHANNEL(channel); } +grpc_compression_options grpc_channel_compression_options( + const grpc_channel *channel) { + return channel->compression_options; +} + grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) { char tmp[GPR_LTOA_MIN_BUFSIZE]; switch (i) { diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h index 22dae930e4..7eff7b8883 100644 --- a/src/core/lib/surface/channel.h +++ b/src/core/lib/surface/channel.h @@ -42,6 +42,11 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, grpc_channel_stack_type channel_stack_type, grpc_transport *optional_transport); +grpc_call *grpc_channel_create_pollset_set_call( + grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, + grpc_pollset_set *pollset_set, const char *method, const char *host, + gpr_timespec deadline, void *reserved); + /** Get a (borrowed) pointer to this channels underlying channel stack */ grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel); @@ -71,4 +76,8 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx, grpc_channel_internal_unref(exec_ctx, channel) #endif +/** Return the channel's compression options. */ +grpc_compression_options grpc_channel_compression_options( + const grpc_channel *channel); + #endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_H */ diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index ecc127876b..2cc6aa74e0 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,6 +50,8 @@ #include "src/core/lib/surface/event_string.h" #include "src/core/lib/surface/surface_trace.h" +int grpc_trace_operation_failures; + typedef struct { grpc_pollset_worker **worker; void *tag; @@ -231,12 +233,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, #endif GPR_TIMER_BEGIN("grpc_cq_end_op", 0); - if (grpc_api_trace) { + if (grpc_api_trace || + (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) { const char *errmsg = grpc_error_string(error); GRPC_API_TRACE( "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, " "done_arg=%p, storage=%p)", 7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage)); + if (grpc_trace_operation_failures) { + gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg); + } grpc_error_free_string(errmsg); } diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index a866858cb5..b9dd1092b6 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -39,6 +39,8 @@ #include <grpc/grpc.h> #include "src/core/lib/iomgr/pollset.h" +extern int grpc_trace_operation_failures; + typedef struct grpc_cq_completion { /** user supplied tag */ void *tag; diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 1c8b709015..f07039cb94 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -165,6 +165,7 @@ void grpc_init(void) { &grpc_trace_channel_stack_builder); grpc_register_tracer("http1", &grpc_http1_trace); grpc_register_tracer("compression", &grpc_compression_trace); + grpc_register_tracer("op_failure", &grpc_trace_operation_failures); grpc_security_pre_init(); grpc_iomgr_init(); grpc_executor_init(); diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c index 5c15b676f8..5ea4cba5d1 100644 --- a/src/core/lib/surface/lame_client.c +++ b/src/core/lib/surface/lame_client.c @@ -92,17 +92,17 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { if (op->on_connectivity_state_change) { - GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE); - *op->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE; - grpc_exec_ctx_push(exec_ctx, op->on_connectivity_state_change, - GRPC_ERROR_NONE, NULL); + GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_SHUTDOWN); + *op->connectivity_state = GRPC_CHANNEL_SHUTDOWN; + grpc_exec_ctx_sched(exec_ctx, op->on_connectivity_state_change, + GRPC_ERROR_NONE, NULL); } if (op->on_consumed != NULL) { - grpc_exec_ctx_push(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL); } if (op->send_ping != NULL) { - grpc_exec_ctx_push(exec_ctx, op->send_ping, - GRPC_ERROR_CREATE("lame client channel"), NULL); + grpc_exec_ctx_sched(exec_ctx, op->send_ping, + GRPC_ERROR_CREATE("lame client channel"), NULL); } GRPC_ERROR_UNREF(op->disconnect_with_error); } @@ -111,6 +111,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_call_element_args *args) {} static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + const grpc_call_stats *stats, void *and_free_memory) { gpr_free(and_free_memory); } @@ -130,7 +131,7 @@ const grpc_channel_filter grpc_lame_filter = { lame_start_transport_op, sizeof(call_data), init_call_elem, - grpc_call_stack_ignore_set_pollset, + grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index f261321750..def6e5068b 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -344,8 +344,8 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, grpc_closure_init( &calld->kill_zombie_closure, kill_zombie, grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0)); - grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE, - NULL); + grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE, + NULL); } } @@ -526,7 +526,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, grpc_closure_init( &calld->kill_zombie_closure, kill_zombie, grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0)); - grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, error, NULL); + grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, error, NULL); return; } @@ -571,8 +571,8 @@ static void finish_start_new_rpc( calld->state = ZOMBIED; gpr_mu_unlock(&calld->mu_state); grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem); - grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE, - NULL); + grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE, + NULL); return; } @@ -757,8 +757,8 @@ static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr, GRPC_ERROR_CREATE_REFERENCING("Missing :authority or :path", &error, 1); } - grpc_exec_ctx_push(exec_ctx, calld->on_done_recv_initial_metadata, error, - NULL); + grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv_initial_metadata, error, + NULL); } static void server_mutate_op(grpc_call_element *elem, @@ -794,8 +794,8 @@ static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr, calld->state = ZOMBIED; gpr_mu_unlock(&calld->mu_state); grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem); - grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE, - NULL); + grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, + GRPC_ERROR_NONE, NULL); } else if (calld->state == PENDING) { calld->state = ZOMBIED; gpr_mu_unlock(&calld->mu_state); @@ -812,9 +812,9 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd, const void *transport_server_data) { channel_data *chand = cd; /* create a call */ - grpc_call *call = - grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data, - NULL, 0, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_call *call = grpc_call_create(chand->channel, NULL, 0, NULL, NULL, + transport_server_data, NULL, 0, + gpr_inf_future(GPR_CLOCK_MONOTONIC)); grpc_call_element *elem = grpc_call_stack_element(grpc_call_get_call_stack(call), 0); call_data *calld = elem->call_data; @@ -831,7 +831,7 @@ static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd, grpc_error *error) { channel_data *chand = cd; grpc_server *server = chand->server; - if (chand->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) { + if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) { grpc_transport_op op; memset(&op, 0, sizeof(op)); op.on_connectivity_state_change = &chand->channel_connectivity_changed, @@ -864,7 +864,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, } static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - void *ignored) { + const grpc_call_stats *stats, void *ignored) { channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; @@ -929,7 +929,7 @@ const grpc_channel_filter grpc_server_top_filter = { grpc_channel_next_op, sizeof(call_data), init_call_elem, - grpc_call_stack_ignore_set_pollset, + grpc_call_stack_ignore_set_pollset_or_pollset_set, destroy_call_elem, sizeof(channel_data), init_channel_elem, @@ -1339,8 +1339,8 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, grpc_closure_init( &calld->kill_zombie_closure, kill_zombie, grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0)); - grpc_exec_ctx_push(exec_ctx, &calld->kill_zombie_closure, - GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, + GRPC_ERROR_NONE, NULL); } else { GPR_ASSERT(calld->state == PENDING); calld->state = ACTIVATED; diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 3286af9fb2..054f112127 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -51,8 +51,8 @@ const char *grpc_connectivity_state_name(grpc_connectivity_state state) { return "READY"; case GRPC_CHANNEL_TRANSIENT_FAILURE: return "TRANSIENT_FAILURE"; - case GRPC_CHANNEL_FATAL_FAILURE: - return "FATAL_FAILURE"; + case GRPC_CHANNEL_SHUTDOWN: + return "SHUTDOWN"; } GPR_UNREACHABLE_CODE(return "UNKNOWN"); } @@ -73,13 +73,13 @@ void grpc_connectivity_state_destroy(grpc_exec_ctx *exec_ctx, while ((w = tracker->watchers)) { tracker->watchers = w->next; - if (GRPC_CHANNEL_FATAL_FAILURE != *w->current) { - *w->current = GRPC_CHANNEL_FATAL_FAILURE; + if (GRPC_CHANNEL_SHUTDOWN != *w->current) { + *w->current = GRPC_CHANNEL_SHUTDOWN; error = GRPC_ERROR_NONE; } else { error = GRPC_ERROR_CREATE("Shutdown connectivity owner"); } - grpc_exec_ctx_push(exec_ctx, w->notify, error, NULL); + grpc_exec_ctx_sched(exec_ctx, w->notify, error, NULL); gpr_free(w); } GRPC_ERROR_UNREF(tracker->current_error); @@ -114,7 +114,7 @@ int grpc_connectivity_state_notify_on_state_change( if (current == NULL) { grpc_connectivity_state_watcher *w = tracker->watchers; if (w != NULL && w->notify == notify) { - grpc_exec_ctx_push(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL); tracker->watchers = w->next; gpr_free(w); return 0; @@ -122,7 +122,7 @@ int grpc_connectivity_state_notify_on_state_change( while (w != NULL) { grpc_connectivity_state_watcher *rm_candidate = w->next; if (rm_candidate != NULL && rm_candidate->notify == notify) { - grpc_exec_ctx_push(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL); w->next = w->next->next; gpr_free(rm_candidate); return 0; @@ -133,8 +133,8 @@ int grpc_connectivity_state_notify_on_state_change( } else { if (tracker->current_state != *current) { *current = tracker->current_state; - grpc_exec_ctx_push(exec_ctx, notify, - GRPC_ERROR_REF(tracker->current_error), NULL); + grpc_exec_ctx_sched(exec_ctx, notify, + GRPC_ERROR_REF(tracker->current_error), NULL); } else { grpc_connectivity_state_watcher *w = gpr_malloc(sizeof(*w)); w->current = current; @@ -164,7 +164,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, case GRPC_CHANNEL_READY: GPR_ASSERT(error == GRPC_ERROR_NONE); break; - case GRPC_CHANNEL_FATAL_FAILURE: + case GRPC_CHANNEL_SHUTDOWN: case GRPC_CHANNEL_TRANSIENT_FAILURE: GPR_ASSERT(error != GRPC_ERROR_NONE); break; @@ -174,13 +174,13 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, if (tracker->current_state == state) { return; } - GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_FATAL_FAILURE); + GPR_ASSERT(tracker->current_state != GRPC_CHANNEL_SHUTDOWN); tracker->current_state = state; while ((w = tracker->watchers) != NULL) { *w->current = tracker->current_state; tracker->watchers = w->next; - grpc_exec_ctx_push(exec_ctx, w->notify, - GRPC_ERROR_REF(tracker->current_error), NULL); + grpc_exec_ctx_sched(exec_ctx, w->notify, + GRPC_ERROR_REF(tracker->current_error), NULL); gpr_free(w); } } diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 82c8e239f6..0677f29766 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -129,7 +129,10 @@ typedef struct mdtab_shard { internal_metadata **elems; size_t count; size_t capacity; - size_t free; + /** Estimate of the number of unreferenced mdelems in the hash table. + This will eventually converge to the exact number, but it's instantaneous + accuracy is not guaranteed */ + gpr_atm free_estimate; } mdtab_shard; #define LOG2_STRTAB_SHARD_COUNT 5 @@ -217,7 +220,7 @@ void grpc_mdctx_global_init(void) { mdtab_shard *shard = &g_mdtab_shard[i]; gpr_mu_init(&shard->mu); shard->count = 0; - shard->free = 0; + gpr_atm_no_barrier_store(&shard->free_estimate, 0); shard->capacity = INITIAL_MDTAB_CAPACITY; shard->elems = gpr_malloc(sizeof(*shard->elems) * shard->capacity); memset(shard->elems, 0, sizeof(*shard->elems) * shard->capacity); @@ -232,7 +235,7 @@ void grpc_mdctx_global_shutdown(void) { gc_mdtab(shard); /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */ if (shard->count != 0) { - gpr_log(GPR_DEBUG, "WARNING: %d metadata elements were leaked", + gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata elements were leaked", shard->count); if (grpc_iomgr_abort_on_leaks()) { abort(); @@ -245,7 +248,7 @@ void grpc_mdctx_global_shutdown(void) { gpr_mu_destroy(&shard->mu); /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */ if (shard->count != 0) { - gpr_log(GPR_DEBUG, "WARNING: %d metadata strings were leaked", + gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked", shard->count); for (size_t j = 0; j < shard->capacity; j++) { for (internal_string *s = shard->strs[j]; s; s = s->bucket_next) { @@ -281,10 +284,8 @@ static void ref_md_locked(mdtab_shard *shard, grpc_mdstr_as_c_string((grpc_mdstr *)md->key), grpc_mdstr_as_c_string((grpc_mdstr *)md->value)); #endif - if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 2)) { - shard->free--; - } else { - GPR_ASSERT(1 != gpr_atm_no_barrier_fetch_add(&md->refcnt, -1)); + if (0 == gpr_atm_no_barrier_fetch_add(&md->refcnt, 1)) { + gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -1); } } @@ -447,6 +448,7 @@ static void gc_mdtab(mdtab_shard *shard) { size_t i; internal_metadata **prev_next; internal_metadata *md, *next; + gpr_atm num_freed = 0; GPR_TIMER_BEGIN("gc_mdtab", 0); for (i = 0; i < shard->capacity; i++) { @@ -463,13 +465,14 @@ static void gc_mdtab(mdtab_shard *shard) { } gpr_free(md); *prev_next = next; - shard->free--; + num_freed++; shard->count--; } else { prev_next = &md->bucket_next; } } } + gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed); GPR_TIMER_END("gc_mdtab", 0); } @@ -504,7 +507,8 @@ static void grow_mdtab(mdtab_shard *shard) { } static void rehash_mdtab(mdtab_shard *shard) { - if (shard->free > shard->capacity / 4) { + if (gpr_atm_no_barrier_load(&shard->free_estimate) > + (gpr_atm)(shard->capacity / 4)) { gc_mdtab(shard); } else { grow_mdtab(shard); @@ -553,7 +557,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *mkey, /* not found: create a new pair */ md = gpr_malloc(sizeof(internal_metadata)); - gpr_atm_rel_store(&md->refcnt, 2); + gpr_atm_rel_store(&md->refcnt, 1); md->key = key; md->value = value; md->user_data = 0; @@ -645,7 +649,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) { this function - meaning that no adjustment to mdtab_free is necessary, simplifying the logic here to be just an atomic increment */ /* use C assert to have this removed in opt builds */ - assert(gpr_atm_no_barrier_load(&md->refcnt) >= 2); + assert(gpr_atm_no_barrier_load(&md->refcnt) >= 1); gpr_atm_no_barrier_fetch_add(&md->refcnt, 1); return gmd; } @@ -662,18 +666,13 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { grpc_mdstr_as_c_string((grpc_mdstr *)md->key), grpc_mdstr_as_c_string((grpc_mdstr *)md->value)); #endif - if (2 == gpr_atm_full_fetch_add(&md->refcnt, -1)) { - uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash); + uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash); + if (1 == gpr_atm_full_fetch_add(&md->refcnt, -1)) { + /* once the refcount hits zero, some other thread can come along and + free md at any time: it's unsafe from this point on to access it */ mdtab_shard *shard = &g_mdtab_shard[SHARD_IDX(hash, LOG2_MDTAB_SHARD_COUNT)]; - GPR_TIMER_BEGIN("grpc_mdelem_unref.to_zero", 0); - gpr_mu_lock(&shard->mu); - if (1 == gpr_atm_no_barrier_load(&md->refcnt)) { - shard->free++; - gpr_atm_no_barrier_store(&md->refcnt, 0); - } - gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdelem_unref.to_zero", 0); + gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1); } } diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 73b0041fd4..c5f16e530d 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -48,7 +48,7 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 6, 2, 4, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] = {11, 35, 10, 35, 12, 35, 12, 49, 13, 35, 14, 35, 15, 35, 16, 35, 17, 35, @@ -56,10 +56,10 @@ const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] = 30, 18, 30, 35, 31, 35, 32, 35, 36, 35, 37, 35, 38, 35, 39, 35, 42, 33, 42, 34, 42, 48, 42, 53, 42, 54, 42, 55, 42, 56, 43, 33, 43, 48, 43, 53, 46, 0, 46, 1, 46, 2, 50, 35, 57, 35, 58, 35, 59, 35, 60, 35, 61, 35, - 62, 35, 63, 35, 64, 35, 65, 35, 66, 40, 66, 68, 66, 71, 67, 79, 67, 80, - 69, 35, 70, 35, 72, 35, 73, 35, 74, 35, 75, 35, 76, 41, 76, 51, 76, 52, - 77, 35, 78, 35, 81, 3, 81, 4, 81, 5, 81, 6, 81, 7, 81, 8, 81, 9, - 82, 35, 83, 84, 85, 35, 86, 35, 87, 35, 88, 35, 89, 35}; + 62, 35, 63, 35, 64, 35, 65, 35, 66, 35, 67, 40, 67, 69, 67, 72, 68, 80, + 68, 81, 70, 35, 71, 35, 73, 35, 74, 35, 75, 35, 76, 35, 77, 41, 77, 51, + 77, 52, 78, 35, 79, 35, 82, 3, 82, 4, 82, 5, 82, 6, 82, 7, 82, 8, + 82, 9, 83, 35, 84, 85, 86, 35, 87, 35, 88, 35, 89, 35, 90, 35}; const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = { "0", @@ -126,6 +126,7 @@ const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = { "if-unmodified-since", "last-modified", "link", + "load-reporting", "location", "max-forwards", ":method", diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index f9d8bcdc8f..5ff0d2f3bc 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -44,7 +44,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 90 +#define GRPC_STATIC_MDSTR_COUNT 91 extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT]; /* "0" */ #define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0]) @@ -175,60 +175,62 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT]; #define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[62]) /* "link" */ #define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[63]) +/* "load-reporting" */ +#define GRPC_MDSTR_LOAD_REPORTING (&grpc_static_mdstr_table[64]) /* "location" */ -#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[64]) +#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[65]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[65]) +#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[66]) /* ":method" */ -#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[66]) +#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[67]) /* ":path" */ -#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[67]) +#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[68]) /* "POST" */ -#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[68]) +#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[69]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[69]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[70]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[70]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[71]) /* "PUT" */ -#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[71]) +#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[72]) /* "range" */ -#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[72]) +#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[73]) /* "referer" */ -#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[73]) +#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[74]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[74]) +#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[75]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[75]) +#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[76]) /* ":scheme" */ -#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[76]) +#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[77]) /* "server" */ -#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[77]) +#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[78]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[78]) +#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[79]) /* "/" */ -#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[79]) +#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[80]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[80]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[81]) /* ":status" */ -#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[81]) +#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[82]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[82]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[83]) /* "te" */ -#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[83]) +#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[84]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[84]) +#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[85]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[85]) +#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[86]) /* "user-agent" */ -#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[86]) +#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[87]) /* "vary" */ -#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[87]) +#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[88]) /* "via" */ -#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[88]) +#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[89]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[89]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[90]) -#define GRPC_STATIC_MDELEM_COUNT 79 +#define GRPC_STATIC_MDELEM_COUNT 80 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "accept-charset": "" */ @@ -333,71 +335,73 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; #define GRPC_MDELEM_LAST_MODIFIED_EMPTY (&grpc_static_mdelem_table[45]) /* "link": "" */ #define GRPC_MDELEM_LINK_EMPTY (&grpc_static_mdelem_table[46]) +/* "load-reporting": "" */ +#define GRPC_MDELEM_LOAD_REPORTING_EMPTY (&grpc_static_mdelem_table[47]) /* "location": "" */ -#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[47]) +#define GRPC_MDELEM_LOCATION_EMPTY (&grpc_static_mdelem_table[48]) /* "max-forwards": "" */ -#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[48]) +#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (&grpc_static_mdelem_table[49]) /* ":method": "GET" */ -#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[49]) +#define GRPC_MDELEM_METHOD_GET (&grpc_static_mdelem_table[50]) /* ":method": "POST" */ -#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[50]) +#define GRPC_MDELEM_METHOD_POST (&grpc_static_mdelem_table[51]) /* ":method": "PUT" */ -#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[51]) +#define GRPC_MDELEM_METHOD_PUT (&grpc_static_mdelem_table[52]) /* ":path": "/" */ -#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[52]) +#define GRPC_MDELEM_PATH_SLASH (&grpc_static_mdelem_table[53]) /* ":path": "/index.html" */ -#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[53]) +#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (&grpc_static_mdelem_table[54]) /* "proxy-authenticate": "" */ -#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[54]) +#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[55]) /* "proxy-authorization": "" */ -#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[55]) +#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY (&grpc_static_mdelem_table[56]) /* "range": "" */ -#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[56]) +#define GRPC_MDELEM_RANGE_EMPTY (&grpc_static_mdelem_table[57]) /* "referer": "" */ -#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[57]) +#define GRPC_MDELEM_REFERER_EMPTY (&grpc_static_mdelem_table[58]) /* "refresh": "" */ -#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[58]) +#define GRPC_MDELEM_REFRESH_EMPTY (&grpc_static_mdelem_table[59]) /* "retry-after": "" */ -#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[59]) +#define GRPC_MDELEM_RETRY_AFTER_EMPTY (&grpc_static_mdelem_table[60]) /* ":scheme": "grpc" */ -#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[60]) +#define GRPC_MDELEM_SCHEME_GRPC (&grpc_static_mdelem_table[61]) /* ":scheme": "http" */ -#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[61]) +#define GRPC_MDELEM_SCHEME_HTTP (&grpc_static_mdelem_table[62]) /* ":scheme": "https" */ -#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[62]) +#define GRPC_MDELEM_SCHEME_HTTPS (&grpc_static_mdelem_table[63]) /* "server": "" */ -#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[63]) +#define GRPC_MDELEM_SERVER_EMPTY (&grpc_static_mdelem_table[64]) /* "set-cookie": "" */ -#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[64]) +#define GRPC_MDELEM_SET_COOKIE_EMPTY (&grpc_static_mdelem_table[65]) /* ":status": "200" */ -#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[65]) +#define GRPC_MDELEM_STATUS_200 (&grpc_static_mdelem_table[66]) /* ":status": "204" */ -#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[66]) +#define GRPC_MDELEM_STATUS_204 (&grpc_static_mdelem_table[67]) /* ":status": "206" */ -#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[67]) +#define GRPC_MDELEM_STATUS_206 (&grpc_static_mdelem_table[68]) /* ":status": "304" */ -#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[68]) +#define GRPC_MDELEM_STATUS_304 (&grpc_static_mdelem_table[69]) /* ":status": "400" */ -#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[69]) +#define GRPC_MDELEM_STATUS_400 (&grpc_static_mdelem_table[70]) /* ":status": "404" */ -#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[70]) +#define GRPC_MDELEM_STATUS_404 (&grpc_static_mdelem_table[71]) /* ":status": "500" */ -#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[71]) +#define GRPC_MDELEM_STATUS_500 (&grpc_static_mdelem_table[72]) /* "strict-transport-security": "" */ #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (&grpc_static_mdelem_table[72]) + (&grpc_static_mdelem_table[73]) /* "te": "trailers" */ -#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[73]) +#define GRPC_MDELEM_TE_TRAILERS (&grpc_static_mdelem_table[74]) /* "transfer-encoding": "" */ -#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[74]) +#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (&grpc_static_mdelem_table[75]) /* "user-agent": "" */ -#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[75]) +#define GRPC_MDELEM_USER_AGENT_EMPTY (&grpc_static_mdelem_table[76]) /* "vary": "" */ -#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[76]) +#define GRPC_MDELEM_VARY_EMPTY (&grpc_static_mdelem_table[77]) /* "via": "" */ -#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[77]) +#define GRPC_MDELEM_VIA_EMPTY (&grpc_static_mdelem_table[78]) /* "www-authenticate": "" */ -#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[78]) +#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[79]) extern const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2]; diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index fdf0f4b2aa..1105494a85 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -60,7 +60,7 @@ void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount) { #endif if (gpr_unref(&refcount->refs)) { - grpc_exec_ctx_push(exec_ctx, &refcount->destroy, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(exec_ctx, &refcount->destroy, GRPC_ERROR_NONE, NULL); } } @@ -125,10 +125,19 @@ void grpc_transport_perform_op(grpc_exec_ctx *exec_ctx, transport->vtable->perform_op(exec_ctx, transport, op); } -void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_transport *transport, grpc_stream *stream, - grpc_pollset *pollset) { - transport->vtable->set_pollset(exec_ctx, transport, stream, pollset); +void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport, + grpc_stream *stream, + grpc_polling_entity *pollent) { + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + if ((pollset = grpc_polling_entity_pollset(pollent)) != NULL) { + transport->vtable->set_pollset(exec_ctx, transport, stream, pollset); + } else if ((pollset_set = grpc_polling_entity_pollset_set(pollent)) != NULL) { + transport->vtable->set_pollset_set(exec_ctx, transport, stream, + pollset_set); + } else { + abort(); + } } void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, @@ -146,11 +155,11 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op, grpc_error *error) { - grpc_exec_ctx_push(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error), - NULL); - grpc_exec_ctx_push(exec_ctx, op->recv_initial_metadata_ready, - GRPC_ERROR_REF(error), NULL); - grpc_exec_ctx_push(exec_ctx, op->on_complete, error, NULL); + grpc_exec_ctx_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error), + NULL); + grpc_exec_ctx_sched(exec_ctx, op->recv_initial_metadata_ready, + GRPC_ERROR_REF(error), NULL); + grpc_exec_ctx_sched(exec_ctx, op->on_complete, error, NULL); } void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op, diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index f2d750e870..a46ccb643c 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -37,6 +37,7 @@ #include <stddef.h> #include "src/core/lib/channel/context.h" +#include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/transport/byte_stream.h" @@ -197,9 +198,8 @@ int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount, const void *server_data); -void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx, - grpc_transport *transport, grpc_stream *stream, - grpc_pollset *pollset); +void grpc_transport_set_pops(grpc_exec_ctx *exec_ctx, grpc_transport *transport, + grpc_stream *stream, grpc_polling_entity *pollent); /* Destroy transport data for a stream. diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index 956155eec8..fc7140671b 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -53,6 +53,10 @@ typedef struct grpc_transport_vtable { void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_transport *self, grpc_stream *stream, grpc_pollset *pollset); + /* implementation of grpc_transport_set_pollset */ + void (*set_pollset_set)(grpc_exec_ctx *exec_ctx, grpc_transport *self, + grpc_stream *stream, grpc_pollset_set *pollset_set); + /* implementation of grpc_transport_perform_stream_op */ void (*perform_stream_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self, grpc_stream *stream, grpc_transport_stream_op *op); |