diff options
Diffstat (limited to 'src/core/lib')
157 files changed, 9847 insertions, 4642 deletions
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 1a099ac437..238d176dfa 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -31,17 +31,18 @@ * */ -#include "src/core/lib/channel/channel_args.h" -#include <grpc/grpc.h> -#include "src/core/lib/support/string.h" +#include <limits.h> +#include <string.h> #include <grpc/compression.h> +#include <grpc/grpc.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> -#include <string.h> +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/support/string.h" static grpc_arg copy_arg(const grpc_arg *src) { grpc_arg dst; @@ -329,7 +330,9 @@ const grpc_arg *grpc_channel_args_find(const grpc_channel_args *args, return NULL; } -int grpc_channel_arg_get_integer(grpc_arg *arg, grpc_integer_options options) { +int grpc_channel_arg_get_integer(const grpc_arg *arg, + const grpc_integer_options options) { + if (arg == NULL) return options.default_value; if (arg->type != GRPC_ARG_INTEGER) { gpr_log(GPR_ERROR, "%s ignored: it must be an integer", arg->key); return options.default_value; @@ -346,3 +349,26 @@ int grpc_channel_arg_get_integer(grpc_arg *arg, grpc_integer_options options) { } return arg->value.integer; } + +bool grpc_channel_arg_get_bool(const grpc_arg *arg, bool default_value) { + if (arg == NULL) return default_value; + if (arg->type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s ignored: it must be an integer", arg->key); + return default_value; + } + switch (arg->value.integer) { + case 0: + return false; + case 1: + return true; + default: + gpr_log(GPR_ERROR, "%s treated as bool but set to %d (assuming true)", + arg->key, arg->value.integer); + return true; + } +} + +bool grpc_channel_args_want_minimal_stack(const grpc_channel_args *args) { + return grpc_channel_arg_get_bool( + grpc_channel_args_find(args, GRPC_ARG_MINIMAL_STACK), false); +} diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 5c7d31f8bb..f0f603e251 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -113,12 +113,18 @@ grpc_channel_args *grpc_channel_args_set_socket_mutator( const grpc_arg *grpc_channel_args_find(const grpc_channel_args *args, const char *name); +bool grpc_channel_args_want_minimal_stack(const grpc_channel_args *args); + typedef struct grpc_integer_options { int default_value; // Return this if value is outside of expected bounds. int min_value; int max_value; } grpc_integer_options; + /** Returns the value of \a arg, subject to the contraints in \a options. */ -int grpc_channel_arg_get_integer(grpc_arg *arg, grpc_integer_options options); +int grpc_channel_arg_get_integer(const grpc_arg *arg, + const grpc_integer_options options); + +bool grpc_channel_arg_get_bool(const grpc_arg *arg, bool default_value); #endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_ARGS_H */ diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 6d53b0576e..7db54d1107 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -38,7 +38,7 @@ #include <stdlib.h> #include <string.h> -int grpc_trace_channel = 0; +grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false); /* Memory layouts. @@ -246,9 +246,9 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, } void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { grpc_call_element *next_elem = elem + 1; - next_elem->filter->start_transport_stream_op(exec_ctx, next_elem, op); + next_elem->filter->start_transport_stream_op_batch(exec_ctx, next_elem, op); } char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, @@ -284,7 +284,8 @@ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem) { void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_error *error) { - grpc_transport_stream_op *op = grpc_make_transport_stream_op(NULL); - op->cancel_error = error; - elem->filter->start_transport_stream_op(exec_ctx, elem, op); + grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(NULL); + op->cancel_stream = true; + op->payload->cancel_stream.cancel_error = error; + elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op); } diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 80e3603e8d..c26d61b2ef 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -112,9 +112,9 @@ typedef struct { typedef struct { /* Called to eg. send/receive data on a call. See grpc_call_next_op on how to call the next element in the stack */ - void (*start_transport_stream_op)(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op); + void (*start_transport_stream_op_batch)(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_transport_stream_op_batch *op); /* Called to handle channel level operations - e.g. new calls, or transport closure. See grpc_channel_next_op on how to call the next element in the stack */ @@ -281,7 +281,7 @@ void grpc_call_stack_ignore_set_pollset_or_pollset_set( grpc_polling_entity *pollent); /* Call the next operation in a call stack */ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op); + grpc_transport_stream_op_batch *op); /* Call the next operation (depending on call directionality) in a channel stack */ void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, @@ -300,16 +300,17 @@ grpc_channel_stack *grpc_channel_stack_from_top_element( grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem); void grpc_call_log_op(char *file, int line, gpr_log_severity severity, - grpc_call_element *elem, grpc_transport_stream_op *op); + grpc_call_element *elem, + grpc_transport_stream_op_batch *op); void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, grpc_call_element *cur_elem, grpc_error *error); -extern int grpc_trace_channel; +extern grpc_tracer_flag grpc_trace_channel; #define GRPC_CALL_LOG_OP(sev, elem, op) \ - if (grpc_trace_channel) grpc_call_log_op(sev, elem, op) + if (GRPC_TRACER_ON(grpc_trace_channel)) grpc_call_log_op(sev, elem, op) #ifdef __cplusplus } diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index b515b7321a..44b030e4d1 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -38,7 +38,8 @@ #include <grpc/support/alloc.h> #include <grpc/support/string_util.h> -int grpc_trace_channel_stack_builder = 0; +grpc_tracer_flag grpc_trace_channel_stack_builder = + GRPC_TRACER_INITIALIZER(false); typedef struct filter_node { struct filter_node *next; @@ -113,6 +114,17 @@ grpc_channel_stack_builder_create_iterator_at_last( return create_iterator_at_filter_node(builder, &builder->end); } +bool grpc_channel_stack_builder_iterator_is_end( + grpc_channel_stack_builder_iterator *iterator) { + return iterator->node == &iterator->builder->end; +} + +const char *grpc_channel_stack_builder_iterator_filter_name( + grpc_channel_stack_builder_iterator *iterator) { + if (iterator->node->filter == NULL) return NULL; + return iterator->node->filter->name; +} + bool grpc_channel_stack_builder_move_next( grpc_channel_stack_builder_iterator *iterator) { if (iterator->node == &iterator->builder->end) return false; diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h index 8adf38e27b..8cb36eb117 100644 --- a/src/core/lib/channel/channel_stack_builder.h +++ b/src/core/lib/channel/channel_stack_builder.h @@ -98,6 +98,10 @@ bool grpc_channel_stack_builder_iterator_is_first( bool grpc_channel_stack_builder_iterator_is_end( grpc_channel_stack_builder_iterator *iterator); +/// What is the name of the filter at this iterator position? +const char *grpc_channel_stack_builder_iterator_filter_name( + grpc_channel_stack_builder_iterator *iterator); + /// Move an iterator to the next item bool grpc_channel_stack_builder_move_next( grpc_channel_stack_builder_iterator *iterator); @@ -161,7 +165,7 @@ grpc_error *grpc_channel_stack_builder_finish( void grpc_channel_stack_builder_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder); -extern int grpc_trace_channel_stack_builder; +extern grpc_tracer_flag grpc_trace_channel_stack_builder; #ifdef __cplusplus } diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c deleted file mode 100644 index 02dc479f3a..0000000000 --- a/src/core/lib/channel/compress_filter.c +++ /dev/null @@ -1,351 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include <assert.h> -#include <string.h> - -#include <grpc/compression.h> -#include <grpc/slice_buffer.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> - -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/compress_filter.h" -#include "src/core/lib/compression/algorithm_metadata.h" -#include "src/core/lib/compression/message_compress.h" -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/transport/static_metadata.h" - -int grpc_compression_trace = 0; - -typedef struct call_data { - grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */ - grpc_linked_mdelem compression_algorithm_storage; - grpc_linked_mdelem accept_encoding_storage; - uint32_t remaining_slice_bytes; - /** Compression algorithm we'll try to use. It may be given by incoming - * metadata, or by the channel's default compression settings. */ - grpc_compression_algorithm compression_algorithm; - /** If true, contents of \a compression_algorithm are authoritative */ - int has_compression_algorithm; - - grpc_transport_stream_op *send_op; - uint32_t send_length; - uint32_t send_flags; - grpc_slice incoming_slice; - grpc_slice_buffer_stream replacement_stream; - grpc_closure *post_send; - grpc_closure send_done; - grpc_closure got_slice; -} call_data; - -typedef struct channel_data { - /** The default, channel-level, compression algorithm */ - grpc_compression_algorithm default_compression_algorithm; - /** Bitset of enabled algorithms */ - uint32_t enabled_algorithms_bitset; - /** Supported compression algorithms */ - uint32_t supported_compression_algorithms; -} channel_data; - -static int skip_compression(grpc_call_element *elem, uint32_t flags) { - call_data *calld = elem->call_data; - channel_data *channeld = elem->channel_data; - - if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { - return 1; - } - if (calld->has_compression_algorithm) { - if (calld->compression_algorithm == GRPC_COMPRESS_NONE) { - return 1; - } - return 0; /* we have an actual call-specific algorithm */ - } - /* no per-call compression override */ - return channeld->default_compression_algorithm == GRPC_COMPRESS_NONE; -} - -/** Filter initial metadata */ -static grpc_error *process_send_initial_metadata( - grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_metadata_batch *initial_metadata) GRPC_MUST_USE_RESULT; -static grpc_error *process_send_initial_metadata( - grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_metadata_batch *initial_metadata) { - call_data *calld = elem->call_data; - channel_data *channeld = elem->channel_data; - /* Parse incoming request for compression. If any, it'll be available - * at calld->compression_algorithm */ - if (initial_metadata->idx.named.grpc_internal_encoding_request != NULL) { - grpc_mdelem md = - initial_metadata->idx.named.grpc_internal_encoding_request->md; - if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md), - &calld->compression_algorithm)) { - char *val = grpc_slice_to_c_string(GRPC_MDVALUE(md)); - gpr_log(GPR_ERROR, - "Invalid compression algorithm: '%s' (unknown). Ignoring.", val); - gpr_free(val); - calld->compression_algorithm = GRPC_COMPRESS_NONE; - } - if (!GPR_BITGET(channeld->enabled_algorithms_bitset, - calld->compression_algorithm)) { - char *val = grpc_slice_to_c_string(GRPC_MDVALUE(md)); - gpr_log(GPR_ERROR, - "Invalid compression algorithm: '%s' (previously disabled). " - "Ignoring.", - val); - gpr_free(val); - calld->compression_algorithm = GRPC_COMPRESS_NONE; - } - calld->has_compression_algorithm = 1; - - grpc_metadata_batch_remove( - exec_ctx, initial_metadata, - initial_metadata->idx.named.grpc_internal_encoding_request); - } else { - /* If no algorithm was found in the metadata and we aren't - * exceptionally skipping compression, fall back to the channel - * default */ - calld->compression_algorithm = channeld->default_compression_algorithm; - calld->has_compression_algorithm = 1; /* GPR_TRUE */ - } - - grpc_error *error = GRPC_ERROR_NONE; - /* hint compression algorithm */ - if (calld->compression_algorithm != GRPC_COMPRESS_NONE) { - error = grpc_metadata_batch_add_tail( - exec_ctx, initial_metadata, &calld->compression_algorithm_storage, - grpc_compression_encoding_mdelem(calld->compression_algorithm)); - } - - if (error != GRPC_ERROR_NONE) return error; - - /* convey supported compression algorithms */ - error = grpc_metadata_batch_add_tail( - exec_ctx, initial_metadata, &calld->accept_encoding_storage, - GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS( - channeld->supported_compression_algorithms)); - - return error; -} - -static void continue_send_message(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem); - -static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { - grpc_call_element *elem = elemp; - call_data *calld = elem->call_data; - grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &calld->slices); - calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error); -} - -static void finish_send_message(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - call_data *calld = elem->call_data; - int did_compress; - grpc_slice_buffer tmp; - grpc_slice_buffer_init(&tmp); - did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm, - &calld->slices, &tmp); - if (did_compress) { - if (grpc_compression_trace) { - char *algo_name; - const size_t before_size = calld->slices.length; - const size_t after_size = tmp.length; - const float savings_ratio = 1.0f - (float)after_size / (float)before_size; - GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, - &algo_name)); - gpr_log(GPR_DEBUG, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR - " bytes (%.2f%% savings)", - algo_name, before_size, after_size, 100 * savings_ratio); - } - grpc_slice_buffer_swap(&calld->slices, &tmp); - calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; - } else { - if (grpc_compression_trace) { - char *algo_name; - GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, - &algo_name)); - gpr_log(GPR_DEBUG, - "Algorithm '%s' enabled but decided not to compress. Input size: " - "%" PRIuPTR, - algo_name, calld->slices.length); - } - } - - grpc_slice_buffer_destroy_internal(exec_ctx, &tmp); - - grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices, - calld->send_flags); - calld->send_op->send_message = &calld->replacement_stream.base; - calld->post_send = calld->send_op->on_complete; - calld->send_op->on_complete = &calld->send_done; - - grpc_call_next_op(exec_ctx, elem, calld->send_op); -} - -static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { - grpc_call_element *elem = elemp; - call_data *calld = elem->call_data; - grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); - if (calld->send_length == calld->slices.length) { - finish_send_message(exec_ctx, elem); - } else { - continue_send_message(exec_ctx, elem); - } -} - -static void continue_send_message(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - call_data *calld = elem->call_data; - while (grpc_byte_stream_next(exec_ctx, calld->send_op->send_message, - &calld->incoming_slice, ~(size_t)0, - &calld->got_slice)) { - grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); - if (calld->send_length == calld->slices.length) { - finish_send_message(exec_ctx, elem); - break; - } - } -} - -static void compress_start_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - call_data *calld = elem->call_data; - - GPR_TIMER_BEGIN("compress_start_transport_stream_op", 0); - - if (op->send_initial_metadata) { - grpc_error *error = process_send_initial_metadata( - exec_ctx, elem, op->send_initial_metadata); - if (error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); - return; - } - } - if (op->send_message != NULL && - !skip_compression(elem, op->send_message->flags)) { - calld->send_op = op; - calld->send_length = op->send_message->length; - calld->send_flags = op->send_message->flags; - continue_send_message(exec_ctx, elem); - } else { - /* pass control down the stack */ - grpc_call_next_op(exec_ctx, elem, op); - } - - GPR_TIMER_END("compress_start_transport_stream_op", 0); -} - -/* Constructor for call_data */ -static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - const grpc_call_element_args *args) { - /* grab pointers to our data from the call element */ - call_data *calld = elem->call_data; - - /* initialize members */ - grpc_slice_buffer_init(&calld->slices); - calld->has_compression_algorithm = 0; - grpc_closure_init(&calld->got_slice, got_slice, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->send_done, send_done, elem, - grpc_schedule_on_exec_ctx); - - return GRPC_ERROR_NONE; -} - -/* Destructor for call_data */ -static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const grpc_call_final_info *final_info, - grpc_closure *ignored) { - /* grab pointers to our data from the call element */ - call_data *calld = elem->call_data; - grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices); -} - -/* Constructor for channel_data */ -static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, - grpc_channel_element_args *args) { - channel_data *channeld = elem->channel_data; - - channeld->enabled_algorithms_bitset = - grpc_channel_args_compression_algorithm_get_states(args->channel_args); - - channeld->default_compression_algorithm = - grpc_channel_args_get_compression_algorithm(args->channel_args); - /* Make sure the default isn't disabled. */ - if (!GPR_BITGET(channeld->enabled_algorithms_bitset, - channeld->default_compression_algorithm)) { - gpr_log(GPR_DEBUG, - "compression algorithm %d not enabled: switching to none", - channeld->default_compression_algorithm); - channeld->default_compression_algorithm = GRPC_COMPRESS_NONE; - } - - channeld->supported_compression_algorithms = 1; /* always support identity */ - for (grpc_compression_algorithm algo_idx = 1; - algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { - /* skip disabled algorithms */ - if (!GPR_BITGET(channeld->enabled_algorithms_bitset, algo_idx)) { - continue; - } - channeld->supported_compression_algorithms |= 1u << algo_idx; - } - - GPR_ASSERT(!args->is_last); - return GRPC_ERROR_NONE; -} - -/* Destructor for channel data */ -static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem) {} - -const grpc_channel_filter grpc_compress_filter = { - compress_start_transport_stream_op, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "compress"}; diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index 75c68a5534..d8985268eb 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -62,9 +62,9 @@ typedef struct connected_channel_call_data { void *unused; } call_data; /* Intercept a call operation and either push it directly up or translate it into transport stream operations */ -static void con_start_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { +static void con_start_transport_stream_op_batch( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_transport_stream_op_batch *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; GRPC_CALL_LOG_OP(GPR_INFO, elem, op); @@ -128,7 +128,9 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem) { channel_data *cd = (channel_data *)elem->channel_data; - grpc_transport_destroy(exec_ctx, cd->transport); + if (cd->transport) { + grpc_transport_destroy(exec_ctx, cd->transport); + } } static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { @@ -142,7 +144,7 @@ static void con_get_channel_info(grpc_exec_ctx *exec_ctx, const grpc_channel_info *channel_info) {} const grpc_channel_filter grpc_connected_filter = { - con_start_transport_stream_op, + con_start_transport_stream_op_batch, con_start_transport_op, sizeof(call_data), init_call_elem, diff --git a/src/core/lib/channel/context.h b/src/core/lib/channel/context.h index 2c1174ce7a..f0a21113c5 100644 --- a/src/core/lib/channel/context.h +++ b/src/core/lib/channel/context.h @@ -50,8 +50,8 @@ typedef enum { /// Reserved for traffic_class_context. GRPC_CONTEXT_TRAFFIC, - /// Costs for Load Reporting. - GRPC_CONTEXT_LR_COST, + /// Value is a \a grpc_grpclb_client_stats. + GRPC_GRPCLB_CLIENT_STATS, GRPC_CONTEXT_COUNT } grpc_context_index; diff --git a/src/core/lib/channel/deadline_filter.c b/src/core/lib/channel/deadline_filter.c deleted file mode 100644 index ca701ed457..0000000000 --- a/src/core/lib/channel/deadline_filter.c +++ /dev/null @@ -1,344 +0,0 @@ -// -// Copyright 2016, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include "src/core/lib/channel/deadline_filter.h" - -#include <stdbool.h> -#include <string.h> - -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/sync.h> -#include <grpc/support/time.h> - -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/slice/slice_internal.h" - -// -// grpc_deadline_state -// - -// Timer callback. -static void timer_callback(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - grpc_call_element* elem = arg; - grpc_deadline_state* deadline_state = elem->call_data; - if (error != GRPC_ERROR_CANCELLED) { - grpc_call_element_signal_error( - exec_ctx, elem, - grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED)); - } - GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer"); -} - -// Starts the deadline timer. -static void start_timer_if_needed(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - gpr_timespec deadline) { - deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); - if (gpr_time_cmp(deadline, gpr_inf_future(GPR_CLOCK_MONOTONIC)) == 0) { - return; - } - grpc_deadline_state* deadline_state = elem->call_data; - grpc_deadline_timer_state cur_state; - grpc_closure* closure = NULL; -retry: - cur_state = - (grpc_deadline_timer_state)gpr_atm_acq_load(&deadline_state->timer_state); - switch (cur_state) { - case GRPC_DEADLINE_STATE_PENDING: - // Note: We do not start the timer if there is already a timer - return; - case GRPC_DEADLINE_STATE_FINISHED: - if (gpr_atm_rel_cas(&deadline_state->timer_state, - GRPC_DEADLINE_STATE_FINISHED, - GRPC_DEADLINE_STATE_PENDING)) { - // If we've already created and destroyed a timer, we always create a - // new closure: we have no other guarantee that the inlined closure is - // not in use (it may hold a pending call to timer_callback) - closure = grpc_closure_create(timer_callback, elem, - grpc_schedule_on_exec_ctx); - } else { - goto retry; - } - break; - case GRPC_DEADLINE_STATE_INITIAL: - if (gpr_atm_rel_cas(&deadline_state->timer_state, - GRPC_DEADLINE_STATE_INITIAL, - GRPC_DEADLINE_STATE_PENDING)) { - closure = - grpc_closure_init(&deadline_state->timer_callback, timer_callback, - elem, grpc_schedule_on_exec_ctx); - } else { - goto retry; - } - break; - } - GPR_ASSERT(closure); - GRPC_CALL_STACK_REF(deadline_state->call_stack, "deadline_timer"); - grpc_timer_init(exec_ctx, &deadline_state->timer, deadline, closure, - gpr_now(GPR_CLOCK_MONOTONIC)); -} - -// Cancels the deadline timer. -static void cancel_timer_if_needed(grpc_exec_ctx* exec_ctx, - grpc_deadline_state* deadline_state) { - if (gpr_atm_rel_cas(&deadline_state->timer_state, GRPC_DEADLINE_STATE_PENDING, - GRPC_DEADLINE_STATE_FINISHED)) { - grpc_timer_cancel(exec_ctx, &deadline_state->timer); - } else { - // timer was either in STATE_INITAL (nothing to cancel) - // OR in STATE_FINISHED (again nothing to cancel) - } -} - -// Callback run when the call is complete. -static void on_complete(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - grpc_deadline_state* deadline_state = arg; - cancel_timer_if_needed(exec_ctx, deadline_state); - // Invoke the next callback. - grpc_closure_run(exec_ctx, deadline_state->next_on_complete, - GRPC_ERROR_REF(error)); -} - -// Inject our own on_complete callback into op. -static void inject_on_complete_cb(grpc_deadline_state* deadline_state, - grpc_transport_stream_op* op) { - deadline_state->next_on_complete = op->on_complete; - grpc_closure_init(&deadline_state->on_complete, on_complete, deadline_state, - grpc_schedule_on_exec_ctx); - op->on_complete = &deadline_state->on_complete; -} - -void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - grpc_call_stack* call_stack) { - grpc_deadline_state* deadline_state = elem->call_data; - deadline_state->call_stack = call_stack; -} - -void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem) { - grpc_deadline_state* deadline_state = elem->call_data; - cancel_timer_if_needed(exec_ctx, deadline_state); -} - -// Callback and associated state for starting the timer after call stack -// initialization has been completed. -struct start_timer_after_init_state { - grpc_call_element* elem; - gpr_timespec deadline; - grpc_closure closure; -}; -static void start_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - struct start_timer_after_init_state* state = arg; - start_timer_if_needed(exec_ctx, state->elem, state->deadline); - gpr_free(state); -} - -void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec deadline) { - // Deadline will always be infinite on servers, so the timer will only be - // set on clients with a finite deadline. - deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC); - if (gpr_time_cmp(deadline, gpr_inf_future(GPR_CLOCK_MONOTONIC)) != 0) { - // When the deadline passes, we indicate the failure by sending down - // an op with cancel_error set. However, we can't send down any ops - // until after the call stack is fully initialized. If we start the - // timer here, we have no guarantee that the timer won't pop before - // call stack initialization is finished. To avoid that problem, we - // create a closure to start the timer, and we schedule that closure - // to be run after call stack initialization is done. - struct start_timer_after_init_state* state = gpr_malloc(sizeof(*state)); - state->elem = elem; - state->deadline = deadline; - grpc_closure_init(&state->closure, start_timer_after_init, state, - grpc_schedule_on_exec_ctx); - grpc_closure_sched(exec_ctx, &state->closure, GRPC_ERROR_NONE); - } -} - -void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec new_deadline) { - grpc_deadline_state* deadline_state = elem->call_data; - cancel_timer_if_needed(exec_ctx, deadline_state); - start_timer_if_needed(exec_ctx, elem, new_deadline); -} - -void grpc_deadline_state_client_start_transport_stream_op( - grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - grpc_transport_stream_op* op) { - grpc_deadline_state* deadline_state = elem->call_data; - if (op->cancel_error != GRPC_ERROR_NONE) { - cancel_timer_if_needed(exec_ctx, deadline_state); - } else { - // Make sure we know when the call is complete, so that we can cancel - // the timer. - if (op->recv_trailing_metadata != NULL) { - inject_on_complete_cb(deadline_state, op); - } - } -} - -// -// filter code -// - -// Constructor for channel_data. Used for both client and server filters. -static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem, - grpc_channel_element_args* args) { - GPR_ASSERT(!args->is_last); - return GRPC_ERROR_NONE; -} - -// Destructor for channel_data. Used for both client and server filters. -static void destroy_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem) {} - -// Call data used for both client and server filter. -typedef struct base_call_data { - grpc_deadline_state deadline_state; -} base_call_data; - -// Additional call data used only for the server filter. -typedef struct server_call_data { - base_call_data base; // Must be first. - // The closure for receiving initial metadata. - grpc_closure recv_initial_metadata_ready; - // Received initial metadata batch. - grpc_metadata_batch* recv_initial_metadata; - // The original recv_initial_metadata_ready closure, which we chain to - // after our own closure is invoked. - grpc_closure* next_recv_initial_metadata_ready; -} server_call_data; - -// Constructor for call_data. Used for both client and server filters. -static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - const grpc_call_element_args* args) { - grpc_deadline_state_init(exec_ctx, elem, args->call_stack); - grpc_deadline_state_start(exec_ctx, elem, args->deadline); - return GRPC_ERROR_NONE; -} - -// Destructor for call_data. Used for both client and server filters. -static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - const grpc_call_final_info* final_info, - grpc_closure* ignored) { - grpc_deadline_state_destroy(exec_ctx, elem); -} - -// Method for starting a call op for client filter. -static void client_start_transport_stream_op(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - grpc_transport_stream_op* op) { - grpc_deadline_state_client_start_transport_stream_op(exec_ctx, elem, op); - // Chain to next filter. - grpc_call_next_op(exec_ctx, elem, op); -} - -// Callback for receiving initial metadata on the server. -static void recv_initial_metadata_ready(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - grpc_call_element* elem = arg; - server_call_data* calld = elem->call_data; - // Get deadline from metadata and start the timer if needed. - start_timer_if_needed(exec_ctx, elem, calld->recv_initial_metadata->deadline); - // Invoke the next callback. - calld->next_recv_initial_metadata_ready->cb( - exec_ctx, calld->next_recv_initial_metadata_ready->cb_arg, error); -} - -// Method for starting a call op for server filter. -static void server_start_transport_stream_op(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - grpc_transport_stream_op* op) { - server_call_data* calld = elem->call_data; - if (op->cancel_error != GRPC_ERROR_NONE) { - cancel_timer_if_needed(exec_ctx, &calld->base.deadline_state); - } else { - // If we're receiving initial metadata, we need to get the deadline - // from the recv_initial_metadata_ready callback. So we inject our - // own callback into that hook. - if (op->recv_initial_metadata_ready != NULL) { - calld->next_recv_initial_metadata_ready = op->recv_initial_metadata_ready; - calld->recv_initial_metadata = op->recv_initial_metadata; - grpc_closure_init(&calld->recv_initial_metadata_ready, - recv_initial_metadata_ready, elem, - grpc_schedule_on_exec_ctx); - op->recv_initial_metadata_ready = &calld->recv_initial_metadata_ready; - } - // Make sure we know when the call is complete, so that we can cancel - // the timer. - // Note that we trigger this on recv_trailing_metadata, even though - // the client never sends trailing metadata, because this is the - // hook that tells us when the call is complete on the server side. - if (op->recv_trailing_metadata != NULL) { - inject_on_complete_cb(&calld->base.deadline_state, op); - } - } - // Chain to next filter. - grpc_call_next_op(exec_ctx, elem, op); -} - -const grpc_channel_filter grpc_client_deadline_filter = { - client_start_transport_stream_op, - grpc_channel_next_op, - sizeof(base_call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - 0, // sizeof(channel_data) - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "deadline", -}; - -const grpc_channel_filter grpc_server_deadline_filter = { - server_start_transport_stream_op, - grpc_channel_next_op, - sizeof(server_call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - 0, // sizeof(channel_data) - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "deadline", -}; diff --git a/src/core/lib/channel/deadline_filter.h b/src/core/lib/channel/deadline_filter.h deleted file mode 100644 index 72cd5cb929..0000000000 --- a/src/core/lib/channel/deadline_filter.h +++ /dev/null @@ -1,102 +0,0 @@ -// -// Copyright 2016, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#ifndef GRPC_CORE_LIB_CHANNEL_DEADLINE_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_DEADLINE_FILTER_H - -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/iomgr/timer.h" - -typedef enum grpc_deadline_timer_state { - GRPC_DEADLINE_STATE_INITIAL, - GRPC_DEADLINE_STATE_PENDING, - GRPC_DEADLINE_STATE_FINISHED -} grpc_deadline_timer_state; - -// State used for filters that enforce call deadlines. -// Must be the first field in the filter's call_data. -typedef struct grpc_deadline_state { - // We take a reference to the call stack for the timer callback. - grpc_call_stack* call_stack; - gpr_atm timer_state; - grpc_timer timer; - grpc_closure timer_callback; - // Closure to invoke when the call is complete. - // We use this to cancel the timer. - grpc_closure on_complete; - // The original on_complete closure, which we chain to after our own - // closure is invoked. - grpc_closure* next_on_complete; -} grpc_deadline_state; - -// -// NOTE: All of these functions require that the first field in -// elem->call_data is a grpc_deadline_state. -// - -// assumes elem->call_data is zero'd -void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - grpc_call_stack* call_stack); -void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem); - -// Starts the timer with the specified deadline. -// Should be called from the filter's init_call_elem() method. -void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec deadline); - -// Cancels the existing timer and starts a new one with new_deadline. -// -// Note: It is generally safe to call this with an earlier deadline -// value than the current one, but not the reverse. No checks are done -// to ensure that the timer callback is not invoked while it is in the -// process of being reset, which means that attempting to increase the -// deadline may result in the timer being called twice. -void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - gpr_timespec new_deadline); - -// To be called from the client-side filter's start_transport_stream_op() -// method. Ensures that the deadline timer is cancelled when the call -// is completed. -// -// Note: It is the caller's responsibility to chain to the next filter if -// necessary after this function returns. -void grpc_deadline_state_client_start_transport_stream_op( - grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - grpc_transport_stream_op* op); - -// Deadline filters for direct client channels and server channels. -// Note: Deadlines for non-direct client channels are handled by the -// client_channel filter. -extern const grpc_channel_filter grpc_client_deadline_filter; -extern const grpc_channel_filter grpc_server_deadline_filter; - -#endif /* GRPC_CORE_LIB_CHANNEL_DEADLINE_FILTER_H */ diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c deleted file mode 100644 index 17af2013d9..0000000000 --- a/src/core/lib/channel/http_client_filter.c +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/channel/http_client_filter.h" -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/string_util.h> -#include <string.h> -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/b64.h" -#include "src/core/lib/slice/percent_encoding.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/transport/static_metadata.h" -#include "src/core/lib/transport/transport_impl.h" - -#define EXPECTED_CONTENT_TYPE "application/grpc" -#define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 - -/* default maximum size of payload eligable for GET request */ -static const size_t kMaxPayloadSizeForGet = 2048; - -typedef struct call_data { - grpc_linked_mdelem method; - grpc_linked_mdelem scheme; - grpc_linked_mdelem authority; - grpc_linked_mdelem te_trailers; - grpc_linked_mdelem content_type; - grpc_linked_mdelem user_agent; - - grpc_metadata_batch *recv_initial_metadata; - grpc_metadata_batch *recv_trailing_metadata; - uint8_t *payload_bytes; - - /* Vars to read data off of send_message */ - grpc_transport_stream_op send_op; - uint32_t send_length; - uint32_t send_flags; - grpc_slice incoming_slice; - grpc_slice_buffer_stream replacement_stream; - grpc_slice_buffer slices; - /* flag that indicates that all slices of send_messages aren't availble */ - bool send_message_blocked; - - /** Closure to call when finished with the hc_on_recv hook */ - grpc_closure *on_done_recv_initial_metadata; - grpc_closure *on_done_recv_trailing_metadata; - grpc_closure *on_complete; - grpc_closure *post_send; - - /** Receive closures are chained: we inject this closure as the on_done_recv - up-call on transport_op, and remember to call our on_done_recv member - after handling it. */ - grpc_closure hc_on_recv_initial_metadata; - grpc_closure hc_on_recv_trailing_metadata; - grpc_closure hc_on_complete; - grpc_closure got_slice; - grpc_closure send_done; -} call_data; - -typedef struct channel_data { - grpc_mdelem static_scheme; - grpc_mdelem user_agent; - size_t max_payload_size_for_get; -} channel_data; - -static grpc_error *client_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_metadata_batch *b) { - if (b->idx.named.status != NULL) { - if (grpc_mdelem_eq(b->idx.named.status->md, GRPC_MDELEM_STATUS_200)) { - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.status); - } else { - char *val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.status->md), - GPR_DUMP_ASCII); - char *msg; - gpr_asprintf(&msg, "Received http2 header with status: %s", val); - grpc_error *e = grpc_error_set_str( - grpc_error_set_int( - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Received http2 :status header with non-200 OK status"), - GRPC_ERROR_STR_VALUE, grpc_slice_from_copied_string(val)), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED), - GRPC_ERROR_STR_GRPC_MESSAGE, grpc_slice_from_copied_string(msg)); - gpr_free(val); - gpr_free(msg); - return e; - } - } - - if (b->idx.named.grpc_message != NULL) { - grpc_slice pct_decoded_msg = grpc_permissive_percent_decode_slice( - GRPC_MDVALUE(b->idx.named.grpc_message->md)); - if (grpc_slice_is_equivalent(pct_decoded_msg, - GRPC_MDVALUE(b->idx.named.grpc_message->md))) { - grpc_slice_unref_internal(exec_ctx, pct_decoded_msg); - } else { - grpc_metadata_batch_set_value(exec_ctx, b->idx.named.grpc_message, - pct_decoded_msg); - } - } - - if (b->idx.named.content_type != NULL) { - if (!grpc_mdelem_eq(b->idx.named.content_type->md, - GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) { - if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md), - EXPECTED_CONTENT_TYPE, - EXPECTED_CONTENT_TYPE_LENGTH) && - (GRPC_SLICE_START_PTR(GRPC_MDVALUE( - b->idx.named.content_type->md))[EXPECTED_CONTENT_TYPE_LENGTH] == - '+' || - GRPC_SLICE_START_PTR(GRPC_MDVALUE( - b->idx.named.content_type->md))[EXPECTED_CONTENT_TYPE_LENGTH] == - ';')) { - /* Although the C implementation doesn't (currently) generate them, - any custom +-suffix is explicitly valid. */ - /* TODO(klempner): We should consider preallocating common values such - as +proto or +json, or at least stashing them if we see them. */ - /* TODO(klempner): Should we be surfacing this to application code? */ - } else { - /* TODO(klempner): We're currently allowing this, but we shouldn't - see it without a proxy so log for now. */ - char *val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.content_type->md), - GPR_DUMP_ASCII); - gpr_log(GPR_INFO, "Unexpected content-type '%s'", val); - gpr_free(val); - } - } - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.content_type); - } - - return GRPC_ERROR_NONE; -} - -static void hc_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, - void *user_data, grpc_error *error) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - if (error == GRPC_ERROR_NONE) { - error = client_filter_incoming_metadata(exec_ctx, elem, - calld->recv_initial_metadata); - } else { - GRPC_ERROR_REF(error); - } - grpc_closure_run(exec_ctx, calld->on_done_recv_initial_metadata, error); -} - -static void hc_on_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, - void *user_data, grpc_error *error) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - if (error == GRPC_ERROR_NONE) { - error = client_filter_incoming_metadata(exec_ctx, elem, - calld->recv_trailing_metadata); - } else { - GRPC_ERROR_REF(error); - } - grpc_closure_run(exec_ctx, calld->on_done_recv_trailing_metadata, error); -} - -static void hc_on_complete(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_error *error) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - if (calld->payload_bytes) { - gpr_free(calld->payload_bytes); - calld->payload_bytes = NULL; - } - calld->on_complete->cb(exec_ctx, calld->on_complete->cb_arg, error); -} - -static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { - grpc_call_element *elem = elemp; - call_data *calld = elem->call_data; - grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &calld->slices); - calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error); -} - -static void remove_if_present(grpc_exec_ctx *exec_ctx, - grpc_metadata_batch *batch, - grpc_metadata_batch_callouts_index idx) { - if (batch->idx.array[idx] != NULL) { - grpc_metadata_batch_remove(exec_ctx, batch, batch->idx.array[idx]); - } -} - -static void continue_send_message(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - call_data *calld = elem->call_data; - uint8_t *wrptr = calld->payload_bytes; - while (grpc_byte_stream_next(exec_ctx, calld->send_op.send_message, - &calld->incoming_slice, ~(size_t)0, - &calld->got_slice)) { - memcpy(wrptr, GRPC_SLICE_START_PTR(calld->incoming_slice), - GRPC_SLICE_LENGTH(calld->incoming_slice)); - wrptr += GRPC_SLICE_LENGTH(calld->incoming_slice); - grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); - if (calld->send_length == calld->slices.length) { - calld->send_message_blocked = false; - break; - } - } -} - -static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { - grpc_call_element *elem = elemp; - call_data *calld = elem->call_data; - calld->send_message_blocked = false; - grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); - if (calld->send_length == calld->slices.length) { - /* Pass down the original send_message op that was blocked.*/ - grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices, - calld->send_flags); - calld->send_op.send_message = &calld->replacement_stream.base; - calld->post_send = calld->send_op.on_complete; - calld->send_op.on_complete = &calld->send_done; - grpc_call_next_op(exec_ctx, elem, &calld->send_op); - } else { - continue_send_message(exec_ctx, elem); - } -} - -static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - /* grab pointers to our data from the call element */ - call_data *calld = elem->call_data; - channel_data *channeld = elem->channel_data; - grpc_error *error; - - if (op->send_initial_metadata != NULL) { - /* Decide which HTTP VERB to use. We use GET if the request is marked - cacheable, and the operation contains both initial metadata and send - message, and the payload is below the size threshold, and all the data - for this request is immediately available. */ - grpc_mdelem method = GRPC_MDELEM_METHOD_POST; - if ((op->send_initial_metadata_flags & - GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) && - op->send_message != NULL && - op->send_message->length < channeld->max_payload_size_for_get) { - method = GRPC_MDELEM_METHOD_GET; - /* The following write to calld->send_message_blocked isn't racy with - reads in hc_start_transport_op (which deals with SEND_MESSAGE ops) because - being here means ops->send_message is not NULL, which is primarily - guarding the read there. */ - calld->send_message_blocked = true; - } else if (op->send_initial_metadata_flags & - GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) { - method = GRPC_MDELEM_METHOD_PUT; - } - - /* Attempt to read the data from send_message and create a header field. */ - if (grpc_mdelem_eq(method, GRPC_MDELEM_METHOD_GET)) { - /* allocate memory to hold the entire payload */ - calld->payload_bytes = gpr_malloc(op->send_message->length); - - /* read slices of send_message and copy into payload_bytes */ - calld->send_op = *op; - calld->send_length = op->send_message->length; - calld->send_flags = op->send_message->flags; - continue_send_message(exec_ctx, elem); - - if (calld->send_message_blocked == false) { - /* when all the send_message data is available, then modify the path - * MDELEM by appending base64 encoded query to the path */ - const int k_url_safe = 1; - const int k_multi_line = 0; - const unsigned char k_query_separator = '?'; - - grpc_slice path_slice = - GRPC_MDVALUE(op->send_initial_metadata->idx.named.path->md); - /* sum up individual component's lengths and allocate enough memory to - * hold combined path+query */ - size_t estimated_len = GRPC_SLICE_LENGTH(path_slice); - estimated_len++; /* for the '?' */ - estimated_len += grpc_base64_estimate_encoded_size( - op->send_message->length, k_url_safe, k_multi_line); - estimated_len += 1; /* for the trailing 0 */ - grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len); - - /* memcopy individual pieces into this slice */ - uint8_t *write_ptr = - (uint8_t *)GRPC_SLICE_START_PTR(path_with_query_slice); - uint8_t *original_path = (uint8_t *)GRPC_SLICE_START_PTR(path_slice); - memcpy(write_ptr, original_path, GRPC_SLICE_LENGTH(path_slice)); - write_ptr += GRPC_SLICE_LENGTH(path_slice); - - *write_ptr = k_query_separator; - write_ptr++; /* for the '?' */ - - grpc_base64_encode_core((char *)write_ptr, calld->payload_bytes, - op->send_message->length, k_url_safe, - k_multi_line); - - /* remove trailing unused memory and add trailing 0 to terminate string - */ - char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice); - /* safe to use strlen since base64_encode will always add '\0' */ - size_t path_length = strlen(t) + 1; - *(t + path_length) = '\0'; - path_with_query_slice = - grpc_slice_sub(path_with_query_slice, 0, path_length); - - /* substitute previous path with the new path+query */ - grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_PATH, path_with_query_slice); - grpc_metadata_batch *b = op->send_initial_metadata; - error = grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, - mdelem_path_and_query); - if (error != GRPC_ERROR_NONE) return error; - - calld->on_complete = op->on_complete; - op->on_complete = &calld->hc_on_complete; - op->send_message = NULL; - grpc_slice_unref_internal(exec_ctx, path_with_query_slice); - } else { - /* Not all data is available. Fall back to POST. */ - gpr_log(GPR_DEBUG, - "Request is marked Cacheable but not all data is available.\ - Falling back to POST"); - method = GRPC_MDELEM_METHOD_POST; - } - } - - remove_if_present(exec_ctx, op->send_initial_metadata, GRPC_BATCH_METHOD); - remove_if_present(exec_ctx, op->send_initial_metadata, GRPC_BATCH_SCHEME); - remove_if_present(exec_ctx, op->send_initial_metadata, GRPC_BATCH_TE); - remove_if_present(exec_ctx, op->send_initial_metadata, - GRPC_BATCH_CONTENT_TYPE); - remove_if_present(exec_ctx, op->send_initial_metadata, - GRPC_BATCH_USER_AGENT); - - /* Send : prefixed headers, which have to be before any application - layer headers. */ - error = grpc_metadata_batch_add_head(exec_ctx, op->send_initial_metadata, - &calld->method, method); - if (error != GRPC_ERROR_NONE) return error; - error = - grpc_metadata_batch_add_head(exec_ctx, op->send_initial_metadata, - &calld->scheme, channeld->static_scheme); - if (error != GRPC_ERROR_NONE) return error; - error = grpc_metadata_batch_add_tail(exec_ctx, op->send_initial_metadata, - &calld->te_trailers, - GRPC_MDELEM_TE_TRAILERS); - if (error != GRPC_ERROR_NONE) return error; - error = grpc_metadata_batch_add_tail( - exec_ctx, op->send_initial_metadata, &calld->content_type, - GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC); - if (error != GRPC_ERROR_NONE) return error; - error = grpc_metadata_batch_add_tail(exec_ctx, op->send_initial_metadata, - &calld->user_agent, - GRPC_MDELEM_REF(channeld->user_agent)); - if (error != GRPC_ERROR_NONE) return error; - } - - if (op->recv_initial_metadata != NULL) { - /* substitute our callback for the higher callback */ - calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready; - op->recv_initial_metadata_ready = &calld->hc_on_recv_initial_metadata; - } - - if (op->recv_trailing_metadata != NULL) { - /* substitute our callback for the higher callback */ - calld->recv_trailing_metadata = op->recv_trailing_metadata; - calld->on_done_recv_trailing_metadata = op->on_complete; - op->on_complete = &calld->hc_on_recv_trailing_metadata; - } - - return GRPC_ERROR_NONE; -} - -static void hc_start_transport_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - GPR_TIMER_BEGIN("hc_start_transport_op", 0); - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - grpc_error *error = hc_mutate_op(exec_ctx, elem, op); - if (error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); - } else { - call_data *calld = elem->call_data; - if (op->send_message != NULL && calld->send_message_blocked) { - /* Don't forward the op. send_message contains slices that aren't ready - yet. The call will be forwarded by the op_complete of slice read call. - */ - } else { - grpc_call_next_op(exec_ctx, elem, op); - } - } - GPR_TIMER_END("hc_start_transport_op", 0); -} - -/* Constructor for call_data */ -static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - const grpc_call_element_args *args) { - call_data *calld = elem->call_data; - calld->on_done_recv_initial_metadata = NULL; - calld->on_done_recv_trailing_metadata = NULL; - calld->on_complete = NULL; - calld->payload_bytes = NULL; - calld->send_message_blocked = false; - grpc_slice_buffer_init(&calld->slices); - grpc_closure_init(&calld->hc_on_recv_initial_metadata, - hc_on_recv_initial_metadata, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->hc_on_recv_trailing_metadata, - hc_on_recv_trailing_metadata, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->hc_on_complete, hc_on_complete, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->got_slice, got_slice, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->send_done, send_done, elem, - grpc_schedule_on_exec_ctx); - return GRPC_ERROR_NONE; -} - -/* Destructor for call_data */ -static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const grpc_call_final_info *final_info, - grpc_closure *ignored) { - call_data *calld = elem->call_data; - grpc_slice_buffer_destroy_internal(exec_ctx, &calld->slices); -} - -static grpc_mdelem scheme_from_args(const grpc_channel_args *args) { - unsigned i; - size_t j; - grpc_mdelem valid_schemes[] = {GRPC_MDELEM_SCHEME_HTTP, - GRPC_MDELEM_SCHEME_HTTPS}; - if (args != NULL) { - for (i = 0; i < args->num_args; ++i) { - if (args->args[i].type == GRPC_ARG_STRING && - strcmp(args->args[i].key, GRPC_ARG_HTTP2_SCHEME) == 0) { - for (j = 0; j < GPR_ARRAY_SIZE(valid_schemes); j++) { - if (0 == grpc_slice_str_cmp(GRPC_MDVALUE(valid_schemes[j]), - args->args[i].value.string)) { - return valid_schemes[j]; - } - } - } - } - } - return GRPC_MDELEM_SCHEME_HTTP; -} - -static size_t max_payload_size_from_args(const grpc_channel_args *args) { - if (args != NULL) { - for (size_t i = 0; i < args->num_args; ++i) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET)) { - if (args->args[i].type != GRPC_ARG_INTEGER) { - gpr_log(GPR_ERROR, "%s: must be an integer", - GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET); - } else { - return (size_t)args->args[i].value.integer; - } - } - } - } - return kMaxPayloadSizeForGet; -} - -static grpc_slice user_agent_from_args(const grpc_channel_args *args, - const char *transport_name) { - gpr_strvec v; - size_t i; - int is_first = 1; - char *tmp; - grpc_slice result; - - gpr_strvec_init(&v); - - for (i = 0; args && i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_PRIMARY_USER_AGENT_STRING)) { - if (args->args[i].type != GRPC_ARG_STRING) { - gpr_log(GPR_ERROR, "Channel argument '%s' should be a string", - GRPC_ARG_PRIMARY_USER_AGENT_STRING); - } else { - if (!is_first) gpr_strvec_add(&v, gpr_strdup(" ")); - is_first = 0; - gpr_strvec_add(&v, gpr_strdup(args->args[i].value.string)); - } - } - } - - gpr_asprintf(&tmp, "%sgrpc-c/%s (%s; %s; %s)", is_first ? "" : " ", - grpc_version_string(), GPR_PLATFORM_STRING, transport_name, - grpc_g_stands_for()); - is_first = 0; - gpr_strvec_add(&v, tmp); - - for (i = 0; args && i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_SECONDARY_USER_AGENT_STRING)) { - if (args->args[i].type != GRPC_ARG_STRING) { - gpr_log(GPR_ERROR, "Channel argument '%s' should be a string", - GRPC_ARG_SECONDARY_USER_AGENT_STRING); - } else { - if (!is_first) gpr_strvec_add(&v, gpr_strdup(" ")); - is_first = 0; - gpr_strvec_add(&v, gpr_strdup(args->args[i].value.string)); - } - } - } - - tmp = gpr_strvec_flatten(&v, NULL); - gpr_strvec_destroy(&v); - result = grpc_slice_intern(grpc_slice_from_static_string(tmp)); - gpr_free(tmp); - - return result; -} - -/* Constructor for channel_data */ -static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, - grpc_channel_element_args *args) { - channel_data *chand = elem->channel_data; - GPR_ASSERT(!args->is_last); - GPR_ASSERT(args->optional_transport != NULL); - chand->static_scheme = scheme_from_args(args->channel_args); - chand->max_payload_size_for_get = - max_payload_size_from_args(args->channel_args); - chand->user_agent = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_USER_AGENT, - user_agent_from_args(args->channel_args, - args->optional_transport->vtable->name)); - return GRPC_ERROR_NONE; -} - -/* Destructor for channel data */ -static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem) { - channel_data *chand = elem->channel_data; - GRPC_MDELEM_UNREF(exec_ctx, chand->user_agent); -} - -const grpc_channel_filter grpc_http_client_filter = { - hc_start_transport_op, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "http-client"}; diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c deleted file mode 100644 index df0f95010b..0000000000 --- a/src/core/lib/channel/http_server_filter.c +++ /dev/null @@ -1,429 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/channel/http_server_filter.h" - -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <string.h> -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/slice/b64.h" -#include "src/core/lib/slice/percent_encoding.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/transport/static_metadata.h" - -#define EXPECTED_CONTENT_TYPE "application/grpc" -#define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 - -extern int grpc_http_trace; - -typedef struct call_data { - grpc_linked_mdelem status; - grpc_linked_mdelem content_type; - - /* did this request come with path query containing request payload */ - bool seen_path_with_query; - /* flag to ensure payload_bin is delivered only once */ - bool payload_bin_delivered; - - grpc_metadata_batch *recv_initial_metadata; - bool *recv_idempotent_request; - bool *recv_cacheable_request; - /** Closure to call when finished with the hs_on_recv hook */ - grpc_closure *on_done_recv; - /** Closure to call when we retrieve read message from the path URI - */ - grpc_closure *recv_message_ready; - grpc_closure *on_complete; - grpc_byte_stream **pp_recv_message; - grpc_slice_buffer read_slice_buffer; - grpc_slice_buffer_stream read_stream; - - /** Receive closures are chained: we inject this closure as the on_done_recv - up-call on transport_op, and remember to call our on_done_recv member - after handling it. */ - grpc_closure hs_on_recv; - grpc_closure hs_on_complete; - grpc_closure hs_recv_message_ready; -} call_data; - -typedef struct channel_data { uint8_t unused; } channel_data; - -static grpc_error *server_filter_outgoing_metadata(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_metadata_batch *b) { - if (b->idx.named.grpc_message != NULL) { - grpc_slice pct_encoded_msg = grpc_percent_encode_slice( - GRPC_MDVALUE(b->idx.named.grpc_message->md), - grpc_compatible_percent_encoding_unreserved_bytes); - if (grpc_slice_is_equivalent(pct_encoded_msg, - GRPC_MDVALUE(b->idx.named.grpc_message->md))) { - grpc_slice_unref_internal(exec_ctx, pct_encoded_msg); - } else { - grpc_metadata_batch_set_value(exec_ctx, b->idx.named.grpc_message, - pct_encoded_msg); - } - } - return GRPC_ERROR_NONE; -} - -static void add_error(const char *error_name, grpc_error **cumulative, - grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; - if (*cumulative == GRPC_ERROR_NONE) { - *cumulative = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_name); - } - *cumulative = grpc_error_add_child(*cumulative, new); -} - -static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_metadata_batch *b) { - call_data *calld = elem->call_data; - grpc_error *error = GRPC_ERROR_NONE; - static const char *error_name = "Failed processing incoming headers"; - - if (b->idx.named.method != NULL) { - if (grpc_mdelem_eq(b->idx.named.method->md, GRPC_MDELEM_METHOD_POST)) { - *calld->recv_idempotent_request = false; - *calld->recv_cacheable_request = false; - } else if (grpc_mdelem_eq(b->idx.named.method->md, - GRPC_MDELEM_METHOD_PUT)) { - *calld->recv_idempotent_request = true; - } else if (grpc_mdelem_eq(b->idx.named.method->md, - GRPC_MDELEM_METHOD_GET)) { - *calld->recv_cacheable_request = true; - } else { - add_error(error_name, &error, - grpc_attach_md_to_error( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"), - b->idx.named.method->md)); - } - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.method); - } else { - add_error( - error_name, &error, - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), - GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":method"))); - } - - if (b->idx.named.te != NULL) { - if (!grpc_mdelem_eq(b->idx.named.te->md, GRPC_MDELEM_TE_TRAILERS)) { - add_error(error_name, &error, - grpc_attach_md_to_error( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"), - b->idx.named.te->md)); - } - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.te); - } else { - add_error(error_name, &error, - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), - GRPC_ERROR_STR_KEY, grpc_slice_from_static_string("te"))); - } - - if (b->idx.named.scheme != NULL) { - if (!grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTP) && - !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTPS) && - !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_GRPC)) { - add_error(error_name, &error, - grpc_attach_md_to_error( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"), - b->idx.named.scheme->md)); - } - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.scheme); - } else { - add_error( - error_name, &error, - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), - GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":scheme"))); - } - - if (b->idx.named.content_type != NULL) { - if (!grpc_mdelem_eq(b->idx.named.content_type->md, - GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) { - if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md), - EXPECTED_CONTENT_TYPE, - EXPECTED_CONTENT_TYPE_LENGTH) && - (GRPC_SLICE_START_PTR(GRPC_MDVALUE( - b->idx.named.content_type->md))[EXPECTED_CONTENT_TYPE_LENGTH] == - '+' || - GRPC_SLICE_START_PTR(GRPC_MDVALUE( - b->idx.named.content_type->md))[EXPECTED_CONTENT_TYPE_LENGTH] == - ';')) { - /* Although the C implementation doesn't (currently) generate them, - any custom +-suffix is explicitly valid. */ - /* TODO(klempner): We should consider preallocating common values such - as +proto or +json, or at least stashing them if we see them. */ - /* TODO(klempner): Should we be surfacing this to application code? */ - } else { - /* TODO(klempner): We're currently allowing this, but we shouldn't - see it without a proxy so log for now. */ - char *val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.content_type->md), - GPR_DUMP_ASCII); - gpr_log(GPR_INFO, "Unexpected content-type '%s'", val); - gpr_free(val); - } - } - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.content_type); - } - - if (b->idx.named.path == NULL) { - add_error(error_name, &error, - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), - GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path"))); - } else if (*calld->recv_cacheable_request == true) { - /* We have a cacheable request made with GET verb. The path contains the - * query parameter which is base64 encoded request payload. */ - const char k_query_separator = '?'; - grpc_slice path_slice = GRPC_MDVALUE(b->idx.named.path->md); - uint8_t *path_ptr = (uint8_t *)GRPC_SLICE_START_PTR(path_slice); - size_t path_length = GRPC_SLICE_LENGTH(path_slice); - /* offset of the character '?' */ - size_t offset = 0; - for (offset = 0; offset < path_length && *path_ptr != k_query_separator; - path_ptr++, offset++) - ; - if (offset < path_length) { - grpc_slice query_slice = - grpc_slice_sub(path_slice, offset + 1, path_length); - - /* substitute path metadata with just the path (not query) */ - grpc_mdelem mdelem_path_without_query = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_PATH, grpc_slice_sub(path_slice, 0, offset)); - - grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, - mdelem_path_without_query); - - /* decode payload from query and add to the slice buffer to be returned */ - const int k_url_safe = 1; - grpc_slice_buffer_add( - &calld->read_slice_buffer, - grpc_base64_decode(exec_ctx, - (const char *)GRPC_SLICE_START_PTR(query_slice), - k_url_safe)); - grpc_slice_buffer_stream_init(&calld->read_stream, - &calld->read_slice_buffer, 0); - calld->seen_path_with_query = true; - grpc_slice_unref_internal(exec_ctx, query_slice); - } else { - gpr_log(GPR_ERROR, "GET request without QUERY"); - } - } - - if (b->idx.named.host != NULL && b->idx.named.authority == NULL) { - grpc_linked_mdelem *el = b->idx.named.host; - grpc_mdelem md = GRPC_MDELEM_REF(el->md); - grpc_metadata_batch_remove(exec_ctx, b, el); - add_error( - error_name, &error, - grpc_metadata_batch_add_head( - exec_ctx, b, el, grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_ref_internal(GRPC_MDVALUE(md))))); - GRPC_MDELEM_UNREF(exec_ctx, md); - } - - if (b->idx.named.authority == NULL) { - add_error( - error_name, &error, - grpc_error_set_str( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), - GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":authority"))); - } - - return error; -} - -static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_error *err) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - if (err == GRPC_ERROR_NONE) { - err = server_filter_incoming_metadata(exec_ctx, elem, - calld->recv_initial_metadata); - } else { - GRPC_ERROR_REF(err); - } - grpc_closure_run(exec_ctx, calld->on_done_recv, err); -} - -static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_error *err) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - /* Call recv_message_ready if we got the payload via the path field */ - if (calld->seen_path_with_query && calld->recv_message_ready != NULL) { - *calld->pp_recv_message = calld->payload_bin_delivered - ? NULL - : (grpc_byte_stream *)&calld->read_stream; - grpc_closure_run(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err)); - calld->recv_message_ready = NULL; - calld->payload_bin_delivered = true; - } - grpc_closure_run(exec_ctx, calld->on_complete, GRPC_ERROR_REF(err)); -} - -static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_error *err) { - grpc_call_element *elem = user_data; - call_data *calld = elem->call_data; - if (calld->seen_path_with_query) { - /* do nothing. This is probably a GET request, and payload will be returned - in hs_on_complete callback. */ - } else { - grpc_closure_run(exec_ctx, calld->recv_message_ready, GRPC_ERROR_REF(err)); - } -} - -static void hs_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op) { - /* grab pointers to our data from the call element */ - call_data *calld = elem->call_data; - - if (op->send_initial_metadata != NULL) { - grpc_error *error = GRPC_ERROR_NONE; - static const char *error_name = "Failed sending initial metadata"; - add_error(error_name, &error, grpc_metadata_batch_add_head( - exec_ctx, op->send_initial_metadata, - &calld->status, GRPC_MDELEM_STATUS_200)); - add_error(error_name, &error, - grpc_metadata_batch_add_tail( - exec_ctx, op->send_initial_metadata, &calld->content_type, - GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)); - add_error(error_name, &error, - server_filter_outgoing_metadata(exec_ctx, elem, - op->send_initial_metadata)); - if (error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); - return; - } - } - - if (op->recv_initial_metadata) { - /* substitute our callback for the higher callback */ - GPR_ASSERT(op->recv_idempotent_request != NULL); - GPR_ASSERT(op->recv_cacheable_request != NULL); - calld->recv_initial_metadata = op->recv_initial_metadata; - calld->recv_idempotent_request = op->recv_idempotent_request; - calld->recv_cacheable_request = op->recv_cacheable_request; - calld->on_done_recv = op->recv_initial_metadata_ready; - op->recv_initial_metadata_ready = &calld->hs_on_recv; - } - - if (op->recv_message) { - calld->recv_message_ready = op->recv_message_ready; - calld->pp_recv_message = op->recv_message; - if (op->recv_message_ready) { - op->recv_message_ready = &calld->hs_recv_message_ready; - } - if (op->on_complete) { - calld->on_complete = op->on_complete; - op->on_complete = &calld->hs_on_complete; - } - } - - if (op->send_trailing_metadata) { - grpc_error *error = server_filter_outgoing_metadata( - exec_ctx, elem, op->send_trailing_metadata); - if (error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); - return; - } - } -} - -static void hs_start_transport_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - GPR_TIMER_BEGIN("hs_start_transport_op", 0); - hs_mutate_op(exec_ctx, elem, op); - grpc_call_next_op(exec_ctx, elem, op); - GPR_TIMER_END("hs_start_transport_op", 0); -} - -/* Constructor for call_data */ -static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - const grpc_call_element_args *args) { - /* grab pointers to our data from the call element */ - call_data *calld = elem->call_data; - /* initialize members */ - grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->hs_on_complete, hs_on_complete, elem, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&calld->hs_recv_message_ready, hs_recv_message_ready, elem, - grpc_schedule_on_exec_ctx); - grpc_slice_buffer_init(&calld->read_slice_buffer); - return GRPC_ERROR_NONE; -} - -/* Destructor for call_data */ -static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const grpc_call_final_info *final_info, - grpc_closure *ignored) { - call_data *calld = elem->call_data; - grpc_slice_buffer_destroy_internal(exec_ctx, &calld->read_slice_buffer); -} - -/* Constructor for channel_data */ -static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, - grpc_channel_element_args *args) { - GPR_ASSERT(!args->is_last); - return GRPC_ERROR_NONE; -} - -/* Destructor for channel data */ -static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem) {} - -const grpc_channel_filter grpc_http_server_filter = { - hs_start_transport_op, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "http-server"}; diff --git a/src/core/lib/channel/max_age_filter.c b/src/core/lib/channel/max_age_filter.c deleted file mode 100644 index c25481486c..0000000000 --- a/src/core/lib/channel/max_age_filter.c +++ /dev/null @@ -1,386 +0,0 @@ -/* - * - * Copyright 2017, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/channel/message_size_filter.h" - -#include <limits.h> -#include <string.h> - -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/transport/http2_errors.h" -#include "src/core/lib/transport/service_config.h" - -#define DEFAULT_MAX_CONNECTION_AGE_MS INT_MAX -#define DEFAULT_MAX_CONNECTION_AGE_GRACE_MS INT_MAX -#define DEFAULT_MAX_CONNECTION_IDLE_MS INT_MAX - -typedef struct channel_data { - /* We take a reference to the channel stack for the timer callback */ - grpc_channel_stack* channel_stack; - /* Guards access to max_age_timer, max_age_timer_pending, max_age_grace_timer - and max_age_grace_timer_pending */ - gpr_mu max_age_timer_mu; - /* True if the max_age timer callback is currently pending */ - bool max_age_timer_pending; - /* True if the max_age_grace timer callback is currently pending */ - bool max_age_grace_timer_pending; - /* The timer for checking if the channel has reached its max age */ - grpc_timer max_age_timer; - /* The timer for checking if the max-aged channel has uesed up the grace - period */ - grpc_timer max_age_grace_timer; - /* The timer for checking if the channel's idle duration reaches - max_connection_idle */ - grpc_timer max_idle_timer; - /* Allowed max time a channel may have no outstanding rpcs */ - gpr_timespec max_connection_idle; - /* Allowed max time a channel may exist */ - gpr_timespec max_connection_age; - /* Allowed grace period after the channel reaches its max age */ - gpr_timespec max_connection_age_grace; - /* Closure to run when the channel's idle duration reaches max_connection_idle - and should be closed gracefully */ - grpc_closure close_max_idle_channel; - /* Closure to run when the channel reaches its max age and should be closed - gracefully */ - grpc_closure close_max_age_channel; - /* Closure to run the channel uses up its max age grace time and should be - closed forcibly */ - grpc_closure force_close_max_age_channel; - /* Closure to run when the init fo channel stack is done and the max_idle - timer should be started */ - grpc_closure start_max_idle_timer_after_init; - /* Closure to run when the init fo channel stack is done and the max_age timer - should be started */ - grpc_closure start_max_age_timer_after_init; - /* Closure to run when the goaway op is finished and the max_age_timer */ - grpc_closure start_max_age_grace_timer_after_goaway_op; - /* Closure to run when the channel connectivity state changes */ - grpc_closure channel_connectivity_changed; - /* Records the current connectivity state */ - grpc_connectivity_state connectivity_state; - /* Number of active calls */ - gpr_atm call_count; -} channel_data; - -/* Increase the nubmer of active calls. Before the increasement, if there are no - calls, the max_idle_timer should be cancelled. */ -static void increase_call_count(grpc_exec_ctx* exec_ctx, channel_data* chand) { - if (gpr_atm_full_fetch_add(&chand->call_count, 1) == 0) { - grpc_timer_cancel(exec_ctx, &chand->max_idle_timer); - } -} - -/* Decrease the nubmer of active calls. After the decrement, if there are no - calls, the max_idle_timer should be started. */ -static void decrease_call_count(grpc_exec_ctx* exec_ctx, channel_data* chand) { - if (gpr_atm_full_fetch_add(&chand->call_count, -1) == 1) { - GRPC_CHANNEL_STACK_REF(chand->channel_stack, "max_age max_idle_timer"); - grpc_timer_init( - exec_ctx, &chand->max_idle_timer, - gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), chand->max_connection_idle), - &chand->close_max_idle_channel, gpr_now(GPR_CLOCK_MONOTONIC)); - } -} - -static void start_max_idle_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - /* Decrease call_count. If there are no active calls at this time, - max_idle_timer will start here. If the number of active calls is not 0, - max_idle_timer will start after all the active calls end. */ - decrease_call_count(exec_ctx, chand); - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age start_max_idle_timer_after_init"); -} - -static void start_max_age_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - gpr_mu_lock(&chand->max_age_timer_mu); - chand->max_age_timer_pending = true; - GRPC_CHANNEL_STACK_REF(chand->channel_stack, "max_age max_age_timer"); - grpc_timer_init( - exec_ctx, &chand->max_age_timer, - gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), chand->max_connection_age), - &chand->close_max_age_channel, gpr_now(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&chand->max_age_timer_mu); - grpc_transport_op* op = grpc_make_transport_op(NULL); - op->on_connectivity_state_change = &chand->channel_connectivity_changed, - op->connectivity_state = &chand->connectivity_state; - grpc_channel_next_op(exec_ctx, - grpc_channel_stack_element(chand->channel_stack, 0), op); - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age start_max_age_timer_after_init"); -} - -static void start_max_age_grace_timer_after_goaway_op(grpc_exec_ctx* exec_ctx, - void* arg, - grpc_error* error) { - channel_data* chand = arg; - gpr_mu_lock(&chand->max_age_timer_mu); - chand->max_age_grace_timer_pending = true; - GRPC_CHANNEL_STACK_REF(chand->channel_stack, "max_age max_age_grace_timer"); - grpc_timer_init(exec_ctx, &chand->max_age_grace_timer, - gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - chand->max_connection_age_grace), - &chand->force_close_max_age_channel, - gpr_now(GPR_CLOCK_MONOTONIC)); - gpr_mu_unlock(&chand->max_age_timer_mu); - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age start_max_age_grace_timer_after_goaway_op"); -} - -static void close_max_idle_channel(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - gpr_atm_no_barrier_fetch_add(&chand->call_count, 1); - if (error == GRPC_ERROR_NONE) { - grpc_transport_op* op = grpc_make_transport_op(NULL); - op->goaway_error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("max_idle"), - GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_NO_ERROR); - grpc_channel_element* elem = - grpc_channel_stack_element(chand->channel_stack, 0); - elem->filter->start_transport_op(exec_ctx, elem, op); - } else if (error != GRPC_ERROR_CANCELLED) { - GRPC_LOG_IF_ERROR("close_max_idle_channel", error); - } - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age max_idle_timer"); -} - -static void close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - gpr_mu_lock(&chand->max_age_timer_mu); - chand->max_age_timer_pending = false; - gpr_mu_unlock(&chand->max_age_timer_mu); - if (error == GRPC_ERROR_NONE) { - GRPC_CHANNEL_STACK_REF(chand->channel_stack, - "max_age start_max_age_grace_timer_after_goaway_op"); - grpc_transport_op* op = grpc_make_transport_op( - &chand->start_max_age_grace_timer_after_goaway_op); - op->goaway_error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("max_age"), - GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_NO_ERROR); - grpc_channel_element* elem = - grpc_channel_stack_element(chand->channel_stack, 0); - elem->filter->start_transport_op(exec_ctx, elem, op); - } else if (error != GRPC_ERROR_CANCELLED) { - GRPC_LOG_IF_ERROR("close_max_age_channel", error); - } - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age max_age_timer"); -} - -static void force_close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - gpr_mu_lock(&chand->max_age_timer_mu); - chand->max_age_grace_timer_pending = false; - gpr_mu_unlock(&chand->max_age_timer_mu); - if (error == GRPC_ERROR_NONE) { - grpc_transport_op* op = grpc_make_transport_op(NULL); - op->disconnect_with_error = - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel reaches max age"); - grpc_channel_element* elem = - grpc_channel_stack_element(chand->channel_stack, 0); - elem->filter->start_transport_op(exec_ctx, elem, op); - } else if (error != GRPC_ERROR_CANCELLED) { - GRPC_LOG_IF_ERROR("force_close_max_age_channel", error); - } - GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->channel_stack, - "max_age max_age_grace_timer"); -} - -static void channel_connectivity_changed(grpc_exec_ctx* exec_ctx, void* arg, - grpc_error* error) { - channel_data* chand = arg; - if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) { - grpc_transport_op* op = grpc_make_transport_op(NULL); - op->on_connectivity_state_change = &chand->channel_connectivity_changed, - op->connectivity_state = &chand->connectivity_state; - grpc_channel_next_op( - exec_ctx, grpc_channel_stack_element(chand->channel_stack, 0), op); - } else { - gpr_mu_lock(&chand->max_age_timer_mu); - if (chand->max_age_timer_pending) { - grpc_timer_cancel(exec_ctx, &chand->max_age_timer); - chand->max_age_timer_pending = false; - } - if (chand->max_age_grace_timer_pending) { - grpc_timer_cancel(exec_ctx, &chand->max_age_grace_timer); - chand->max_age_grace_timer_pending = false; - } - gpr_mu_unlock(&chand->max_age_timer_mu); - /* If there are no active calls, this increasement will cancel - max_idle_timer, and prevent max_idle_timer from being started in the - future. */ - increase_call_count(exec_ctx, chand); - } -} - -/* Constructor for call_data. */ -static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - const grpc_call_element_args* args) { - channel_data* chand = elem->channel_data; - increase_call_count(exec_ctx, chand); - return GRPC_ERROR_NONE; -} - -/* Destructor for call_data. */ -static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - const grpc_call_final_info* final_info, - grpc_closure* ignored) { - channel_data* chand = elem->channel_data; - decrease_call_count(exec_ctx, chand); -} - -/* Constructor for channel_data. */ -static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem, - grpc_channel_element_args* args) { - channel_data* chand = elem->channel_data; - gpr_mu_init(&chand->max_age_timer_mu); - chand->max_age_timer_pending = false; - chand->max_age_grace_timer_pending = false; - chand->channel_stack = args->channel_stack; - chand->max_connection_age = - DEFAULT_MAX_CONNECTION_AGE_MS == INT_MAX - ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(DEFAULT_MAX_CONNECTION_AGE_MS, GPR_TIMESPAN); - chand->max_connection_age_grace = - DEFAULT_MAX_CONNECTION_AGE_GRACE_MS == INT_MAX - ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(DEFAULT_MAX_CONNECTION_AGE_GRACE_MS, - GPR_TIMESPAN); - chand->max_connection_idle = - DEFAULT_MAX_CONNECTION_IDLE_MS == INT_MAX - ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(DEFAULT_MAX_CONNECTION_IDLE_MS, GPR_TIMESPAN); - for (size_t i = 0; i < args->channel_args->num_args; ++i) { - if (0 == strcmp(args->channel_args->args[i].key, - GRPC_ARG_MAX_CONNECTION_AGE_MS)) { - const int value = grpc_channel_arg_get_integer( - &args->channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_CONNECTION_AGE_MS, 1, INT_MAX}); - chand->max_connection_age = - value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(value, GPR_TIMESPAN); - } else if (0 == strcmp(args->channel_args->args[i].key, - GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS)) { - const int value = grpc_channel_arg_get_integer( - &args->channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_CONNECTION_AGE_GRACE_MS, 0, - INT_MAX}); - chand->max_connection_age_grace = - value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(value, GPR_TIMESPAN); - } else if (0 == strcmp(args->channel_args->args[i].key, - GRPC_ARG_MAX_CONNECTION_IDLE_MS)) { - const int value = grpc_channel_arg_get_integer( - &args->channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_CONNECTION_IDLE_MS, 1, INT_MAX}); - chand->max_connection_idle = - value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(value, GPR_TIMESPAN); - } - } - grpc_closure_init(&chand->close_max_idle_channel, close_max_idle_channel, - chand, grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->close_max_age_channel, close_max_age_channel, chand, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->force_close_max_age_channel, - force_close_max_age_channel, chand, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->start_max_idle_timer_after_init, - start_max_idle_timer_after_init, chand, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->start_max_age_timer_after_init, - start_max_age_timer_after_init, chand, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->start_max_age_grace_timer_after_goaway_op, - start_max_age_grace_timer_after_goaway_op, chand, - grpc_schedule_on_exec_ctx); - grpc_closure_init(&chand->channel_connectivity_changed, - channel_connectivity_changed, chand, - grpc_schedule_on_exec_ctx); - - if (gpr_time_cmp(chand->max_connection_age, gpr_inf_future(GPR_TIMESPAN)) != - 0) { - /* When the channel reaches its max age, we send down an op with - goaway_error set. However, we can't send down any ops until after the - channel stack is fully initialized. If we start the timer here, we have - no guarantee that the timer won't pop before channel stack initialization - is finished. To avoid that problem, we create a closure to start the - timer, and we schedule that closure to be run after call stack - initialization is done. */ - GRPC_CHANNEL_STACK_REF(chand->channel_stack, - "max_age start_max_age_timer_after_init"); - grpc_closure_sched(exec_ctx, &chand->start_max_age_timer_after_init, - GRPC_ERROR_NONE); - } - - /* Initialize the number of calls as 1, so that the max_idle_timer will not - start until start_max_idle_timer_after_init is invoked. */ - gpr_atm_rel_store(&chand->call_count, 1); - if (gpr_time_cmp(chand->max_connection_idle, gpr_inf_future(GPR_TIMESPAN)) != - 0) { - GRPC_CHANNEL_STACK_REF(chand->channel_stack, - "max_age start_max_idle_timer_after_init"); - grpc_closure_sched(exec_ctx, &chand->start_max_idle_timer_after_init, - GRPC_ERROR_NONE); - } - return GRPC_ERROR_NONE; -} - -/* Destructor for channel_data. */ -static void destroy_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem) {} - -const grpc_channel_filter grpc_max_age_filter = { - grpc_call_next_op, - grpc_channel_next_op, - 0, /* sizeof_call_data */ - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "max_age"}; diff --git a/src/core/lib/channel/max_age_filter.h b/src/core/lib/channel/max_age_filter.h deleted file mode 100644 index 93e357a88e..0000000000 --- a/src/core/lib/channel/max_age_filter.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright 2017, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#ifndef GRPC_CORE_LIB_CHANNEL_MAX_AGE_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_MAX_AGE_FILTER_H - -#include "src/core/lib/channel/channel_stack.h" - -extern const grpc_channel_filter grpc_max_age_filter; - -#endif /* GRPC_CORE_LIB_CHANNEL_MAX_AGE_FILTER_H */ diff --git a/src/core/lib/channel/message_size_filter.c b/src/core/lib/channel/message_size_filter.c deleted file mode 100644 index 63136650a5..0000000000 --- a/src/core/lib/channel/message_size_filter.c +++ /dev/null @@ -1,267 +0,0 @@ -// -// Copyright 2016, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#include "src/core/lib/channel/message_size_filter.h" - -#include <limits.h> -#include <string.h> - -#include <grpc/impl/codegen/grpc_types.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/string_util.h> - -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/transport/service_config.h" - -typedef struct message_size_limits { - int max_send_size; - int max_recv_size; -} message_size_limits; - -static void* message_size_limits_copy(void* value) { - void* new_value = gpr_malloc(sizeof(message_size_limits)); - memcpy(new_value, value, sizeof(message_size_limits)); - return new_value; -} - -static void message_size_limits_free(grpc_exec_ctx* exec_ctx, void* value) { - gpr_free(value); -} - -static const grpc_slice_hash_table_vtable message_size_limits_vtable = { - message_size_limits_free, message_size_limits_copy}; - -static void* message_size_limits_create_from_json(const grpc_json* json) { - int max_request_message_bytes = -1; - int max_response_message_bytes = -1; - for (grpc_json* field = json->child; field != NULL; field = field->next) { - if (field->key == NULL) continue; - if (strcmp(field->key, "maxRequestMessageBytes") == 0) { - if (max_request_message_bytes >= 0) return NULL; // Duplicate. - if (field->type != GRPC_JSON_STRING && field->type != GRPC_JSON_NUMBER) { - return NULL; - } - max_request_message_bytes = gpr_parse_nonnegative_int(field->value); - if (max_request_message_bytes == -1) return NULL; - } else if (strcmp(field->key, "maxResponseMessageBytes") == 0) { - if (max_response_message_bytes >= 0) return NULL; // Duplicate. - if (field->type != GRPC_JSON_STRING && field->type != GRPC_JSON_NUMBER) { - return NULL; - } - max_response_message_bytes = gpr_parse_nonnegative_int(field->value); - if (max_response_message_bytes == -1) return NULL; - } - } - message_size_limits* value = gpr_malloc(sizeof(message_size_limits)); - value->max_send_size = max_request_message_bytes; - value->max_recv_size = max_response_message_bytes; - return value; -} - -typedef struct call_data { - int max_send_size; - int max_recv_size; - // Receive closures are chained: we inject this closure as the - // recv_message_ready up-call on transport_stream_op, and remember to - // call our next_recv_message_ready member after handling it. - grpc_closure recv_message_ready; - // Used by recv_message_ready. - grpc_byte_stream** recv_message; - // Original recv_message_ready callback, invoked after our own. - grpc_closure* next_recv_message_ready; -} call_data; - -typedef struct channel_data { - int max_send_size; - int max_recv_size; - // Maps path names to message_size_limits structs. - grpc_slice_hash_table* method_limit_table; -} channel_data; - -// Callback invoked when we receive a message. Here we check the max -// receive message size. -static void recv_message_ready(grpc_exec_ctx* exec_ctx, void* user_data, - grpc_error* error) { - grpc_call_element* elem = user_data; - call_data* calld = elem->call_data; - if (*calld->recv_message != NULL && calld->max_recv_size >= 0 && - (*calld->recv_message)->length > (size_t)calld->max_recv_size) { - char* message_string; - gpr_asprintf(&message_string, - "Received message larger than max (%u vs. %d)", - (*calld->recv_message)->length, calld->max_recv_size); - grpc_error* new_error = grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INVALID_ARGUMENT); - if (error == GRPC_ERROR_NONE) { - error = new_error; - } else { - error = grpc_error_add_child(error, new_error); - GRPC_ERROR_UNREF(new_error); - } - gpr_free(message_string); - } - // Invoke the next callback. - grpc_closure_sched(exec_ctx, calld->next_recv_message_ready, error); -} - -// Start transport stream op. -static void start_transport_stream_op(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - grpc_transport_stream_op* op) { - call_data* calld = elem->call_data; - // Check max send message size. - if (op->send_message != NULL && calld->max_send_size >= 0 && - op->send_message->length > (size_t)calld->max_send_size) { - char* message_string; - gpr_asprintf(&message_string, "Sent message larger than max (%u vs. %d)", - op->send_message->length, calld->max_send_size); - grpc_transport_stream_op_finish_with_failure( - exec_ctx, op, - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string), - GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_INVALID_ARGUMENT)); - gpr_free(message_string); - return; - } - // Inject callback for receiving a message. - if (op->recv_message_ready != NULL) { - calld->next_recv_message_ready = op->recv_message_ready; - calld->recv_message = op->recv_message; - op->recv_message_ready = &calld->recv_message_ready; - } - // Chain to the next filter. - grpc_call_next_op(exec_ctx, elem, op); -} - -// Constructor for call_data. -static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, - grpc_call_element* elem, - const grpc_call_element_args* args) { - channel_data* chand = elem->channel_data; - call_data* calld = elem->call_data; - calld->next_recv_message_ready = NULL; - grpc_closure_init(&calld->recv_message_ready, recv_message_ready, elem, - grpc_schedule_on_exec_ctx); - // Get max sizes from channel data, then merge in per-method config values. - // Note: Per-method config is only available on the client, so we - // apply the max request size to the send limit and the max response - // size to the receive limit. - calld->max_send_size = chand->max_send_size; - calld->max_recv_size = chand->max_recv_size; - if (chand->method_limit_table != NULL) { - message_size_limits* limits = grpc_method_config_table_get( - exec_ctx, chand->method_limit_table, args->path); - if (limits != NULL) { - if (limits->max_send_size >= 0 && - (limits->max_send_size < calld->max_send_size || - calld->max_send_size < 0)) { - calld->max_send_size = limits->max_send_size; - } - if (limits->max_recv_size >= 0 && - (limits->max_recv_size < calld->max_recv_size || - calld->max_recv_size < 0)) { - calld->max_recv_size = limits->max_recv_size; - } - } - } - return GRPC_ERROR_NONE; -} - -// Destructor for call_data. -static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, - const grpc_call_final_info* final_info, - grpc_closure* ignored) {} - -// Constructor for channel_data. -static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem, - grpc_channel_element_args* args) { - GPR_ASSERT(!args->is_last); - channel_data* chand = elem->channel_data; - chand->max_send_size = GRPC_DEFAULT_MAX_SEND_MESSAGE_LENGTH; - chand->max_recv_size = GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH; - for (size_t i = 0; i < args->channel_args->num_args; ++i) { - if (strcmp(args->channel_args->args[i].key, - GRPC_ARG_MAX_SEND_MESSAGE_LENGTH) == 0) { - const grpc_integer_options options = { - GRPC_DEFAULT_MAX_SEND_MESSAGE_LENGTH, 0, INT_MAX}; - chand->max_send_size = - grpc_channel_arg_get_integer(&args->channel_args->args[i], options); - } - if (strcmp(args->channel_args->args[i].key, - GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH) == 0) { - const grpc_integer_options options = { - GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH, 0, INT_MAX}; - chand->max_recv_size = - grpc_channel_arg_get_integer(&args->channel_args->args[i], options); - } - } - // Get method config table from channel args. - const grpc_arg* channel_arg = - grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG); - if (channel_arg != NULL) { - GPR_ASSERT(channel_arg->type == GRPC_ARG_STRING); - grpc_service_config* service_config = - grpc_service_config_create(channel_arg->value.string); - if (service_config != NULL) { - chand->method_limit_table = - grpc_service_config_create_method_config_table( - exec_ctx, service_config, message_size_limits_create_from_json, - &message_size_limits_vtable); - grpc_service_config_destroy(service_config); - } - } - return GRPC_ERROR_NONE; -} - -// Destructor for channel_data. -static void destroy_channel_elem(grpc_exec_ctx* exec_ctx, - grpc_channel_element* elem) { - channel_data* chand = elem->channel_data; - grpc_slice_hash_table_unref(exec_ctx, chand->method_limit_table); -} - -const grpc_channel_filter grpc_message_size_filter = { - start_transport_stream_op, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - grpc_channel_next_get_info, - "message_size"}; diff --git a/src/core/lib/channel/message_size_filter.h b/src/core/lib/channel/message_size_filter.h deleted file mode 100644 index a88ff7f81a..0000000000 --- a/src/core/lib/channel/message_size_filter.h +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright 2016, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// - -#ifndef GRPC_CORE_LIB_CHANNEL_MESSAGE_SIZE_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_MESSAGE_SIZE_FILTER_H - -#include "src/core/lib/channel/channel_stack.h" - -extern const grpc_channel_filter grpc_message_size_filter; - -#endif /* GRPC_CORE_LIB_CHANNEL_MESSAGE_SIZE_FILTER_H */ diff --git a/src/core/lib/compression/message_compress.c b/src/core/lib/compression/message_compress.c index 49beb953b0..fd3d1e6fcc 100644 --- a/src/core/lib/compression/message_compress.c +++ b/src/core/lib/compression/message_compress.c @@ -50,7 +50,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs, int r; int flush; size_t i; - grpc_slice outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); + grpc_slice outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE); const uInt uint_max = ~(uInt)0; GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); @@ -65,7 +65,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs, do { if (zs->avail_out == 0) { grpc_slice_buffer_add_indexed(output, outbuf); - outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); + outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE); GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); zs->avail_out = (uInt)GRPC_SLICE_LENGTH(outbuf); zs->next_out = GRPC_SLICE_START_PTR(outbuf); diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c index c56046785b..4dfea44c57 100644 --- a/src/core/lib/debug/trace.c +++ b/src/core/lib/debug/trace.c @@ -35,24 +35,31 @@ #include <string.h> -#include <grpc/grpc.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include "src/core/lib/support/env.h" +int grpc_tracer_set_enabled(const char *name, int enabled); + typedef struct tracer { const char *name; - int *flag; + grpc_tracer_flag *flag; struct tracer *next; } tracer; static tracer *tracers; -void grpc_register_tracer(const char *name, int *flag) { +#ifdef GRPC_THREADSAFE_TRACER +#define TRACER_SET(flag, on) gpr_atm_no_barrier_store(&(flag).value, (on)) +#else +#define TRACER_SET(flag, on) (flag).value = (on) +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag) { tracer *t = gpr_malloc(sizeof(*t)); t->name = name; t->flag = flag; t->next = tracers; - *flag = 0; + TRACER_SET(*flag, false); tracers = t; } @@ -121,13 +128,13 @@ int grpc_tracer_set_enabled(const char *name, int enabled) { tracer *t; if (0 == strcmp(name, "all")) { for (t = tracers; t; t = t->next) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); } } else { int found = 0; for (t = tracers; t; t = t->next) { if (0 == strcmp(name, t->name)) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); found = 1; } } diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index 7afc38db7e..ba432574d0 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -34,9 +34,35 @@ #ifndef GRPC_CORE_LIB_DEBUG_TRACE_H #define GRPC_CORE_LIB_DEBUG_TRACE_H +#include <grpc/support/atm.h> #include <grpc/support/port_platform.h> +#include <stdbool.h> -void grpc_register_tracer(const char *name, int *flag); +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define GRPC_THREADSAFE_TRACER +#endif +#endif + +typedef struct { +#ifdef GRPC_THREADSAFE_TRACER + gpr_atm value; +#else + bool value; +#endif +} grpc_tracer_flag; + +#ifdef GRPC_THREADSAFE_TRACER +#define GRPC_TRACER_ON(flag) (gpr_atm_no_barrier_load(&(flag).value) != 0) +#define GRPC_TRACER_INITIALIZER(on) \ + { (gpr_atm)(on) } +#else +#define GRPC_TRACER_ON(flag) ((flag).value) +#define GRPC_TRACER_INITIALIZER(on) \ + { (on) } +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag); void grpc_tracer_init(const char *env_var_name); void grpc_tracer_shutdown(void); diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 453a64b049..0ac2c2ad52 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -105,7 +105,7 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, grpc_error *error) { grpc_polling_entity_del_from_pollset_set(exec_ctx, req->pollent, req->context->pollset_set); - grpc_closure_sched(exec_ctx, req->on_done, error); + grpc_closure_sched(exec_ctx, req->on_done, GRPC_ERROR_REF(error)); grpc_http_parser_destroy(&req->parser); if (req->addresses != NULL) { grpc_resolved_addresses_destroy(req->addresses); diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index fc338342e4..76946434f0 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -47,7 +47,7 @@ typedef struct { grpc_channel_security_connector base; - tsi_ssl_handshaker_factory *handshaker_factory; + tsi_ssl_client_handshaker_factory *handshaker_factory; char *secure_peer_name; } grpc_httpcli_ssl_channel_security_connector; @@ -56,7 +56,7 @@ static void httpcli_ssl_destroy(grpc_exec_ctx *exec_ctx, grpc_httpcli_ssl_channel_security_connector *c = (grpc_httpcli_ssl_channel_security_connector *)sc; if (c->handshaker_factory != NULL) { - tsi_ssl_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); } if (c->secure_peer_name != NULL) gpr_free(c->secure_peer_name); gpr_free(sc); @@ -69,7 +69,7 @@ static void httpcli_ssl_add_handshakers(grpc_exec_ctx *exec_ctx, (grpc_httpcli_ssl_channel_security_connector *)sc; tsi_handshaker *handshaker = NULL; if (c->handshaker_factory != NULL) { - tsi_result result = tsi_ssl_handshaker_factory_create_handshaker( + tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker( c->handshaker_factory, c->secure_peer_name, &handshaker); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", @@ -106,9 +106,8 @@ static grpc_security_connector_vtable httpcli_ssl_vtable = { httpcli_ssl_destroy, httpcli_ssl_check_peer}; static grpc_security_status httpcli_ssl_channel_security_connector_create( - grpc_exec_ctx *exec_ctx, const unsigned char *pem_root_certs, - size_t pem_root_certs_size, const char *secure_peer_name, - grpc_channel_security_connector **sc) { + grpc_exec_ctx *exec_ctx, const char *pem_root_certs, + const char *secure_peer_name, grpc_channel_security_connector **sc) { tsi_result result = TSI_OK; grpc_httpcli_ssl_channel_security_connector *c; @@ -126,8 +125,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create( c->secure_peer_name = gpr_strdup(secure_peer_name); } result = tsi_create_ssl_client_handshaker_factory( - NULL, 0, NULL, 0, pem_root_certs, pem_root_certs_size, NULL, NULL, NULL, - 0, &c->handshaker_factory); + NULL, pem_root_certs, NULL, NULL, 0, &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -173,10 +171,9 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint)) { grpc_channel_security_connector *sc = NULL; - const unsigned char *pem_root_certs = NULL; on_done_closure *c = gpr_malloc(sizeof(*c)); - size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); - if (pem_root_certs == NULL || pem_root_certs_size == 0) { + const char *pem_root_certs = grpc_get_default_ssl_roots(); + if (pem_root_certs == NULL) { gpr_log(GPR_ERROR, "Could not get default pem root certs."); on_done(exec_ctx, arg, NULL); gpr_free(c); @@ -186,8 +183,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, c->arg = arg; c->handshake_mgr = grpc_handshake_manager_create(); GPR_ASSERT(httpcli_ssl_channel_security_connector_create( - exec_ctx, pem_root_certs, pem_root_certs_size, host, &sc) == - GRPC_SECURITY_OK); + exec_ctx, pem_root_certs, host, &sc) == GRPC_SECURITY_OK); grpc_channel_security_connector_add_handshakers(exec_ctx, sc, c->handshake_mgr); grpc_handshake_manager_do_handshake( diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index aac506b800..a4357978e4 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -40,7 +40,7 @@ #include <grpc/support/log.h> #include <grpc/support/useful.h> -int grpc_http1_trace = 0; +grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false); static char *buf2str(void *buffer, size_t length) { char *out = gpr_malloc(length + 1); @@ -308,7 +308,7 @@ static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte, case GRPC_HTTP_FIRST_LINE: case GRPC_HTTP_HEADERS: if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) { - if (grpc_http1_trace) + if (GRPC_TRACER_ON(grpc_http1_trace)) gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded", GRPC_HTTP_PARSER_MAX_HEADER_LENGTH); return GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h index a68011dd43..a155fecf11 100644 --- a/src/core/lib/http/parser.h +++ b/src/core/lib/http/parser.h @@ -36,6 +36,7 @@ #include <grpc/slice.h> #include <grpc/support/port_platform.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/error.h" /* Maximum length of a header string of the form 'Key: Value\r\n' */ @@ -121,6 +122,6 @@ grpc_error *grpc_http_parser_eof(grpc_http_parser *parser); void grpc_http_request_destroy(grpc_http_request *request); void grpc_http_response_destroy(grpc_http_response *response); -extern int grpc_http1_trace; +extern grpc_tracer_flag grpc_http1_trace; #endif /* GRPC_CORE_LIB_HTTP_PARSER_H */ diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 6633fb68ec..8ef0b210ad 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -45,6 +45,9 @@ grpc_closure *grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb, closure->cb = cb; closure->cb_arg = cb_arg; closure->scheduler = scheduler; +#ifndef NDEBUG + closure->scheduled = false; +#endif return closure; } @@ -137,6 +140,10 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, grpc_error *error) { GPR_TIMER_BEGIN("grpc_closure_sched", 0); if (c != NULL) { +#ifndef NDEBUG + GPR_ASSERT(!c->scheduled); + c->scheduled = true; +#endif assert(c->cb); c->scheduler->vtable->sched(exec_ctx, c, error); } else { @@ -149,6 +156,10 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { grpc_closure *c = list->head; while (c != NULL) { grpc_closure *next = c->next_data.next; +#ifndef NDEBUG + GPR_ASSERT(!c->scheduled); + c->scheduled = true; +#endif assert(c->cb); c->scheduler->vtable->sched(exec_ctx, c, c->error_data.error); c = next; diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 2510d50b42..2bedbf00d6 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -99,6 +99,10 @@ struct grpc_closure { grpc_error *error; uintptr_t scratch; } error_data; + +#ifndef NDEBUG + bool scheduled; +#endif }; /** Initializes \a closure with \a cb and \a cb_arg. Returns \a closure. */ diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 2bc476bbef..863f22c614 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -42,13 +42,13 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" -int grpc_combiner_trace = 0; +grpc_tracer_flag grpc_combiner_trace = GRPC_TRACER_INITIALIZER(false); -#define GRPC_COMBINER_TRACE(fn) \ - do { \ - if (grpc_combiner_trace) { \ - fn; \ - } \ +#define GRPC_COMBINER_TRACE(fn) \ + do { \ + if (GRPC_TRACER_ON(grpc_combiner_trace)) { \ + fn; \ + } \ } while (0) #define STATE_UNORPHANED 1 @@ -319,6 +319,9 @@ bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) { GPR_TIMER_BEGIN("combiner.exec1", 0); grpc_closure *cl = (grpc_closure *)n; error_data err = unpack_error_data(cl->error_data.scratch); +#ifndef NDEBUG + cl->scheduled = false; +#endif cl->cb(exec_ctx, cl->cb_arg, err.error); if (err.covered_by_poller) { gpr_atm_no_barrier_fetch_add(&lock->elements_covered_by_poller, -1); @@ -337,6 +340,9 @@ bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) { gpr_log(GPR_DEBUG, "C:%p execute_final[%d] c=%p", lock, loops, c)); grpc_closure *next = c->next_data.next; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); c = next; diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 75dcb0b70a..6ab7a2b26b 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -37,6 +37,7 @@ #include <stddef.h> #include <grpc/support/atm.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/mpscq.h" @@ -78,6 +79,6 @@ grpc_closure_scheduler *grpc_combiner_finally_scheduler(grpc_combiner *lock, bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx); -extern int grpc_combiner_trace; +extern grpc_tracer_flag grpc_combiner_trace; #endif /* GRPC_CORE_LIB_IOMGR_COMBINER_H */ diff --git a/src/core/lib/iomgr/endpoint_pair.h b/src/core/lib/iomgr/endpoint_pair.h index f9de0c715e..6407a6ad3f 100644 --- a/src/core/lib/iomgr/endpoint_pair.h +++ b/src/core/lib/iomgr/endpoint_pair.h @@ -41,8 +41,7 @@ typedef struct { grpc_endpoint *server; } grpc_endpoint_pair; -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size); +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args); #endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H */ diff --git a/src/core/lib/iomgr/endpoint_pair_posix.c b/src/core/lib/iomgr/endpoint_pair_posix.c index b9ff969e81..5542a372d8 100644 --- a/src/core/lib/iomgr/endpoint_pair_posix.c +++ b/src/core/lib/iomgr/endpoint_pair_posix.c @@ -62,22 +62,25 @@ static void create_sockets(int sv[2]) { GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]) == GRPC_ERROR_NONE); } -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args) { int sv[2]; grpc_endpoint_pair p; char *final_name; create_sockets(sv); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_asprintf(&final_name, "%s:client", name); - p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), resource_quota, - read_slice_size, "socketpair-server"); + p.client = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], final_name), args, + "socketpair-server"); gpr_free(final_name); gpr_asprintf(&final_name, "%s:server", name); - p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), resource_quota, - read_slice_size, "socketpair-client"); + p.server = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[0], final_name), args, + "socketpair-client"); gpr_free(final_name); + + grpc_exec_ctx_finish(&exec_ctx); return p; } diff --git a/src/core/lib/iomgr/endpoint_pair_uv.c b/src/core/lib/iomgr/endpoint_pair_uv.c index ff24894c6d..9718eb0523 100644 --- a/src/core/lib/iomgr/endpoint_pair_uv.c +++ b/src/core/lib/iomgr/endpoint_pair_uv.c @@ -41,9 +41,8 @@ #include "src/core/lib/iomgr/endpoint_pair.h" -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args) { grpc_endpoint_pair endpoint_pair; // TODO(mlumish): implement this properly under libuv GPR_ASSERT(false && diff --git a/src/core/lib/iomgr/endpoint_pair_windows.c b/src/core/lib/iomgr/endpoint_pair_windows.c index 93f71b745c..25d6264dfb 100644 --- a/src/core/lib/iomgr/endpoint_pair_windows.c +++ b/src/core/lib/iomgr/endpoint_pair_windows.c @@ -83,15 +83,18 @@ static void create_sockets(SOCKET sv[2]) { } grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { + const char *name, grpc_channel_args *channel_args) { SOCKET sv[2]; grpc_endpoint_pair p; create_sockets(sv); - p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"), - resource_quota, "endpoint:server"); - p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"), - resource_quota, "endpoint:client"); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + p.client = grpc_tcp_create(&exec_ctx, + grpc_winsocket_create(sv[1], "endpoint:client"), + channel_args, "endpoint:server"); + p.server = grpc_tcp_create(&exec_ctx, + grpc_winsocket_create(sv[0], "endpoint:server"), + channel_args, "endpoint:client"); + grpc_exec_ctx_finish(&exec_ctx); return p; } diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 1dbb64e8f3..5f2c989aad 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -212,9 +212,19 @@ static uint8_t get_placement(grpc_error **err, size_t size) { GPR_ASSERT(*err); uint8_t slots = (uint8_t)(size / sizeof(intptr_t)); if ((*err)->arena_size + slots > (*err)->arena_capacity) { - (*err)->arena_capacity = (uint8_t)(3 * (*err)->arena_capacity / 2); + (*err)->arena_capacity = + (uint8_t)GPR_MIN(UINT8_MAX - 1, (3 * (*err)->arena_capacity / 2)); + if ((*err)->arena_size + slots > (*err)->arena_capacity) { + return UINT8_MAX; + } +#ifdef GRPC_ERROR_REFCOUNT_DEBUG + grpc_error *orig = *err; +#endif *err = gpr_realloc( *err, sizeof(grpc_error) + (*err)->arena_capacity * sizeof(intptr_t)); +#ifdef GRPC_ERROR_REFCOUNT_DEBUG + if (*err != orig) gpr_log(GPR_DEBUG, "realloc %p -> %p", orig, *err); +#endif } uint8_t placement = (*err)->arena_size; (*err)->arena_size = (uint8_t)((*err)->arena_size + slots); @@ -223,10 +233,14 @@ static uint8_t get_placement(grpc_error **err, size_t size) { static void internal_set_int(grpc_error **err, grpc_error_ints which, intptr_t value) { - // GPR_ASSERT((*err)->ints[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->ints[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + gpr_log(GPR_ERROR, "Error %p is full, dropping int {\"%s\":%" PRIiPTR "}", + *err, error_int_name(which), value); + return; + } } (*err)->ints[which] = slot; (*err)->arena[slot] = value; @@ -234,10 +248,16 @@ static void internal_set_int(grpc_error **err, grpc_error_ints which, static void internal_set_str(grpc_error **err, grpc_error_strs which, grpc_slice value) { - // GPR_ASSERT((*err)->strs[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->strs[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + const char *str = grpc_slice_to_c_string(value); + gpr_log(GPR_ERROR, "Error %p is full, dropping string {\"%s\":\"%s\"}", + *err, error_str_name(which), str); + gpr_free((void *)str); + return; + } } else { unref_slice(*(grpc_slice *)((*err)->arena + slot)); } @@ -245,12 +265,19 @@ static void internal_set_str(grpc_error **err, grpc_error_strs which, memcpy((*err)->arena + slot, &value, sizeof(value)); } +static char *fmt_time(gpr_timespec tm); static void internal_set_time(grpc_error **err, grpc_error_times which, gpr_timespec value) { - // GPR_ASSERT((*err)->times[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->times[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + const char *time_str = fmt_time(value); + gpr_log(GPR_ERROR, "Error %p is full, dropping \"%s\":\"%s\"}", *err, + error_time_name(which), time_str); + gpr_free((void *)time_str); + return; + } } (*err)->times[which] = slot; memcpy((*err)->arena + slot, &value, sizeof(value)); @@ -259,6 +286,12 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, static void internal_add_error(grpc_error **err, grpc_error *new) { grpc_linked_error new_last = {new, UINT8_MAX}; uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); + if (slot == UINT8_MAX) { + gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new, + grpc_error_string(new)); + GRPC_ERROR_UNREF(new); + return; + } if ((*err)->first_err == UINT8_MAX) { GPR_ASSERT((*err)->last_err == UINT8_MAX); (*err)->last_err = slot; @@ -286,7 +319,7 @@ static void internal_add_error(grpc_error **err, grpc_error *new) { // It is very common to include and extra int and string in an error #define SURPLUS_CAPACITY (2 * SLOTS_PER_INT + SLOTS_PER_TIME) -grpc_error *grpc_error_create(grpc_slice file, int line, grpc_slice desc, +grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc, grpc_error **referencing, size_t num_referencing) { GPR_TIMER_BEGIN("grpc_error_create", 0); @@ -312,7 +345,8 @@ grpc_error *grpc_error_create(grpc_slice file, int line, grpc_slice desc, memset(err->times, UINT8_MAX, GRPC_ERROR_TIME_MAX); internal_set_int(&err, GRPC_ERROR_INT_FILE_LINE, line); - internal_set_str(&err, GRPC_ERROR_STR_FILE, file); + internal_set_str(&err, GRPC_ERROR_STR_FILE, + grpc_slice_from_static_string(file)); internal_set_str(&err, GRPC_ERROR_STR_DESCRIPTION, desc); for (size_t i = 0; i < num_referencing; ++i) { @@ -729,7 +763,7 @@ grpc_error *grpc_os_error(const char *file, int line, int err, return grpc_error_set_str( grpc_error_set_str( grpc_error_set_int( - grpc_error_create(grpc_slice_from_static_string(file), line, + grpc_error_create(file, line, grpc_slice_from_static_string("OS Error"), NULL, 0), GRPC_ERROR_INT_ERRNO, err), @@ -745,7 +779,7 @@ grpc_error *grpc_wsa_error(const char *file, int line, int err, grpc_error *error = grpc_error_set_str( grpc_error_set_str( grpc_error_set_int( - grpc_error_create(grpc_slice_from_static_string(file), line, + grpc_error_create(file, line, grpc_slice_from_static_string("OS Error"), NULL, 0), GRPC_ERROR_INT_WSA_ERROR, err), diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 2a44fcfe25..34b24d9263 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -138,7 +138,7 @@ typedef enum { const char *grpc_error_string(grpc_error *error); /// Create an error - but use GRPC_ERROR_CREATE instead -grpc_error *grpc_error_create(grpc_slice file, int line, grpc_slice desc, +grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc, grpc_error **referencing, size_t num_referencing); /// Create an error (this is the preferred way of generating an error that is /// not due to a system call - for system calls, use GRPC_OS_ERROR or @@ -148,21 +148,21 @@ grpc_error *grpc_error_create(grpc_slice file, int line, grpc_slice desc, /// err = grpc_error_create(x, y, z, r, nr) is equivalent to: /// err = grpc_error_create(x, y, z, NULL, 0); /// for (i=0; i<nr; i++) err = grpc_error_add_child(err, r[i]); -#define GRPC_ERROR_CREATE_FROM_STATIC_STRING(desc) \ - grpc_error_create(grpc_slice_from_static_string(__FILE__), __LINE__, \ - grpc_slice_from_static_string(desc), NULL, 0) -#define GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc) \ - grpc_error_create(grpc_slice_from_static_string(__FILE__), __LINE__, \ - grpc_slice_from_copied_string(desc), NULL, 0) +#define GRPC_ERROR_CREATE_FROM_STATIC_STRING(desc) \ + grpc_error_create(__FILE__, __LINE__, grpc_slice_from_static_string(desc), \ + NULL, 0) +#define GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc) \ + grpc_error_create(__FILE__, __LINE__, grpc_slice_from_copied_string(desc), \ + NULL, 0) // Create an error that references some other errors. This function adds a // reference to each error in errs - it does not consume an existing reference -#define GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(desc, errs, count) \ - grpc_error_create(grpc_slice_from_static_string(__FILE__), __LINE__, \ - grpc_slice_from_static_string(desc), errs, count) -#define GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(desc, errs, count) \ - grpc_error_create(grpc_slice_from_static_string(__FILE__), __LINE__, \ - grpc_slice_from_copied_string(desc), errs, count) +#define GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(desc, errs, count) \ + grpc_error_create(__FILE__, __LINE__, grpc_slice_from_static_string(desc), \ + errs, count) +#define GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING(desc, errs, count) \ + grpc_error_create(__FILE__, __LINE__, grpc_slice_from_copied_string(desc), \ + errs, count) //#define GRPC_ERROR_REFCOUNT_DEBUG #ifdef GRPC_ERROR_REFCOUNT_DEBUG diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c new file mode 100644 index 0000000000..ad69f808cd --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -0,0 +1,984 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll1_linux.h" + +#include <assert.h> +#include <errno.h> +#include <poll.h> +#include <pthread.h> +#include <string.h> +#include <sys/epoll.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/tls.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +static grpc_wakeup_fd global_wakeup_fd; +static int g_epfd; + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + int fd; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Pollset Declarations + */ + +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; + +struct grpc_pollset_worker { + kick_state kick_state; + bool initialized_cv; + grpc_pollset_worker *next; + grpc_pollset_worker *prev; + gpr_cv cv; + grpc_closure_list schedule_on_end_work; +}; + +#define MAX_NEIGHBOURHOODS 1024 + +typedef struct pollset_neighbourhood { + gpr_mu mu; + grpc_pollset *active_root; + char pad[GPR_CACHELINE_SIZE]; +} pollset_neighbourhood; + +struct grpc_pollset { + gpr_mu mu; + pollset_neighbourhood *neighbourhood; + bool reassigning_neighbourhood; + grpc_pollset_worker *root_worker; + bool kicked_without_poller; + bool seen_inactive; + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ + int begin_refs; + + grpc_pollset *next; + grpc_pollset *prev; +}; + +/******************************************************************************* + * Pollset-set Declarations + */ + +struct grpc_pollset_set {}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + } + + new_fd->fd = fd; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), + .data.ptr = new_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { + gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); + } + + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + grpc_error *error = GRPC_ERROR_NONE; + + if (!grpc_lfev_is_shutdown(&fd->read_closure)) { + fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason)); + } + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + } + + grpc_closure_sched(exec_ctx, on_done, GRPC_ERROR_REF(error)); + + grpc_iomgr_unregister_object(&fd->iomgr_object); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + gpr_mu_unlock(&fd_freelist_mu); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + return (grpc_workqueue *)0xb0b51ed; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +/******************************************************************************* + * Pollset Definitions + */ + +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static gpr_atm g_active_poller; +static pollset_neighbourhood *g_neighbourhoods; +static size_t g_num_neighbourhoods; +static gpr_mu g_wq_mu; +static grpc_closure_list g_wq_items; + +/* Return true if first in list */ +static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { + if (pollset->root_worker == NULL) { + pollset->root_worker = worker; + worker->next = worker->prev = worker; + return true; + } else { + worker->next = pollset->root_worker; + worker->prev = worker->next->prev; + worker->next->prev = worker; + worker->prev->next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset *pollset, + grpc_pollset_worker *worker) { + if (worker == pollset->root_worker) { + if (worker == worker->next) { + pollset->root_worker = NULL; + return EMPTIED; + } else { + pollset->root_worker = worker->next; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + return NEW_ROOT; + } + } else { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + return REMOVED; + } +} + +static size_t choose_neighbourhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +} + +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + gpr_atm_no_barrier_store(&g_active_poller, 0); + global_wakeup_fd.read_fd = -1; + grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); + gpr_mu_init(&g_wq_mu); + g_wq_items = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; + if (err != GRPC_ERROR_NONE) return err; + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { + return GRPC_OS_ERROR(errno, "epoll_ctl"); + } + g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); + g_neighbourhoods = + gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_init(&g_neighbourhoods[i].mu); + } + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); + gpr_mu_destroy(&g_wq_mu); + if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_destroy(&g_neighbourhoods[i].mu); + } + gpr_free(g_neighbourhoods); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + pollset->seen_inactive = true; +} + +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + gpr_mu_lock(&pollset->mu); + if (!pollset->seen_inactive) { + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + retry_lock_neighbourhood: + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (!pollset->seen_inactive) { + if (pollset->neighbourhood != neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } + } + gpr_mu_unlock(&pollset->neighbourhood->mu); + } + gpr_mu_unlock(&pollset->mu); + gpr_mu_destroy(&pollset->mu); +} + +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->initialized_cv) { + worker->kick_state = KICKED; + gpr_cv_signal(&worker->cv); + } else { + worker->kick_state = KICKED; + append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), + "pollset_shutdown"); + } + + worker = worker->next; + } while (worker != pollset->root_worker); + } + return error; +} + +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL && + pollset->begin_refs == 0) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; + } +} + +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + pollset_maybe_finish_shutdown(exec_ctx, pollset); +} + +#define MAX_EPOLL_EVENTS 100 + +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, now) <= 0) { + return 0; + } + + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); + return millis >= 1 ? millis : 1; +} + +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r; + do { + r = epoll_wait(g_epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_move(&g_wq_items, &exec_ctx->closure_list); + gpr_mu_unlock(&g_wq_mu); + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else { + grpc_fd *fd = (grpc_fd *)(data_ptr); + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + + return error; +} + +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, + gpr_timespec deadline) { + if (worker_hdl != NULL) *worker_hdl = worker; + worker->initialized_cv = false; + worker->kick_state = UNKICKED; + worker->schedule_on_end_work = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; + pollset->begin_refs++; + + if (pollset->seen_inactive) { + // pollset has been observed to be inactive, we need to move back to the + // active list + bool is_reassigning = false; + if (!pollset->reassigning_neighbourhood) { + is_reassigning = true; + pollset->reassigning_neighbourhood = true; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + } + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + // pollset unlocked: state may change (even worker->kick_state) + retry_lock_neighbourhood: + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (pollset->seen_inactive) { + if (neighbourhood != pollset->neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } + pollset->seen_inactive = false; + if (neighbourhood->active_root == NULL) { + neighbourhood->active_root = pollset->next = pollset->prev = pollset; + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { + worker->kick_state = DESIGNATED_POLLER; + } + } else { + pollset->next = neighbourhood->active_root; + pollset->prev = pollset->next->prev; + pollset->next->prev = pollset->prev->next = pollset; + } + } + if (is_reassigning) { + GPR_ASSERT(pollset->reassigning_neighbourhood); + pollset->reassigning_neighbourhood = false; + } + gpr_mu_unlock(&neighbourhood->mu); + } + worker_insert(pollset, worker); + pollset->begin_refs--; + if (worker->kick_state == UNKICKED) { + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (worker->kick_state == UNKICKED && + pollset->shutdown_closure == NULL) { + if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && + worker->kick_state == UNKICKED) { + worker->kick_state = KICKED; + } + } + *now = gpr_now(now->clock_type); + } + + return worker->kick_state == DESIGNATED_POLLER && + pollset->shutdown_closure == NULL; +} + +static bool check_neighbourhood_for_available_poller( + pollset_neighbourhood *neighbourhood) { + bool found_worker = false; + do { + grpc_pollset *inspect = neighbourhood->active_root; + if (inspect == NULL) { + break; + } + gpr_mu_lock(&inspect->mu); + GPR_ASSERT(!inspect->seen_inactive); + grpc_pollset_worker *inspect_worker = inspect->root_worker; + if (inspect_worker != NULL) { + do { + switch (inspect_worker->kick_state) { + case UNKICKED: + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = DESIGNATED_POLLER; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + break; + case KICKED: + break; + case DESIGNATED_POLLER: + found_worker = true; // ok, so someone else found the worker, but + // we'll accept that + break; + } + inspect_worker = inspect_worker->next; + } while (inspect_worker != inspect->root_worker); + } + if (!found_worker) { + inspect->seen_inactive = true; + if (inspect == neighbourhood->active_root) { + neighbourhood->active_root = + inspect->next == inspect ? NULL : inspect->next; + } + inspect->next->prev = inspect->prev; + inspect->prev->next = inspect->next; + inspect->next = inspect->prev = NULL; + } + gpr_mu_unlock(&inspect->mu); + } while (!found_worker); + return found_worker; +} + +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (worker_hdl != NULL) *worker_hdl = NULL; + worker->kick_state = KICKED; + grpc_closure_list_move(&worker->schedule_on_end_work, + &exec_ctx->closure_list); + if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { + if (worker->next != worker && worker->next->kick_state == UNKICKED) { + GPR_ASSERT(worker->next->initialized_cv); + gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); + worker->next->kick_state = DESIGNATED_POLLER; + gpr_cv_signal(&worker->next->cv); + if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + } else { + gpr_atm_no_barrier_store(&g_active_poller, 0); + gpr_mu_unlock(&pollset->mu); + size_t poller_neighbourhood_idx = + (size_t)(pollset->neighbourhood - g_neighbourhoods); + bool found_worker = false; + bool scan_state[MAX_NEIGHBOURHOODS]; + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + found_worker = + check_neighbourhood_for_available_poller(neighbourhood); + gpr_mu_unlock(&neighbourhood->mu); + scan_state[i] = true; + } else { + scan_state[i] = false; + } + } + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + if (scan_state[i]) continue; + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + gpr_mu_lock(&neighbourhood->mu); + found_worker = check_neighbourhood_for_available_poller(neighbourhood); + gpr_mu_unlock(&neighbourhood->mu); + } + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + } else if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (EMPTIED == worker_remove(pollset, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + GPR_ASSERT(!pollset->shutdown_closure); + GPR_ASSERT(!pollset->seen_inactive); + gpr_mu_unlock(&pollset->mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), + err_desc); + gpr_mu_lock(&pollset->mu); + gpr_tls_set(&g_current_thread_worker, 0); + } + end_worker(exec_ctx, pollset, &worker, worker_hdl); + gpr_tls_set(&g_current_thread_pollset, 0); + return error; +} + +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { + grpc_pollset_worker *root_worker = pollset->root_worker; + if (root_worker == NULL) { + pollset->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } + grpc_pollset_worker *next_worker = root_worker->next; + if (root_worker == next_worker && + root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( + &g_active_poller)) { + root_worker->kick_state = KICKED; + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else if (next_worker->kick_state == UNKICKED) { + GPR_ASSERT(next_worker->initialized_cv); + next_worker->kick_state = KICKED; + gpr_cv_signal(&next_worker->cv); + return GRPC_ERROR_NONE; + } else { + return GRPC_ERROR_NONE; + } + } else { + return GRPC_ERROR_NONE; + } + } else if (specific_worker->kick_state == KICKED) { + return GRPC_ERROR_NONE; + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + specific_worker->kick_state = KICKED; + return GRPC_ERROR_NONE; + } else if (specific_worker == + (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + specific_worker->kick_state = KICKED; + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else if (specific_worker->initialized_cv) { + specific_worker->kick_state = KICKED; + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } else { + specific_worker->kick_state = KICKED; + return GRPC_ERROR_NONE; + } +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) {} + +/******************************************************************************* + * Workqueue Definitions + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) {} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) {} +#endif + +static void wq_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + // find a neighbourhood to wakeup + bool scheduled = false; + size_t initial_neighbourhood = choose_neighbourhood(); + for (size_t i = 0; !scheduled && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(initial_neighbourhood + i) % g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + if (neighbourhood->active_root != NULL) { + grpc_pollset *inspect = neighbourhood->active_root; + do { + if (gpr_mu_trylock(&inspect->mu)) { + if (inspect->root_worker != NULL) { + grpc_pollset_worker *inspect_worker = inspect->root_worker; + do { + if (inspect_worker->kick_state == UNKICKED) { + inspect_worker->kick_state = KICKED; + grpc_closure_list_append( + &inspect_worker->schedule_on_end_work, closure, error); + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + scheduled = true; + } + inspect_worker = inspect_worker->next; + } while (!scheduled && inspect_worker != inspect->root_worker); + } + gpr_mu_unlock(&inspect->mu); + } + inspect = inspect->next; + } while (!scheduled && inspect != neighbourhood->active_root); + } + gpr_mu_unlock(&neighbourhood->mu); + } + } + if (!scheduled) { + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_append(&g_wq_items, closure, error); + gpr_mu_unlock(&g_wq_mu); + GRPC_LOG_IF_ERROR("workqueue_scheduler", + grpc_wakeup_fd_wakeup(&global_wakeup_fd)); + } +} + +static const grpc_closure_scheduler_vtable + singleton_workqueue_scheduler_vtable = {wq_sched, wq_sched, + "epoll1_workqueue"}; + +static grpc_closure_scheduler singleton_workqueue_scheduler = { + &singleton_workqueue_scheduler_vtable}; + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return &singleton_workqueue_scheduler; +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + return (grpc_pollset_set *)((intptr_t)0xdeafbeef); +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) {} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + /* TODO(ctiller): temporary, until this stabilizes */ + if (!explicit_request) return NULL; + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + g_epfd = epoll_create1(EPOLL_CLOEXEC); + if (g_epfd < 0) { + gpr_log(GPR_ERROR, "epoll unavailable"); + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + close(g_epfd); + fd_global_shutdown(); + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h new file mode 100644 index 0000000000..bd52478a7c --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -0,0 +1,44 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +// a polling engine that utilizes a singleton epoll set and turnstile polling + +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c new file mode 100644 index 0000000000..d23bf6c06c --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -0,0 +1,2146 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" + +#include <assert.h> +#include <errno.h> +#include <limits.h> +#include <poll.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include <sys/epoll.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/tls.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/env.h" + +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +/* The maximum number of polling threads per polling island. By default no + limit */ +static int g_max_pollers_per_pi = INT_MAX; + +static int grpc_wakeup_signal = -1; +static bool is_grpc_wakeup_signal_initialized = false; + +/* Implements the function defined in grpc_posix.h. This function might be + * called before even calling grpc_init() to set either a different signal to + * use. If signum == -1, then the use of signals is disabled */ +static void grpc_use_signal(int signum) { + grpc_wakeup_signal = signum; + is_grpc_wakeup_signal_initialized = true; + + if (grpc_wakeup_signal < 0) { + gpr_log(GPR_INFO, + "Use of signals is disabled. Epoll engine will not be used"); + } else { + gpr_log(GPR_INFO, "epoll engine will be using signal: %d", + grpc_wakeup_signal); + } +} + +struct polling_island; + +typedef enum { + POLL_OBJ_FD, + POLL_OBJ_POLLSET, + POLL_OBJ_POLLSET_SET +} poll_obj_type; + +typedef struct poll_obj { +#ifdef PO_DEBUG + poll_obj_type obj_type; +#endif + gpr_mu mu; + struct polling_island *pi; +} poll_obj; + +static const char *poll_obj_string(poll_obj_type po_type) { + switch (po_type) { + case POLL_OBJ_FD: + return "fd"; + case POLL_OBJ_POLLSET: + return "pollset"; + case POLL_OBJ_POLLSET_SET: + return "pollset_set"; + } + + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + poll_obj po; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +typedef struct worker_node { + struct worker_node *next; + struct worker_node *prev; +} worker_node; + +/* This is also used as grpc_workqueue (by directly casing it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Pointer to the polling_island this merged into. + * merged_to value is only set once in polling_island's lifetime (and that too + * only if the island is merged with another island). Because of this, we can + * use gpr_atm type here so that we can do atomic access on this and reduce + * lock contention on 'mu' mutex. + * + * Note that if this field is not NULL (i.e not 0), all the remaining fields + * (except mu and ref_count) are invalid and must be ignored. */ + gpr_atm merged_to; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The list of workers waiting to do polling on this polling island */ + gpr_mu worker_list_mu; + worker_node worker_list_head; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +#define WORKER_FROM_WORKER_LIST_NODE(p) \ + (struct grpc_pollset_worker *)(((char *)(p)) - \ + offsetof(grpc_pollset_worker, pi_list_link)) +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; + + /* Indicates if it is this worker's turn to do epoll */ + gpr_atm is_polling_turn; + + /* Node in the polling island's worker list. */ + worker_node pi_list_link; +}; + +struct grpc_pollset { + poll_obj po; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + poll_obj po; +}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. + Note that this deletion not be done under a lock. Once the ref count goes + to zero, we are guaranteed that no one else holds a reference to the + polling island (and that there is no racing pi_add_ref() call either). + + Also, if we are deleting the polling island and the merged_to field is + non-empty, we should remove a ref to the merged_to polling island + */ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + polling_island_delete(exec_ctx, pi); + if (next != NULL) { + PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */ + } + } +} + +static void worker_node_init(worker_node *node) { + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void push_back_worker_node(worker_node *head, worker_node *node) { + node->next = head; + node->prev = head->prev; + head->prev->next = node; + head->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void remove_worker_node(worker_node *node) { + node->next->prev = node->prev; + node->prev->next = node->next; + /* If node's next and prev point to itself, the node is considered detached + * from the list*/ + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static worker_node *pop_front_worker_node(worker_node *head) { + worker_node *node = head->next; + if (node != head) { + remove_worker_node(node); + } else { + node = NULL; + } + + return node; +} + +/* Returns true if the node's next and prev are pointing to itself (which + indicates that the node is not in the list */ +static bool is_worker_node_detached(worker_node *node) { + return (node->next == node->prev && node->next == node); +} + +/* The caller is expected to hold pi->mu lock before calling this function + */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_all_fds_locked(polling_island *pi, + bool remove_fd_refs, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fds"; + + for (i = 0; i < pi->fd_cnt; i++) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " + "error: %d (%s)", + pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + if (remove_fd_refs) { + GRPC_FD_UNREF(pi->fds[i], "polling_island"); + } + } + + pi->fd_cnt = 0; +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + + gpr_mu_init(&pi->worker_list_mu); + worker_node_init(&pi->worker_list_head); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_mu_destroy(&pi->worker_list_mu); + GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); + + gpr_free(pi->fds); + gpr_free(pi); +} + +/* Attempts to gets the last polling island in the linked list (liked by the + * 'merged_to' field). Since this does not lock the polling island, there are no + * guarantees that the island returned is the last island */ +static polling_island *polling_island_maybe_get_latest(polling_island *pi) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + while (next != NULL) { + pi = next; + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + } + + return pi; +} + +/* Gets the lock on the *latest* polling island i.e the last polling island in + the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the + returned polling island's mu. + Usage: To lock/unlock polling island "pi", do the following: + polling_island *pi_latest = polling_island_lock(pi); + ... + ... critical section .. + ... + gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ +static polling_island *polling_island_lock(polling_island *pi) { + polling_island *next = NULL; + + while (true) { + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* Looks like 'pi' is the last node in the linked list but unless we check + this by holding the pi->mu lock, we cannot be sure (i.e without the + pi->mu lock, we don't prevent island merges). + To be absolutely sure, check once more by holding the pi->mu lock */ + gpr_mu_lock(&pi->mu); + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* pi is infact the last node and we have the pi->mu lock. we're done */ + break; + } + + /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu + * isn't the lock we are interested in. Continue traversing the list */ + gpr_mu_unlock(&pi->mu); + } + + pi = next; + } + + return pi; +} + +/* Gets the lock on the *latest* polling islands in the linked lists pointed by + *p and *q (and also updates *p and *q to point to the latest polling islands) + + This function is needed because calling the following block of code to obtain + locks on polling islands (*p and *q) is prone to deadlocks. + { + polling_island_lock(*p, true); + polling_island_lock(*q, true); + } + + Usage/example: + polling_island *p1; + polling_island *p2; + .. + polling_island_lock_pair(&p1, &p2); + .. + .. Critical section with both p1 and p2 locked + .. + // Release locks: Always call polling_island_unlock_pair() to release locks + polling_island_unlock_pair(p1, p2); +*/ +static void polling_island_lock_pair(polling_island **p, polling_island **q) { + polling_island *pi_1 = *p; + polling_island *pi_2 = *q; + polling_island *next_1 = NULL; + polling_island *next_2 = NULL; + + /* The algorithm is simple: + - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and + keep updating pi_1 and pi_2) + - Then obtain locks on the islands by following a lock order rule of + locking polling_island with lower address first + Special case: Before obtaining the locks, check if pi_1 and pi_2 are + pointing to the same island. If that is the case, we can just call + polling_island_lock() + - After obtaining both the locks, double check that the polling islands + are still the last polling islands in their respective linked lists + (this is because there might have been polling island merges before + we got the lock) + - If the polling islands are the last islands, we are done. If not, + release the locks and continue the process from the first step */ + while (true) { + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + while (next_1 != NULL) { + pi_1 = next_1; + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + } + + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + while (next_2 != NULL) { + pi_2 = next_2; + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + } + + if (pi_1 == pi_2) { + pi_1 = pi_2 = polling_island_lock(pi_1); + break; + } + + if (pi_1 < pi_2) { + gpr_mu_lock(&pi_1->mu); + gpr_mu_lock(&pi_2->mu); + } else { + gpr_mu_lock(&pi_2->mu); + gpr_mu_lock(&pi_1->mu); + } + + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + if (next_1 == NULL && next_2 == NULL) { + break; + } + + gpr_mu_unlock(&pi_1->mu); + gpr_mu_unlock(&pi_2->mu); + } + + *p = pi_1; + *q = pi_2; +} + +static void polling_island_unlock_pair(polling_island *p, polling_island *q) { + if (p == q) { + gpr_mu_unlock(&p->mu); + } else { + gpr_mu_unlock(&p->mu); + gpr_mu_unlock(&q->mu); + } +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_move_items_to_parent(polling_island *q) { + polling_island *p = (polling_island *)gpr_atm_no_barrier_load(&q->merged_to); + if (p == NULL) { + return; + } + gpr_mu_lock(&q->workqueue_read_mu); + int num_added = 0; + while (gpr_atm_no_barrier_load(&q->workqueue_item_count) > 0) { + gpr_mpscq_node *n = gpr_mpscq_pop(&q->workqueue_items); + if (n != NULL) { + gpr_atm_no_barrier_fetch_add(&q->workqueue_item_count, -1); + gpr_atm_no_barrier_fetch_add(&p->workqueue_item_count, 1); + gpr_mpscq_push(&p->workqueue_items, n); + num_added++; + } + } + gpr_mu_unlock(&q->workqueue_read_mu); + if (num_added > 0) { + workqueue_maybe_wakeup(p); + } + workqueue_move_items_to_parent(p); +} + +static polling_island *polling_island_merge(polling_island *p, + polling_island *q, + grpc_error **error) { + /* Get locks on both the polling islands */ + polling_island_lock_pair(&p, &q); + + if (p != q) { + /* Make sure that p points to the polling island with fewer fds than q */ + if (p->fd_cnt > q->fd_cnt) { + GPR_SWAP(polling_island *, p, q); + } + + /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q + Note that the refcounts on the fds being moved will not change here. + This is why the last param in the following two functions is 'false') */ + polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); + polling_island_remove_all_fds_locked(p, false, error); + + /* Wakeup all the pollers (if any) on p so that they pickup this change */ + polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); + + /* Add the 'merged_to' link from p --> q */ + gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); + PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ + + workqueue_move_items_to_parent(p); + } + /* else if p == q, nothing needs to be done */ + + polling_island_unlock_pair(p, q); + + /* Return the merged polling island (Note that no merge would have happened + if p == q which is ok) */ + return q; +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + workqueue_move_items_to_parent(pi); + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static __thread bool g_initialized_sigmask; +static __thread sigset_t g_orig_sigmask; +static __thread sigset_t g_wakeup_sig_set; + +static void sig_handler(int sig_num) { +#ifdef GRPC_EPOLL_DEBUG + gpr_log(GPR_INFO, "Received signal %d", sig_num); +#endif +} + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->pt_id = pthread_self(); + worker->next = worker->prev = NULL; + gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); + gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); + worker_node_init(&worker->pi_list_link); +} + +static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *worker_kick(grpc_pollset_worker *worker, + gpr_atm *is_kicked) { + grpc_error *err = GRPC_ERROR_NONE; + + /* Kick the worker only if it was not already kicked */ + if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { + GRPC_POLLING_TRACE( + "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", + (void *)worker, (long int)worker->pt_id); + int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); + if (err_num != 0) { + err = GRPC_OS_ERROR(err_num, "pthread_kill"); + } + } + return err; +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_kicked); +} + +static grpc_error *poller_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_polling_turn); +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->po.mu); + *mu = &pollset->po.mu; + pollset->po.pi = NULL; +#ifdef PO_DEBUG + pollset->po.obj_type = POLL_OBJ_POLLSET; +#endif + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; +} + +/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->po.pi != NULL) { + PI_UNREF(exec_ctx, ps->po.pi, reason); + } + ps->po.pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->po.pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->po.mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +/* NOTE: This function may modify 'now' */ +static bool acquire_polling_lease(grpc_pollset_worker *worker, + polling_island *pi, gpr_timespec deadline, + gpr_timespec *now) { + bool is_lease_acquired = false; + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + + if (num_pollers >= g_max_pollers_per_pi) { + push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + + bool is_timeout = false; + int ret; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); + if (timeout_ms == -1) { + ret = sigwaitinfo(&g_wakeup_sig_set, NULL); + } else { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + GRPC_SCHEDULING_START_BLOCKING_REGION; + ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (ret == -1) { + if (errno == EAGAIN) { + is_timeout = true; + } else { + /* NOTE: This should not happen. If we see these log messages, it means + we are most likely doing something incorrect in the setup * needed + for sigwaitinfo/sigtimedwait */ + gpr_log(GPR_ERROR, + "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, + timeout_ms); + } + } + + /* Did the worker come out of sigtimedwait due to a thread that just + exited epoll and kicking it (in release_polling_lease function). */ + bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); + + /* Did the worker come out of sigtimedwait due to a thread alerting it that + some completion event was (likely) available in the completion queue */ + bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + + if (is_kicked || is_timeout) { + *now = deadline; /* Essentially make the epoll timeout = 0 */ + } else if (is_polling_turn) { + *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ + } + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + /* The node might have already been removed from the list by the poller + that kicked this. However it is safe to call 'remove_worker_node' on + an already detached node */ + remove_worker_node(&worker->pi_list_link); + /* It is important to read the num_pollers again under the lock so that we + * have the latest num_pollers value that doesn't change while we are doing + * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ + num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + } + + if (num_pollers < g_max_pollers_per_pi) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + is_lease_acquired = true; + } + + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + return is_lease_acquired; +} + +static void release_polling_lease(polling_island *pi, grpc_error **error) { + gpr_mu_lock(&pi->worker_list_mu); + + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + worker_node *node = pop_front_worker_node(&pi->worker_list_head); + if (node != NULL) { + grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); + append_error(error, poller_kick(next_worker), "poller kick error"); + } + + gpr_mu_unlock(&pi->worker_list_mu); +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + /* Only g_max_pollers_per_pi threads can be doing polling in parallel. + If we cannot get a lease, we cannot continue to do epoll_pwait() */ + if (!acquire_polling_lease(worker, pi, deadline, &now)) { + return; + } + + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + /* timeout_ms is the time between 'now' and 'deadline' */ + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = + epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + /* Give back the lease right away so that some other thread can enter */ + release_polling_lease(pi, error); + + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + int epoll_fd = -1; + polling_island *pi = NULL; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the + latest polling island pointed by pollset->po.pi + + Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. There is however a possibility that the polling island from + which we got the epoll_fd, got merged with another island in the meantime. + This is okay because in such a case, we will wakeup right-away from + epoll_pwait() (because any merge will poison the old polling island's epoll + set 'polling_island_wakeup_fd') and then pick up the latest polling_island + the next time this function - pollset_work_and_unlock()) is called */ + + if (pollset->po.pi == NULL) { + pollset->po.pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->po.pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. Cannot continue */ + } + + PI_ADD_REF(pollset->po.pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->po.pi); + } + + pi = polling_island_maybe_get_latest(pollset->po.pi); + epoll_fd = pi->epoll_fd; + + /* Update the pollset->po.pi since the island being pointed by + pollset->po.pi maybe older than the one pointed by pi) */ + if (pollset->po.pi != pi) { + /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the + polling island to be deleted */ + PI_ADD_REF(pi, "ps"); + PI_UNREF(exec_ctx, pollset->po.pi, "ps"); + pollset->po.pi = pi; + } + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->po.mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + g_current_thread_polling_island = pi; + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, + deadline, sig_mask, error); + g_current_thread_polling_island = NULL; + } + + GPR_ASSERT(pi != NULL); + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->po.pi + that we got before releasing the polling island lock). This is because + pollset->po.pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up + (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the + worker that there is some pending work that needs immediate attention + (like an event on the completion queue, or a polling island merge that + results in a new epoll-fd to wait on) and that the worker should not + spend time waiting in epoll_pwait(). + + A worker can be kicked anytime from the point it is added to the pollset + via push_front_worker() (or push_back_worker()) to the point it is + removed via remove_worker(). + If the worker is kicked before/during it calls epoll_pwait(), it should + immediately exit from epoll_wait(). If the worker is kicked after it + returns from epoll_wait(), then nothing really needs to be done. + + To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all + times *except* when it is in epoll_pwait(). This way, the worker never + misses acting on a kick */ + + if (!g_initialized_sigmask) { + sigemptyset(&g_wakeup_sig_set); + sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); + sigdelset(&g_orig_sigmask, grpc_wakeup_signal); + g_initialized_sigmask = true; + /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. + This is the mask used at all times *except during + epoll_wait()*" + g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and + this is the mask to use *during epoll_wait()* + + The new_mask is set on the worker before it is added to the pollset + (i.e before it can be kicked) */ + } + + push_front_worker(pollset, &worker); /* Add worker to pollset */ + + pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, + &g_orig_sigmask, &error); + grpc_exec_ctx_flush(exec_ctx); + + gpr_mu_lock(&pollset->po.mu); + + /* Note: There is no need to reset worker.is_kicked to 0 since we are no + longer going to use this worker */ + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->po.mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag, + poll_obj_type bag_type, poll_obj *item, + poll_obj_type item_type) { + GPR_TIMER_BEGIN("add_poll_object", 0); + +#ifdef PO_DEBUG + GPR_ASSERT(item->obj_type == item_type); + GPR_ASSERT(bag->obj_type == bag_type); +#endif + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&bag->mu); + gpr_mu_lock(&item->mu); + +retry: + /* + * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing + * 2) If item->pi and bag->pi are both NULL, create a new polling island (with + * a refcount of 2) and point item->pi and bag->pi to the new island + * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to + * the other's non-NULL pi + * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the + * polling islands and update item->pi and bag->pi to point to the new + * island + */ + + /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already + * orphaned */ + if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + return; + } + + if (item->pi == bag->pi) { + pi_new = item->pi; + if (pi_new == NULL) { + /* GPR_ASSERT(item->pi == bag->pi == NULL) */ + + /* If we are adding an fd to a bag (i.e pollset or pollset_set), then + * we need to do some extra work to make TSAN happy */ + if (item_type == POLL_OBJ_FD) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&item->mu); + pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error); + gpr_mu_lock(&item->mu); + + /* Need to reverify any assumptions made between the initial lock and + getting to this branch: if they've changed, we need to throw away our + work and figure things out again. */ + if (item->pi != NULL) { + GRPC_POLLING_TRACE( + "add_poll_object: Raced creating new polling island. pi_new: %p " + "(fd: %d, %s: %p)", + (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), + (void *)bag); + /* No need to lock 'pi_new' here since this is a new polling island + and no one has a reference to it yet */ + polling_island_remove_all_fds_locked(pi_new, true, &error); + + /* Ref and unref so that the polling island gets deleted during unref + */ + PI_ADD_REF(pi_new, "dance_of_destruction"); + PI_UNREF(exec_ctx, pi_new, "dance_of_destruction"); + goto retry; + } + } else { + pi_new = polling_island_create(exec_ctx, NULL, &error); + } + + GRPC_POLLING_TRACE( + "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " + "%s: %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + GRPC_POLLING_TRACE( + "add_poll_object: Same polling island. pi: %p (%s, %s)", + (void *)pi_new, poll_obj_string(item_type), + poll_obj_string(bag_type)); + } + } else if (item->pi == NULL) { + /* GPR_ASSERT(bag->pi != NULL) */ + /* Make pi_new point to latest pi*/ + pi_new = polling_island_lock(bag->pi); + + if (item_type == POLL_OBJ_FD) { + grpc_fd *fd = FD_FROM_PO(item); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + } + + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else if (bag->pi == NULL) { + /* GPR_ASSERT(item->pi != NULL) */ + /* Make pi_new to point to latest pi */ + pi_new = polling_island_lock(item->pi); + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + pi_new = polling_island_merge(item->pi, bag->pi, &error); + GRPC_POLLING_TRACE( + "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } + + /* At this point, pi_new is the polling island that both item->pi and bag->pi + MUST be pointing to */ + + if (item->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(item_type)); + if (item->pi != NULL) { + PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type)); + } + item->pi = pi_new; + } + + if (bag->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(bag_type)); + if (bag->pi != NULL) { + PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type)); + } + bag->pi = pi_new; + } + + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + + GRPC_LOG_IF_ERROR("add_poll_object", error); + GPR_TIMER_END("add_poll_object", 0); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po, + POLL_OBJ_FD); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->po.pi = NULL; +#ifdef PO_DEBUG + pss->po.obj_type = POLL_OBJ_POLLSET_SET; +#endif + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + + if (pss->po.pi != NULL) { + PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); + } + + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po, + POLL_OBJ_FD); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po, + POLL_OBJ_POLLSET); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po, + POLL_OBJ_POLLSET_SET); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +/* This is mainly for testing purposes. Checks to see if environment variable + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to + * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ +static void set_max_pollers_per_island() { + char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); + if (s) { + g_max_pollers_per_pi = (int)strtol(s, NULL, 10); + if (g_max_pollers_per_pi < 0) { + g_max_pollers_per_pi = INT_MAX; + } + } else { + g_max_pollers_per_pi = INT_MAX; + } + + gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", + g_max_pollers_per_pi); +} + +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux( + bool explicitly_requested) { + if (!explicitly_requested) { + return NULL; + } + + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + grpc_use_signal(SIGRTMIN + 6); + } + + set_max_pollers_per_island(); + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux( + bool explicitly_requested) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/channel/http_client_filter.h b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h index 9e6e106e9c..379e1ded3b 100644 --- a/src/core/lib/channel/http_client_filter.h +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h @@ -1,4 +1,5 @@ /* + * * Copyright 2015, Google Inc. * All rights reserved. * @@ -30,18 +31,13 @@ * */ -#ifndef GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H - -#include "src/core/lib/channel/channel_stack.h" - -/* Processes metadata on the client side for HTTP2 transports */ -extern const grpc_channel_filter grpc_http_client_filter; +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H -/* Channel arg to override the http2 :scheme header */ -#define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" -/* Channel arg to determine maximum size of payload eligable for GET request */ -#define GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET "grpc.max_payload_size_for_get" +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux( + bool explicitly_requested); -#endif /* GRPC_CORE_LIB_CHANNEL_HTTP_CLIENT_FILTER_H */ +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c new file mode 100644 index 0000000000..bb44321922 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -0,0 +1,1337 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" + +#include <assert.h> +#include <errno.h> +#include <limits.h> +#include <poll.h> +#include <pthread.h> +#include <string.h> +#include <sys/epoll.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/thd.h> +#include <grpc/support/tls.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +struct epoll_set; + +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + +/******************************************************************************* + * Fd Declarations + */ +struct grpc_fd { + gpr_mu mu; + struct epoll_set *eps; + + int fd; + + /* The fd is either closed or we relinquished control of it. In either cases, + this indicates that the 'fd' on this structure is no longer valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + grpc_iomgr_object iomgr_object; +}; + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * epoll set Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define EPS_ADD_REF(p, r) eps_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define EPS_UNREF(exec_ctx, p, r) \ + eps_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define EPS_ADD_REF(p, r) eps_add_ref((p)) +#define EPS_UNREF(exec_ctx, p, r) eps_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_EPS_REF_COUNT_DEBUG) */ + +/* This is also used as grpc_workqueue (by directly casting it) */ +typedef struct epoll_set { + grpc_closure_scheduler workqueue_scheduler; + + /* Mutex poller should acquire to poll this. This enforces that only one + * poller can be polling on epoll_set at any time */ + gpr_mu mu; + + /* Ref count. Use EPS_ADD_REF() and EPS_UNREF() macros to increment/decrement + the refcount. Once the ref count becomes zero, this structure is destroyed + which means we should ensure that there is never a scenario where a + EPS_ADD_REF() is racing with a EPS_UNREF() that just made the ref_count + zero. */ + gpr_atm ref_count; + + /* Number of threads currently polling on this epoll set*/ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* Is the epoll set shutdown */ + gpr_atm is_shutdown; + + /* The fd of the underlying epoll set */ + int epoll_fd; +} epoll_set; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + gpr_cv kick_cv; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset { + gpr_mu mu; + struct epoll_set *eps; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set {}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Declarations + */ + +size_t g_num_eps = 1; +struct epoll_set **g_epoll_sets = NULL; +gpr_atm g_next_eps; +size_t g_num_threads_per_eps = 1; +gpr_thd_id *g_poller_threads = NULL; + +/* Used as read-notifier pollsets for fds. We won't be using read notifier + * pollsets with this polling engine. So it does not matter what pollset we + * return */ +grpc_pollset g_read_notifier; + +static void add_fd_to_eps(grpc_fd *fd); +static bool init_epoll_sets(); +static void shutdown_epoll_sets(); +static void poller_thread_loop(void *arg); +static void start_poller_threads(); +static void shutdown_poller_threads(); + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * epoll set Definitions + */ + +/* The wakeup fd that is used to wake up all threads in an epoll_set informing + that the epoll set is shutdown. This wakeup fd initialized to be readable + and MUST NOT be consumed i.e the threads that woke up MUST NOT call + grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd epoll_set_wakeup_fd; + +/* The epoll set being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread epoll_set *g_current_thread_epoll_set; + +/* Forward declaration */ +static void epoll_set_delete(epoll_set *eps); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void eps_add_ref(epoll_set *eps); +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void eps_add_ref_dbg(epoll_set *eps, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_add_ref(eps); + gpr_log(GPR_DEBUG, "Add ref eps: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, old_cnt + 1, reason, file, line); +} + +static void eps_unref_dbg(grpc_exec_ctx *exec_ctx, epoll_set *eps, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_unref(exec_ctx, eps); + gpr_log(GPR_DEBUG, "Unref eps: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + eps_add_ref_dbg((epoll_set *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + eps_unref_dbg(exec_ctx, (epoll_set *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + eps_add_ref((epoll_set *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + eps_unref(exec_ctx, (epoll_set *)workqueue); + } +} +#endif + +static void eps_add_ref(epoll_set *eps) { + gpr_atm_no_barrier_fetch_add(&eps->ref_count, 1); +} + +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { + /* If ref count went to zero, delete the epoll set. This deletion is + not done under a lock since once the ref count goes to zero, we are + guaranteed that no one else holds a reference to the epoll set (and + that there is no racing eps_add_ref() call either).*/ + if (1 == gpr_atm_full_fetch_add(&eps->ref_count, -1)) { + epoll_set_delete(eps); + } +} + +static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, + grpc_error **error) { + int err; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "epoll_set_add_fd_locked"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fd; + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + eps->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "epoll_set_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, bool is_fd_closed, + grpc_error **error) { + int err; + char *err_msg; + const char *err_desc = "epoll_set_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + eps->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } +} + +/* Might return NULL in case of an error */ +static epoll_set *epoll_set_create(grpc_error **error) { + epoll_set *eps = NULL; + const char *err_desc = "epoll_set_create"; + + *error = GRPC_ERROR_NONE; + + eps = gpr_malloc(sizeof(*eps)); + eps->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + eps->epoll_fd = -1; + + gpr_mu_init(&eps->mu); + gpr_mu_init(&eps->workqueue_read_mu); + gpr_mpscq_init(&eps->workqueue_items); + gpr_atm_rel_store(&eps->workqueue_item_count, 0); + + gpr_atm_rel_store(&eps->ref_count, 0); + gpr_atm_rel_store(&eps->poller_count, 0); + + gpr_atm_rel_store(&eps->is_shutdown, false); + + if (!append_error(error, grpc_wakeup_fd_init(&eps->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + eps->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (eps->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + epoll_set_add_wakeup_fd_locked(eps, &eps->workqueue_wakeup_fd, error); + +done: + if (*error != GRPC_ERROR_NONE) { + epoll_set_delete(eps); + eps = NULL; + } + return eps; +} + +static void epoll_set_delete(epoll_set *eps) { + if (eps->epoll_fd >= 0) { + close(eps->epoll_fd); + } + + GPR_ASSERT(gpr_atm_no_barrier_load(&eps->workqueue_item_count) == 0); + gpr_mu_destroy(&eps->mu); + gpr_mu_destroy(&eps->workqueue_read_mu); + gpr_mpscq_destroy(&eps->workqueue_items); + grpc_wakeup_fd_destroy(&eps->workqueue_wakeup_fd); + + gpr_free(eps); +} + +static void workqueue_maybe_wakeup(epoll_set *eps) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_epoll_set == eps); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&eps->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&eps->workqueue_wakeup_fd)); + } +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + epoll_set *eps = (epoll_set *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&eps->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&eps->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(eps); + } + + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + epoll_set *eps = (epoll_set *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &eps->workqueue_scheduler; +} + +static grpc_error *epoll_set_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&epoll_set_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&epoll_set_wakeup_fd); + } + + return error; +} + +static void epoll_set_global_shutdown() { + grpc_wakeup_fd_destroy(&epoll_set_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +static grpc_fd *get_fd_from_freelist() { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + return new_fd; +} + +static void add_fd_to_freelist(grpc_fd *fd) { + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); +} + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = get_fd_from_freelist(); + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->mu); + } + + /* Note: It is not really needed to get the new_fd->mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->mu); + new_fd->eps = NULL; + + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); + gpr_free(fd_name); + + /* Associate the fd with one of the eps */ + add_fd_to_eps(new_fd); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + epoll_set *unref_eps = NULL; + + gpr_mu_lock(&fd->mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the fd from the epoll set */ + if (fd->eps != NULL) { + epoll_set_remove_fd(fd->eps, fd, is_fd_closed, &error); + unref_eps = fd->eps; + fd->eps = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->mu); + + /* We are done with this fd. Release it (i.e add back to freelist) */ + add_fd_to_freelist(fd); + + if (unref_eps != NULL) { + /* Unref stale epoll set here, outside the fd lock above. + The epoll set owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + EPS_UNREF(exec_ctx, unref_eps, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +/* This polling engine doesn't really need the read notifier functionality. So + * it just returns a dummy read notifier pollset */ +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + return &g_read_notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } + +/******************************************************************************* + * Pollset Definitions + */ +/* TODO: sreek - Not needed anymore */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->next = worker->prev = NULL; + gpr_cv_init(&worker->kick_cv); +} + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + gpr_cv_signal(&worker->kick_cv); + return GRPC_ERROR_NONE; +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->eps = NULL; + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, + char *reason) { + if (ps->eps != NULL) { + EPS_UNREF(exec_ctx, ps->eps, reason); + } + ps->eps = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->eps to NULL */ + pollset_release_epoll_set(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying epoll set. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, epoll_set *eps) { + if (gpr_mu_trylock(&eps->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&eps->workqueue_items); + gpr_mu_unlock(&eps->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&eps->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(eps); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&eps->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(eps); + } + } + return false; +} + +/* Blocking call */ +static void acquire_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_lock(&eps->mu); + } +} + +static void release_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_unlock(&eps->mu); + } +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, + grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "do_epoll_wait"; + + int timeout_ms = -1; + + GRPC_SCHEDULING_START_BLOCKING_REGION; + acquire_epoll_lease(eps); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); + release_epoll_lease(eps); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + if (ep_rv < 0) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &eps->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&eps->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, eps); + } else if (data_ptr == &epoll_set_wakeup_fd) { + gpr_atm_rel_store(&eps->is_shutdown, 1); + gpr_log(GPR_INFO, "pollset poller: shutdown set"); + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, + grpc_error **error) { + int epoll_fd = -1; + GPR_TIMER_BEGIN("epoll_set_work", 0); + + /* Since epoll_fd is immutable, it is safe to read it without a lock on the + epoll set. */ + epoll_fd = eps->epoll_fd; + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, eps)) { + gpr_atm_no_barrier_fetch_add(&eps->poller_count, 1); + g_current_thread_epoll_set = eps; + + do_epoll_wait(exec_ctx, epoll_fd, eps, error); + + g_current_thread_epoll_set = NULL; + gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); + } + + GPR_TIMER_END("epoll_set_work", 0); +} + +/* pollset->mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + push_front_worker(pollset, &worker); + + gpr_cv_wait(&worker.kick_cv, &pollset->mu, + gpr_convert_clock_type(deadline, GPR_CLOCK_REALTIME)); + /* pollset->mu locked here */ + + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + /* Nothing to do */ +} + +/******************************************************************************* + * Pollset-set Definitions + */ +grpc_pollset_set g_dummy_pollset_set; +static grpc_pollset_set *pollset_set_create(void) { + return &g_dummy_pollset_set; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + /* Nothing to do */ +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + shutdown_poller_threads(); + shutdown_epoll_sets(); + fd_global_shutdown(); + pollset_global_shutdown(); + epoll_set_global_shutdown(); + gpr_log(GPR_INFO, "ev-epoll-threadpool engine shutdown complete"); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Definitions + */ +static void add_fd_to_eps(grpc_fd *fd) { + GPR_ASSERT(fd->eps == NULL); + GPR_TIMER_BEGIN("add_fd_to_eps", 0); + + grpc_error *error = GRPC_ERROR_NONE; + size_t idx = (size_t)gpr_atm_no_barrier_fetch_add(&g_next_eps, 1) % g_num_eps; + epoll_set *eps = g_epoll_sets[idx]; + + gpr_mu_lock(&fd->mu); + + if (fd->orphaned) { + gpr_mu_unlock(&fd->mu); + return; /* Early out */ + } + + epoll_set_add_fd_locked(eps, fd, &error); + EPS_ADD_REF(eps, "fd"); + fd->eps = eps; + + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %" PRIdPTR ")", fd->fd, + idx); + gpr_mu_unlock(&fd->mu); + + GRPC_LOG_IF_ERROR("add_fd_to_eps", error); + GPR_TIMER_END("add_fd_to_eps", 0); +} + +static bool init_epoll_sets() { + grpc_error *error = GRPC_ERROR_NONE; + bool is_success = true; + + g_epoll_sets = (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); + + for (size_t i = 0; i < g_num_eps; i++) { + g_epoll_sets[i] = epoll_set_create(&error); + if (g_epoll_sets[i] == NULL) { + gpr_log(GPR_ERROR, "Error in creating a epoll set"); + g_num_eps = i; /* Helps cleanup */ + shutdown_epoll_sets(); + is_success = false; + goto done; + } + + EPS_ADD_REF(g_epoll_sets[i], "init_epoll_sets"); + } + + gpr_atm_no_barrier_store(&g_next_eps, 0); + gpr_mu *mu; + pollset_init(&g_read_notifier, &mu); + +done: + GRPC_LOG_IF_ERROR("init_epoll_sets", error); + return is_success; +} + +static void shutdown_epoll_sets() { + if (!g_epoll_sets) { + return; + } + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + for (size_t i = 0; i < g_num_eps; i++) { + EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_epoll_sets"); + } + grpc_exec_ctx_flush(&exec_ctx); + + gpr_free(g_epoll_sets); + g_epoll_sets = NULL; + pollset_destroy(&exec_ctx, &g_read_notifier); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void poller_thread_loop(void *arg) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_error *error = GRPC_ERROR_NONE; + epoll_set *eps = (epoll_set *)arg; + + while (!gpr_atm_acq_load(&eps->is_shutdown)) { + epoll_set_work(&exec_ctx, eps, &error); + grpc_exec_ctx_flush(&exec_ctx); + } + + grpc_exec_ctx_finish(&exec_ctx); + GRPC_LOG_IF_ERROR("poller_thread_loop", error); +} + +/* g_epoll_sets MUST be initialized before calling this */ +static void start_poller_threads() { + GPR_ASSERT(g_epoll_sets); + + gpr_log(GPR_INFO, "Starting poller threads"); + + size_t num_threads = g_num_eps * g_num_threads_per_eps; + g_poller_threads = (gpr_thd_id *)malloc(num_threads * sizeof(gpr_thd_id)); + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + + for (size_t i = 0; i < num_threads; i++) { + gpr_thd_new(&g_poller_threads[i], poller_thread_loop, + (void *)g_epoll_sets[i % g_num_eps], &options); + } +} + +static void shutdown_poller_threads() { + GPR_ASSERT(g_poller_threads); + GPR_ASSERT(g_epoll_sets); + grpc_error *error = GRPC_ERROR_NONE; + + gpr_log(GPR_INFO, "Shutting down pollers"); + + epoll_set *eps = NULL; + size_t num_threads = g_num_eps * g_num_threads_per_eps; + for (size_t i = 0; i < num_threads; i++) { + eps = g_epoll_sets[i]; + epoll_set_add_wakeup_fd_locked(eps, &epoll_set_wakeup_fd, &error); + } + + for (size_t i = 0; i < g_num_eps; i++) { + gpr_thd_join(g_poller_threads[i]); + } + + GRPC_LOG_IF_ERROR("shutdown_poller_threads", error); + gpr_free(g_poller_threads); + g_poller_threads = NULL; +} + +/****************************************************************************/ + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly) { + if (!requested_explicitly) return NULL; + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", epoll_set_global_init())) { + return NULL; + } + + if (!init_epoll_sets()) { + return NULL; + } + + /* TODO (sreek): Maynot be a good idea to start threads here (especially if + * this engine doesn't get picked. Consider introducing an engine_init + * function in the vtable */ + start_poller_threads(); + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h new file mode 100644 index 0000000000..9af776a52e --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c new file mode 100644 index 0000000000..7cb6085e25 --- /dev/null +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -0,0 +1,1511 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epollex_linux.h" + +#include <assert.h> +#include <errno.h> +#include <poll.h> +#include <pthread.h> +#include <string.h> +#include <sys/socket.h> +#include <unistd.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/tls.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/is_epollexclusive_available.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/spinlock.h" + +/******************************************************************************* + * Pollset-set sibling link + */ + +typedef enum { + PO_POLLING_GROUP, + PO_POLLSET_SET, + PO_POLLSET, + PO_FD, /* ordering is important: we always want to lock pollsets before fds: + this guarantees that using an fd as a pollable is safe */ + PO_EMPTY_POLLABLE, + PO_COUNT +} polling_obj_type; + +typedef struct polling_obj polling_obj; +typedef struct polling_group polling_group; + +struct polling_obj { + gpr_mu mu; + polling_obj_type type; + polling_group *group; + struct polling_obj *next; + struct polling_obj *prev; +}; + +struct polling_group { + polling_obj po; + gpr_refcount refs; +}; + +static void po_init(polling_obj *po, polling_obj_type type); +static void po_destroy(polling_obj *po); +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b); +static int po_cmp(polling_obj *a, polling_obj *b); + +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count); +static polling_group *pg_ref(polling_group *pg); +static void pg_unref(polling_group *pg); +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b); +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po); + +/******************************************************************************* + * pollable Declarations + */ + +typedef struct pollable { + polling_obj po; + int epfd; + grpc_wakeup_fd wakeup; + grpc_pollset_worker *root_worker; +} pollable; + +static pollable g_empty_pollable; + +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p); + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + pollable pollable; + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + grpc_closure_scheduler workqueue_scheduler; + /* Spinlock guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_spinlock workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + gpr_mu orphaned_mu; + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +/******************************************************************************* + * Pollset Declarations + */ + +typedef struct pollset_worker_link { + grpc_pollset_worker *next; + grpc_pollset_worker *prev; +} pollset_worker_link; + +typedef enum { + PWL_POLLSET, + PWL_POLLABLE, + POLLSET_WORKER_LINK_COUNT +} pollset_worker_links; + +struct grpc_pollset_worker { + bool kicked; + bool initialized_cv; + pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; + gpr_cv cv; + grpc_pollset *pollset; + pollable *pollable; +}; + +struct grpc_pollset { + pollable pollable; + pollable *current_pollable; + bool kicked_without_poller; + grpc_closure *shutdown_closure; + grpc_pollset_worker *root_worker; +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + polling_obj po; +}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(ec, fd, n, reason) \ + unref_by(ec, fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_fd *fd = arg; + /* Add the fd to the freelist */ + grpc_iomgr_unregister_object(&fd->iomgr_object); + pollable_destroy(&fd->pollable); + gpr_mu_destroy(&fd->orphaned_mu); + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, + const char *reason, const char *file, int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); + } else { + GPR_ASSERT(old > n); + } +} + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + } + + pollable_init(&new_fd->pollable, PO_FD); + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + gpr_mu_init(&new_fd->orphaned_mu); + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + GRPC_LOG_IF_ERROR("fd_create", + grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); + new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + new_fd->workqueue_read_mu = GPR_SPINLOCK_INITIALIZER; + gpr_mpscq_init(&new_fd->workqueue_items); + gpr_atm_no_barrier_store(&new_fd->workqueue_item_count, 0); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->orphaned_mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->orphaned_mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + + gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->orphaned_mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + if (!is_fd_closed) { + gpr_log(GPR_DEBUG, "TODO: handle fd removal?"); + } + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->orphaned_mu); + gpr_mu_unlock(&fd->pollable.po.mu); + UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + REF_BY(fd, 2, "return_workqueue"); + return (grpc_workqueue *)fd; +} + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2, file, line, reason); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + unref_by(exec_ctx, (grpc_fd *)workqueue, 2, file, line, reason); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + unref_by(exec_ctx, (grpc_fd *)workqueue, 2); + } +} +#endif + +static void workqueue_wakeup(grpc_fd *fd) { + GRPC_LOG_IF_ERROR("workqueue_enqueue", + grpc_wakeup_fd_wakeup(&fd->workqueue_wakeup_fd)); +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - + offsetof(grpc_fd, workqueue_scheduler)); + REF_BY(fd, 2, "workqueue_enqueue"); + gpr_atm last = gpr_atm_no_barrier_fetch_add(&fd->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&fd->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_wakeup(fd); + } + UNREF_BY(exec_ctx, fd, 2, "workqueue_enqueue"); +} + +static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + /* handle spurious wakeups */ + if (!gpr_spinlock_trylock(&fd->workqueue_read_mu)) return; + gpr_mpscq_node *n = gpr_mpscq_pop(&fd->workqueue_items); + gpr_spinlock_unlock(&fd->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&fd->workqueue_item_count, -1) > 1) { + workqueue_wakeup(fd); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + } else if (gpr_atm_no_barrier_load(&fd->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_wakeup(fd); + } +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return &((grpc_fd *)workqueue)->workqueue_scheduler; +} + +/******************************************************************************* + * Pollable Definitions + */ + +static void pollable_init(pollable *p, polling_obj_type type) { + po_init(&p->po, type); + p->root_worker = NULL; + p->epfd = -1; +} + +static void pollable_destroy(pollable *p) { + po_destroy(&p->po); + if (p->epfd != -1) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + } +} + +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p) { + if (p->epfd == -1) { + int new_epfd = epoll_create1(EPOLL_CLOEXEC); + if (new_epfd < 0) { + return GRPC_OS_ERROR(errno, "epoll_create1"); + } + grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); + if (err != GRPC_ERROR_NONE) { + close(new_epfd); + return err; + } + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), + .data.ptr = &p->wakeup}; + if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { + err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(new_epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + return err; + } + + p->epfd = new_epfd; + } + return GRPC_ERROR_NONE; +} + +/* pollable must be materialized */ +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollable_add_fd"; + const int epfd = p->epfd; + GPR_ASSERT(epfd != -1); + + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "add fd %p to pollable %p", fd, p); + } + + gpr_mu_lock(&fd->orphaned_mu); + if (fd->orphaned) { + gpr_mu_unlock(&fd->orphaned_mu); + return GRPC_ERROR_NONE; + } + struct epoll_event ev_fd = { + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE), + .data.ptr = fd}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { + switch (errno) { + case EEXIST: /* if this fd is already in the epoll set, the workqueue fd + must also be - just return */ + gpr_mu_unlock(&fd->orphaned_mu); + return GRPC_ERROR_NONE; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + struct epoll_event ev_wq = { + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE), + .data.ptr = (void *)(1 + (intptr_t)fd)}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != + 0) { + switch (errno) { + case EEXIST: /* if the workqueue fd is already in the epoll set we're ok + - no need to do anything special */ + break; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + gpr_mu_unlock(&fd->orphaned_mu); + + return error; +} + +/******************************************************************************* + * Pollset Definitions + */ + +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + pollable_init(&g_empty_pollable, PO_EMPTY_POLLABLE); + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + pollable_destroy(&g_empty_pollable); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->pollable != &pollset->pollable) { + gpr_mu_lock(&worker->pollable->po.mu); + } + if (worker->initialized_cv) { + worker->kicked = true; + gpr_cv_signal(&worker->cv); + } else { + append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + "pollset_shutdown"); + } + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + } + + worker = worker->links[PWL_POLLSET].next; + } while (worker != pollset->root_worker); + } + return error; +} + +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, + grpc_pollset_worker *specific_worker) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, + "PS:%p kick %p tls_pollset=%p tls_worker=%p " + "root_worker=(pollset:%p pollable:%p)", + p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), + (void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker, + p->root_worker); + } + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { + if (pollset->root_worker == NULL) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); + } + pollset->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } else { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p); + } + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + return grpc_wakeup_fd_wakeup(&p->wakeup); + } + } else { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p); + } + return GRPC_ERROR_NONE; + } + } else if (specific_worker->kicked) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p); + } + return GRPC_ERROR_NONE; + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); + } + specific_worker->kicked = true; + return GRPC_ERROR_NONE; + } else if (specific_worker == p->root_worker) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p); + } + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + specific_worker->kicked = true; + return grpc_wakeup_fd_wakeup(&p->wakeup); + } else { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); + } + specific_worker->kicked = true; + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } +} + +/* p->po.mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable) { + gpr_mu_lock(&p->po.mu); + } + grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + if (p != &pollset->pollable) { + gpr_mu_unlock(&p->po.mu); + } + return error; +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + pollable_init(&pollset->pollable, PO_POLLSET); + pollset->current_pollable = &g_empty_pollable; + pollset->kicked_without_poller = false; + pollset->shutdown_closure = NULL; + pollset->root_worker = NULL; + *mu = &pollset->pollable.po.mu; +} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, now) <= 0) { + return 0; + } + + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "fd_become_pollable"; + if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + } + return error; +} + +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; + } +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + pollset_maybe_finish_shutdown(exec_ctx, pollset); +} + +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable; +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. */ +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + pollable_destroy(&pollset->pollable); + if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + "pollset_pollable"); + } +} + +#define MAX_EPOLL_EVENTS 100 + +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + pollable *p, gpr_timespec now, + gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout); + } + + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r; + do { + r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); + } + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &p->wakeup) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p poll %p got pollset_wakeup", pollset, p); + } + append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); + } else { + grpc_fd *fd = (grpc_fd *)(((intptr_t)data_ptr) & ~(intptr_t)1); + bool is_workqueue = (((intptr_t)data_ptr) & 1) != 0; + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, + "PS:%p poll %p got fd %p: is_wq=%d cancel=%d read=%d " + "write=%d", + pollset, p, fd, is_workqueue, cancel, read_ev, write_ev); + } + if (is_workqueue) { + append_error(&error, + grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), + err_desc); + fd_invoke_workqueue(exec_ctx, fd); + } else { + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + } + + return error; +} + +/* Return true if first in list */ +static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, + grpc_pollset_worker *worker) { + if (*root == NULL) { + *root = worker; + worker->links[link].next = worker->links[link].prev = worker; + return true; + } else { + worker->links[link].next = *root; + worker->links[link].prev = worker->links[link].next->links[link].prev; + worker->links[link].next->links[link].prev = worker; + worker->links[link].prev->links[link].next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset_worker **root, + pollset_worker_links link, + grpc_pollset_worker *worker) { + if (worker == *root) { + if (worker == worker->links[link].next) { + *root = NULL; + return EMPTIED; + } else { + *root = worker->links[link].next; + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return NEW_ROOT; + } + } else { + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return REMOVED; + } +} + +/* Return true if this thread should poll */ +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, + gpr_timespec deadline) { + bool do_poll = true; + if (worker_hdl != NULL) *worker_hdl = worker; + worker->initialized_cv = false; + worker->kicked = false; + worker->pollset = pollset; + worker->pollable = pollset->current_pollable; + + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + } + + worker_insert(&pollset->root_worker, PWL_POLLSET, worker); + if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&pollset->pollable.po.mu); + } + if (GRPC_TRACER_ON(grpc_polling_trace) && + worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, + worker->pollable, worker, + poll_deadline_to_millis_timeout(deadline, *now)); + } + while (do_poll && worker->pollable->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, + worker->pollable, worker); + } + do_poll = false; + } else if (worker->kicked) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, + worker); + } + do_poll = false; + } else if (GRPC_TRACER_ON(grpc_polling_trace) && + worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, + worker->pollable, worker); + } + } + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&worker->pollable->po.mu); + } + *now = gpr_now(now->clock_type); + } + + return do_poll && pollset->shutdown_closure == NULL && + pollset->current_pollable == worker->pollable; +} + +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (NEW_ROOT == + worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable->root_worker->cv); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); + } + if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + if (0 && GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 + ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", + pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, + deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, + pollset->root_worker); + } + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } + if (pollset->current_pollable != &pollset->pollable) { + gpr_mu_lock(&pollset->current_pollable->po.mu); + } + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + GPR_ASSERT(!pollset->shutdown_closure); + append_error(&error, pollable_materialize(worker.pollable), err_desc); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } + gpr_mu_unlock(&pollset->pollable.po.mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, now, + deadline), + err_desc); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->pollable.po.mu); + if (worker.pollable != &pollset->pollable) { + gpr_mu_lock(&worker.pollable->po.mu); + } + gpr_tls_set(&g_current_thread_pollset, 0); + gpr_tls_set(&g_current_thread_worker, 0); + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } + end_worker(exec_ctx, pollset, &worker, worker_hdl); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } + return error; +} + +static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_fd *fd = arg; + UNREF_BY(exec_ctx, fd, 2, "pollset_pollable"); +} + +/* expects pollsets locked, flag whether fd is locked or not */ +static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, + bool fd_locked) { + static const char *err_desc = "pollset_add_fd"; + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->current_pollable == &g_empty_pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from empty to fd", pollset, + fd); + /* empty pollable --> single fd pollable */ + append_error(&error, pollset_kick_all(pollset), err_desc); + pollset->current_pollable = &fd->pollable; + if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + append_error(&error, fd_become_pollable_locked(fd), err_desc); + if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); + REF_BY(fd, 2, "pollset_pollable"); + } else if (pollset->current_pollable == &pollset->pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); + append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + err_desc); + } else if (pollset->current_pollable != &fd->pollable) { + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from fd %p to multipoller", + pollset, fd, had_fd); + append_error(&error, pollset_kick_all(pollset), err_desc); + pollset->current_pollable = &pollset->pollable; + if (append_error(&error, pollable_materialize(&pollset->pollable), + err_desc)) { + pollable_add_fd(&pollset->pollable, had_fd); + pollable_add_fd(&pollset->pollable, fd); + } + grpc_closure_sched(exec_ctx, + grpc_closure_create(unref_fd_no_longer_poller, had_fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); + } + return error; +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + gpr_mu_lock(&pollset->pollable.po.mu); + grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); + gpr_mu_unlock(&pollset->pollable.po.mu); + GRPC_LOG_IF_ERROR("pollset_add_fd", error); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); + po_init(&pss->po, PO_POLLSET_SET); + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + po_destroy(&pss->po); + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + po_join(exec_ctx, &pss->po, &fd->pollable.po); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + po_join(exec_ctx, &pss->po, &ps->pollable.po); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + po_join(exec_ctx, &bag->po, &item->po); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +static void po_init(polling_obj *po, polling_obj_type type) { + gpr_mu_init(&po->mu); + po->type = type; + po->group = NULL; + po->next = po; + po->prev = po; +} + +static polling_group *pg_lock_latest(polling_group *pg) { + /* assumes pg unlocked; consumes ref, returns ref */ + gpr_mu_lock(&pg->po.mu); + while (pg->po.group != NULL) { + polling_group *new_pg = pg_ref(pg->po.group); + gpr_mu_unlock(&pg->po.mu); + pg_unref(pg); + pg = new_pg; + gpr_mu_lock(&pg->po.mu); + } + return pg; +} + +static void po_destroy(polling_obj *po) { + if (po->group != NULL) { + polling_group *pg = pg_lock_latest(po->group); + po->prev->next = po->next; + po->next->prev = po->prev; + gpr_mu_unlock(&pg->po.mu); + pg_unref(pg); + } + gpr_mu_destroy(&po->mu); +} + +static polling_group *pg_ref(polling_group *pg) { + gpr_ref(&pg->refs); + return pg; +} + +static void pg_unref(polling_group *pg) { + if (gpr_unref(&pg->refs)) { + po_destroy(&pg->po); + gpr_free(pg); + } +} + +static int po_cmp(polling_obj *a, polling_obj *b) { + if (a == b) return 0; + if (a->type < b->type) return -1; + if (a->type > b->type) return 1; + if (a < b) return -1; + assert(a > b); + return 1; +} + +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + switch (po_cmp(a, b)) { + case 0: + return; + case 1: + GPR_SWAP(polling_obj *, a, b); + /* fall through */ + case -1: + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + + if (a->group == NULL) { + if (b->group == NULL) { + polling_obj *initial_po[] = {a, b}; + pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + pg_join(exec_ctx, b_group, a); + } + } else if (b->group == NULL) { + polling_group *a_group = pg_ref(a->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_join(exec_ctx, a_group, b); + } else if (a->group == b->group) { + /* nothing to do */ + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *a_group = pg_ref(a->group); + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_merge(exec_ctx, a_group, b_group); + } + } +} + +static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + if (a->type == PO_FD && b->type == PO_POLLSET) { + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a, true); + } else if (a->type == PO_POLLSET && b->type == PO_FD) { + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b, true); + } +} + +static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, + polling_group *to) { + for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { + for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { + if (po_cmp(a, b) < 0) { + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + } else { + GPR_ASSERT(po_cmp(a, b) != 0); + gpr_mu_lock(&b->mu); + gpr_mu_lock(&a->mu); + } + pg_notify(exec_ctx, a, b); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } + } +} + +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count) { + /* assumes all polling objects in initial_po are locked */ + polling_group *pg = gpr_malloc(sizeof(*pg)); + po_init(&pg->po, PO_POLLING_GROUP); + gpr_ref_init(&pg->refs, (int)initial_po_count); + for (size_t i = 0; i < initial_po_count; i++) { + GPR_ASSERT(initial_po[i]->group == NULL); + initial_po[i]->group = pg; + } + for (size_t i = 1; i < initial_po_count; i++) { + initial_po[i]->prev = initial_po[i - 1]; + } + for (size_t i = 0; i < initial_po_count - 1; i++) { + initial_po[i]->next = initial_po[i + 1]; + } + initial_po[0]->prev = &pg->po; + initial_po[initial_po_count - 1]->next = &pg->po; + pg->po.next = initial_po[0]; + pg->po.prev = initial_po[initial_po_count - 1]; + for (size_t i = 1; i < initial_po_count; i++) { + for (size_t j = 0; j < i; j++) { + pg_notify(exec_ctx, initial_po[i], initial_po[j]); + } + } +} + +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po) { + /* assumes neither pg nor po are locked; consumes one ref to pg */ + pg = pg_lock_latest(pg); + /* pg locked */ + for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; + existing != &pg->po; existing = existing->next) { + if (po_cmp(po, existing) < 0) { + gpr_mu_lock(&po->mu); + gpr_mu_lock(&existing->mu); + } else { + GPR_ASSERT(po_cmp(po, existing) != 0); + gpr_mu_lock(&existing->mu); + gpr_mu_lock(&po->mu); + } + /* pg, po, existing locked */ + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group during joining: we needed + to do a full merge */ + return; + } + pg_notify(exec_ctx, po, existing); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + } + gpr_mu_lock(&po->mu); + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group during joining: we needed + to do a full merge */ + return; + } + po->group = pg; + po->next = &pg->po; + po->prev = pg->po.prev; + po->prev->next = po->next->prev = po; + gpr_mu_unlock(&pg->po.mu); + gpr_mu_unlock(&po->mu); +} + +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b) { + for (;;) { + if (a == b) { + pg_unref(a); + pg_unref(b); + return; + } + if (a > b) GPR_SWAP(polling_group *, a, b); + gpr_mu_lock(&a->po.mu); + gpr_mu_lock(&b->po.mu); + if (a->po.group != NULL) { + polling_group *m2 = pg_ref(a->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(a); + a = m2; + } else if (b->po.group != NULL) { + polling_group *m2 = pg_ref(b->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(b); + b = m2; + } else { + break; + } + } + polling_group **unref = NULL; + size_t unref_count = 0; + size_t unref_cap = 0; + b->po.group = a; + pg_broadcast(exec_ctx, a, b); + pg_broadcast(exec_ctx, b, a); + while (b->po.next != &b->po) { + polling_obj *po = b->po.next; + gpr_mu_lock(&po->mu); + if (unref_count == unref_cap) { + unref_cap = GPR_MAX(8, 3 * unref_cap / 2); + unref = gpr_realloc(unref, unref_cap * sizeof(*unref)); + } + unref[unref_count++] = po->group; + po->group = pg_ref(a); + // unlink from b + po->prev->next = po->next; + po->next->prev = po->prev; + // link to a + po->next = &a->po; + po->prev = a->po.prev; + po->next->prev = po->prev->next = po; + gpr_mu_unlock(&po->mu); + } + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + for (size_t i = 0; i < unref_count; i++) { + pg_unref(unref[i]); + } + gpr_free(unref); + pg_unref(b); +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { + if (!explicitly_requested) return NULL; + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!grpc_is_epollexclusive_available()) { + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + pollset_global_shutdown(); + fd_global_shutdown(); + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ + +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/channel/http_server_filter.h b/src/core/lib/iomgr/ev_epollex_linux.h index 77ba2d263d..a078a7f19a 100644 --- a/src/core/lib/channel/http_server_filter.h +++ b/src/core/lib/iomgr/ev_epollex_linux.h @@ -31,12 +31,13 @@ * */ -#ifndef GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H -#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" -/* Processes metadata on the client side for HTTP2 transports */ -extern const grpc_channel_filter grpc_http_server_filter; +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested); -#endif /* GRPC_CORE_LIB_CHANNEL_HTTP_SERVER_FILTER_H */ +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index f6372c0f3f..52362a62f4 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -36,7 +36,7 @@ /* This polling engine is only relevant on linux kernels supporting epoll() */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include <assert.h> #include <errno.h> @@ -56,16 +56,18 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } /* Uncomment the following to enable extra checks on poll_object operations */ @@ -74,11 +76,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */ static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - /* Implements the function defined in grpc_posix.h. This function might be * called before even calling grpc_init() to set either a different signal to * use. If signum == -1, then the use of signals is disabled */ @@ -140,52 +137,11 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; - /* Internally stores data of type (grpc_error *). If the FD is shutdown, this - contains reason for shutdown (i.e a pointer to grpc_error) ORed with - FD_SHUTDOWN_BIT. Since address allocations are word-aligned, the lower bit - of (grpc_error *) addresses is guaranteed to be zero. Even if the - (grpc_error *), is of special types like GRPC_ERROR_NONE, GRPC_ERROR_OOM - etc, the lower bit is guaranteed to be zero. - - Once an fd is shutdown, any pending or future read/write closures on the - fd should fail */ - gpr_atm shutdown_error; - /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ bool orphaned; - /* Closures to call when the fd is readable or writable respectively. These - fields contain one of the following values: - CLOSURE_READY : The fd has an I/O event of interest but there is no - closure yet to execute - - CLOSURE_NOT_READY : The fd has no I/O event of interest - - closure ptr : The closure to be executed when the fd has an I/O - event of interest - - shutdown_error | FD_SHUTDOWN_BIT : - 'shutdown_error' field ORed with FD_SHUTDOWN_BIT. - This indicates that the fd is shutdown. Since all - memory allocations are word-aligned, the lower two - bits of the shutdown_error pointer are always 0. So - it is safe to OR these with FD_SHUTDOWN_BIT - - Valid state transitions: - - <closure ptr> <-----3------ CLOSURE_NOT_READY ----1----> CLOSURE_READY - | | ^ | ^ | | - | | | | | | | - | +--------------4----------+ 6 +---------2---------------+ | - | | | - | v | - +-----5-------> [shutdown_error | FD_SHUTDOWN_BIT] <----7---------+ - - For 1, 4 : See set_ready() function - For 2, 3 : See notify_on() function - For 5,6,7: See set_shutdown() function */ gpr_atm read_closure; gpr_atm write_closure; @@ -217,11 +173,6 @@ static void fd_unref(grpc_fd *fd); static void fd_global_init(void); static void fd_global_shutdown(void); -#define CLOSURE_NOT_READY ((gpr_atm)0) -#define CLOSURE_READY ((gpr_atm)2) - -#define FD_SHUTDOWN_BIT 1 - /******************************************************************************* * Polling island Declarations */ @@ -498,8 +449,8 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), - errno, strerror(errno)); + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -602,7 +553,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, goto done; } - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); if (initial_fd != NULL) { @@ -948,10 +898,8 @@ static void unref_by(grpc_fd *fd, int n) { fd_freelist = fd; grpc_iomgr_unregister_object(&fd->iomgr_object); - grpc_error *err = (grpc_error *)gpr_atm_acq_load(&fd->shutdown_error); - /* Clear the least significant bit if it set (in case fd was shutdown) */ - err = (grpc_error *)((intptr_t)err & ~FD_SHUTDOWN_BIT); - GRPC_ERROR_UNREF(err); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); gpr_mu_unlock(&fd_freelist_mu); } else { @@ -1015,10 +963,9 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; - gpr_atm_no_barrier_store(&new_fd->shutdown_error, (gpr_atm)GRPC_ERROR_NONE); new_fd->orphaned = false; - gpr_atm_no_barrier_store(&new_fd->read_closure, CLOSURE_NOT_READY); - gpr_atm_no_barrier_store(&new_fd->write_closure, CLOSURE_NOT_READY); + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; @@ -1104,153 +1051,6 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_ERROR_UNREF(error); } -static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, - grpc_closure *closure) { - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - switch (curr) { - case CLOSURE_NOT_READY: { - /* CLOSURE_NOT_READY -> <closure>. - - We're guaranteed by API that there's an acquire barrier before here, - so there's no need to double-dip and this can be a release-only. - - The release itself pairs with the acquire half of a set_ready full - barrier. */ - if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { - return; /* Successful. Return */ - } - - break; /* retry */ - } - - case CLOSURE_READY: { - /* Change the state to CLOSURE_NOT_READY. Schedule the closure if - successful. If not, the state most likely transitioned to shutdown. - We should retry. - - This can be a no-barrier cas since the state is being transitioned to - CLOSURE_NOT_READY; set_ready and set_shutdown do not schedule any - closure when transitioning out of CLOSURE_NO_READY state (i.e there - is no other code that needs to 'happen-after' this) */ - if (gpr_atm_no_barrier_cas(state, CLOSURE_READY, CLOSURE_NOT_READY)) { - grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); - return; /* Successful. Return */ - } - - break; /* retry */ - } - - default: { - /* 'curr' is either a closure or the fd is shutdown(in which case 'curr' - contains a pointer to the shutdown-error). If the fd is shutdown, - schedule the closure with the shutdown error */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - grpc_error *shutdown_err = (grpc_error *)(curr & ~FD_SHUTDOWN_BIT); - grpc_closure_sched(exec_ctx, closure, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "FD Shutdown", &shutdown_err, 1)); - return; - } - - /* There is already a closure!. This indicates a bug in the code */ - gpr_log(GPR_ERROR, - "notify_on called with a previous callback still pending"); - abort(); - } - } - } - - GPR_UNREACHABLE_CODE(return ); -} - -static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, - grpc_error *shutdown_err) { - gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; - - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - switch (curr) { - case CLOSURE_READY: - case CLOSURE_NOT_READY: - /* Need a full barrier here so that the initial load in notify_on - doesn't need a barrier */ - if (gpr_atm_full_cas(state, curr, new_state)) { - return; /* early out */ - } - break; /* retry */ - - default: { - /* 'curr' is either a closure or the fd is already shutdown */ - - /* If fd is already shutdown, we are done */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - return; - } - - /* Fd is not shutdown. Schedule the closure and move the state to - shutdown state. - Needs an acquire to pair with setting the closure (and get a - happens-after on that edge), and a release to pair with anything - loading the shutdown state. */ - if (gpr_atm_full_cas(state, curr, new_state)) { - grpc_closure_sched(exec_ctx, (grpc_closure *)curr, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "FD Shutdown", &shutdown_err, 1)); - return; - } - - /* 'curr' was a closure but now changed to a different state. We will - have to retry */ - break; - } - } - } - - GPR_UNREACHABLE_CODE(return ); -} - -static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state) { - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - - switch (curr) { - case CLOSURE_READY: { - /* Already ready. We are done here */ - return; - } - - case CLOSURE_NOT_READY: { - /* No barrier required as we're transitioning to a state that does not - involve a closure */ - if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { - return; /* early out */ - } - break; /* retry */ - } - - default: { - /* 'curr' is either a closure or the fd is shutdown */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - /* The fd is shutdown. Do nothing */ - return; - } - /* Full cas: acquire pairs with this cas' release in the event of a - spurious set_ready; release pairs with this or the acquire in - notify_on (or set_shutdown) */ - else if (gpr_atm_full_cas(state, curr, CLOSURE_NOT_READY)) { - grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); - return; - } - /* else the state changed again (only possible by either a racing - set_ready or set_shutdown functions. In both these cases, the closure - would have been scheduled for execution. So we are done here */ - return; - } - } - } -} - static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); @@ -1258,33 +1058,27 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, } static bool fd_is_shutdown(grpc_fd *fd) { - grpc_error *err = (grpc_error *)gpr_atm_acq_load(&fd->shutdown_error); - return (((intptr_t)err & FD_SHUTDOWN_BIT) > 0); + return grpc_lfev_is_shutdown(&fd->read_closure); } /* Might be called multiple times */ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - /* Store the shutdown error ORed with FD_SHUTDOWN_BIT in fd->shutdown_error */ - if (gpr_atm_rel_cas(&fd->shutdown_error, (gpr_atm)GRPC_ERROR_NONE, - (gpr_atm)why | FD_SHUTDOWN_BIT)) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { shutdown(fd->fd, SHUT_RDWR); - - set_shutdown(exec_ctx, fd, &fd->read_closure, why); - set_shutdown(exec_ctx, fd, &fd->write_closure, why); - } else { - /* Shutdown already called */ - GRPC_ERROR_UNREF(why); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); } + GRPC_ERROR_UNREF(why); } static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { - notify_on(exec_ctx, fd, &fd->read_closure, closure); + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); } static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { - notify_on(exec_ctx, fd, &fd->write_closure, closure); + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { @@ -1316,11 +1110,10 @@ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); poller_kick_init(); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } @@ -1332,7 +1125,7 @@ static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { GRPC_POLLING_TRACE( "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", - (void *)worker, worker->pt_id); + (void *)worker, (long int)worker->pt_id); int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); if (err_num != 0) { err = GRPC_OS_ERROR(err_num, "pthread_kill"); @@ -1426,10 +1219,6 @@ static grpc_error *pollset_kick(grpc_pollset *p, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->po.mu); *mu = &pollset->po.mu; @@ -1467,13 +1256,14 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } timeout = gpr_time_sub(deadline, now); - return gpr_time_to_millis(gpr_time_add( + int millis = gpr_time_to_millis(gpr_time_add( timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; } static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *notifier) { - set_ready(exec_ctx, fd, &fd->read_closure); + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); /* Note, it is possible that fd_become_readable might be called twice with different 'notifier's when an fd becomes readable and it is in two epoll @@ -1485,7 +1275,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - set_ready(exec_ctx, fd, &fd->write_closure); + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, @@ -1531,7 +1321,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other * than destroying the mutexes, there is nothing special that needs to be done * here */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); gpr_mu_destroy(&pollset->po.mu); } @@ -1547,6 +1337,9 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); return true; @@ -1649,10 +1442,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, for (int i = 0; i < ep_rv; ++i) { void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { + if (data_ptr == &pi->workqueue_wakeup_fd) { append_error(error, grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), err_desc); @@ -1714,7 +1504,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, worker.pt_id = pthread_self(); gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); @@ -1792,7 +1582,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->po.mu); } - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); gpr_tls_set(&g_current_thread_worker, (intptr_t)0); @@ -2092,8 +1882,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, @@ -2116,7 +1904,8 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -2131,7 +1920,13 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { } if (!is_grpc_wakeup_signal_initialized) { - grpc_use_signal(SIGRTMIN + 6); + /* TODO(ctiller): when other epoll engines are ready, remove the true || to + * force this to be explitly chosen if needed */ + if (true || explicit_request) { + grpc_use_signal(SIGRTMIN + 6); + } else { + return NULL; + } } fd_global_init(); @@ -2153,7 +1948,10 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_epoll_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h index 8fc3ff59a3..9e4034f2a7 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.h +++ b/src/core/lib/iomgr/ev_epollsig_linux.h @@ -31,13 +31,13 @@ * */ -#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H -#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_linux(void); +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request); #ifdef GRPC_LINUX_EPOLL void *grpc_fd_get_polling_island(grpc_fd *fd); @@ -45,4 +45,4 @@ void *grpc_pollset_get_polling_island(grpc_pollset *ps); bool grpc_are_polling_islands_equal(void *p, void *q); #endif /* defined(GRPC_LINUX_EPOLL) */ -#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H */ +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index ca6e855611..3a7648ac32 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -52,11 +52,14 @@ #include <grpc/support/useful.h> #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_cv.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + /******************************************************************************* * FD declarations */ @@ -121,8 +124,6 @@ struct grpc_fd { grpc_pollset *read_notifier_pollset; }; -static grpc_wakeup_fd global_wakeup_fd; - /* Begin polling on an fd. Registers that the given pollset is interested in this fd - so that if read or writability interest changes, the pollset can be kicked to pick up that @@ -783,19 +784,14 @@ static grpc_error *pollset_kick(grpc_pollset *p, static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_poller); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_poller); gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - /* main interface */ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -814,7 +810,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->pollset_set_count = 0; } -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); while (pollset->local_wakeup_cache) { @@ -870,7 +866,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; grpc_error *error = GRPC_ERROR_NONE; /* Avoid malloc for small number of elements. */ @@ -951,13 +947,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } fd_count = 0; - pfd_count = 2; - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd); + pfd_count = 1; + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); pfds[0].events = POLLIN; pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); - pfds[1].events = POLLIN; - pfds[1].revents = 0; for (i = 0; i < pollset->fd_count; i++) { if (fd_is_orphaned(pollset->fds[i])) { GRPC_FD_UNREF(pollset->fds[i], "multipoller"); @@ -973,7 +966,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->fd_count = fd_count; gpr_mu_unlock(&pollset->mu); - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { grpc_fd *fd = watchers[i].fd; pfds[i].events = (short)fd_begin_poll(fd, pollset, &worker, POLLIN, POLLOUT, &watchers[i]); @@ -991,7 +984,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1001,19 +994,15 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } } else if (r == 0) { - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } } else { if (pfds[0].revents & POLLIN_CHECK) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd)); - } - if (pfds[1].revents & POLLIN_CHECK) { work_combine_error( &error, grpc_wakeup_fd_consume_wakeup(&worker.wakeup_fd->fd)); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1090,7 +1079,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->mu); } } - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; GPR_TIMER_END("pollset_work", 0); GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; @@ -1558,8 +1547,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, @@ -1567,7 +1554,7 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -const grpc_event_engine_vtable *grpc_init_poll_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1577,7 +1564,7 @@ const grpc_event_engine_vtable *grpc_init_poll_posix(void) { return &vtable; } -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request) { global_cv_fd_table_init(); grpc_enable_cv_wakeup_fds(1); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { diff --git a/src/core/lib/iomgr/ev_poll_posix.h b/src/core/lib/iomgr/ev_poll_posix.h index 202ffca14c..2890e93ead 100644 --- a/src/core/lib/iomgr/ev_poll_posix.h +++ b/src/core/lib/iomgr/ev_poll_posix.h @@ -36,7 +36,7 @@ #include "src/core/lib/iomgr/ev_posix.h" -const grpc_event_engine_vtable *grpc_init_poll_posix(void); -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void); +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request); +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request); #endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index b5be5504b9..c4d2f23e29 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,10 +44,18 @@ #include <grpc/support/string_util.h> #include <grpc/support/useful.h> -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/ev_epoll1_linux.h" +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" +#include "src/core/lib/iomgr/ev_epollex_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" +grpc_tracer_flag grpc_polling_trace = + GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ + /** Default poll() function - a pointer so that it can be overridden by some * tests */ grpc_poll_function_type grpc_poll_function = poll; @@ -57,7 +65,8 @@ grpc_wakeup_fd grpc_global_wakeup_fd; static const grpc_event_engine_vtable *g_event_engine; static const char *g_poll_strategy_name = NULL; -typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(void); +typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)( + bool explicit_request); typedef struct { const char *name; @@ -65,7 +74,11 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { - {"epoll", grpc_init_epoll_linux}, + {"epollex", grpc_init_epollex_linux}, + {"epollsig", grpc_init_epollsig_linux}, + {"epoll1", grpc_init_epoll1_linux}, + {"epoll-threadpool", grpc_init_epoll_thread_pool_linux}, + {"epoll-limited", grpc_init_epoll_limited_pollers_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, }; @@ -102,7 +115,8 @@ static bool is(const char *want, const char *have) { static void try_engine(const char *engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { if (is(engine, g_factories[i].name)) { - if ((g_event_engine = g_factories[i].factory())) { + if ((g_event_engine = g_factories[i].factory( + 0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name); return; @@ -111,10 +125,18 @@ static void try_engine(const char *engine) { } } +/* This should be used for testing purposes ONLY */ +void grpc_set_event_engine_test_only( + const grpc_event_engine_vtable *ev_engine) { + g_event_engine = ev_engine; +} + /* Call this only after calling grpc_event_engine_init() */ const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; } void grpc_event_engine_init(void) { + grpc_register_tracer("polling", &grpc_polling_trace); + char *s = gpr_getenv("GRPC_POLL_STRATEGY"); if (s == NULL) { s = gpr_strdup("all"); @@ -191,8 +213,8 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_event_engine->pollset_shutdown(exec_ctx, pollset, closure); } -void grpc_pollset_destroy(grpc_pollset *pollset) { - g_event_engine->pollset_destroy(pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + g_event_engine->pollset_destroy(exec_ctx, pollset); } grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -254,8 +276,6 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, g_event_engine->pollset_set_del_fd(exec_ctx, pollset_set, fd); } -grpc_error *grpc_kick_poller(void) { return g_event_engine->kick_poller(); } - #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG grpc_workqueue *grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, const char *reason) { diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1a9e5c115a..80619aab5f 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -36,12 +36,15 @@ #include <poll.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" +extern grpc_tracer_flag grpc_polling_trace; /* Disabled by default */ + typedef struct grpc_fd grpc_fd; typedef struct grpc_event_engine_vtable { @@ -64,7 +67,7 @@ typedef struct grpc_event_engine_vtable { void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu); void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); - void (*pollset_destroy)(grpc_pollset *pollset); + void (*pollset_destroy)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); @@ -93,8 +96,6 @@ typedef struct grpc_event_engine_vtable { void (*pollset_set_del_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); - grpc_error *(*kick_poller)(void); - void (*shutdown_engine)(void); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG @@ -183,4 +184,7 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int); extern grpc_poll_function_type grpc_poll_function; +/* This should be used for testing purposes ONLY */ +void grpc_set_event_engine_test_only(const grpc_event_engine_vtable *); + #endif /* GRPC_CORE_LIB_IOMGR_EV_POSIX_H */ diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c index 83bb436bd0..318bb2b713 100644 --- a/src/core/lib/iomgr/exec_ctx.c +++ b/src/core/lib/iomgr/exec_ctx.c @@ -62,6 +62,11 @@ bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) { return true; } +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) { + return exec_ctx->active_combiner != NULL || + !grpc_closure_list_empty(exec_ctx->closure_list); +} + bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { bool did_something = 0; GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); @@ -73,6 +78,9 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { grpc_closure *next = c->next_data.next; grpc_error *error = c->error_data.error; did_something = true; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); c = next; @@ -93,6 +101,9 @@ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) { static void exec_ctx_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error) { +#ifndef NDEBUG + closure->scheduled = false; +#endif closure->cb(exec_ctx, closure->cb_arg, error); GRPC_ERROR_UNREF(error); } diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h index f99a0fee5f..759a3ae2d5 100644 --- a/src/core/lib/iomgr/exec_ctx.h +++ b/src/core/lib/iomgr/exec_ctx.h @@ -93,6 +93,8 @@ struct grpc_exec_ctx { extern grpc_closure_scheduler *grpc_schedule_on_exec_ctx; +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx); + /** Flush any work that has been enqueued onto this grpc_exec_ctx. * Caller must guarantee that no interfering locks are held. * Returns true if work was performed, false otherwise. */ diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index ae3e2eabc3..75fd5b1c50 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -83,6 +83,9 @@ static void closure_exec_thread_func(void *ignored) { while (c != NULL) { grpc_closure *next = c->next_data.next; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(&exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); c = next; @@ -146,6 +149,9 @@ void grpc_executor_shutdown(grpc_exec_ctx *exec_ctx) { while (c != NULL) { grpc_closure *next = c->next_data.next; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); c = next; diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 001e528409..1fd41c2f88 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -47,6 +47,7 @@ #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/network_status_tracker.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" @@ -67,6 +68,8 @@ void grpc_iomgr_init(void) { grpc_iomgr_platform_init(); } +void grpc_iomgr_start(void) { grpc_timer_manager_init(); } + static size_t count_objects(void) { grpc_iomgr_object *obj; size_t n = 0; @@ -88,6 +91,7 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) { gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN)); gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME); + grpc_timer_manager_shutdown(); grpc_iomgr_platform_flush(); gpr_mu_lock(&g_mu); diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 245a1e08aa..6e2e023615 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -40,6 +40,9 @@ /** Initializes the iomgr. */ void grpc_iomgr_init(void); +/** Starts any background threads for iomgr. */ +void grpc_iomgr_start(void); + /** Signals the intention to shutdown the iomgr. Expects to be able to flush * exec_ctx. */ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx); diff --git a/src/core/lib/iomgr/is_epollexclusive_available.c b/src/core/lib/iomgr/is_epollexclusive_available.c new file mode 100644 index 0000000000..34fcf313e5 --- /dev/null +++ b/src/core/lib/iomgr/is_epollexclusive_available.c @@ -0,0 +1,116 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#include "src/core/lib/iomgr/is_epollexclusive_available.h" + +#ifdef GRPC_LINUX_EPOLL + +#include <grpc/support/log.h> + +#include <errno.h> +#include <sys/eventfd.h> +#include <unistd.h> + +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +bool grpc_is_epollexclusive_available(void) { + static bool logged_why_not = false; + + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + return false; + } + int evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (evfd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "eventfd failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + close(fd); + return false; + } + struct epoll_event ev = { + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), + .data.ptr = NULL}; + if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) { + if (errno != EINVAL) { + if (!logged_why_not) { + gpr_log( + GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + } else { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + close(evfd); + close(fd); + return true; +} + +#else + +bool grpc_is_epollexclusive_available(void) { return false; } + +#endif diff --git a/src/core/lib/channel/compress_filter.h b/src/core/lib/iomgr/is_epollexclusive_available.h index e4a2a829d5..b65b819e74 100644 --- a/src/core/lib/channel/compress_filter.h +++ b/src/core/lib/iomgr/is_epollexclusive_available.h @@ -31,37 +31,11 @@ * */ -#ifndef GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H -#define GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H +#ifndef GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H +#define GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H -#include <grpc/impl/codegen/compression_types.h> +#include <stdbool.h> -#include "src/core/lib/channel/channel_stack.h" +bool grpc_is_epollexclusive_available(void); -extern int grpc_compression_trace; - -/** Compression filter for outgoing data. - * - * See <grpc/compression.h> for the available compression settings. - * - * Compression settings may come from: - * - Channel configuration, as established at channel creation time. - * - The metadata accompanying the outgoing data to be compressed. This is - * taken as a request only. We may choose not to honor it. The metadata key - * is given by \a GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY. - * - * Compression can be disabled for concrete messages (for instance in order to - * prevent CRIME/BEAST type attacks) by having the GRPC_WRITE_NO_COMPRESS set in - * the BEGIN_MESSAGE flags. - * - * The attempted compression mechanism is added to the resulting initial - * metadata under the'grpc-encoding' key. - * - * If compression is actually performed, BEGIN_MESSAGE's flag is modified to - * incorporate GRPC_WRITE_INTERNAL_COMPRESS. Otherwise, and regardless of the - * aforementioned 'grpc-encoding' metadata value, data will pass through - * uncompressed. */ - -extern const grpc_channel_filter grpc_compress_filter; - -#endif /* GRPC_CORE_LIB_CHANNEL_COMPRESS_FILTER_H */ +#endif /* GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H */ diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c new file mode 100644 index 0000000000..898ec1cb1b --- /dev/null +++ b/src/core/lib/iomgr/lockfree_event.c @@ -0,0 +1,254 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/lockfree_event.h" + +#include <grpc/support/log.h> + +#include "src/core/lib/debug/trace.h" + +extern grpc_tracer_flag grpc_polling_trace; + +/* 'state' holds the to call when the fd is readable or writable respectively. + It can contain one of the following values: + CLOSURE_READY : The fd has an I/O event of interest but there is no + closure yet to execute + + CLOSURE_NOT_READY : The fd has no I/O event of interest + + closure ptr : The closure to be executed when the fd has an I/O + event of interest + + shutdown_error | FD_SHUTDOWN_BIT : + 'shutdown_error' field ORed with FD_SHUTDOWN_BIT. + This indicates that the fd is shutdown. Since all + memory allocations are word-aligned, the lower two + bits of the shutdown_error pointer are always 0. So + it is safe to OR these with FD_SHUTDOWN_BIT + + Valid state transitions: + + <closure ptr> <-----3------ CLOSURE_NOT_READY ----1----> CLOSURE_READY + | | ^ | ^ | | + | | | | | | | + | +--------------4----------+ 6 +---------2---------------+ | + | | | + | v | + +-----5-------> [shutdown_error | FD_SHUTDOWN_BIT] <----7---------+ + + For 1, 4 : See grpc_lfev_set_ready() function + For 2, 3 : See grpc_lfev_notify_on() function + For 5,6,7: See grpc_lfev_set_shutdown() function */ + +#define CLOSURE_NOT_READY ((gpr_atm)0) +#define CLOSURE_READY ((gpr_atm)2) + +#define FD_SHUTDOWN_BIT ((gpr_atm)1) + +void grpc_lfev_init(gpr_atm *state) { + gpr_atm_no_barrier_store(state, CLOSURE_NOT_READY); +} + +void grpc_lfev_destroy(gpr_atm *state) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + if (curr & FD_SHUTDOWN_BIT) { + GRPC_ERROR_UNREF((grpc_error *)(curr & ~FD_SHUTDOWN_BIT)); + } else { + GPR_ASSERT(curr == CLOSURE_NOT_READY || curr == CLOSURE_READY); + } +} + +bool grpc_lfev_is_shutdown(gpr_atm *state) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + return (curr & FD_SHUTDOWN_BIT) != 0; +} + +void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_closure *closure) { + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state, + (void *)curr, closure); + } + switch (curr) { + case CLOSURE_NOT_READY: { + /* CLOSURE_NOT_READY -> <closure>. + + We're guaranteed by API that there's an acquire barrier before here, + so there's no need to double-dip and this can be a release-only. + + The release itself pairs with the acquire half of a set_ready full + barrier. */ + if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { + return; /* Successful. Return */ + } + + break; /* retry */ + } + + case CLOSURE_READY: { + /* Change the state to CLOSURE_NOT_READY. Schedule the closure if + successful. If not, the state most likely transitioned to shutdown. + We should retry. + + This can be a no-barrier cas since the state is being transitioned to + CLOSURE_NOT_READY; set_ready and set_shutdown do not schedule any + closure when transitioning out of CLOSURE_NO_READY state (i.e there + is no other code that needs to 'happen-after' this) */ + if (gpr_atm_no_barrier_cas(state, CLOSURE_READY, CLOSURE_NOT_READY)) { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + return; /* Successful. Return */ + } + + break; /* retry */ + } + + default: { + /* 'curr' is either a closure or the fd is shutdown(in which case 'curr' + contains a pointer to the shutdown-error). If the fd is shutdown, + schedule the closure with the shutdown error */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + grpc_error *shutdown_err = (grpc_error *)(curr & ~FD_SHUTDOWN_BIT); + grpc_closure_sched(exec_ctx, closure, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "FD Shutdown", &shutdown_err, 1)); + return; + } + + /* There is already a closure!. This indicates a bug in the code */ + gpr_log(GPR_ERROR, + "notify_on called with a previous callback still pending"); + abort(); + } + } + } + + GPR_UNREACHABLE_CODE(return ); +} + +bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_error *shutdown_err) { + gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; + + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state, + (void *)curr, grpc_error_string(shutdown_err)); + } + switch (curr) { + case CLOSURE_READY: + case CLOSURE_NOT_READY: + /* Need a full barrier here so that the initial load in notify_on + doesn't need a barrier */ + if (gpr_atm_full_cas(state, curr, new_state)) { + return true; /* early out */ + } + break; /* retry */ + + default: { + /* 'curr' is either a closure or the fd is already shutdown */ + + /* If fd is already shutdown, we are done */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + GRPC_ERROR_UNREF(shutdown_err); + return false; + } + + /* Fd is not shutdown. Schedule the closure and move the state to + shutdown state. + Needs an acquire to pair with setting the closure (and get a + happens-after on that edge), and a release to pair with anything + loading the shutdown state. */ + if (gpr_atm_full_cas(state, curr, new_state)) { + grpc_closure_sched(exec_ctx, (grpc_closure *)curr, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "FD Shutdown", &shutdown_err, 1)); + return true; + } + + /* 'curr' was a closure but now changed to a different state. We will + have to retry */ + break; + } + } + } + + GPR_UNREACHABLE_CODE(return false); +} + +void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) { + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr); + } + + switch (curr) { + case CLOSURE_READY: { + /* Already ready. We are done here */ + return; + } + + case CLOSURE_NOT_READY: { + /* No barrier required as we're transitioning to a state that does not + involve a closure */ + if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { + return; /* early out */ + } + break; /* retry */ + } + + default: { + /* 'curr' is either a closure or the fd is shutdown */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + /* The fd is shutdown. Do nothing */ + return; + } + /* Full cas: acquire pairs with this cas' release in the event of a + spurious set_ready; release pairs with this or the acquire in + notify_on (or set_shutdown) */ + else if (gpr_atm_full_cas(state, curr, CLOSURE_NOT_READY)) { + grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); + return; + } + /* else the state changed again (only possible by either a racing + set_ready or set_shutdown functions. In both these cases, the closure + would have been scheduled for execution. So we are done here */ + return; + } + } + } +} diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h new file mode 100644 index 0000000000..1d9119204c --- /dev/null +++ b/src/core/lib/iomgr/lockfree_event.h @@ -0,0 +1,54 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H +#define GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H + +/* Lock free event notification for file descriptors */ + +#include <grpc/support/atm.h> + +#include "src/core/lib/iomgr/exec_ctx.h" + +void grpc_lfev_init(gpr_atm *state); +void grpc_lfev_destroy(gpr_atm *state); +bool grpc_lfev_is_shutdown(gpr_atm *state); + +void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_closure *closure); +/* Returns true on first successful shutdown */ +bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_error *shutdown_err); +void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state); + +#endif /* GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H */ diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index e19ce697b8..69e20098d7 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -40,8 +40,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) - /* A grpc_pollset is a set of file descriptors that a higher level item is interested in. For example: - a server will typically keep a pollset containing all connected channels, @@ -59,7 +57,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); * pollset's mutex must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); -void grpc_pollset_destroy(grpc_pollset *pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); /* Do some work on a pollset. May involve invoking asynchronous callbacks, or actually polling file @@ -75,6 +73,10 @@ void grpc_pollset_destroy(grpc_pollset *pollset); and it is guaranteed that it will not be released by grpc_pollset_work AFTER worker has been destroyed. + It's legal for worker to be NULL: in that case, this specific thread can not + be directly woken with a kick, but maybe be indirectly (with a kick against + the pollset as a whole). + Tries not to block past deadline. May call grpc_closure_list_run on grpc_closure_list, without holding the pollset @@ -84,8 +86,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec deadline) GRPC_MUST_USE_RESULT; /* Break one polling thread out of polling work for this pollset. - If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers. - Otherwise, if specific_worker is non-NULL, then kick that worker. */ + If specific_worker is non-NULL, then kick that worker. */ grpc_error *grpc_pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index a2f81bcd78..5923da98e2 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -106,7 +106,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } -void grpc_pollset_destroy(grpc_pollset *pollset) { +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { uv_close((uv_handle_t *)&pollset->timer, timer_close_cb); // timer.data is a boolean indicating that the timer has finished closing pollset->timer.data = (void *)0; diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 17043c1ea1..b5f454cfa9 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -43,6 +43,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_windows.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + gpr_mu grpc_polling_mu; static grpc_pollset_worker *g_active_poller; static grpc_pollset_worker g_global_root_worker; @@ -114,13 +116,13 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } -void grpc_pollset_destroy(grpc_pollset *pollset) {} +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {} grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; int added_worker = 0; worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = @@ -185,7 +187,7 @@ done: remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } gpr_cv_destroy(&worker.cv); - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; return GRPC_ERROR_NONE; } @@ -227,6 +229,4 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, return GRPC_ERROR_NONE; } -void grpc_kick_poller(void) { grpc_iocp_kick(); } - #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index 94a454c0b7..2a553f4114 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -85,6 +85,11 @@ #define GRPC_LINUX_SOCKETUTILS 1 #endif #endif +#ifndef __GLIBC__ +#define GRPC_LINUX_EPOLL 1 +#define GRPC_LINUX_EVENTFD 1 +#define GRPC_MSG_IOVLEN_TYPE int +#endif #ifndef GRPC_LINUX_EVENTFD #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 #endif diff --git a/src/core/lib/iomgr/resolve_address_uv.c b/src/core/lib/iomgr/resolve_address_uv.c index 102d1aa290..6b468764fc 100644 --- a/src/core/lib/iomgr/resolve_address_uv.c +++ b/src/core/lib/iomgr/resolve_address_uv.c @@ -69,8 +69,9 @@ static int retry_named_port_failure(int status, request *r, int retry_status; uv_getaddrinfo_t *req = gpr_malloc(sizeof(uv_getaddrinfo_t)); req->data = r; + r->port = svc[i][1]; retry_status = uv_getaddrinfo(uv_default_loop(), req, getaddrinfo_cb, - r->host, svc[i][1], r->hints); + r->host, r->port, r->hints); if (retry_status < 0 || getaddrinfo_cb == NULL) { // The callback will not be called gpr_free(req); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 8dcd80d001..6b2b85cce0 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,7 +44,7 @@ #include "src/core/lib/iomgr/combiner.h" -int grpc_resource_quota_trace = 0; +grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false); #define MEMORY_USAGE_ESTIMATION_MAX 65536 @@ -142,6 +142,8 @@ struct grpc_resource_quota { /* Amount of free memory in the resource quota */ int64_t free_pool; + gpr_atm last_size; + /* Has rq_step been scheduled to occur? */ bool step_scheduled; /* Are we currently reclaiming memory */ @@ -305,13 +307,14 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, resource_user->free_pool = 0; resource_quota->free_pool -= amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, resource_quota->free_pool); } - } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) { + } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && + resource_user->free_pool >= 0) { gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", resource_quota->name, resource_user->name); } @@ -340,7 +343,7 @@ static bool rq_reclaim_from_per_user_free_pool( resource_user->free_pool = 0; resource_quota->free_pool += amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, @@ -363,7 +366,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx, : GRPC_RULIST_RECLAIMER_BENIGN; grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list); if (resource_user == NULL) return false; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation", resource_quota->name, resource_user->name, destructive ? "destructive" : "benign"); @@ -581,6 +584,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { resource_quota->combiner = grpc_combiner_create(NULL); resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; + gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -643,11 +647,17 @@ void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, rq_resize_args *a = gpr_malloc(sizeof(*a)); a->resource_quota = grpc_resource_quota_ref_internal(resource_quota); a->size = (int64_t)size; + gpr_atm_no_barrier_store(&resource_quota->last_size, + (gpr_atm)GPR_MIN((size_t)GPR_ATM_MAX, size)); grpc_closure_init(&a->closure, rq_resize, a, grpc_schedule_on_exec_ctx); grpc_closure_sched(&exec_ctx, &a->closure, GRPC_ERROR_NONE); grpc_exec_ctx_finish(&exec_ctx); } +size_t grpc_resource_quota_peek_size(grpc_resource_quota *resource_quota) { + return (size_t)gpr_atm_no_barrier_load(&resource_quota->last_size); +} + /******************************************************************************* * grpc_resource_user channel args api */ @@ -777,7 +787,7 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); ru_ref_by(resource_user, (gpr_atm)size); resource_user->free_pool -= (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -801,7 +811,7 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); bool was_zero_or_negative = resource_user->free_pool <= 0; resource_user->free_pool += (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -830,7 +840,7 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user) { - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete", resource_user->resource_quota->name, resource_user->name); } diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index b9f62cbf83..51122dad01 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -36,6 +36,7 @@ #include <grpc/grpc.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" /** \file Tracks resource usage against a pool. @@ -75,7 +76,7 @@ maintain lists of users (which users arrange to leave before they are destroyed) */ -extern int grpc_resource_quota_trace; +extern grpc_tracer_flag grpc_resource_quota_trace; grpc_resource_quota *grpc_resource_quota_ref_internal( grpc_resource_quota *resource_quota); @@ -90,6 +91,8 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( double grpc_resource_quota_get_memory_pressure( grpc_resource_quota *resource_quota); +size_t grpc_resource_quota_peek_size(grpc_resource_quota *resource_quota); + typedef struct grpc_resource_user grpc_resource_user; grpc_resource_user *grpc_resource_user_create( diff --git a/src/core/lib/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c index 9d2732666b..a6a4cac3e2 100644 --- a/src/core/lib/iomgr/sockaddr_utils.c +++ b/src/core/lib/iomgr/sockaddr_utils.c @@ -55,7 +55,9 @@ int grpc_sockaddr_is_v4mapped(const grpc_resolved_address *resolved_addr, GPR_ASSERT(resolved_addr != resolved_addr4_out); const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr; struct sockaddr_in *addr4_out = - (struct sockaddr_in *)resolved_addr4_out->addr; + resolved_addr4_out == NULL + ? NULL + : (struct sockaddr_in *)resolved_addr4_out->addr; if (addr->sa_family == AF_INET6) { const struct sockaddr_in6 *addr6 = (const struct sockaddr_in6 *)addr; if (memcmp(addr6->sin6_addr.s6_addr, kV4MappedPrefix, diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h index 2b22f11b49..be3ea2038f 100644 --- a/src/core/lib/iomgr/sockaddr_utils.h +++ b/src/core/lib/iomgr/sockaddr_utils.h @@ -50,7 +50,7 @@ int grpc_sockaddr_to_v4mapped(const grpc_resolved_address *addr, grpc_resolved_address *addr6_out); /* If addr is ::, 0.0.0.0, or ::ffff:0.0.0.0, writes the port number to - *port_out (if not NULL) and returns true, otherwise returns false. */ + *port_out (if not NULL) and returns true, otherwise returns false. */ int grpc_sockaddr_is_wildcard(const grpc_resolved_address *addr, int *port_out); /* Writes 0.0.0.0:port and [::]:port to separate sockaddrs. */ diff --git a/src/core/lib/iomgr/socket_mutator.h b/src/core/lib/iomgr/socket_mutator.h index 2f5b6c248e..28b1710ec4 100644 --- a/src/core/lib/iomgr/socket_mutator.h +++ b/src/core/lib/iomgr/socket_mutator.h @@ -37,6 +37,8 @@ #include <grpc/impl/codegen/grpc_types.h> #include <grpc/support/sync.h> +#include <stdbool.h> + #ifdef __cplusplus extern "C" { #endif diff --git a/src/core/lib/iomgr/sys_epoll_wrapper.h b/src/core/lib/iomgr/sys_epoll_wrapper.h new file mode 100644 index 0000000000..2f08423193 --- /dev/null +++ b/src/core/lib/iomgr/sys_epoll_wrapper.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H +#define GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H + +#include <sys/epoll.h> + +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE (1 << 28) +#endif + +#endif /* GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H */ diff --git a/src/core/lib/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h index 0485661316..bc367bdfa5 100644 --- a/src/core/lib/iomgr/tcp_client.h +++ b/src/core/lib/iomgr/tcp_client.h @@ -40,10 +40,6 @@ #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/resolve_address.h" -/* Channel arg (integer) setting how large a slice to try and read from the wire - each time recvmsg (or equivalent) is called */ -#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" - /* Asynchronously connect to an address (specified as (addr, len)), and call cb with arg and the completed connection when done (or call cb with arg and NULL on failure). diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index a108b10da6..5c7da999e0 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -58,7 +58,7 @@ #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/support/string.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct { gpr_mu mu; @@ -114,7 +114,7 @@ done: static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; async_connect *ac = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", ac->addr_str, str); @@ -137,29 +137,7 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { grpc_endpoint *grpc_tcp_client_create_from_fd( grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args, const char *addr_str) { - size_t tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE; - grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); - if (channel_args != NULL) { - for (size_t i = 0; i < channel_args->num_args; i++) { - if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) { - grpc_integer_options options = {(int)tcp_read_chunk_size, 1, - 8 * 1024 * 1024}; - tcp_read_chunk_size = (size_t)grpc_channel_arg_get_integer( - &channel_args->args[i], options); - } else if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - resource_quota = grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); - } - } - } - - grpc_endpoint *ep = - grpc_tcp_create(fd, resource_quota, tcp_read_chunk_size, addr_str); - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - return ep; + return grpc_tcp_create(exec_ctx, fd, channel_args, addr_str); } static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { @@ -174,7 +152,7 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { GRPC_ERROR_REF(error); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: error=%s", ac->addr_str, str); @@ -352,9 +330,9 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); ac->channel_args = grpc_channel_args_copy(channel_args); - if (grpc_tcp_trace) { - gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", - ac->addr_str); + if (GRPC_TRACER_ON(grpc_tcp_trace)) { + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting fd %p", + ac->addr_str, fdobj); } gpr_mu_lock(&ac->mu); diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index 682c24ed56..f0856a76d4 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -46,7 +46,7 @@ #include "src/core/lib/iomgr/tcp_uv.h" #include "src/core/lib/iomgr/timer.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct grpc_uv_tcp_connect { uv_connect_t connect_req; @@ -72,7 +72,7 @@ static void uv_tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; grpc_uv_tcp_connect *connect = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", connect->addr_name, str); @@ -156,7 +156,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, uv_tcp_init(uv_default_loop(), connect->tcp_handle); connect->connect_req.data = connect; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", connect->addr_name); } diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index a356564766..d6baca50ba 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -63,7 +63,7 @@ typedef struct { int refs; grpc_closure on_connect; grpc_endpoint **endpoint; - grpc_resource_quota *resource_quota; + grpc_channel_args *channel_args; } async_connect; static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, @@ -72,7 +72,7 @@ static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, int done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { - grpc_resource_quota_unref_internal(exec_ctx, ac->resource_quota); + grpc_channel_args_destroy(exec_ctx, ac->channel_args); gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_name); gpr_free(ac); @@ -119,7 +119,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (!wsa_success) { error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx"); } else { - *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name); + *ep = + grpc_tcp_create(exec_ctx, socket, ac->channel_args, ac->addr_name); socket = NULL; } } else { @@ -152,17 +153,6 @@ static void tcp_client_connect_impl( grpc_winsocket_callback_info *info; grpc_error *error = GRPC_ERROR_NONE; - grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); - if (channel_args != NULL) { - for (size_t i = 0; i < channel_args->num_args; i++) { - if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - resource_quota = grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); - } - } - } - *endpoint = NULL; /* Use dualstack sockets where available. */ @@ -225,7 +215,7 @@ static void tcp_client_connect_impl( ac->refs = 2; ac->addr_name = grpc_sockaddr_to_uri(addr); ac->endpoint = endpoint; - ac->resource_quota = resource_quota; + ac->channel_args = grpc_channel_args_copy(channel_args); grpc_closure_init(&ac->on_connect, on_connect, ac, grpc_schedule_on_exec_ctx); grpc_closure_init(&ac->on_alarm, on_alarm, ac, grpc_schedule_on_exec_ctx); @@ -247,7 +237,6 @@ failure: } else if (sock != INVALID_SOCKET) { closesocket(sock); } - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); grpc_closure_sched(exec_ctx, on_done, final_error); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 4d7cf3ff51..5d360b0b80 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -52,7 +52,9 @@ #include <grpc/support/string_util.h> #include <grpc/support/sync.h> #include <grpc/support/time.h> +#include <grpc/support/useful.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/profiling/timers.h" @@ -72,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type; typedef size_t msg_iovlen_type; #endif -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; @@ -80,10 +82,14 @@ typedef struct { int fd; bool finished_edge; msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */ - size_t slice_size; + double target_length; + double bytes_read_this_round; gpr_refcount refcount; gpr_atm shutdown_count; + int min_read_chunk_size; + int max_read_chunk_size; + /* garbage after the last read */ grpc_slice_buffer last_read_buffer; @@ -108,6 +114,42 @@ typedef struct { grpc_resource_user_slice_allocator slice_allocator; } grpc_tcp; +static void add_to_estimate(grpc_tcp *tcp, size_t bytes) { + tcp->bytes_read_this_round += (double)bytes; +} + +static void finish_estimate(grpc_tcp *tcp) { + /* If we read >80% of the target buffer in one read loop, increase the size + of the target buffer to either the amount read, or twice its previous + value */ + if (tcp->bytes_read_this_round > tcp->target_length * 0.8) { + tcp->target_length = + GPR_MAX(2 * tcp->target_length, tcp->bytes_read_this_round); + } else { + tcp->target_length = + 0.99 * tcp->target_length + 0.01 * tcp->bytes_read_this_round; + } + tcp->bytes_read_this_round = 0; +} + +static size_t get_target_read_size(grpc_tcp *tcp) { + grpc_resource_quota *rq = grpc_resource_user_quota(tcp->resource_user); + double pressure = grpc_resource_quota_get_memory_pressure(rq); + double target = + tcp->target_length * (pressure > 0.8 ? (1.0 - pressure) / 0.2 : 1.0); + size_t sz = (((size_t)GPR_CLAMP(target, tcp->min_read_chunk_size, + tcp->max_read_chunk_size)) + + 255) & + ~(size_t)255; + /* don't use more than 1/16th of the overall resource quota for a single read + * alloc */ + size_t rqmax = grpc_resource_quota_peek_size(rq); + if (sz > rqmax / 16 && rqmax > 1024) { + sz = rqmax / 16; + } + return sz; +} + static grpc_error *tcp_annotate_error(grpc_error *src_error, grpc_tcp *tcp) { return grpc_error_set_str( grpc_error_set_int(src_error, GRPC_ERROR_INT_FD, tcp->fd), @@ -179,7 +221,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, grpc_error *error) { grpc_closure *cb = tcp->read_cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -232,9 +274,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { /* NB: After calling call_read_cb a parallel call of the read handler may * be running. */ if (errno == EAGAIN) { - if (tcp->iov_size > 1) { - tcp->iov_size /= 2; - } + finish_estimate(tcp); /* We've consumed the edge, request a new one */ grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { @@ -253,14 +293,13 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { GRPC_ERROR_CREATE_FROM_STATIC_STRING("Socket closed"), tcp)); TCP_UNREF(exec_ctx, tcp, "read"); } else { + add_to_estimate(tcp, (size_t)read_bytes); GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); if ((size_t)read_bytes < tcp->incoming_buffer->length) { grpc_slice_buffer_trim_end( tcp->incoming_buffer, tcp->incoming_buffer->length - (size_t)read_bytes, &tcp->last_read_buffer); - } else if (tcp->iov_size < MAX_READ_IOVEC) { - ++tcp->iov_size; } GPR_ASSERT((size_t)read_bytes == tcp->incoming_buffer->length); call_read_cb(exec_ctx, tcp, GRPC_ERROR_NONE); @@ -285,11 +324,11 @@ static void tcp_read_allocation_done(grpc_exec_ctx *exec_ctx, void *tcpp, } static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { - if (tcp->incoming_buffer->count < (size_t)tcp->iov_size) { - grpc_resource_user_alloc_slices( - exec_ctx, &tcp->slice_allocator, tcp->slice_size, - (size_t)tcp->iov_size - tcp->incoming_buffer->count, - tcp->incoming_buffer); + size_t target_read_size = get_target_read_size(tcp); + if (tcp->incoming_buffer->length < target_read_size && + tcp->incoming_buffer->count < MAX_READ_IOVEC) { + grpc_resource_user_alloc_slices(exec_ctx, &tcp->slice_allocator, + target_read_size, 1, tcp->incoming_buffer); } else { tcp_do_read(exec_ctx, tcp); } @@ -429,14 +468,14 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, } if (!tcp_flush(tcp, &error)) { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { cb = tcp->write_cb; tcp->write_cb = NULL; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } @@ -451,7 +490,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_tcp *tcp = (grpc_tcp *)ep; grpc_error *error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; for (i = 0; i < buf->count; i++) { @@ -482,12 +521,12 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, if (!tcp_flush(tcp, &error)) { TCP_REF(tcp, "write"); tcp->write_cb = cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } @@ -540,9 +579,50 @@ static const grpc_endpoint_vtable vtable = {tcp_read, tcp_get_peer, tcp_get_fd}; -grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, - grpc_resource_quota *resource_quota, - size_t slice_size, const char *peer_string) { +#define MAX_CHUNK_SIZE 32 * 1024 * 1024 + +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_fd *em_fd, + const grpc_channel_args *channel_args, + const char *peer_string) { + int tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE; + int tcp_max_read_chunk_size = 4 * 1024 * 1024; + int tcp_min_read_chunk_size = 256; + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_min_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_max_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_ref_internal( + channel_args->args[i].value.pointer.p); + } + } + } + + if (tcp_min_read_chunk_size > tcp_max_read_chunk_size) { + tcp_min_read_chunk_size = tcp_max_read_chunk_size; + } + tcp_read_chunk_size = GPR_CLAMP(tcp_read_chunk_size, tcp_min_read_chunk_size, + tcp_max_read_chunk_size); + grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->peer_string = gpr_strdup(peer_string); @@ -552,7 +632,10 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, tcp->release_fd_cb = NULL; tcp->release_fd = NULL; tcp->incoming_buffer = NULL; - tcp->slice_size = slice_size; + tcp->target_length = (double)tcp_read_chunk_size; + tcp->min_read_chunk_size = tcp_min_read_chunk_size; + tcp->max_read_chunk_size = tcp_max_read_chunk_size; + tcp->bytes_read_this_round = 0; tcp->iov_size = 1; tcp->finished_edge = true; /* paired with unref in grpc_tcp_destroy */ @@ -569,6 +652,7 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, &tcp->slice_allocator, tcp->resource_user, tcp_read_allocation_done, tcp); /* Tell network status tracker about new endpoint */ grpc_network_status_register_endpoint(&tcp->base); + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); return &tcp->base; } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 1c0d13f96e..4ad60c116e 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -44,17 +44,17 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/ev_posix.h" -#define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 - -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. Takes ownership of fd. */ -grpc_endpoint *grpc_tcp_create(grpc_fd *fd, grpc_resource_quota *resource_quota, - size_t read_slice_size, const char *peer_string); +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + const grpc_channel_args *args, + const char *peer_string); /* Return the tcp endpoint's fd, or -1 if this is not available. Does not release the fd. diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index d6a017cf7f..08997b5e2b 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -59,6 +59,7 @@ #include <grpc/support/time.h> #include <grpc/support/useful.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -90,7 +91,6 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s = gpr_zalloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; - s->resource_quota = grpc_resource_quota_create(NULL); s->expand_wildcard_addrs = false; for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) { @@ -98,27 +98,14 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, s->so_reuseport = has_so_reuseport && (args->args[i].value.integer != 0); } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE_FROM_STATIC_STRING(GRPC_ARG_ALLOW_REUSEPORT " must be an integer"); } - } else if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { - if (args->args[i].type == GRPC_ARG_POINTER) { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - s->resource_quota = - grpc_resource_quota_ref_internal(args->args[i].value.pointer.p); - } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - gpr_free(s); - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); - } } else if (0 == strcmp(GRPC_ARG_EXPAND_WILDCARD_ADDRS, args->args[i].key)) { if (args->args[i].type == GRPC_ARG_INTEGER) { s->expand_wildcard_addrs = (args->args[i].value.integer != 0); } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE_FROM_STATIC_STRING( GRPC_ARG_EXPAND_WILDCARD_ADDRS " must be an integer"); @@ -138,6 +125,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, s->head = NULL; s->tail = NULL; s->nports = 0; + s->channel_args = grpc_channel_args_copy(args); gpr_atm_no_barrier_store(&s->next_pollset_to_assign, 0); *server = s; return GRPC_ERROR_NONE; @@ -158,8 +146,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { s->head = sp->next; gpr_free(sp); } - - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); + grpc_channel_args_destroy(exec_ctx, s->channel_args); gpr_free(s); } @@ -270,7 +257,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { addr_str = grpc_sockaddr_to_uri(&addr); gpr_asprintf(&name, "tcp-server-connection:%s", addr_str); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str); } @@ -286,8 +273,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { sp->server->on_accept_cb( exec_ctx, sp->server->on_accept_cb_arg, - grpc_tcp_create(fdobj, sp->server->resource_quota, - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), + grpc_tcp_create(exec_ctx, fdobj, sp->server->channel_args, addr_str), read_notifier_pollset, acceptor); gpr_free(name); diff --git a/src/core/lib/iomgr/tcp_server_utils_posix.h b/src/core/lib/iomgr/tcp_server_utils_posix.h index f5dc8532f9..c15e2e1493 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix.h +++ b/src/core/lib/iomgr/tcp_server_utils_posix.h @@ -103,7 +103,8 @@ struct grpc_tcp_server { /* next pollset to assign a channel to */ gpr_atm next_pollset_to_assign; - grpc_resource_quota *resource_quota; + /* channel args for this server */ + grpc_channel_args *channel_args; }; /* If successful, add a listener to \a s for \a addr, set \a dsmode for the diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index 12ce7d3fdd..4c17f08918 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -46,6 +46,7 @@ #include <grpc/support/sync.h> #include <grpc/support/time.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/iocp_windows.h" #include "src/core/lib/iomgr/pollset_windows.h" #include "src/core/lib/iomgr/resolve_address.h" @@ -102,7 +103,7 @@ struct grpc_tcp_server { /* shutdown callback */ grpc_closure *shutdown_complete; - grpc_resource_quota *resource_quota; + grpc_channel_args *channel_args; }; /* Public function. Allocates the proper data structures to hold a @@ -112,21 +113,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, const grpc_channel_args *args, grpc_tcp_server **server) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); - s->resource_quota = grpc_resource_quota_create(NULL); - for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { - if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { - if (args->args[i].type == GRPC_ARG_POINTER) { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - s->resource_quota = - grpc_resource_quota_ref_internal(args->args[i].value.pointer.p); - } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - gpr_free(s); - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); - } - } - } + s->channel_args = grpc_channel_args_copy(args); gpr_ref_init(&s->refs, 1); gpr_mu_init(&s->mu); s->active_ports = 0; @@ -155,7 +142,7 @@ static void destroy_server(grpc_exec_ctx *exec_ctx, void *arg, grpc_winsocket_destroy(sp->socket); gpr_free(sp); } - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); + grpc_channel_args_destroy(exec_ctx, s->channel_args); gpr_free(s); } @@ -383,8 +370,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_free(utf8_message); } gpr_asprintf(&fd_name, "tcp_server:%s", peer_name_string); - ep = grpc_tcp_create(grpc_winsocket_create(sock, fd_name), - sp->server->resource_quota, peer_name_string); + ep = grpc_tcp_create(exec_ctx, grpc_winsocket_create(sock, fd_name), + sp->server->channel_args, peer_name_string); gpr_free(fd_name); gpr_free(peer_name_string); } else { diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index 8e8db9f7b4..e7157537f6 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -52,7 +52,7 @@ #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; @@ -158,7 +158,7 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread); grpc_slice_buffer_add(tcp->read_slices, sub); error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -199,7 +199,7 @@ static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice_from_static_string(uv_strerror(status))); grpc_closure_sched(exec_ctx, cb, error); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "Initiating read on %p: error=%s", tcp, str); } @@ -217,7 +217,7 @@ static void write_callback(uv_write_t *req, int status) { } else { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write complete on %p: error=%s", tcp, str); } @@ -238,7 +238,7 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice *slice; uv_write_t *write_req; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t j; for (j = 0; j < write_slices->count; j++) { @@ -346,7 +346,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "Creating TCP endpoint %p", tcp); } diff --git a/src/core/lib/iomgr/tcp_uv.h b/src/core/lib/iomgr/tcp_uv.h index 970fcafe4a..106bec5eca 100644 --- a/src/core/lib/iomgr/tcp_uv.h +++ b/src/core/lib/iomgr/tcp_uv.h @@ -44,11 +44,12 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include <uv.h> -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; #define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index 9134883226..bdd4dd07af 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -219,7 +219,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->read_slices = read_slices; grpc_slice_buffer_reset_and_unref_internal(exec_ctx, read_slices); - tcp->read_slice = grpc_slice_malloc(8192); + tcp->read_slice = GRPC_SLICE_MALLOC(8192); buffer.len = (ULONG)GRPC_SLICE_LENGTH( tcp->read_slice); // we know slice size fits in 32bit. @@ -430,9 +430,19 @@ static grpc_endpoint_vtable vtable = {win_read, win_get_peer, win_get_fd}; -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, - grpc_resource_quota *resource_quota, +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, + grpc_channel_args *channel_args, char *peer_string) { + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_ref_internal( + channel_args->args[i].value.pointer.p); + } + } + } grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); memset(tcp, 0, sizeof(grpc_tcp)); tcp->base.vtable = &vtable; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 4402de1c38..abafdb22d2 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -50,8 +50,8 @@ /* Create a tcp endpoint given a winsock handle. * Takes ownership of the handle. */ -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, - grpc_resource_quota *resource_quota, +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, + grpc_channel_args *channel_args, char *peer_string); grpc_error *grpc_tcp_prepare_socket(SOCKET sock); diff --git a/src/core/lib/iomgr/timer.h b/src/core/lib/iomgr/timer.h index d84a278b18..e0338f93c7 100644 --- a/src/core/lib/iomgr/timer.h +++ b/src/core/lib/iomgr/timer.h @@ -101,6 +101,9 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, void grpc_timer_list_init(gpr_timespec now); void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx); +/* Consume a kick issued by grpc_kick_poller */ +void grpc_timer_consume_kick(void); + /* the following must be implemented by each iomgr implementation */ void grpc_kick_poller(void); diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index e53c801929..b28340b71c 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -37,9 +37,13 @@ #include "src/core/lib/iomgr/timer.h" +#include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> #include <grpc/support/sync.h> +#include <grpc/support/tls.h> #include <grpc/support/useful.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/time_averaged_stats.h" #include "src/core/lib/iomgr/timer_heap.h" #include "src/core/lib/support/spinlock.h" @@ -52,12 +56,15 @@ #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); + typedef struct { gpr_mu mu; grpc_time_averaged_stats stats; /* All and only timers with deadlines <= this will be in the heap. */ - gpr_timespec queue_deadline_cap; - gpr_timespec min_deadline; + gpr_atm queue_deadline_cap; + gpr_atm min_deadline; /* Index in the g_shard_queue */ uint32_t shard_queue_index; /* This holds all timers with deadlines < queue_deadline_cap. Timers in this @@ -67,38 +74,92 @@ typedef struct { grpc_timer list; } shard_type; -/* Protects g_shard_queue */ -static gpr_mu g_mu; -/* Allow only one run_some_expired_timers at once */ -static gpr_spinlock g_checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER; +struct shared_mutables { + gpr_atm min_timer; + /* Allow only one run_some_expired_timers at once */ + gpr_spinlock checker_mu; + bool initialized; + /* Protects g_shard_queue */ + gpr_mu mu; +} GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); + +static struct shared_mutables g_shared_mutables = { + .checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER, .initialized = false, +}; static gpr_clock_type g_clock_type; static shard_type g_shards[NUM_SHARDS]; -/* Protected by g_mu */ +/* Protected by g_shared_mutables.mu */ static shard_type *g_shard_queue[NUM_SHARDS]; -static bool g_initialized = false; +static gpr_timespec g_start_time; + +GPR_TLS_DECL(g_last_seen_min_timer); + +static gpr_atm saturating_add(gpr_atm a, gpr_atm b) { + if (a > GPR_ATM_MAX - b) { + return GPR_ATM_MAX; + } + return a + b; +} + +static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, + gpr_atm *next, grpc_error *error); + +static gpr_timespec dbl_to_ts(double d) { + gpr_timespec ts; + ts.tv_sec = (int64_t)d; + ts.tv_nsec = (int32_t)(1e9 * (d - (double)ts.tv_sec)); + ts.clock_type = GPR_TIMESPAN; + return ts; +} + +static gpr_atm timespec_to_atm_round_up(gpr_timespec ts) { + ts = gpr_time_sub(ts, g_start_time); + double x = GPR_MS_PER_SEC * (double)ts.tv_sec + + (double)ts.tv_nsec / GPR_NS_PER_MS + + (double)(GPR_NS_PER_SEC - 1) / (double)GPR_NS_PER_SEC; + if (x < 0) return 0; + if (x > GPR_ATM_MAX) return GPR_ATM_MAX; + return (gpr_atm)x; +} + +static gpr_atm timespec_to_atm_round_down(gpr_timespec ts) { + ts = gpr_time_sub(ts, g_start_time); + double x = + GPR_MS_PER_SEC * (double)ts.tv_sec + (double)ts.tv_nsec / GPR_NS_PER_MS; + if (x < 0) return 0; + if (x > GPR_ATM_MAX) return GPR_ATM_MAX; + return (gpr_atm)x; +} -static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, - gpr_timespec *next, grpc_error *error); +static gpr_timespec atm_to_timespec(gpr_atm x) { + return gpr_time_add(g_start_time, dbl_to_ts((double)x / 1000.0)); +} -static gpr_timespec compute_min_deadline(shard_type *shard) { +static gpr_atm compute_min_deadline(shard_type *shard) { return grpc_timer_heap_is_empty(&shard->heap) - ? shard->queue_deadline_cap + ? saturating_add(shard->queue_deadline_cap, 1) : grpc_timer_heap_top(&shard->heap)->deadline; } void grpc_timer_list_init(gpr_timespec now) { uint32_t i; - g_initialized = true; - gpr_mu_init(&g_mu); + g_shared_mutables.initialized = true; + gpr_mu_init(&g_shared_mutables.mu); g_clock_type = now.clock_type; + g_start_time = now; + g_shared_mutables.min_timer = timespec_to_atm_round_down(now); + gpr_tls_init(&g_last_seen_min_timer); + gpr_tls_set(&g_last_seen_min_timer, 0); + grpc_register_tracer("timer", &grpc_timer_trace); + grpc_register_tracer("timer_check", &grpc_timer_check_trace); for (i = 0; i < NUM_SHARDS; i++) { shard_type *shard = &g_shards[i]; gpr_mu_init(&shard->mu); grpc_time_averaged_stats_init(&shard->stats, 1.0 / ADD_DEADLINE_SCALE, 0.1, 0.5); - shard->queue_deadline_cap = now; + shard->queue_deadline_cap = g_shared_mutables.min_timer; shard->shard_queue_index = i; grpc_timer_heap_init(&shard->heap); shard->list.next = shard->list.prev = &shard->list; @@ -110,29 +171,23 @@ void grpc_timer_list_init(gpr_timespec now) { void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) { int i; run_some_expired_timers( - exec_ctx, gpr_inf_future(g_clock_type), NULL, + exec_ctx, GPR_ATM_MAX, NULL, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Timer list shutdown")); for (i = 0; i < NUM_SHARDS; i++) { shard_type *shard = &g_shards[i]; gpr_mu_destroy(&shard->mu); grpc_timer_heap_destroy(&shard->heap); } - gpr_mu_destroy(&g_mu); - g_initialized = false; + gpr_mu_destroy(&g_shared_mutables.mu); + gpr_tls_destroy(&g_last_seen_min_timer); + g_shared_mutables.initialized = false; } static double ts_to_dbl(gpr_timespec ts) { return (double)ts.tv_sec + 1e-9 * ts.tv_nsec; } -static gpr_timespec dbl_to_ts(double d) { - gpr_timespec ts; - ts.tv_sec = (int64_t)d; - ts.tv_nsec = (int32_t)(1e9 * (d - (double)ts.tv_sec)); - ts.clock_type = GPR_TIMESPAN; - return ts; -} - +/* returns true if the first element in the list */ static void list_join(grpc_timer *head, grpc_timer *timer) { timer->next = head; timer->prev = head->prev; @@ -158,15 +213,13 @@ static void swap_adjacent_shards_in_queue(uint32_t first_shard_queue_index) { static void note_deadline_change(shard_type *shard) { while (shard->shard_queue_index > 0 && - gpr_time_cmp( - shard->min_deadline, - g_shard_queue[shard->shard_queue_index - 1]->min_deadline) < 0) { + shard->min_deadline < + g_shard_queue[shard->shard_queue_index - 1]->min_deadline) { swap_adjacent_shards_in_queue(shard->shard_queue_index - 1); } while (shard->shard_queue_index < NUM_SHARDS - 1 && - gpr_time_cmp( - shard->min_deadline, - g_shard_queue[shard->shard_queue_index + 1]->min_deadline) > 0) { + shard->min_deadline > + g_shard_queue[shard->shard_queue_index + 1]->min_deadline) { swap_adjacent_shards_in_queue(shard->shard_queue_index); } } @@ -179,9 +232,16 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, GPR_ASSERT(deadline.clock_type == g_clock_type); GPR_ASSERT(now.clock_type == g_clock_type); timer->closure = closure; - timer->deadline = deadline; + gpr_atm deadline_atm = timer->deadline = timespec_to_atm_round_up(deadline); + + if (GRPC_TRACER_ON(grpc_timer_trace)) { + gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 ".%09d [%" PRIdPTR + "] now %" PRId64 ".%09d [%" PRIdPTR "] call %p[%p]", + timer, deadline.tv_sec, deadline.tv_nsec, deadline_atm, now.tv_sec, + now.tv_nsec, timespec_to_atm_round_down(now), closure, closure->cb); + } - if (!g_initialized) { + if (!g_shared_mutables.initialized) { timer->pending = false; grpc_closure_sched(exec_ctx, timer->closure, GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -201,12 +261,18 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, grpc_time_averaged_stats_add_sample(&shard->stats, ts_to_dbl(gpr_time_sub(deadline, now))); - if (gpr_time_cmp(deadline, shard->queue_deadline_cap) < 0) { + if (deadline_atm < shard->queue_deadline_cap) { is_first_timer = grpc_timer_heap_add(&shard->heap, timer); } else { timer->heap_index = INVALID_HEAP_INDEX; list_join(&shard->list, timer); } + if (GRPC_TRACER_ON(grpc_timer_trace)) { + gpr_log(GPR_DEBUG, " .. add to shard %d with queue_deadline_cap=%" PRIdPTR + " => is_first_timer=%s", + (int)(shard - g_shards), shard->queue_deadline_cap, + is_first_timer ? "true" : "false"); + } gpr_mu_unlock(&shard->mu); /* Deadline may have decreased, we need to adjust the master queue. Note @@ -221,28 +287,41 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, In that case, the timer will simply have to wait for the next grpc_timer_check. */ if (is_first_timer) { - gpr_mu_lock(&g_mu); - if (gpr_time_cmp(deadline, shard->min_deadline) < 0) { - gpr_timespec old_min_deadline = g_shard_queue[0]->min_deadline; - shard->min_deadline = deadline; + gpr_mu_lock(&g_shared_mutables.mu); + if (GRPC_TRACER_ON(grpc_timer_trace)) { + gpr_log(GPR_DEBUG, " .. old shard min_deadline=%" PRIdPTR, + shard->min_deadline); + } + if (deadline_atm < shard->min_deadline) { + gpr_atm old_min_deadline = g_shard_queue[0]->min_deadline; + shard->min_deadline = deadline_atm; note_deadline_change(shard); - if (shard->shard_queue_index == 0 && - gpr_time_cmp(deadline, old_min_deadline) < 0) { + if (shard->shard_queue_index == 0 && deadline_atm < old_min_deadline) { + gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, deadline_atm); grpc_kick_poller(); } } - gpr_mu_unlock(&g_mu); + gpr_mu_unlock(&g_shared_mutables.mu); } } +void grpc_timer_consume_kick(void) { + /* force re-evaluation of last seeen min */ + gpr_tls_set(&g_last_seen_min_timer, 0); +} + void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { - if (!g_initialized) { + if (!g_shared_mutables.initialized) { /* must have already been cancelled, also the shard mutex is invalid */ return; } shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)]; gpr_mu_lock(&shard->mu); + if (GRPC_TRACER_ON(grpc_timer_trace)) { + gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer, + timer->pending ? "true" : "false"); + } if (timer->pending) { grpc_closure_sched(exec_ctx, timer->closure, GRPC_ERROR_CANCELLED); timer->pending = false; @@ -260,7 +339,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { for timers that fall at or under it. Returns true if the queue is no longer empty. REQUIRES: shard->mu locked */ -static int refill_queue(shard_type *shard, gpr_timespec now) { +static int refill_queue(shard_type *shard, gpr_atm now) { /* Compute the new queue window width and bound by the limits: */ double computed_deadline_delta = grpc_time_averaged_stats_update_average(&shard->stats) * @@ -271,12 +350,22 @@ static int refill_queue(shard_type *shard, gpr_timespec now) { grpc_timer *timer, *next; /* Compute the new cap and put all timers under it into the queue: */ - shard->queue_deadline_cap = gpr_time_add( - gpr_time_max(now, shard->queue_deadline_cap), dbl_to_ts(deadline_delta)); + shard->queue_deadline_cap = + saturating_add(GPR_MAX(now, shard->queue_deadline_cap), + (gpr_atm)(deadline_delta * 1000.0)); + + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, " .. shard[%d]->queue_deadline_cap --> %" PRIdPTR, + (int)(shard - g_shards), shard->queue_deadline_cap); + } for (timer = shard->list.next; timer != &shard->list; timer = next) { next = timer->next; - if (gpr_time_cmp(timer->deadline, shard->queue_deadline_cap) < 0) { + if (timer->deadline < shard->queue_deadline_cap) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, " .. add timer with deadline %" PRIdPTR " to heap", + timer->deadline); + } list_remove(timer); grpc_timer_heap_add(&shard->heap, timer); } @@ -287,15 +376,29 @@ static int refill_queue(shard_type *shard, gpr_timespec now) { /* This pops the next non-cancelled timer with deadline <= now from the queue, or returns NULL if there isn't one. REQUIRES: shard->mu locked */ -static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) { +static grpc_timer *pop_one(shard_type *shard, gpr_atm now) { grpc_timer *timer; for (;;) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, " .. shard[%d]: heap_empty=%s", + (int)(shard - g_shards), + grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false"); + } if (grpc_timer_heap_is_empty(&shard->heap)) { - if (gpr_time_cmp(now, shard->queue_deadline_cap) < 0) return NULL; + if (now < shard->queue_deadline_cap) return NULL; if (!refill_queue(shard, now)) return NULL; } timer = grpc_timer_heap_top(&shard->heap); - if (gpr_time_cmp(timer->deadline, now) > 0) return NULL; + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, + " .. check top timer deadline=%" PRIdPTR " now=%" PRIdPTR, + timer->deadline, now); + } + if (timer->deadline > now) return NULL; + if (GRPC_TRACER_ON(grpc_timer_trace)) { + gpr_log(GPR_DEBUG, "TIMER %p: FIRE %" PRIdPTR "ms late", timer, + now - timer->deadline); + } timer->pending = false; grpc_timer_heap_pop(&shard->heap); return timer; @@ -304,7 +407,7 @@ static grpc_timer *pop_one(shard_type *shard, gpr_timespec now) { /* REQUIRES: shard->mu unlocked */ static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard, - gpr_timespec now, gpr_timespec *new_min_deadline, + gpr_atm now, gpr_atm *new_min_deadline, grpc_error *error) { size_t n = 0; grpc_timer *timer; @@ -318,17 +421,29 @@ static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard, return n; } -static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, - gpr_timespec *next, grpc_error *error) { +static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, + gpr_atm *next, grpc_error *error) { size_t n = 0; - /* TODO(ctiller): verify that there are any timers (atomically) here */ + gpr_atm min_timer = gpr_atm_no_barrier_load(&g_shared_mutables.min_timer); + gpr_tls_set(&g_last_seen_min_timer, min_timer); + if (now < min_timer) { + if (next != NULL) *next = GPR_MIN(*next, min_timer); + return 0; + } + + if (gpr_spinlock_trylock(&g_shared_mutables.checker_mu)) { + gpr_mu_lock(&g_shared_mutables.mu); - if (gpr_spinlock_trylock(&g_checker_mu)) { - gpr_mu_lock(&g_mu); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, " .. shard[%d]->min_deadline = %" PRIdPTR, + (int)(g_shard_queue[0] - g_shards), + g_shard_queue[0]->min_deadline); + } - while (gpr_time_cmp(g_shard_queue[0]->min_deadline, now) < 0) { - gpr_timespec new_min_deadline; + while (g_shard_queue[0]->min_deadline < now || + (now != GPR_ATM_MAX && g_shard_queue[0]->min_deadline == now)) { + gpr_atm new_min_deadline; /* For efficiency, we pop as many available timers as we can from the shard. This may violate perfect timer deadline ordering, but that @@ -336,6 +451,14 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, n += pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, error); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, " .. popped --> %" PRIdPTR + ", shard[%d]->min_deadline %" PRIdPTR + " --> %" PRIdPTR ", now=%" PRIdPTR, + n, (int)(g_shard_queue[0] - g_shards), + g_shard_queue[0]->min_deadline, new_min_deadline, now); + } + /* An grpc_timer_init() on the shard could intervene here, adding a new timer that is earlier than new_min_deadline. However, grpc_timer_init() will block on the master_lock before it can call @@ -346,23 +469,24 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, } if (next) { - *next = gpr_time_min(*next, g_shard_queue[0]->min_deadline); + *next = GPR_MIN(*next, g_shard_queue[0]->min_deadline); } - gpr_mu_unlock(&g_mu); - gpr_spinlock_unlock(&g_checker_mu); + gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, + g_shard_queue[0]->min_deadline); + gpr_mu_unlock(&g_shared_mutables.mu); + gpr_spinlock_unlock(&g_shared_mutables.checker_mu); } else if (next != NULL) { /* TODO(ctiller): this forces calling code to do an short poll, and then retry the timer check (because this time through the timer list was contended). - We could reduce the cost here dramatically by keeping a count of how many - currently active pollers got through the uncontended case above + We could reduce the cost here dramatically by keeping a count of how + many currently active pollers got through the uncontended case above successfully, and waking up other pollers IFF that count drops to zero. Once that count is in place, this entire else branch could disappear. */ - *next = gpr_time_min( - *next, gpr_time_add(now, gpr_time_from_millis(1, GPR_TIMESPAN))); + *next = GPR_MIN(*next, now + 1); } GRPC_ERROR_UNREF(error); @@ -372,12 +496,71 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_timespec now, bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec *next) { + // prelude GPR_ASSERT(now.clock_type == g_clock_type); - return run_some_expired_timers( - exec_ctx, now, next, + gpr_atm now_atm = timespec_to_atm_round_down(now); + + /* fetch from a thread-local first: this avoids contention on a globally + mutable cacheline in the common case */ + gpr_atm min_timer = gpr_tls_get(&g_last_seen_min_timer); + if (now_atm < min_timer) { + if (next != NULL) { + *next = + atm_to_timespec(GPR_MIN(timespec_to_atm_round_up(*next), min_timer)); + } + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, + "TIMER CHECK SKIP: now_atm=%" PRIdPTR " min_timer=%" PRIdPTR, + now_atm, min_timer); + } + return 0; + } + + grpc_error *shutdown_error = gpr_time_cmp(now, gpr_inf_future(now.clock_type)) != 0 ? GRPC_ERROR_NONE - : GRPC_ERROR_CREATE_FROM_STATIC_STRING("Shutting down timer system")); + : GRPC_ERROR_CREATE_FROM_STATIC_STRING("Shutting down timer system"); + + // tracing + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + char *next_str; + if (next == NULL) { + next_str = gpr_strdup("NULL"); + } else { + gpr_asprintf(&next_str, "%" PRId64 ".%09d [%" PRIdPTR "]", next->tv_sec, + next->tv_nsec, timespec_to_atm_round_down(*next)); + } + gpr_log(GPR_DEBUG, "TIMER CHECK BEGIN: now=%" PRId64 ".%09d [%" PRIdPTR + "] next=%s tls_min=%" PRIdPTR " glob_min=%" PRIdPTR, + now.tv_sec, now.tv_nsec, now_atm, next_str, + gpr_tls_get(&g_last_seen_min_timer), + gpr_atm_no_barrier_load(&g_shared_mutables.min_timer)); + gpr_free(next_str); + } + // actual code + bool r; + gpr_atm next_atm; + if (next == NULL) { + r = run_some_expired_timers(exec_ctx, now_atm, NULL, shutdown_error); + } else { + next_atm = timespec_to_atm_round_down(*next); + r = run_some_expired_timers(exec_ctx, now_atm, &next_atm, shutdown_error); + *next = atm_to_timespec(next_atm); + } + // tracing + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + char *next_str; + if (next == NULL) { + next_str = gpr_strdup("NULL"); + } else { + gpr_asprintf(&next_str, "%" PRId64 ".%09d [%" PRIdPTR "]", next->tv_sec, + next->tv_nsec, next_atm); + } + gpr_log(GPR_DEBUG, "TIMER CHECK END: %d timers triggered; next=%s", r, + next_str); + gpr_free(next_str); + } + return r > 0; } #endif /* GRPC_TIMER_USE_GENERIC */ diff --git a/src/core/lib/iomgr/timer_generic.h b/src/core/lib/iomgr/timer_generic.h index 1608dce9fb..c79a431aa0 100644 --- a/src/core/lib/iomgr/timer_generic.h +++ b/src/core/lib/iomgr/timer_generic.h @@ -38,7 +38,7 @@ #include "src/core/lib/iomgr/exec_ctx.h" struct grpc_timer { - gpr_timespec deadline; + gpr_atm deadline; uint32_t heap_index; /* INVALID_HEAP_INDEX if not in heap */ bool pending; struct grpc_timer *next; diff --git a/src/core/lib/iomgr/timer_heap.c b/src/core/lib/iomgr/timer_heap.c index f736d335e6..03ccfe023a 100644 --- a/src/core/lib/iomgr/timer_heap.c +++ b/src/core/lib/iomgr/timer_heap.c @@ -50,7 +50,7 @@ static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) { while (i > 0) { uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break; + if (first[parent]->deadline <= t->deadline) break; first[i] = first[parent]; first[i]->heap_index = i; i = parent; @@ -68,12 +68,12 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length, uint32_t left_child = 1u + 2u * i; if (left_child >= length) break; uint32_t right_child = left_child + 1; - uint32_t next_i = right_child < length && - gpr_time_cmp(first[left_child]->deadline, - first[right_child]->deadline) > 0 - ? right_child - : left_child; - if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break; + uint32_t next_i = + right_child < length && + first[left_child]->deadline > first[right_child]->deadline + ? right_child + : left_child; + if (t->deadline <= first[next_i]->deadline) break; first[i] = first[next_i]; first[i]->heap_index = i; i = next_i; @@ -97,7 +97,7 @@ static void maybe_shrink(grpc_timer_heap *heap) { static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) { uint32_t i = timer->heap_index; uint32_t parent = (uint32_t)(((int)i - 1) / 2); - if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) { + if (heap->timers[parent]->deadline > timer->deadline) { adjust_upwards(heap->timers, i, timer); } else { adjust_downwards(heap->timers, i, heap->timer_count, timer); diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c new file mode 100644 index 0000000000..24085093e7 --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.c @@ -0,0 +1,276 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/timer_manager.h" + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/thd.h> + +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/timer.h" + +typedef struct completed_thread { + gpr_thd_id t; + struct completed_thread *next; +} completed_thread; + +extern grpc_tracer_flag grpc_timer_check_trace; + +// global mutex +static gpr_mu g_mu; +// are we multi-threaded +static bool g_threaded; +// cv to wait until a thread is needed +static gpr_cv g_cv_wait; +// cv for notification when threading ends +static gpr_cv g_cv_shutdown; +// number of threads in the system +static int g_thread_count; +// number of threads sitting around waiting +static int g_waiter_count; +// linked list of threads that have completed (and need joining) +static completed_thread *g_completed_threads; +// was the manager kicked by the timer system +static bool g_kicked; +// is there a thread waiting until the next timer should fire? +static bool g_has_timed_waiter; +// generation counter to track which thread is waiting for the next timer +static uint64_t g_timed_waiter_generation; + +static void timer_thread(void *unused); + +static void gc_completed_threads(void) { + if (g_completed_threads != NULL) { + completed_thread *to_gc = g_completed_threads; + g_completed_threads = NULL; + gpr_mu_unlock(&g_mu); + while (to_gc != NULL) { + gpr_thd_join(to_gc->t); + completed_thread *next = to_gc->next; + gpr_free(to_gc); + to_gc = next; + } + gpr_mu_lock(&g_mu); + } +} + +static void start_timer_thread_and_unlock(void) { + GPR_ASSERT(g_threaded); + ++g_waiter_count; + ++g_thread_count; + gpr_mu_unlock(&g_mu); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "Spawn timer thread"); + } + gpr_thd_id thd; + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&opt); + gpr_thd_new(&thd, timer_thread, NULL, &opt); +} + +void grpc_timer_manager_tick() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + grpc_timer_check(&exec_ctx, now, &next); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void timer_thread(void *unused) { + // this threads exec_ctx: we try to run things through to completion here + // since it's easy to spin up new threads + grpc_exec_ctx exec_ctx = + GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + const gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_MONOTONIC); + for (;;) { + gpr_timespec next = inf_future; + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + // check timer state, updates next to the next time to run a check + if (grpc_timer_check(&exec_ctx, now, &next)) { + // if there's something to execute... + gpr_mu_lock(&g_mu); + // remove a waiter from the pool, and start another thread if necessary + --g_waiter_count; + if (g_waiter_count == 0 && g_threaded) { + start_timer_thread_and_unlock(); + } else { + // if there's no thread waiting with a timeout, kick an existing waiter + // so that the next deadline is not missed + if (!g_has_timed_waiter) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "kick untimed waiter"); + } + gpr_cv_signal(&g_cv_wait); + } + gpr_mu_unlock(&g_mu); + } + // without our lock, flush the exec_ctx + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(&g_mu); + // garbage collect any threads hanging out that are dead + gc_completed_threads(); + // get ready to wait again + ++g_waiter_count; + gpr_mu_unlock(&g_mu); + } else { + gpr_mu_lock(&g_mu); + // if we're not threaded anymore, leave + if (!g_threaded) break; + // if there's no timed waiter, we should become one: that waiter waits + // only until the next timer should expire + // all other timers wait forever + uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1; + if (!g_has_timed_waiter) { + g_has_timed_waiter = true; + // we use a generation counter to track the timed waiter so we can + // cancel an existing one quickly (and when it actually times out it'll + // figure stuff out instead of incurring a wakeup) + my_timed_waiter_generation = ++g_timed_waiter_generation; + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep for a while"); + } + } else { + next = inf_future; + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep until kicked"); + } + } + gpr_cv_wait(&g_cv_wait, &g_mu, next); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", + my_timed_waiter_generation == g_timed_waiter_generation, + g_kicked); + } + // if this was the timed waiter, then we need to check timers, and flag + // that there's now no timed waiter... we'll look for a replacement if + // there's work to do after checking timers (code above) + if (my_timed_waiter_generation == g_timed_waiter_generation) { + g_has_timed_waiter = false; + } + // if this was a kick from the timer system, consume it (and don't stop + // this thread yet) + if (g_kicked) { + grpc_timer_consume_kick(); + g_kicked = false; + } + gpr_mu_unlock(&g_mu); + } + } + // terminate the thread: drop the waiter count, thread count, and let whomever + // stopped the threading stuff know that we're done + --g_waiter_count; + --g_thread_count; + if (0 == g_thread_count) { + gpr_cv_signal(&g_cv_shutdown); + } + completed_thread *ct = gpr_malloc(sizeof(*ct)); + ct->t = gpr_thd_currentid(); + ct->next = g_completed_threads; + g_completed_threads = ct; + gpr_mu_unlock(&g_mu); + grpc_exec_ctx_finish(&exec_ctx); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "End timer thread"); + } +} + +static void start_threads(void) { + gpr_mu_lock(&g_mu); + if (!g_threaded) { + g_threaded = true; + start_timer_thread_and_unlock(); + } else { + g_threaded = false; + gpr_mu_unlock(&g_mu); + } +} + +void grpc_timer_manager_init(void) { + gpr_mu_init(&g_mu); + gpr_cv_init(&g_cv_wait); + gpr_cv_init(&g_cv_shutdown); + g_threaded = false; + g_thread_count = 0; + g_waiter_count = 0; + g_completed_threads = NULL; + + start_threads(); +} + +static void stop_threads(void) { + gpr_mu_lock(&g_mu); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "stop timer threads: threaded=%d", g_threaded); + } + if (g_threaded) { + g_threaded = false; + gpr_cv_broadcast(&g_cv_wait); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } + while (g_thread_count > 0) { + gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } + gc_completed_threads(); + } + } + gpr_mu_unlock(&g_mu); +} + +void grpc_timer_manager_shutdown(void) { + stop_threads(); + + gpr_mu_destroy(&g_mu); + gpr_cv_destroy(&g_cv_wait); + gpr_cv_destroy(&g_cv_shutdown); +} + +void grpc_timer_manager_set_threading(bool threaded) { + if (threaded) { + start_threads(); + } else { + stop_threads(); + } +} + +void grpc_kick_poller(void) { + gpr_mu_lock(&g_mu); + g_kicked = true; + g_has_timed_waiter = false; + ++g_timed_waiter_generation; + gpr_cv_signal(&g_cv_wait); + gpr_mu_unlock(&g_mu); +} diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h new file mode 100644 index 0000000000..46729ccea6 --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.h @@ -0,0 +1,52 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H +#define GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H + +#include <stdbool.h> + +/* Timer Manager tries to keep one thread waiting for the next timeout at all + times */ + +void grpc_timer_manager_init(void); +void grpc_timer_manager_shutdown(void); + +/* enable/disable threading - must be called after grpc_timer_manager_init and + * before grpc_timer_manager_shutdown */ +void grpc_timer_manager_set_threading(bool enabled); +/* explicitly perform one tick of the timer system - for when threading is + * disabled */ +void grpc_timer_manager_tick(void); + +#endif /* GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H */ diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 8e8a07578c..2952e44b58 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -38,10 +38,14 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/timer.h" #include <uv.h> +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); + static void timer_close_callback(uv_handle_t *handle) { gpr_free(handle); } static void stop_uv_timer(uv_timer_t *handle) { @@ -100,4 +104,6 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, void grpc_timer_list_init(gpr_timespec now) {} void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {} +void grpc_timer_consume_kick(void) {} + #endif /* GRPC_UV */ diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 60579e18ba..af70746064 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -79,14 +79,24 @@ struct grpc_udp_listener { grpc_resolved_address addr; grpc_closure read_closure; grpc_closure write_closure; + // To be called when corresponding QuicGrpcServer closes all active + // connections. + grpc_closure orphan_fd_closure; grpc_closure destroyed_closure; grpc_udp_server_read_cb read_cb; grpc_udp_server_write_cb write_cb; grpc_udp_server_orphan_cb orphan_cb; + // True if orphan_cb is trigered. + bool orphan_notified; struct grpc_udp_listener *next; }; +struct shutdown_fd_args { + grpc_fd *fd; + gpr_mu *server_mu; +}; + /* the overall server */ struct grpc_udp_server { gpr_mu mu; @@ -146,6 +156,19 @@ grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args) { return s; } +static void shutdown_fd(grpc_exec_ctx *exec_ctx, void *args, + grpc_error *error) { + struct shutdown_fd_args *shutdown_args = (struct shutdown_fd_args *)args; + gpr_mu_lock(shutdown_args->server_mu); + grpc_fd_shutdown(exec_ctx, shutdown_args->fd, GRPC_ERROR_REF(error)); + gpr_mu_unlock(shutdown_args->server_mu); + gpr_free(shutdown_args); +} + +static void dummy_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + // No-op. +} + static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { if (s->shutdown_complete != NULL) { grpc_closure_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE); @@ -195,12 +218,16 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { grpc_closure_init(&sp->destroyed_closure, destroyed_port, s, grpc_schedule_on_exec_ctx); - - /* Call the orphan_cb to signal that the FD is about to be closed and - * should no longer be used. */ - GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); - + if (!sp->orphan_notified) { + /* Call the orphan_cb to signal that the FD is about to be closed and + * should no longer be used. Because at this point, all listening ports + * have been shutdown already, no need to shutdown again.*/ + grpc_closure_init(&sp->orphan_fd_closure, dummy_cb, sp->emfd, + grpc_schedule_on_exec_ctx); + GPR_ASSERT(sp->orphan_cb); + sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure, + sp->server->user_data); + } grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); } @@ -225,9 +252,14 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); - grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Server destroyed")); + struct shutdown_fd_args *args = gpr_malloc(sizeof(*args)); + args->fd = sp->emfd; + args->server_mu = &s->mu; + grpc_closure_init(&sp->orphan_fd_closure, shutdown_fd, args, + grpc_schedule_on_exec_ctx); + sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure, + sp->server->user_data); + sp->orphan_notified = true; } gpr_mu_unlock(&s->mu); } else { @@ -391,6 +423,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, sp->read_cb = read_cb; sp->write_cb = write_cb; sp->orphan_cb = orphan_cb; + sp->orphan_notified = false; GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); gpr_free(name); diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index 9df3fe4d1f..8006037644 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -55,7 +55,9 @@ typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd, void *user_data); + grpc_fd *emfd, + grpc_closure *shutdown_fd_callback, + void *user_data); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args); diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c index 52b80141d1..d89da47fc1 100644 --- a/src/core/lib/security/credentials/credentials.c +++ b/src/core/lib/security/credentials/credentials.c @@ -37,7 +37,6 @@ #include <string.h> #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/http_client_filter.h" #include "src/core/lib/http/httpcli.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/executor.h" diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 510b79552a..89b8e3c0b3 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -71,7 +71,7 @@ typedef enum { #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60 -#define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata" +#define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata.google.internal" #define GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH \ "/computeMetadata/v1/instance/service-accounts/default/token" diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c index 68636ba208..3fdb67fb91 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.c +++ b/src/core/lib/security/credentials/fake/fake_credentials.c @@ -39,11 +39,15 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/support/string.h" /* -- Fake transport security credentials. -- */ +#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ + "grpc.fake_security.expected_targets" + static grpc_security_status fake_transport_security_create_security_connector( grpc_exec_ctx *exec_ctx, grpc_channel_credentials *c, grpc_call_credentials *call_creds, const char *target, @@ -88,6 +92,25 @@ grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( return c; } +grpc_arg grpc_fake_transport_expected_targets_arg(char *expected_targets) { + grpc_arg arg; + arg.type = GRPC_ARG_STRING; + arg.key = GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS; + arg.value.string = expected_targets; + return arg; +} + +const char *grpc_fake_transport_get_expected_targets( + const grpc_channel_args *args) { + const grpc_arg *expected_target_arg = + grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); + if (expected_target_arg != NULL && + expected_target_arg->type == GRPC_ARG_STRING) { + return expected_target_arg->value.string; + } + return NULL; +} + /* -- Metadata-only test credentials. -- */ static void md_only_test_destruct(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index 0fe98417c6..a28b545a67 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -38,10 +38,17 @@ /* -- Fake transport security credentials. -- */ +/* Creates a fake transport security credentials object for testing. */ +grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); + +/* Creates a fake server transport security credentials object for testing. */ +grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( + void); + /* Used to verify the target names given to the fake transport security * connector. * - * Its syntax by example: + * The syntax of \a expected_targets by example: * For LB channels: * "backend_target_1,backend_target_2,...;lb_target_1,lb_target_2,..." * For regular channels: @@ -50,15 +57,11 @@ * That is to say, LB channels have a heading list of LB targets separated from * the list of backend targets by a semicolon. For non-LB channels, only the * latter is present. */ -#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ - "grpc.test_only.fake_security.expected_target" +grpc_arg grpc_fake_transport_expected_targets_arg(char *expected_targets); -/* Creates a fake transport security credentials object for testing. */ -grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); - -/* Creates a fake server transport security credentials object for testing. */ -grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( - void); +/* Return the value associated with the expected targets channel arg or NULL */ +const char *grpc_fake_transport_get_expected_targets( + const grpc_channel_args *args); /* -- Metadata-only Test credentials. -- */ diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index 97501e6788..4d8c451ea8 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -99,7 +99,7 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *e) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static int is_stack_running_on_compute_engine(grpc_exec_ctx *exec_ctx) { diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c index 178ce89aa6..0e7c1afb02 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.c +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c @@ -171,7 +171,7 @@ static char *redact_private_key(const char *json_key) { grpc_call_credentials *grpc_service_account_jwt_access_credentials_create( const char *json_key, gpr_timespec token_lifetime, void *reserved) { - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *clean_json = redact_private_key(json_key); gpr_log(GPR_INFO, "grpc_service_account_jwt_access_credentials_create(" diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index ccfb3566c1..29235b6eb3 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -412,7 +412,7 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create( const char *json_refresh_token, void *reserved) { grpc_auth_refresh_token token = grpc_auth_refresh_token_create_from_string(json_refresh_token); - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *loggable_token = create_loggable_refresh_token(&token); gpr_log(GPR_INFO, "grpc_refresh_token_credentials_create(json_refresh_token=%s, " diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c index 4b17ac8098..7c35ebe684 100644 --- a/src/core/lib/security/credentials/ssl/ssl_credentials.c +++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c @@ -36,33 +36,28 @@ #include <string.h> #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/http_client_filter.h" #include "src/core/lib/surface/api_trace.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> // -// Utils +// SSL Channel Credentials. // -static void ssl_copy_key_material(const char *input, unsigned char **output, - size_t *output_size) { - *output_size = strlen(input); - *output = gpr_malloc(*output_size); - memcpy(*output, input, *output_size); +static void ssl_config_pem_key_cert_pair_destroy( + tsi_ssl_pem_key_cert_pair *kp) { + if (kp == NULL) return; + gpr_free((void *)kp->private_key); + gpr_free((void *)kp->cert_chain); } -// -// SSL Channel Credentials. -// - static void ssl_destruct(grpc_exec_ctx *exec_ctx, grpc_channel_credentials *creds) { grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds; - if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); - if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key); - if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain); + gpr_free(c->config.pem_root_certs); + ssl_config_pem_key_cert_pair_destroy(&c->config.pem_key_cert_pair); } static grpc_security_status ssl_create_security_connector( @@ -103,18 +98,15 @@ static void ssl_build_config(const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, grpc_ssl_config *config) { if (pem_root_certs != NULL) { - ssl_copy_key_material(pem_root_certs, &config->pem_root_certs, - &config->pem_root_certs_size); + config->pem_root_certs = gpr_strdup(pem_root_certs); } if (pem_key_cert_pair != NULL) { GPR_ASSERT(pem_key_cert_pair->private_key != NULL); GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL); - ssl_copy_key_material(pem_key_cert_pair->private_key, - &config->pem_private_key, - &config->pem_private_key_size); - ssl_copy_key_material(pem_key_cert_pair->cert_chain, - &config->pem_cert_chain, - &config->pem_cert_chain_size); + config->pem_key_cert_pair.cert_chain = + gpr_strdup(pem_key_cert_pair->cert_chain); + config->pem_key_cert_pair.private_key = + gpr_strdup(pem_key_cert_pair->private_key); } } @@ -144,22 +136,10 @@ static void ssl_server_destruct(grpc_exec_ctx *exec_ctx, grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds; size_t i; for (i = 0; i < c->config.num_key_cert_pairs; i++) { - if (c->config.pem_private_keys[i] != NULL) { - gpr_free(c->config.pem_private_keys[i]); - } - if (c->config.pem_cert_chains[i] != NULL) { - gpr_free(c->config.pem_cert_chains[i]); - } - } - if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys); - if (c->config.pem_private_keys_sizes != NULL) { - gpr_free(c->config.pem_private_keys_sizes); - } - if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains); - if (c->config.pem_cert_chains_sizes != NULL) { - gpr_free(c->config.pem_cert_chains_sizes); + ssl_config_pem_key_cert_pair_destroy(&c->config.pem_key_cert_pairs[i]); } - if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); + gpr_free(c->config.pem_key_cert_pairs); + gpr_free(c->config.pem_root_certs); } static grpc_security_status ssl_server_create_security_connector( @@ -180,30 +160,21 @@ static void ssl_build_server_config( size_t i; config->client_certificate_request = client_certificate_request; if (pem_root_certs != NULL) { - ssl_copy_key_material(pem_root_certs, &config->pem_root_certs, - &config->pem_root_certs_size); + config->pem_root_certs = gpr_strdup(pem_root_certs); } if (num_key_cert_pairs > 0) { GPR_ASSERT(pem_key_cert_pairs != NULL); - config->pem_private_keys = - gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *)); - config->pem_cert_chains = - gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *)); - config->pem_private_keys_sizes = - gpr_malloc(num_key_cert_pairs * sizeof(size_t)); - config->pem_cert_chains_sizes = - gpr_malloc(num_key_cert_pairs * sizeof(size_t)); + config->pem_key_cert_pairs = + gpr_zalloc(num_key_cert_pairs * sizeof(tsi_ssl_pem_key_cert_pair)); } config->num_key_cert_pairs = num_key_cert_pairs; for (i = 0; i < num_key_cert_pairs; i++) { GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL); GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL); - ssl_copy_key_material(pem_key_cert_pairs[i].private_key, - &config->pem_private_keys[i], - &config->pem_private_keys_sizes[i]); - ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain, - &config->pem_cert_chains[i], - &config->pem_cert_chains_sizes[i]); + config->pem_key_cert_pairs[i].cert_chain = + gpr_strdup(pem_key_cert_pairs[i].cert_chain); + config->pem_key_cert_pairs[i].private_key = + gpr_strdup(pem_key_cert_pairs[i].private_key); } } diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index 8f321b9911..dff05633ec 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -64,7 +64,7 @@ typedef struct { pollset_set so that work can progress when this call wants work to progress */ grpc_polling_entity *pollent; - grpc_transport_stream_op op; + grpc_transport_stream_op_batch op; uint8_t security_context_set; grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT]; grpc_auth_metadata_context auth_md_context; @@ -108,7 +108,7 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, const char *error_details) { grpc_call_element *elem = (grpc_call_element *)user_data; call_data *calld = elem->call_data; - grpc_transport_stream_op *op = &calld->op; + grpc_transport_stream_op_batch *op = &calld->op; grpc_metadata_batch *mdb; size_t i; reset_auth_metadata_context(&calld->auth_md_context); @@ -122,8 +122,8 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED); } else { GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT); - GPR_ASSERT(op->send_initial_metadata != NULL); - mdb = op->send_initial_metadata; + GPR_ASSERT(op->send_initial_metadata); + mdb = op->payload->send_initial_metadata.send_initial_metadata; for (i = 0; i < num_md; i++) { add_error(&error, grpc_metadata_batch_add_tail( @@ -136,7 +136,7 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, if (error == GRPC_ERROR_NONE) { grpc_call_next_op(exec_ctx, elem, op); } else { - grpc_transport_stream_op_finish_with_failure(exec_ctx, op, error); + grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error); } } @@ -172,11 +172,13 @@ void build_auth_metadata_context(grpc_security_connector *sc, static void send_security_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; grpc_client_security_context *ctx = - (grpc_client_security_context *)op->context[GRPC_CONTEXT_SECURITY].value; + (grpc_client_security_context *)op->payload + ->context[GRPC_CONTEXT_SECURITY] + .value; grpc_call_credentials *channel_call_creds = chand->security_connector->request_metadata_creds; int call_creds_has_md = (ctx != NULL) && (ctx->creds != NULL); @@ -191,7 +193,7 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx, calld->creds = grpc_composite_call_credentials_create(channel_call_creds, ctx->creds, NULL); if (calld->creds == NULL) { - grpc_transport_stream_op_finish_with_failure( + grpc_transport_stream_op_batch_finish_with_failure( exec_ctx, op, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -242,7 +244,7 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data, that is being sent or received. */ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { GPR_TIMER_BEGIN("auth_start_transport_op", 0); /* grab pointers to our data from the call element */ @@ -251,23 +253,25 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_linked_mdelem *l; grpc_client_security_context *sec_ctx = NULL; - if (calld->security_context_set == 0 && op->cancel_error == GRPC_ERROR_NONE) { + if (!op->cancel_stream && calld->security_context_set == 0) { calld->security_context_set = 1; - GPR_ASSERT(op->context); - if (op->context[GRPC_CONTEXT_SECURITY].value == NULL) { - op->context[GRPC_CONTEXT_SECURITY].value = + GPR_ASSERT(op->payload->context != NULL); + if (op->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) { + op->payload->context[GRPC_CONTEXT_SECURITY].value = grpc_client_security_context_create(); - op->context[GRPC_CONTEXT_SECURITY].destroy = + op->payload->context[GRPC_CONTEXT_SECURITY].destroy = grpc_client_security_context_destroy; } - sec_ctx = op->context[GRPC_CONTEXT_SECURITY].value; + sec_ctx = op->payload->context[GRPC_CONTEXT_SECURITY].value; GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter"); sec_ctx->auth_context = GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter"); } - if (op->send_initial_metadata != NULL) { - for (l = op->send_initial_metadata->list.head; l != NULL; l = l->next) { + if (op->send_initial_metadata) { + for (l = op->payload->send_initial_metadata.send_initial_metadata->list + .head; + l != NULL; l = l->next) { grpc_mdelem md = l->md; /* Pointer comparison is OK for md_elems created from the same context. */ @@ -339,8 +343,16 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element_args *args) { grpc_security_connector *sc = grpc_security_connector_find_in_args(args->channel_args); + if (sc == NULL) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Security connector missing from client auth filter args"); + } grpc_auth_context *auth_context = grpc_find_auth_context_in_args(args->channel_args); + if (auth_context == NULL) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Auth context missing from client auth filter args"); + } /* grab pointers to our data from the channel element */ channel_data *chand = elem->channel_data; @@ -349,8 +361,6 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, handle the case that there's no 'next' filter to call on the up or down path */ GPR_ASSERT(!args->is_last); - GPR_ASSERT(sc != NULL); - GPR_ASSERT(auth_context != NULL); /* initialize members */ chand->security_connector = diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 24da949e48..48d368a2a7 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -75,7 +75,7 @@ typedef struct { gpr_refcount ref; } secure_endpoint; -int grpc_trace_secure_endpoint = 0; +grpc_tracer_flag grpc_trace_secure_endpoint = GRPC_TRACER_INITIALIZER(false); static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; @@ -130,14 +130,14 @@ static void secure_endpoint_ref(secure_endpoint *ep) { gpr_ref(&ep->ref); } static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { grpc_slice_buffer_add(ep->read_buffer, ep->read_staging_buffer); - ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); *cur = GRPC_SLICE_START_PTR(ep->read_staging_buffer); *end = GRPC_SLICE_END_PTR(ep->read_staging_buffer); } static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, grpc_error *error) { - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { size_t i; for (i = 0; i < ep->read_buffer->count; i++) { char *data = grpc_dump_slice(ep->read_buffer->slices[i], @@ -252,7 +252,7 @@ static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, static void flush_write_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { grpc_slice_buffer_add(&ep->output_buffer, ep->write_staging_buffer); - ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); *cur = GRPC_SLICE_START_PTR(ep->write_staging_buffer); *end = GRPC_SLICE_END_PTR(ep->write_staging_buffer); } @@ -269,7 +269,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &ep->output_buffer); - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { for (i = 0; i < slices->count; i++) { char *data = grpc_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -415,8 +415,8 @@ grpc_endpoint *grpc_secure_endpoint_create( grpc_slice_buffer_add(&ep->leftover_bytes, grpc_slice_ref_internal(leftover_slices[i])); } - ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); - ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); + ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); grpc_slice_buffer_init(&ep->output_buffer); grpc_slice_buffer_init(&ep->source_buffer); ep->read_buffer = NULL; diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h index a61f40a4fa..f1a5c8cb6d 100644 --- a/src/core/lib/security/transport/secure_endpoint.h +++ b/src/core/lib/security/transport/secure_endpoint.h @@ -39,7 +39,7 @@ struct tsi_frame_protector; -extern int grpc_trace_secure_endpoint; +extern grpc_tracer_flag grpc_trace_secure_endpoint; /* Takes ownership of protector and to_wrap, and refs leftover_slices. */ grpc_endpoint *grpc_secure_endpoint_create( diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index 2b51706161..30431a4e4a 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -78,9 +78,8 @@ void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb) { /* Defines the cipher suites that we accept by default. All these cipher suites are compliant with HTTP2. */ -#define GRPC_SSL_CIPHER_SUITES \ - "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-" \ - "SHA384:ECDHE-RSA-AES256-GCM-SHA384" +#define GRPC_SSL_CIPHER_SUITES \ + "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384" static gpr_once cipher_suites_once = GPR_ONCE_INIT; static const char *cipher_suites = NULL; @@ -423,12 +422,8 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( c->base.check_call_host = fake_channel_check_call_host; c->base.add_handshakers = fake_channel_add_handshakers; c->target = gpr_strdup(target); - const grpc_arg *expected_target_arg = - grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); - if (expected_target_arg != NULL) { - GPR_ASSERT(expected_target_arg->type == GRPC_ARG_STRING); - c->expected_targets = gpr_strdup(expected_target_arg->value.string); - } + const char *expected_targets = grpc_fake_transport_get_expected_targets(args); + c->expected_targets = gpr_strdup(expected_targets); c->is_lb_channel = (grpc_lb_targets_info_find_in_args(args) != NULL); return &c->base; } @@ -448,14 +443,14 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( typedef struct { grpc_channel_security_connector base; - tsi_ssl_handshaker_factory *handshaker_factory; + tsi_ssl_client_handshaker_factory *handshaker_factory; char *target_name; char *overridden_target_name; } grpc_ssl_channel_security_connector; typedef struct { grpc_server_security_connector base; - tsi_ssl_handshaker_factory *handshaker_factory; + tsi_ssl_server_handshaker_factory *handshaker_factory; } grpc_ssl_server_security_connector; static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, @@ -464,7 +459,7 @@ static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, (grpc_ssl_channel_security_connector *)sc; grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); if (c->handshaker_factory != NULL) { - tsi_ssl_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); } if (c->target_name != NULL) gpr_free(c->target_name); if (c->overridden_target_name != NULL) gpr_free(c->overridden_target_name); @@ -476,26 +471,11 @@ static void ssl_server_destroy(grpc_exec_ctx *exec_ctx, grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; if (c->handshaker_factory != NULL) { - tsi_ssl_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_server_handshaker_factory_destroy(c->handshaker_factory); } gpr_free(sc); } -static grpc_security_status ssl_create_handshaker( - tsi_ssl_handshaker_factory *handshaker_factory, bool is_client, - const char *peer_name, tsi_handshaker **handshaker) { - tsi_result result = TSI_OK; - if (handshaker_factory == NULL) return GRPC_SECURITY_ERROR; - result = tsi_ssl_handshaker_factory_create_handshaker( - handshaker_factory, is_client ? peer_name : NULL, handshaker); - if (result != TSI_OK) { - gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", - tsi_result_to_string(result)); - return GRPC_SECURITY_ERROR; - } - return GRPC_SECURITY_OK; -} - static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_handshake_manager *handshake_mgr) { @@ -503,11 +483,17 @@ static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx, (grpc_ssl_channel_security_connector *)sc; // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; - ssl_create_handshaker(c->handshaker_factory, true /* is_client */, - c->overridden_target_name != NULL - ? c->overridden_target_name - : c->target_name, - &tsi_hs); + tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker( + c->handshaker_factory, + c->overridden_target_name != NULL ? c->overridden_target_name + : c->target_name, + &tsi_hs); + if (result != TSI_OK) { + gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", + tsi_result_to_string(result)); + return; + } + // Create handshakers. grpc_handshake_manager_add(handshake_mgr, grpc_security_handshaker_create( exec_ctx, tsi_hs, &sc->base)); @@ -520,8 +506,14 @@ static void ssl_server_add_handshakers(grpc_exec_ctx *exec_ctx, (grpc_ssl_server_security_connector *)sc; // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; - ssl_create_handshaker(c->handshaker_factory, false /* is_client */, - NULL /* peer_name */, &tsi_hs); + tsi_result result = tsi_ssl_server_handshaker_factory_create_handshaker( + c->handshaker_factory, &tsi_hs); + if (result != TSI_OK) { + gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", + tsi_result_to_string(result)); + return; + } + // Create handshakers. grpc_handshake_manager_add(handshake_mgr, grpc_security_handshaker_create( exec_ctx, tsi_hs, &sc->base)); @@ -702,6 +694,7 @@ static grpc_security_connector_vtable ssl_channel_vtable = { static grpc_security_connector_vtable ssl_server_vtable = { ssl_server_destroy, ssl_server_check_peer}; +/* returns a NULL terminated slice. */ static grpc_slice compute_default_pem_root_certs_once(void) { grpc_slice result = grpc_empty_slice(); @@ -710,7 +703,7 @@ static grpc_slice compute_default_pem_root_certs_once(void) { gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); if (default_root_certs_path != NULL) { GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(default_root_certs_path, 0, &result)); + grpc_load_file(default_root_certs_path, 1, &result)); gpr_free(default_root_certs_path); } @@ -721,15 +714,18 @@ static grpc_slice compute_default_pem_root_certs_once(void) { ovrd_res = ssl_roots_override_cb(&pem_root_certs); if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) { GPR_ASSERT(pem_root_certs != NULL); - result = grpc_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free); + result = grpc_slice_from_copied_buffer( + pem_root_certs, + strlen(pem_root_certs) + 1); // NULL terminator. } + gpr_free(pem_root_certs); } /* Fall back to installed certs if needed. */ if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(installed_roots_path, 0, &result)); + grpc_load_file(installed_roots_path, 1, &result)); } return result; } @@ -769,13 +765,14 @@ get_tsi_client_certificate_request_type( } } -size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) { +const char *grpc_get_default_ssl_roots(void) { /* TODO(jboeuf@google.com): Maybe revisit the approach which consists in loading all the roots once for the lifetime of the process. */ static gpr_once once = GPR_ONCE_INIT; gpr_once_init(&once, init_default_pem_root_certs); - *pem_root_certs = GRPC_SLICE_START_PTR(default_pem_root_certs); - return GRPC_SLICE_LENGTH(default_pem_root_certs); + return GRPC_SLICE_IS_EMPTY(default_pem_root_certs) + ? NULL + : (const char *)GRPC_SLICE_START_PTR(default_pem_root_certs); } grpc_security_status grpc_ssl_channel_security_connector_create( @@ -783,22 +780,16 @@ grpc_security_status grpc_ssl_channel_security_connector_create( const grpc_ssl_config *config, const char *target_name, const char *overridden_target_name, grpc_channel_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); - const unsigned char **alpn_protocol_strings = + const char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); - unsigned char *alpn_protocol_string_lengths = - gpr_malloc(sizeof(unsigned char) * num_alpn_protocols); tsi_result result = TSI_OK; grpc_ssl_channel_security_connector *c; size_t i; - const unsigned char *pem_root_certs; - size_t pem_root_certs_size; + const char *pem_root_certs; char *port; for (i = 0; i < num_alpn_protocols; i++) { - alpn_protocol_strings[i] = - (const unsigned char *)grpc_chttp2_get_alpn_version_index(i); - alpn_protocol_string_lengths[i] = - (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i)); + alpn_protocol_strings[i] = grpc_chttp2_get_alpn_version_index(i); } if (config == NULL || target_name == NULL) { @@ -806,14 +797,13 @@ grpc_security_status grpc_ssl_channel_security_connector_create( goto error; } if (config->pem_root_certs == NULL) { - pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); - if (pem_root_certs == NULL || pem_root_certs_size == 0) { + pem_root_certs = grpc_get_default_ssl_roots(); + if (pem_root_certs == NULL) { gpr_log(GPR_ERROR, "Could not get default pem root certs."); goto error; } } else { pem_root_certs = config->pem_root_certs; - pem_root_certs_size = config->pem_root_certs_size; } c = gpr_zalloc(sizeof(grpc_ssl_channel_security_connector)); @@ -830,11 +820,12 @@ grpc_security_status grpc_ssl_channel_security_connector_create( if (overridden_target_name != NULL) { c->overridden_target_name = gpr_strdup(overridden_target_name); } + + bool has_key_cert_pair = config->pem_key_cert_pair.private_key != NULL && + config->pem_key_cert_pair.cert_chain != NULL; result = tsi_create_ssl_client_handshaker_factory( - config->pem_private_key, config->pem_private_key_size, - config->pem_cert_chain, config->pem_cert_chain_size, pem_root_certs, - pem_root_certs_size, ssl_cipher_suites(), alpn_protocol_strings, - alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols, + has_key_cert_pair ? &config->pem_key_cert_pair : NULL, pem_root_certs, + ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", @@ -845,12 +836,10 @@ grpc_security_status grpc_ssl_channel_security_connector_create( } *sc = &c->base; gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_OK; error: gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_ERROR; } @@ -858,19 +847,14 @@ grpc_security_status grpc_ssl_server_security_connector_create( grpc_exec_ctx *exec_ctx, const grpc_ssl_server_config *config, grpc_server_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); - const unsigned char **alpn_protocol_strings = + const char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); - unsigned char *alpn_protocol_string_lengths = - gpr_malloc(sizeof(unsigned char) * num_alpn_protocols); tsi_result result = TSI_OK; grpc_ssl_server_security_connector *c; size_t i; for (i = 0; i < num_alpn_protocols; i++) { - alpn_protocol_strings[i] = - (const unsigned char *)grpc_chttp2_get_alpn_version_index(i); - alpn_protocol_string_lengths[i] = - (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i)); + alpn_protocol_strings[i] = grpc_chttp2_get_alpn_version_index(i); } if (config == NULL || config->num_key_cert_pairs == 0) { @@ -883,15 +867,11 @@ grpc_security_status grpc_ssl_server_security_connector_create( c->base.base.url_scheme = GRPC_SSL_URL_SCHEME; c->base.base.vtable = &ssl_server_vtable; result = tsi_create_ssl_server_handshaker_factory_ex( - (const unsigned char **)config->pem_private_keys, - config->pem_private_keys_sizes, - (const unsigned char **)config->pem_cert_chains, - config->pem_cert_chains_sizes, config->num_key_cert_pairs, - config->pem_root_certs, config->pem_root_certs_size, - get_tsi_client_certificate_request_type( - config->client_certificate_request), - ssl_cipher_suites(), alpn_protocol_strings, alpn_protocol_string_lengths, - (uint16_t)num_alpn_protocols, &c->handshaker_factory); + config->pem_key_cert_pairs, config->num_key_cert_pairs, + config->pem_root_certs, get_tsi_client_certificate_request_type( + config->client_certificate_request), + ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, + &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -902,11 +882,9 @@ grpc_security_status grpc_ssl_server_security_connector_create( c->base.add_handshakers = ssl_server_add_handshakers; *sc = &c->base; gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_OK; error: gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_ERROR; } diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index cf56cb3183..d74f6739c0 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -34,11 +34,14 @@ #ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H #define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H +#include <stdbool.h> + #include <grpc/grpc_security.h> #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/tcp_server.h" +#include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security_interface.h" /* --- status enum. --- */ @@ -184,13 +187,10 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( void); /* Config for ssl clients. */ + typedef struct { - unsigned char *pem_private_key; - size_t pem_private_key_size; - unsigned char *pem_cert_chain; - size_t pem_cert_chain_size; - unsigned char *pem_root_certs; - size_t pem_root_certs_size; + tsi_ssl_pem_key_cert_pair pem_key_cert_pair; + char *pem_root_certs; } grpc_ssl_config; /* Creates an SSL channel_security_connector. @@ -211,21 +211,17 @@ grpc_security_status grpc_ssl_channel_security_connector_create( const grpc_ssl_config *config, const char *target_name, const char *overridden_target_name, grpc_channel_security_connector **sc); -/* Gets the default ssl roots. */ -size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs); +/* Gets the default ssl roots. Returns NULL if not found. */ +const char *grpc_get_default_ssl_roots(void); /* Exposed for TESTING ONLY!. */ grpc_slice grpc_get_default_ssl_roots_for_testing(void); /* Config for ssl servers. */ typedef struct { - unsigned char **pem_private_keys; - size_t *pem_private_keys_sizes; - unsigned char **pem_cert_chains; - size_t *pem_cert_chains_sizes; + tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs; size_t num_key_cert_pairs; - unsigned char *pem_root_certs; - size_t pem_root_certs_size; + char *pem_root_certs; grpc_ssl_client_certificate_request_type client_certificate_request; } grpc_ssl_server_config; diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index 2f39327670..509b4b556d 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -287,12 +287,11 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, if (num_left_overs > 0) { /* Put the leftovers in our buffer (ownership transfered). */ if (has_left_overs_in_current_slice) { - grpc_slice_buffer_add( - &h->left_overs, - grpc_slice_split_tail(&h->args->read_buffer->slices[i], - consumed_slice_size)); + grpc_slice tail = grpc_slice_split_tail(&h->args->read_buffer->slices[i], + consumed_slice_size); + grpc_slice_buffer_add(&h->left_overs, tail); /* split_tail above increments refcount. */ - grpc_slice_unref_internal(exec_ctx, h->args->read_buffer->slices[i]); + grpc_slice_unref_internal(exec_ctx, tail); } grpc_slice_buffer_addn( &h->left_overs, &h->args->read_buffer->slices[i + 1], diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index 3cf0632220..1aca76f9e8 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -49,7 +49,7 @@ typedef struct call_data { up-call on transport_op, and remember to call our on_done_recv member after handling it. */ grpc_closure auth_on_recv; - grpc_transport_stream_op *transport_op; + grpc_transport_stream_op_batch *transport_op; grpc_metadata_array md; const grpc_metadata *consumed_md; size_t num_consumed_md; @@ -138,12 +138,11 @@ static void on_md_processing_done( error_details = error_details != NULL ? error_details : "Authentication metadata processing failed."; - calld->transport_op->send_initial_metadata = NULL; - if (calld->transport_op->send_message != NULL) { - grpc_byte_stream_destroy(&exec_ctx, calld->transport_op->send_message); - calld->transport_op->send_message = NULL; + if (calld->transport_op->send_message) { + grpc_byte_stream_destroy( + &exec_ctx, calld->transport_op->payload->send_message.send_message); + calld->transport_op->payload->send_message.send_message = NULL; } - calld->transport_op->send_trailing_metadata = NULL; grpc_closure_sched( &exec_ctx, calld->on_done_recv, grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details), @@ -171,14 +170,17 @@ static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, } static void set_recv_ops_md_callbacks(grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { call_data *calld = elem->call_data; - if (op->recv_initial_metadata != NULL) { + if (op->recv_initial_metadata) { /* substitute our callback for the higher callback */ - calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->recv_initial_metadata_ready; - op->recv_initial_metadata_ready = &calld->auth_on_recv; + calld->recv_initial_metadata = + op->payload->recv_initial_metadata.recv_initial_metadata; + calld->on_done_recv = + op->payload->recv_initial_metadata.recv_initial_metadata_ready; + op->payload->recv_initial_metadata.recv_initial_metadata_ready = + &calld->auth_on_recv; calld->transport_op = op; } } @@ -190,7 +192,7 @@ static void set_recv_ops_md_callbacks(grpc_call_element *elem, that is being sent or received. */ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { set_recv_ops_md_callbacks(elem, op); grpc_call_next_op(exec_ctx, elem, op); } diff --git a/src/core/lib/slice/b64.c b/src/core/lib/slice/b64.c index 2007cc4810..d9091646e0 100644 --- a/src/core/lib/slice/b64.c +++ b/src/core/lib/slice/b64.c @@ -202,7 +202,7 @@ static int decode_group(const unsigned char *codes, size_t num_codes, grpc_slice grpc_base64_decode_with_len(grpc_exec_ctx *exec_ctx, const char *b64, size_t b64_len, int url_safe) { - grpc_slice result = grpc_slice_malloc(b64_len); + grpc_slice result = GRPC_SLICE_MALLOC(b64_len); unsigned char *current = GRPC_SLICE_START_PTR(result); size_t result_size = 0; unsigned char codes[4]; diff --git a/src/core/lib/slice/percent_encoding.c b/src/core/lib/slice/percent_encoding.c index c76c58d371..a77c69763f 100644 --- a/src/core/lib/slice/percent_encoding.c +++ b/src/core/lib/slice/percent_encoding.c @@ -71,7 +71,7 @@ grpc_slice grpc_percent_encode_slice(grpc_slice slice, return grpc_slice_ref_internal(slice); } // second pass: actually encode - grpc_slice out = grpc_slice_malloc(output_length); + grpc_slice out = GRPC_SLICE_MALLOC(output_length); uint8_t *q = GRPC_SLICE_START_PTR(out); for (p = slice_start; p < slice_end; p++) { if (is_unreserved_character(*p, unreserved_bytes)) { @@ -125,7 +125,7 @@ bool grpc_strict_percent_decode_slice(grpc_slice slice_in, return true; } p = GRPC_SLICE_START_PTR(slice_in); - *slice_out = grpc_slice_malloc(out_length); + *slice_out = GRPC_SLICE_MALLOC(out_length); uint8_t *q = GRPC_SLICE_START_PTR(*slice_out); while (p != in_end) { if (*p == '%') { @@ -163,7 +163,7 @@ grpc_slice grpc_permissive_percent_decode_slice(grpc_slice slice_in) { return grpc_slice_ref_internal(slice_in); } p = GRPC_SLICE_START_PTR(slice_in); - grpc_slice out = grpc_slice_malloc(out_length); + grpc_slice out = GRPC_SLICE_MALLOC(out_length); uint8_t *q = GRPC_SLICE_START_PTR(out); while (p != in_end) { if (*p == '%') { diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 1cddf062cd..b90738fd1a 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -55,6 +55,13 @@ grpc_slice grpc_empty_slice(void) { return out; } +grpc_slice grpc_slice_copy(grpc_slice s) { + grpc_slice out = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(s)); + memcpy(GRPC_SLICE_START_PTR(out), GRPC_SLICE_START_PTR(s), + GRPC_SLICE_LENGTH(s)); + return out; +} + grpc_slice grpc_slice_ref_internal(grpc_slice slice) { if (slice.refcount) { slice.refcount->vtable->ref(slice.refcount); @@ -197,7 +204,8 @@ grpc_slice grpc_slice_new_with_len(void *p, size_t len, } grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t length) { - grpc_slice slice = grpc_slice_malloc(length); + if (length == 0) return grpc_empty_slice(); + grpc_slice slice = GRPC_SLICE_MALLOC(length); memcpy(GRPC_SLICE_START_PTR(slice), source, length); return slice; } @@ -227,35 +235,42 @@ static const grpc_slice_refcount_vtable malloc_vtable = { malloc_ref, malloc_unref, grpc_slice_default_eq_impl, grpc_slice_default_hash_impl}; +grpc_slice grpc_slice_malloc_large(size_t length) { + grpc_slice slice; + + /* Memory layout used by the slice created here: + + +-----------+----------------------------------------------------------+ + | refcount | bytes | + +-----------+----------------------------------------------------------+ + + refcount is a malloc_refcount + bytes is an array of bytes of the requested length + Both parts are placed in the same allocation returned from gpr_malloc */ + malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length); + + /* Initial refcount on rc is 1 - and it's up to the caller to release + this reference. */ + gpr_ref_init(&rc->refs, 1); + + rc->base.vtable = &malloc_vtable; + rc->base.sub_refcount = &rc->base; + + /* Build up the slice to be returned. */ + /* The slices refcount points back to the allocated block. */ + slice.refcount = &rc->base; + /* The data bytes are placed immediately after the refcount struct */ + slice.data.refcounted.bytes = (uint8_t *)(rc + 1); + /* And the length of the block is set to the requested length */ + slice.data.refcounted.length = length; + return slice; +} + grpc_slice grpc_slice_malloc(size_t length) { grpc_slice slice; if (length > sizeof(slice.data.inlined.bytes)) { - /* Memory layout used by the slice created here: - - +-----------+----------------------------------------------------------+ - | refcount | bytes | - +-----------+----------------------------------------------------------+ - - refcount is a malloc_refcount - bytes is an array of bytes of the requested length - Both parts are placed in the same allocation returned from gpr_malloc */ - malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length); - - /* Initial refcount on rc is 1 - and it's up to the caller to release - this reference. */ - gpr_ref_init(&rc->refs, 1); - - rc->base.vtable = &malloc_vtable; - rc->base.sub_refcount = &rc->base; - - /* Build up the slice to be returned. */ - /* The slices refcount points back to the allocated block. */ - slice.refcount = &rc->base; - /* The data bytes are placed immediately after the refcount struct */ - slice.data.refcounted.bytes = (uint8_t *)(rc + 1); - /* And the length of the block is set to the requested length */ - slice.data.refcounted.length = length; + return grpc_slice_malloc_large(length); } else { /* small slice: just inline the data */ slice.refcount = NULL; @@ -305,7 +320,8 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { return subset; } -grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { +grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, + grpc_slice_ref_whom ref_whom) { grpc_slice tail; if (source->refcount == NULL) { @@ -319,28 +335,46 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { } else { size_t tail_length = source->data.refcounted.length - split; GPR_ASSERT(source->data.refcounted.length >= split); - if (tail_length < sizeof(tail.data.inlined.bytes)) { + if (tail_length < sizeof(tail.data.inlined.bytes) && + ref_whom != GRPC_SLICE_REF_TAIL) { /* Copy out the bytes - it'll be cheaper than refcounting */ tail.refcount = NULL; tail.data.inlined.length = (uint8_t)tail_length; memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split, tail_length); + source->refcount = source->refcount->sub_refcount; } else { /* Build the result */ - tail.refcount = source->refcount->sub_refcount; - /* Bump the refcount */ - tail.refcount->vtable->ref(tail.refcount); + switch (ref_whom) { + case GRPC_SLICE_REF_TAIL: + tail.refcount = source->refcount->sub_refcount; + source->refcount = &noop_refcount; + break; + case GRPC_SLICE_REF_HEAD: + tail.refcount = &noop_refcount; + source->refcount = source->refcount->sub_refcount; + break; + case GRPC_SLICE_REF_BOTH: + tail.refcount = source->refcount->sub_refcount; + source->refcount = source->refcount->sub_refcount; + /* Bump the refcount */ + tail.refcount->vtable->ref(tail.refcount); + break; + } /* Point into the source array */ tail.data.refcounted.bytes = source->data.refcounted.bytes + split; tail.data.refcounted.length = tail_length; } - source->refcount = source->refcount->sub_refcount; source->data.refcounted.length = split; } return tail; } +grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { + return grpc_slice_split_tail_maybe_ref(source, split, GRPC_SLICE_REF_BOTH); +} + grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { grpc_slice head; @@ -382,8 +416,9 @@ grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { } int grpc_slice_default_eq_impl(grpc_slice a, grpc_slice b) { - return GRPC_SLICE_LENGTH(a) == GRPC_SLICE_LENGTH(b) && - 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b), + if (GRPC_SLICE_LENGTH(a) != GRPC_SLICE_LENGTH(b)) return false; + if (GRPC_SLICE_LENGTH(a) == 0) return true; + return 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b), GRPC_SLICE_LENGTH(a)); } @@ -457,7 +492,7 @@ int grpc_slice_slice(grpc_slice haystack, grpc_slice needle) { } grpc_slice grpc_slice_dup(grpc_slice a) { - grpc_slice copy = grpc_slice_malloc(GRPC_SLICE_LENGTH(a)); + grpc_slice copy = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(a)); memcpy(GRPC_SLICE_START_PTR(copy), GRPC_SLICE_START_PTR(a), GRPC_SLICE_LENGTH(a)); return copy; diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 9176dc8a42..e8d41ca0f7 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -46,27 +46,29 @@ #define GROW(x) (3 * (x) / 2) static void maybe_embiggen(grpc_slice_buffer *sb) { - if (sb->base_slices != sb->slices) { - memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); - sb->slices = sb->base_slices; - } - /* How far away from sb->base_slices is sb->slices pointer */ size_t slice_offset = (size_t)(sb->slices - sb->base_slices); size_t slice_count = sb->count + slice_offset; if (slice_count == sb->capacity) { - sb->capacity = GROW(sb->capacity); - GPR_ASSERT(sb->capacity > slice_count); - if (sb->base_slices == sb->inlined) { - sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); - memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); + if (sb->base_slices != sb->slices) { + /* Make room by moving elements if there's still space unused */ + memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); + sb->slices = sb->base_slices; } else { - sb->base_slices = - gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); - } + /* Allocate more memory if no more space is available */ + sb->capacity = GROW(sb->capacity); + GPR_ASSERT(sb->capacity > slice_count); + if (sb->base_slices == sb->inlined) { + sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); + memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); + } else { + sb->base_slices = + gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); + } - sb->slices = sb->base_slices + slice_offset; + sb->slices = sb->base_slices + slice_offset; + } } } @@ -251,16 +253,18 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src, src->length = 0; } -void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, - grpc_slice_buffer *dst) { - size_t output_len = dst->length + n; - size_t new_input_len = src->length - n; +static void slice_buffer_move_first_maybe_ref(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst, + bool incref) { GPR_ASSERT(src->length >= n); if (src->length == n) { grpc_slice_buffer_move_into(src, dst); return; } + size_t output_len = dst->length + n; + size_t new_input_len = src->length - n; + while (src->count > 0) { grpc_slice slice = grpc_slice_buffer_take_first(src); size_t slice_len = GRPC_SLICE_LENGTH(slice); @@ -270,11 +274,18 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, } else if (n == slice_len) { grpc_slice_buffer_add(dst, slice); break; - } else { /* n < slice_len */ - grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n)); + } else if (incref) { /* n < slice_len */ + grpc_slice_buffer_undo_take_first( + src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_BOTH)); GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); grpc_slice_buffer_add(dst, slice); break; + } else { /* n < slice_len */ + grpc_slice_buffer_undo_take_first( + src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_TAIL)); + GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); + grpc_slice_buffer_add_indexed(dst, slice); + break; } } GPR_ASSERT(dst->length == output_len); @@ -282,6 +293,16 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, GPR_ASSERT(src->count > 0); } +void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst) { + slice_buffer_move_first_maybe_ref(src, n, dst, true); +} + +void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst) { + slice_buffer_move_first_maybe_ref(src, n, dst, false); +} + void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *src, size_t n, void *dst) { diff --git a/src/core/lib/slice/slice_hash_table.c b/src/core/lib/slice/slice_hash_table.c index 219567f36f..444f22aa19 100644 --- a/src/core/lib/slice/slice_hash_table.c +++ b/src/core/lib/slice/slice_hash_table.c @@ -42,56 +42,47 @@ struct grpc_slice_hash_table { gpr_refcount refs; + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value); size_t size; + size_t max_num_probes; grpc_slice_hash_table_entry* entries; }; static bool is_empty(grpc_slice_hash_table_entry* entry) { - return entry->vtable == NULL; + return entry->value == NULL; } -// Helper function for insert and get operations that performs quadratic -// probing (https://en.wikipedia.org/wiki/Quadratic_probing). -static size_t grpc_slice_hash_table_find_index( - const grpc_slice_hash_table* table, const grpc_slice key, bool find_empty) { - size_t hash = grpc_slice_hash(key); - for (size_t i = 0; i < table->size; ++i) { - const size_t idx = (hash + i * i) % table->size; +static void grpc_slice_hash_table_add(grpc_slice_hash_table* table, + grpc_slice key, void* value) { + GPR_ASSERT(value != NULL); + const size_t hash = grpc_slice_hash(key); + for (size_t offset = 0; offset < table->size; ++offset) { + const size_t idx = (hash + offset) % table->size; if (is_empty(&table->entries[idx])) { - return find_empty ? idx : table->size; - } - if (grpc_slice_eq(table->entries[idx].key, key)) { - return idx; + table->entries[idx].key = key; + table->entries[idx].value = value; + // Keep track of the maximum number of probes needed, since this + // provides an upper bound for lookups. + if (offset > table->max_num_probes) table->max_num_probes = offset; + return; } } - return table->size; // Not found. -} - -static void grpc_slice_hash_table_add( - grpc_slice_hash_table* table, grpc_slice key, void* value, - const grpc_slice_hash_table_vtable* vtable) { - GPR_ASSERT(value != NULL); - const size_t idx = - grpc_slice_hash_table_find_index(table, key, true /* find_empty */); - GPR_ASSERT(idx != table->size); // Table should never be full. - grpc_slice_hash_table_entry* entry = &table->entries[idx]; - entry->key = grpc_slice_ref_internal(key); - entry->value = vtable->copy_value(value); - entry->vtable = vtable; + GPR_ASSERT(false); // Table should never be full. } grpc_slice_hash_table* grpc_slice_hash_table_create( - size_t num_entries, grpc_slice_hash_table_entry* entries) { + size_t num_entries, grpc_slice_hash_table_entry* entries, + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)) { grpc_slice_hash_table* table = gpr_zalloc(sizeof(*table)); gpr_ref_init(&table->refs, 1); - // Quadratic probing gets best performance when the table is no more - // than half full. + table->destroy_value = destroy_value; + // Keep load factor low to improve performance of lookups. table->size = num_entries * 2; const size_t entry_size = sizeof(grpc_slice_hash_table_entry) * table->size; table->entries = gpr_zalloc(entry_size); for (size_t i = 0; i < num_entries; ++i) { grpc_slice_hash_table_entry* entry = &entries[i]; - grpc_slice_hash_table_add(table, entry->key, entry->value, entry->vtable); + grpc_slice_hash_table_add(table, entry->key, entry->value); } return table; } @@ -108,7 +99,7 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx, grpc_slice_hash_table_entry* entry = &table->entries[i]; if (!is_empty(entry)) { grpc_slice_unref_internal(exec_ctx, entry->key); - entry->vtable->destroy_value(exec_ctx, entry->value); + table->destroy_value(exec_ctx, entry->value); } } gpr_free(table->entries); @@ -118,8 +109,15 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx, void* grpc_slice_hash_table_get(const grpc_slice_hash_table* table, const grpc_slice key) { - const size_t idx = - grpc_slice_hash_table_find_index(table, key, false /* find_empty */); - if (idx == table->size) return NULL; // Not found. - return table->entries[idx].value; + const size_t hash = grpc_slice_hash(key); + // We cap the number of probes at the max number recorded when + // populating the table. + for (size_t offset = 0; offset <= table->max_num_probes; ++offset) { + const size_t idx = (hash + offset) % table->size; + if (is_empty(&table->entries[idx])) break; + if (grpc_slice_eq(table->entries[idx].key, key)) { + return table->entries[idx].value; + } + } + return NULL; // Not found. } diff --git a/src/core/lib/slice/slice_hash_table.h b/src/core/lib/slice/slice_hash_table.h index d0c27122d7..1e61c5eb11 100644 --- a/src/core/lib/slice/slice_hash_table.h +++ b/src/core/lib/slice/slice_hash_table.h @@ -37,33 +37,28 @@ /** Hash table implementation. * * This implementation uses open addressing - * (https://en.wikipedia.org/wiki/Open_addressing) with quadratic - * probing (https://en.wikipedia.org/wiki/Quadratic_probing). + * (https://en.wikipedia.org/wiki/Open_addressing) with linear + * probing (https://en.wikipedia.org/wiki/Linear_probing). * * The keys are \a grpc_slice objects. The values are arbitrary pointers - * with a common vtable. + * with a common destroy function. * * Hash tables are intentionally immutable, to avoid the need for locking. */ typedef struct grpc_slice_hash_table grpc_slice_hash_table; -typedef struct grpc_slice_hash_table_vtable { - void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value); - void *(*copy_value)(void *value); -} grpc_slice_hash_table_vtable; - typedef struct grpc_slice_hash_table_entry { grpc_slice key; void *value; /* Must not be NULL. */ - const grpc_slice_hash_table_vtable *vtable; } grpc_slice_hash_table_entry; /** Creates a new hash table of containing \a entries, which is an array - of length \a num_entries. - Creates its own copy of all keys and values from \a entries. */ + of length \a num_entries. Takes ownership of all keys and values in + \a entries. Values will be cleaned up via \a destroy_value(). */ grpc_slice_hash_table *grpc_slice_hash_table_create( - size_t num_entries, grpc_slice_hash_table_entry *entries); + size_t num_entries, grpc_slice_hash_table_entry *entries, + void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value)); grpc_slice_hash_table *grpc_slice_hash_table_ref(grpc_slice_hash_table *table); void grpc_slice_hash_table_unref(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/support/atomic.h b/src/core/lib/support/atomic.h new file mode 100644 index 0000000000..2226189b68 --- /dev/null +++ b/src/core/lib/support/atomic.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_H + +#include <grpc/support/port_platform.h> + +#ifdef GPR_HAS_CXX11_ATOMIC +#include "src/core/lib/support/atomic_with_std.h" +#else +#include "src/core/lib/support/atomic_with_atm.h" +#endif + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */ diff --git a/src/core/lib/support/atomic_with_atm.h b/src/core/lib/support/atomic_with_atm.h new file mode 100644 index 0000000000..55727f1dee --- /dev/null +++ b/src/core/lib/support/atomic_with_atm.h @@ -0,0 +1,70 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H + +#include <grpc/support/atm.h> + +namespace grpc_core { + +enum MemoryOrderRelaxed { memory_order_relaxed }; + +template <class T> +class atomic; + +template <> +class atomic<bool> { + public: + atomic() { gpr_atm_no_barrier_store(&x_, static_cast<gpr_atm>(false)); } + explicit atomic(bool x) { + gpr_atm_no_barrier_store(&x_, static_cast<gpr_atm>(x)); + } + + bool compare_exchange_strong(bool& expected, bool update, MemoryOrderRelaxed, + MemoryOrderRelaxed) { + if (!gpr_atm_no_barrier_cas(&x_, static_cast<gpr_atm>(expected), + static_cast<gpr_atm>(update))) { + expected = gpr_atm_no_barrier_load(&x_) != 0; + return false; + } + return true; + } + + private: + gpr_atm x_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H */ diff --git a/src/core/lib/support/atomic_with_std.h b/src/core/lib/support/atomic_with_std.h new file mode 100644 index 0000000000..7e9c19efe8 --- /dev/null +++ b/src/core/lib/support/atomic_with_std.h @@ -0,0 +1,48 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H + +#include <atomic> + +namespace grpc_core { + +template <class T> +using atomic = std::atomic<T>; + +typedef std::memory_order memory_order; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H */ diff --git a/src/core/lib/support/avl.c b/src/core/lib/support/avl.c index acf8fd5a55..ffa10c1e4f 100644 --- a/src/core/lib/support/avl.c +++ b/src/core/lib/support/avl.c @@ -205,8 +205,8 @@ static gpr_avl_node *rebalance(const gpr_avl_vtable *vtable, void *key, } } -static gpr_avl_node *add(const gpr_avl_vtable *vtable, gpr_avl_node *node, - void *key, void *value) { +static gpr_avl_node *add_key(const gpr_avl_vtable *vtable, gpr_avl_node *node, + void *key, void *value) { long cmp; if (node == NULL) { return new_node(key, value, NULL, NULL); @@ -217,17 +217,17 @@ static gpr_avl_node *add(const gpr_avl_vtable *vtable, gpr_avl_node *node, } else if (cmp > 0) { return rebalance( vtable, vtable->copy_key(node->key), vtable->copy_value(node->value), - add(vtable, node->left, key, value), ref_node(node->right)); + add_key(vtable, node->left, key, value), ref_node(node->right)); } else { return rebalance(vtable, vtable->copy_key(node->key), vtable->copy_value(node->value), ref_node(node->left), - add(vtable, node->right, key, value)); + add_key(vtable, node->right, key, value)); } } gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value) { gpr_avl_node *old_root = avl.root; - avl.root = add(avl.vtable, avl.root, key, value); + avl.root = add_key(avl.vtable, avl.root, key, value); assert_invariants(avl.root); unref_node(avl.vtable, old_root); return avl; @@ -247,8 +247,8 @@ static gpr_avl_node *in_order_tail(gpr_avl_node *node) { return node; } -static gpr_avl_node *remove(const gpr_avl_vtable *vtable, gpr_avl_node *node, - void *key) { +static gpr_avl_node *remove_key(const gpr_avl_vtable *vtable, + gpr_avl_node *node, void *key) { long cmp; if (node == NULL) { return NULL; @@ -263,27 +263,27 @@ static gpr_avl_node *remove(const gpr_avl_vtable *vtable, gpr_avl_node *node, gpr_avl_node *h = in_order_head(node->right); return rebalance(vtable, vtable->copy_key(h->key), vtable->copy_value(h->value), ref_node(node->left), - remove(vtable, node->right, h->key)); + remove_key(vtable, node->right, h->key)); } else { gpr_avl_node *h = in_order_tail(node->left); return rebalance( vtable, vtable->copy_key(h->key), vtable->copy_value(h->value), - remove(vtable, node->left, h->key), ref_node(node->right)); + remove_key(vtable, node->left, h->key), ref_node(node->right)); } } else if (cmp > 0) { - return rebalance(vtable, vtable->copy_key(node->key), - vtable->copy_value(node->value), - remove(vtable, node->left, key), ref_node(node->right)); + return rebalance( + vtable, vtable->copy_key(node->key), vtable->copy_value(node->value), + remove_key(vtable, node->left, key), ref_node(node->right)); } else { return rebalance(vtable, vtable->copy_key(node->key), vtable->copy_value(node->value), ref_node(node->left), - remove(vtable, node->right, key)); + remove_key(vtable, node->right, key)); } } gpr_avl gpr_avl_remove(gpr_avl avl, void *key) { gpr_avl_node *old_root = avl.root; - avl.root = remove(avl.vtable, avl.root, key); + avl.root = remove_key(avl.vtable, avl.root, key); assert_invariants(avl.root); unref_node(avl.vtable, old_root); return avl; diff --git a/src/core/lib/support/cpu_linux.c b/src/core/lib/support/cpu_linux.c index d6f7e7d3da..b826dde160 100644 --- a/src/core/lib/support/cpu_linux.c +++ b/src/core/lib/support/cpu_linux.c @@ -67,12 +67,17 @@ unsigned gpr_cpu_num_cores(void) { } unsigned gpr_cpu_current_cpu(void) { +#ifdef GPR_MUSL_LIBC_COMPAT + // sched_getcpu() is undefined on musl + return 0; +#else int cpu = sched_getcpu(); if (cpu < 0) { gpr_log(GPR_ERROR, "Error determining current CPU: %s\n", strerror(errno)); return 0; } return (unsigned)cpu; +#endif } #endif /* GPR_CPU_LINUX */ diff --git a/src/core/lib/support/memory.h b/src/core/lib/support/memory.h new file mode 100644 index 0000000000..6eff94eff7 --- /dev/null +++ b/src/core/lib/support/memory.h @@ -0,0 +1,74 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_MEMORY_H +#define GRPC_CORE_LIB_SUPPORT_MEMORY_H + +#include <grpc/support/alloc.h> + +#include <memory> +#include <utility> + +namespace grpc_core { + +// Alternative to new, since we cannot use it (for fear of libstdc++) +template <typename T, typename... Args> +inline T* New(Args&&... args) { + void* p = gpr_malloc(sizeof(T)); + return new (p) T(std::forward<Args>(args)...); +} + +// Alternative to delete, since we cannot use it (for fear of libstdc++) +template <typename T> +inline void Delete(T* p) { + p->~T(); + gpr_free(p); +} + +template <typename T> +class DefaultDelete { + public: + void operator()(T* p) { Delete(p); } +}; + +template <typename T, typename Deleter = DefaultDelete<T>> +using UniquePtr = std::unique_ptr<T, Deleter>; + +template <typename T, typename... Args> +inline UniquePtr<T> MakeUnique(Args&&... args) { + return UniquePtr<T>(New<T>(std::forward<Args>(args)...)); +} + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SUPPORT_MEMORY_H */ diff --git a/src/core/lib/support/mpscq.c b/src/core/lib/support/mpscq.c index 5b9323275a..1015cc6776 100644 --- a/src/core/lib/support/mpscq.c +++ b/src/core/lib/support/mpscq.c @@ -54,21 +54,31 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) { } gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { + bool empty; + return gpr_mpscq_pop_and_check_end(q, &empty); +} + +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty) { gpr_mpscq_node *tail = q->tail; gpr_mpscq_node *next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); if (tail == &q->stub) { // indicates the list is actually (ephemerally) empty - if (next == NULL) return NULL; + if (next == NULL) { + *empty = true; + return NULL; + } q->tail = next; tail = next; next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); } if (next != NULL) { + *empty = false; q->tail = next; return tail; } gpr_mpscq_node *head = (gpr_mpscq_node *)gpr_atm_acq_load(&q->head); if (tail != head) { + *empty = false; // indicates a retry is in order: we're still adding return NULL; } @@ -79,5 +89,6 @@ gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { return tail; } // indicates a retry is in order: we're still adding + *empty = false; return NULL; } diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/support/mpscq.h index 977a117952..24c89f90c9 100644 --- a/src/core/lib/support/mpscq.h +++ b/src/core/lib/support/mpscq.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_SUPPORT_MPSCQ_H #include <grpc/support/atm.h> +#include <stdbool.h> #include <stddef.h> // Multiple-producer single-consumer lock free queue, based upon the @@ -62,4 +63,7 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n); // the queue is empty!!) gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q); +// Pop a node; sets *empty to true if the queue is empty, or false if it is not +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty); + #endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */ diff --git a/src/core/lib/support/stack_lockfree.c b/src/core/lib/support/stack_lockfree.c index 9d7c9e5a38..dfbd3fb125 100644 --- a/src/core/lib/support/stack_lockfree.c +++ b/src/core/lib/support/stack_lockfree.c @@ -72,28 +72,20 @@ typedef union lockfree_node { struct gpr_stack_lockfree { lockfree_node *entries; lockfree_node head; /* An atomic entry describing curr head */ - -#ifndef NDEBUG - /* Bitmap of pushed entries to check for double-push or pop */ - gpr_atm pushed[(INVALID_ENTRY_INDEX + 1) / (8 * sizeof(gpr_atm))]; -#endif }; gpr_stack_lockfree *gpr_stack_lockfree_create(size_t entries) { gpr_stack_lockfree *stack; - stack = gpr_malloc(sizeof(*stack)); + stack = (gpr_stack_lockfree *)gpr_malloc(sizeof(*stack)); /* Since we only allocate 16 bits to represent an entry number, * make sure that we are within the desired range */ /* Reserve the highest entry number as a dummy */ GPR_ASSERT(entries < INVALID_ENTRY_INDEX); - stack->entries = gpr_malloc_aligned(entries * sizeof(stack->entries[0]), - ENTRY_ALIGNMENT_BITS); + stack->entries = (lockfree_node *)gpr_malloc_aligned( + entries * sizeof(stack->entries[0]), ENTRY_ALIGNMENT_BITS); /* Clear out all entries */ memset(stack->entries, 0, entries * sizeof(stack->entries[0])); memset(&stack->head, 0, sizeof(stack->head)); -#ifndef NDEBUG - memset(&stack->pushed, 0, sizeof(stack->pushed)); -#endif GPR_ASSERT(sizeof(stack->entries->atm) == sizeof(stack->entries->contents)); @@ -130,19 +122,6 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) { newhead.contents.aba_ctr = ++curent.contents.aba_ctr; gpr_atm_no_barrier_store(&stack->entries[entry].atm, curent.atm); -#ifndef NDEBUG - /* Check for double push */ - { - int pushed_index = entry / (int)(8 * sizeof(gpr_atm)); - int pushed_bit = entry % (int)(8 * sizeof(gpr_atm)); - gpr_atm old_val; - - old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index], - ((gpr_atm)1 << pushed_bit)); - GPR_ASSERT((old_val & (((gpr_atm)1) << pushed_bit)) == 0); - } -#endif - do { /* Atomically get the existing head value for use */ head.atm = gpr_atm_no_barrier_load(&(stack->head.atm)); @@ -168,18 +147,6 @@ int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) { gpr_atm_no_barrier_load(&(stack->entries[head.contents.index].atm)); } while (!gpr_atm_no_barrier_cas(&(stack->head.atm), head.atm, newhead.atm)); -#ifndef NDEBUG - /* Check for valid pop */ - { - int pushed_index = head.contents.index / (8 * sizeof(gpr_atm)); - int pushed_bit = head.contents.index % (8 * sizeof(gpr_atm)); - gpr_atm old_val; - - old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index], - -((gpr_atm)1 << pushed_bit)); - GPR_ASSERT((old_val & (((gpr_atm)1) << pushed_bit)) != 0); - } -#endif return head.contents.index; } diff --git a/src/core/lib/support/time_posix.c b/src/core/lib/support/time_posix.c index a69c501e9f..9bfec7782a 100644 --- a/src/core/lib/support/time_posix.c +++ b/src/core/lib/support/time_posix.c @@ -42,6 +42,7 @@ #ifdef __linux__ #include <sys/syscall.h> #endif +#include <grpc/support/atm.h> #include <grpc/support/log.h> #include <grpc/support/time.h> #include "src/core/lib/support/block_annotate.h" @@ -144,7 +145,14 @@ static gpr_timespec now_impl(gpr_clock_type clock) { gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type) = now_impl; +#ifdef GPR_LOW_LEVEL_COUNTERS +gpr_atm gpr_now_call_count; +#endif + gpr_timespec gpr_now(gpr_clock_type clock_type) { +#ifdef GPR_LOW_LEVEL_COUNTERS + __atomic_fetch_add(&gpr_now_call_count, 1, __ATOMIC_RELAXED); +#endif return gpr_now_impl(clock_type); } diff --git a/src/core/lib/support/tmpfile_posix.c b/src/core/lib/support/tmpfile_posix.c index 0cd4bb6fc3..5771c158e0 100644 --- a/src/core/lib/support/tmpfile_posix.c +++ b/src/core/lib/support/tmpfile_posix.c @@ -50,34 +50,34 @@ FILE *gpr_tmpfile(const char *prefix, char **tmp_filename) { FILE *result = NULL; - char *template; + char *filename_template; int fd; if (tmp_filename != NULL) *tmp_filename = NULL; - gpr_asprintf(&template, "/tmp/%s_XXXXXX", prefix); - GPR_ASSERT(template != NULL); + gpr_asprintf(&filename_template, "/tmp/%s_XXXXXX", prefix); + GPR_ASSERT(filename_template != NULL); - fd = mkstemp(template); + fd = mkstemp(filename_template); if (fd == -1) { - gpr_log(GPR_ERROR, "mkstemp failed for template %s with error %s.", - template, strerror(errno)); + gpr_log(GPR_ERROR, "mkstemp failed for filename_template %s with error %s.", + filename_template, strerror(errno)); goto end; } result = fdopen(fd, "w+"); if (result == NULL) { gpr_log(GPR_ERROR, "Could not open file %s from fd %d (error = %s).", - template, fd, strerror(errno)); - unlink(template); + filename_template, fd, strerror(errno)); + unlink(filename_template); close(fd); goto end; } end: if (result != NULL && tmp_filename != NULL) { - *tmp_filename = template; + *tmp_filename = filename_template; } else { - gpr_free(template); + gpr_free(filename_template); } return result; } diff --git a/src/core/lib/support/wrap_memcpy.c b/src/core/lib/support/wrap_memcpy.c index 15c289f7b8..deb8d6b198 100644 --- a/src/core/lib/support/wrap_memcpy.c +++ b/src/core/lib/support/wrap_memcpy.c @@ -31,6 +31,8 @@ * */ +#include <grpc/support/port_platform.h> + #include <string.h> /* Provide a wrapped memcpy for targets that need to be backwards @@ -40,7 +42,7 @@ */ #ifdef __linux__ -#ifdef __x86_64__ +#if defined(__x86_64__) && !defined(GPR_MUSL_LIBC_COMPAT) __asm__(".symver memcpy,memcpy@GLIBC_2.2.5"); void *__wrap_memcpy(void *destination, const void *source, size_t num) { return memcpy(destination, source, num); diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c index e71c0ebfc5..b72d534b7e 100644 --- a/src/core/lib/surface/alarm.c +++ b/src/core/lib/surface/alarm.c @@ -81,7 +81,9 @@ void grpc_alarm_cancel(grpc_alarm *alarm) { } void grpc_alarm_destroy(grpc_alarm *alarm) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_alarm_cancel(alarm); - GRPC_CQ_INTERNAL_UNREF(alarm->cq, "alarm"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm"); gpr_free(alarm); + grpc_exec_ctx_finish(&exec_ctx); } diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 79e3e5ca9b..d8941cdf42 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -32,5 +32,6 @@ */ #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/debug/trace.h" -int grpc_api_trace = 0; +grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false); diff --git a/src/core/lib/surface/api_trace.h b/src/core/lib/surface/api_trace.h index c60aaba5e9..d4fbc8d90d 100644 --- a/src/core/lib/surface/api_trace.h +++ b/src/core/lib/surface/api_trace.h @@ -37,7 +37,7 @@ #include <grpc/support/log.h> #include "src/core/lib/debug/trace.h" -extern int grpc_api_trace; +extern grpc_tracer_flag grpc_api_trace; /* Provide unwrapping macros because we're in C89 and variadic macros weren't introduced until C99... */ @@ -58,7 +58,7 @@ extern int grpc_api_trace; /* Due to the limitations of C89's preprocessor, the arity of the var-arg list 'nargs' must be specified. */ #define GRPC_API_TRACE(fmt, nargs, args) \ - if (grpc_api_trace) { \ + if (GRPC_TRACER_ON(grpc_api_trace)) { \ gpr_log(GPR_INFO, fmt GRPC_API_TRACE_UNWRAP##nargs args); \ } diff --git a/src/core/lib/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c index 1a6ccdaddb..539b014278 100644 --- a/src/core/lib/surface/byte_buffer_reader.c +++ b/src/core/lib/surface/byte_buffer_reader.c @@ -124,7 +124,7 @@ grpc_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) { grpc_slice in_slice; size_t bytes_read = 0; const size_t input_size = grpc_byte_buffer_length(reader->buffer_out); - grpc_slice out_slice = grpc_slice_malloc(input_size); + grpc_slice out_slice = GRPC_SLICE_MALLOC(input_size); uint8_t *const outbuf = GRPC_SLICE_START_PTR(out_slice); /* just an alias */ grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index a9317a4694..201969cd45 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -117,41 +117,61 @@ static received_status unpack_received_status(gpr_atm atm) { typedef struct batch_control { grpc_call *call; - grpc_cq_completion cq_completion; + /* Share memory for cq_completion and notify_tag as they are never needed + simultaneously. Each byte used in this data structure count as six bytes + per call, so any savings we can make are worthwhile, + + We use notify_tag to determine whether or not to send notification to the + completion queue. Once we've made that determination, we can reuse the + memory for cq_completion. */ + union { + grpc_cq_completion cq_completion; + struct { + /* Any given op indicates completion by either (a) calling a closure or + (b) sending a notification on the call's completion queue. If + \a is_closure is true, \a tag indicates a closure to be invoked; + otherwise, \a tag indicates the tag to be used in the notification to + be sent to the completion queue. */ + void *tag; + bool is_closure; + } notify_tag; + } completion_data; grpc_closure finish_batch; - void *notify_tag; gpr_refcount steps_to_complete; grpc_error *errors[MAX_ERRORS_PER_BATCH]; gpr_atm num_errors; - uint8_t send_initial_metadata; - uint8_t send_message; - uint8_t send_final_op; - uint8_t recv_initial_metadata; - uint8_t recv_message; - uint8_t recv_final_op; - uint8_t is_notify_tag_closure; - - /* TODO(ctiller): now that this is inlined, figure out how much of the above - state can be eliminated */ - grpc_transport_stream_op op; + grpc_transport_stream_op_batch op; } batch_control; +typedef struct { + gpr_mu child_list_mu; + grpc_call *first_child; +} parent_call; + +typedef struct { + grpc_call *parent; + /** siblings: children of the same parent form a list, and this list is + protected under + parent->mu */ + grpc_call *sibling_next; + grpc_call *sibling_prev; +} child_call; + struct grpc_call { + gpr_refcount ext_ref; gpr_arena *arena; grpc_completion_queue *cq; grpc_polling_entity pollent; grpc_channel *channel; - grpc_call *parent; - grpc_call *first_child; gpr_timespec start_time; - /* protects first_child, and child next/prev links */ - gpr_mu child_list_mu; + /* parent_call* */ gpr_atm parent_call_atm; + child_call *child_call; /* client or server call */ bool is_client; - /** has grpc_call_destroy been called */ + /** has grpc_call_unref been called */ bool destroy_called; /** flag indicating that cancellation is inherited */ bool cancellation_is_inherited; @@ -168,7 +188,8 @@ struct grpc_call { /* have we received initial metadata */ bool has_initial_md_been_received; - batch_control active_batches[MAX_CONCURRENT_BATCHES]; + batch_control *active_batches[MAX_CONCURRENT_BATCHES]; + grpc_transport_stream_op_batch_payload stream_op_payload; /* first idx: is_receiving, second idx: is_trailing */ grpc_metadata_batch metadata_batch[2][2]; @@ -198,12 +219,6 @@ struct grpc_call { int send_extra_metadata_count; gpr_timespec send_deadline; - /** siblings: children of the same parent form a list, and this list is - protected under - parent->mu */ - grpc_call *sibling_next; - grpc_call *sibling_prev; - grpc_slice_buffer_stream sending_stream; grpc_byte_stream *receiving_stream; @@ -229,7 +244,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; -int grpc_call_error_trace = 0; +grpc_tracer_flag grpc_call_error_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_compression_trace = GRPC_TRACER_INITIALIZER(false); #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) @@ -239,7 +255,7 @@ int grpc_call_error_trace = 0; CALL_FROM_CALL_STACK(grpc_call_stack_from_top_element(top_elem)) static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_transport_stream_op *op); + grpc_transport_stream_op_batch *op); static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_status_code status, const char *description); @@ -268,6 +284,27 @@ static void add_init_error(grpc_error **composite, grpc_error *new) { *composite = grpc_error_add_child(*composite, new); } +void *grpc_call_arena_alloc(grpc_call *call, size_t size) { + return gpr_arena_alloc(call->arena, size); +} + +static parent_call *get_or_create_parent_call(grpc_call *call) { + parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + if (p == NULL) { + p = gpr_arena_alloc(call->arena, sizeof(*p)); + gpr_mu_init(&p->child_list_mu); + if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { + gpr_mu_destroy(&p->child_list_mu); + p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + } + } + return p; +} + +static parent_call *get_parent_call(grpc_call *call) { + return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +} + grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, const grpc_call_create_args *args, grpc_call **out_call) { @@ -281,16 +318,16 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); call = gpr_arena_alloc(arena, sizeof(grpc_call) + channel_stack->call_stack_size); + gpr_ref_init(&call->ext_ref, 1); call->arena = arena; *out_call = call; - gpr_mu_init(&call->child_list_mu); call->channel = args->channel; call->cq = args->cq; - call->parent = args->parent_call; call->start_time = gpr_now(GPR_CLOCK_MONOTONIC); /* Always support no compression */ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); call->is_client = args->server_transport_data == NULL; + call->stream_op_payload.context = call->context; grpc_slice path = grpc_empty_slice(); if (call->is_client) { GPR_ASSERT(args->add_initial_metadata_count < @@ -316,12 +353,20 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, gpr_timespec send_deadline = gpr_convert_clock_type(args->send_deadline, GPR_CLOCK_MONOTONIC); + bool immediately_cancel = false; + if (args->parent_call != NULL) { + child_call *cc = call->child_call = + gpr_arena_alloc(arena, sizeof(child_call)); + call->child_call->parent = args->parent_call; + GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - gpr_mu_lock(&args->parent_call->child_list_mu); + parent_call *pc = get_or_create_parent_call(args->parent_call); + + gpr_mu_lock(&pc->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -350,28 +395,27 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { - cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_CANCELLED); + immediately_cancel = true; } } - if (args->parent_call->first_child == NULL) { - args->parent_call->first_child = call; - call->sibling_next = call->sibling_prev = call; + if (pc->first_child == NULL) { + pc->first_child = call; + cc->sibling_next = cc->sibling_prev = call; } else { - call->sibling_next = args->parent_call->first_child; - call->sibling_prev = args->parent_call->first_child->sibling_prev; - call->sibling_next->sibling_prev = call->sibling_prev->sibling_next = - call; + cc->sibling_next = pc->first_child; + cc->sibling_prev = pc->first_child->child_call->sibling_prev; + cc->sibling_next->child_call->sibling_prev = + cc->sibling_prev->child_call->sibling_next = call; } - gpr_mu_unlock(&args->parent_call->child_list_mu); + gpr_mu_unlock(&pc->child_list_mu); } call->send_deadline = send_deadline; GRPC_CHANNEL_INTERNAL_REF(args->channel, "call"); - /* initial refcount dropped by grpc_call_destroy */ + /* initial refcount dropped by grpc_call_unref */ grpc_call_element_args call_args = { .call_stack = CALL_STACK_FROM_CALL(call), .server_transport_data = args->server_transport_data, @@ -386,6 +430,10 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, cancel_with_error(exec_ctx, call, STATUS_FROM_SURFACE, GRPC_ERROR_REF(error)); } + if (immediately_cancel) { + cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + } if (args->cq != NULL) { GPR_ASSERT( args->pollset_set_alternative == NULL && @@ -460,7 +508,10 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - gpr_mu_destroy(&c->child_list_mu); + parent_call *pc = get_parent_call(c); + if (pc != NULL) { + gpr_mu_destroy(&pc->child_list_mu); + } for (ii = 0; ii < c->send_extra_metadata_count; ii++) { GRPC_MDELEM_UNREF(exec_ctx, c->send_extra_metadata[ii].md); } @@ -470,7 +521,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, } } if (c->cq) { - GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } get_final_status(call, set_status_value_directly, &c->final_info.final_status, @@ -489,39 +540,43 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GPR_TIMER_END("destroy_call", 0); } -void grpc_call_destroy(grpc_call *c) { - int cancel; - grpc_call *parent = c->parent; +void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } + +void grpc_call_unref(grpc_call *c) { + if (!gpr_unref(&c->ext_ref)) return; + + child_call *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - GPR_TIMER_BEGIN("grpc_call_destroy", 0); - GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); + GPR_TIMER_BEGIN("grpc_call_unref", 0); + GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); - if (parent) { - gpr_mu_lock(&parent->child_list_mu); - if (c == parent->first_child) { - parent->first_child = c->sibling_next; - if (c == parent->first_child) { - parent->first_child = NULL; + if (cc) { + parent_call *pc = get_parent_call(cc->parent); + gpr_mu_lock(&pc->child_list_mu); + if (c == pc->first_child) { + pc->first_child = cc->sibling_next; + if (c == pc->first_child) { + pc->first_child = NULL; } } - c->sibling_prev->sibling_next = c->sibling_next; - c->sibling_next->sibling_prev = c->sibling_prev; - gpr_mu_unlock(&parent->child_list_mu); - GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); + cc->sibling_prev->child_call->sibling_next = cc->sibling_next; + cc->sibling_next->child_call->sibling_prev = cc->sibling_prev; + gpr_mu_unlock(&pc->child_list_mu); + GRPC_CALL_INTERNAL_UNREF(&exec_ctx, cc->parent, "child"); } GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && - !gpr_atm_acq_load(&c->received_final_op_atm); + bool cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) != 0 && + gpr_atm_acq_load(&c->received_final_op_atm) == 0; if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); } GRPC_CALL_INTERNAL_UNREF(&exec_ctx, c, "destroy"); grpc_exec_ctx_finish(&exec_ctx); - GPR_TIMER_END("grpc_call_destroy", 0); + GPR_TIMER_END("grpc_call_unref", 0); } grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) { @@ -535,13 +590,12 @@ grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) { } static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { grpc_call_element *elem; GPR_TIMER_BEGIN("execute_op", 0); elem = CALL_ELEM_FROM_CALL(call, 0); - op->context = call->context; - elem->filter->start_transport_stream_op(exec_ctx, elem, op); + elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op); GPR_TIMER_END("execute_op", 0); } @@ -594,9 +648,10 @@ static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_error *error) { GRPC_CALL_INTERNAL_REF(c, "termination"); set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error)); - grpc_transport_stream_op *op = grpc_make_transport_stream_op( + grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op( grpc_closure_create(done_termination, c, grpc_schedule_on_exec_ctx)); - op->cancel_error = error; + op->cancel_stream = true; + op->payload->cancel_stream.cancel_error = error; execute_op(exec_ctx, c, op); } @@ -647,7 +702,7 @@ static void get_final_status(grpc_call *call, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { status[i] = unpack_received_status(gpr_atm_acq_load(&call->status[i])); } - if (grpc_call_error_trace) { + if (GRPC_TRACER_ON(grpc_call_error_trace)) { gpr_log(GPR_DEBUG, "get_final_status %s", call->is_client ? "CLI" : "SVR"); for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (status[i].is_set) { @@ -1025,16 +1080,17 @@ static batch_control *allocate_batch_control(grpc_call *call, const grpc_op *ops, size_t num_ops) { int slot = batch_slot_for_op(ops[0].op); - for (size_t i = 1; i < num_ops; i++) { - int op_slot = batch_slot_for_op(ops[i].op); - slot = GPR_MIN(slot, op_slot); + batch_control **pslot = &call->active_batches[slot]; + if (*pslot == NULL) { + *pslot = gpr_arena_alloc(call->arena, sizeof(batch_control)); } - batch_control *bctl = &call->active_batches[slot]; + batch_control *bctl = *pslot; if (bctl->call != NULL) { return NULL; } memset(bctl, 0, sizeof(*bctl)); bctl->call = call; + bctl->op.payload = &call->stream_op_payload; return bctl; } @@ -1069,46 +1125,49 @@ static grpc_error *consolidate_batch_errors(batch_control *bctl) { static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl) { - grpc_call *child_call; grpc_call *next_child_call; grpc_call *call = bctl->call; grpc_error *error = consolidate_batch_errors(bctl); - if (bctl->send_initial_metadata) { + if (bctl->op.send_initial_metadata) { grpc_metadata_batch_destroy( exec_ctx, &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]); } - if (bctl->send_message) { + if (bctl->op.send_message) { call->sending_message = false; } - if (bctl->send_final_op) { + if (bctl->op.send_trailing_metadata) { grpc_metadata_batch_destroy( exec_ctx, &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]); } - if (bctl->recv_final_op) { + if (bctl->op.recv_trailing_metadata) { grpc_metadata_batch *md = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; recv_trailing_filter(exec_ctx, call, md); /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - gpr_mu_lock(&call->child_list_mu); - child_call = call->first_child; - if (child_call != NULL) { - do { - next_child_call = child_call->sibling_next; - if (child_call->cancellation_is_inherited) { - GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - cancel_with_error(exec_ctx, child_call, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_CANCELLED); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); - } - child_call = next_child_call; - } while (child_call != call->first_child); + parent_call *pc = get_parent_call(call); + if (pc != NULL) { + grpc_call *child; + gpr_mu_lock(&pc->child_list_mu); + child = pc->first_child; + if (child != NULL) { + do { + next_child_call = child->child_call->sibling_next; + if (child->cancellation_is_inherited) { + GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); + cancel_with_error(exec_ctx, child, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, child, "propagate_cancel"); + } + child = next_child_call; + } while (child != pc->first_child); + } + gpr_mu_unlock(&pc->child_list_mu); } - gpr_mu_unlock(&call->child_list_mu); if (call->is_client) { get_final_status(call, set_status_value_directly, @@ -1123,15 +1182,16 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, error = GRPC_ERROR_NONE; } - if (bctl->is_notify_tag_closure) { + if (bctl->completion_data.notify_tag.is_closure) { /* unrefs bctl->error */ bctl->call = NULL; - grpc_closure_run(exec_ctx, bctl->notify_tag, error); + grpc_closure_run(exec_ctx, bctl->completion_data.notify_tag.tag, error); GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); } else { /* unrefs bctl->error */ - grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, error, - finish_batch_completion, bctl, &bctl->cq_completion); + grpc_cq_end_op( + exec_ctx, bctl->call->cq, bctl->completion_data.notify_tag.tag, error, + finish_batch_completion, bctl, &bctl->completion_data.cq_completion); } } @@ -1143,6 +1203,7 @@ static void finish_batch_step(grpc_exec_ctx *exec_ctx, batch_control *bctl) { static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, batch_control *bctl) { + grpc_error *error; grpc_call *call = bctl->call; for (;;) { size_t remaining = call->receiving_stream->length - @@ -1154,11 +1215,22 @@ static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, finish_batch_step(exec_ctx, bctl); return; } - if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, - &call->receiving_slice, remaining, + if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, remaining, &call->receiving_slice_ready)) { - grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); + error = grpc_byte_stream_pull(exec_ctx, call->receiving_stream, + &call->receiving_slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); + } else { + grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); + call->receiving_stream = NULL; + grpc_byte_buffer_destroy(*call->receiving_buffer); + *call->receiving_buffer = NULL; + call->receiving_message = 0; + finish_batch_step(exec_ctx, bctl); + return; + } } else { return; } @@ -1169,20 +1241,36 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; grpc_call *call = bctl->call; + grpc_byte_stream *bs = call->receiving_stream; + bool release_error = false; if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); - continue_receiving_slices(exec_ctx, bctl); - } else { - if (grpc_trace_operation_failures) { + grpc_slice slice; + error = grpc_byte_stream_pull(exec_ctx, bs, &slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + slice); + continue_receiving_slices(exec_ctx, bctl); + } else { + /* Error returned by grpc_byte_stream_pull needs to be released manually + */ + release_error = true; + } + } + + if (error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures)) { GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); } grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); call->receiving_stream = NULL; grpc_byte_buffer_destroy(*call->receiving_buffer); *call->receiving_buffer = NULL; + call->receiving_message = 0; finish_batch_step(exec_ctx, bctl); + if (release_error) { + GRPC_ERROR_UNREF(error); + } } } @@ -1267,8 +1355,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, GPR_ASSERT(call->encodings_accepted_by_peer != 0); if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { - extern int grpc_compression_trace; - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); @@ -1374,11 +1461,13 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (bctl == NULL) { return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; } - bctl->notify_tag = notify_tag; - bctl->is_notify_tag_closure = (uint8_t)(is_notify_tag_closure != 0); + bctl->completion_data.notify_tag.tag = notify_tag; + bctl->completion_data.notify_tag.is_closure = + (uint8_t)(is_notify_tag_closure != 0); - grpc_transport_stream_op *stream_op = &bctl->op; - memset(stream_op, 0, sizeof(*stream_op)); + grpc_transport_stream_op_batch *stream_op = &bctl->op; + grpc_transport_stream_op_batch_payload *stream_op_payload = + &call->stream_op_payload; stream_op->covered_by_poller = true; /* rewrite batch ops into a transport op */ @@ -1432,8 +1521,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } - bctl->send_initial_metadata = 1; - call->sent_initial_metadata = 1; + stream_op->send_initial_metadata = true; + call->sent_initial_metadata = true; if (!prepare_application_metadata( exec_ctx, call, (int)op->data.send_initial_metadata.count, op->data.send_initial_metadata.metadata, 0, call->is_client, @@ -1443,9 +1532,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } /* TODO(ctiller): just make these the same variable? */ call->metadata_batch[0][0].deadline = call->send_deadline; - stream_op->send_initial_metadata = + stream_op_payload->send_initial_metadata.send_initial_metadata = &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]; - stream_op->send_initial_metadata_flags = op->flags; + stream_op_payload->send_initial_metadata.send_initial_metadata_flags = + op->flags; break; case GRPC_OP_SEND_MESSAGE: if (!are_write_flags_valid(op->flags)) { @@ -1460,8 +1550,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - bctl->send_message = 1; - call->sending_message = 1; + stream_op->send_message = true; + call->sending_message = true; grpc_slice_buffer_stream_init( &call->sending_stream, &op->data.send_message.send_message->data.raw.slice_buffer, @@ -1473,7 +1563,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, GRPC_COMPRESS_NONE) { call->sending_stream.base.flags |= GRPC_WRITE_INTERNAL_COMPRESS; } - stream_op->send_message = &call->sending_stream.base; + stream_op_payload->send_message.send_message = + &call->sending_stream.base; break; case GRPC_OP_SEND_CLOSE_FROM_CLIENT: /* Flag validation: currently allow no flags */ @@ -1489,9 +1580,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - bctl->send_final_op = 1; - call->sent_final_op = 1; - stream_op->send_trailing_metadata = + stream_op->send_trailing_metadata = true; + call->sent_final_op = true; + stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_SEND_STATUS_FROM_SERVER: @@ -1513,8 +1604,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } - bctl->send_final_op = 1; - call->sent_final_op = 1; + stream_op->send_trailing_metadata = true; + call->sent_final_op = true; GPR_ASSERT(call->send_extra_metadata_count == 0); call->send_extra_metadata_count = 1; call->send_extra_metadata[0].md = grpc_channel_get_reffed_status_elem( @@ -1553,7 +1644,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } - stream_op->send_trailing_metadata = + stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_RECV_INITIAL_METADATA: @@ -1570,16 +1661,16 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, from server.c. In that case, it's coming from accept_stream, and in that case we're not necessarily covered by a poller. */ stream_op->covered_by_poller = call->is_client; - call->received_initial_metadata = 1; + call->received_initial_metadata = true; call->buffered_metadata[0] = op->data.recv_initial_metadata.recv_initial_metadata; grpc_closure_init(&call->receiving_initial_metadata_ready, receiving_initial_metadata_ready, bctl, grpc_schedule_on_exec_ctx); - bctl->recv_initial_metadata = 1; - stream_op->recv_initial_metadata = + stream_op->recv_initial_metadata = true; + stream_op_payload->recv_initial_metadata.recv_initial_metadata = &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; - stream_op->recv_initial_metadata_ready = + stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready = &call->receiving_initial_metadata_ready; num_completion_callbacks_needed++; break; @@ -1593,13 +1684,14 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - call->receiving_message = 1; - bctl->recv_message = 1; + call->receiving_message = true; + stream_op->recv_message = true; call->receiving_buffer = op->data.recv_message.recv_message; - stream_op->recv_message = &call->receiving_stream; + stream_op_payload->recv_message.recv_message = &call->receiving_stream; grpc_closure_init(&call->receiving_stream_ready, receiving_stream_ready, bctl, grpc_schedule_on_exec_ctx); - stream_op->recv_message_ready = &call->receiving_stream_ready; + stream_op_payload->recv_message.recv_message_ready = + &call->receiving_stream_ready; num_completion_callbacks_needed++; break; case GRPC_OP_RECV_STATUS_ON_CLIENT: @@ -1616,16 +1708,17 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - call->requested_final_op = 1; + call->requested_final_op = true; call->buffered_metadata[1] = op->data.recv_status_on_client.trailing_metadata; call->final_op.client.status = op->data.recv_status_on_client.status; call->final_op.client.status_details = op->data.recv_status_on_client.status_details; - bctl->recv_final_op = 1; - stream_op->recv_trailing_metadata = + stream_op->recv_trailing_metadata = true; + stream_op->collect_stats = true; + stream_op_payload->recv_trailing_metadata.recv_trailing_metadata = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; - stream_op->collect_stats = + stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; case GRPC_OP_RECV_CLOSE_ON_SERVER: @@ -1642,13 +1735,14 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; goto done_with_error; } - call->requested_final_op = 1; + call->requested_final_op = true; call->final_op.server.cancelled = op->data.recv_close_on_server.cancelled; - bctl->recv_final_op = 1; - stream_op->recv_trailing_metadata = + stream_op->recv_trailing_metadata = true; + stream_op->collect_stats = true; + stream_op_payload->recv_trailing_metadata.recv_trailing_metadata = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; - stream_op->collect_stats = + stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; } @@ -1660,7 +1754,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } gpr_ref_init(&bctl->steps_to_complete, num_completion_callbacks_needed); - stream_op->context = call->context; grpc_closure_init(&bctl->finish_batch, finish_batch, bctl, grpc_schedule_on_exec_ctx); stream_op->on_complete = &bctl->finish_batch; @@ -1674,26 +1767,26 @@ done: done_with_error: /* reverse any mutations that occured */ - if (bctl->send_initial_metadata) { - call->sent_initial_metadata = 0; + if (stream_op->send_initial_metadata) { + call->sent_initial_metadata = false; grpc_metadata_batch_clear(exec_ctx, &call->metadata_batch[0][0]); } - if (bctl->send_message) { - call->sending_message = 0; + if (stream_op->send_message) { + call->sending_message = false; grpc_byte_stream_destroy(exec_ctx, &call->sending_stream.base); } - if (bctl->send_final_op) { - call->sent_final_op = 0; + if (stream_op->send_trailing_metadata) { + call->sent_final_op = false; grpc_metadata_batch_clear(exec_ctx, &call->metadata_batch[0][1]); } - if (bctl->recv_initial_metadata) { - call->received_initial_metadata = 0; + if (stream_op->recv_initial_metadata) { + call->received_initial_metadata = false; } - if (bctl->recv_message) { - call->receiving_message = 0; + if (stream_op->recv_message) { + call->receiving_message = false; } - if (bctl->recv_final_op) { - call->requested_final_op = 0; + if (stream_op->recv_trailing_metadata) { + call->requested_final_op = false; } goto done; } diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 7d4d0db28d..256a5fa2fe 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -117,7 +117,8 @@ void grpc_call_context_set(grpc_call *call, grpc_context_index elem, void *grpc_call_context_get(grpc_call *call, grpc_context_index elem); #define GRPC_CALL_LOG_BATCH(sev, call, ops, nops, tag) \ - if (grpc_api_trace) grpc_call_log_batch(sev, call, ops, nops, tag) + if (GRPC_TRACER_ON(grpc_api_trace)) \ + grpc_call_log_batch(sev, call, ops, nops, tag) uint8_t grpc_call_is_client(grpc_call *call); @@ -126,7 +127,8 @@ uint8_t grpc_call_is_client(grpc_call *call); grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level); -extern int grpc_call_error_trace; +extern grpc_tracer_flag grpc_call_error_trace; +extern grpc_tracer_flag grpc_compression_trace; #ifdef __cplusplus } diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index b4bfb92042..b3ba826bbc 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -150,17 +150,20 @@ grpc_channel *grpc_channel_create_with_builder( } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) { channel->compression_options.default_level.is_set = true; - GPR_ASSERT(args->args[i].value.integer >= 0 && - args->args[i].value.integer < GRPC_COMPRESS_LEVEL_COUNT); channel->compression_options.default_level.level = - (grpc_compression_level)args->args[i].value.integer; + (grpc_compression_level)grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){GRPC_COMPRESS_LEVEL_NONE, + GRPC_COMPRESS_LEVEL_NONE, + GRPC_COMPRESS_LEVEL_COUNT - 1}); } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { channel->compression_options.default_algorithm.is_set = true; - GPR_ASSERT(args->args[i].value.integer >= 0 && - args->args[i].value.integer < GRPC_COMPRESS_ALGORITHMS_COUNT); channel->compression_options.default_algorithm.algorithm = - (grpc_compression_algorithm)args->args[i].value.integer; + (grpc_compression_algorithm)grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){GRPC_COMPRESS_NONE, GRPC_COMPRESS_NONE, + GRPC_COMPRESS_ALGORITHMS_COUNT - 1}); } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { diff --git a/src/core/lib/surface/channel_init.c b/src/core/lib/surface/channel_init.c index 7acb444d9b..20f5753004 100644 --- a/src/core/lib/surface/channel_init.c +++ b/src/core/lib/surface/channel_init.c @@ -104,30 +104,13 @@ void grpc_channel_init_shutdown(void) { } } -static const char *name_for_type(grpc_channel_stack_type type) { - switch (type) { - case GRPC_CLIENT_CHANNEL: - return "CLIENT_CHANNEL"; - case GRPC_CLIENT_SUBCHANNEL: - return "CLIENT_SUBCHANNEL"; - case GRPC_SERVER_CHANNEL: - return "SERVER_CHANNEL"; - case GRPC_CLIENT_LAME_CHANNEL: - return "CLIENT_LAME_CHANNEL"; - case GRPC_CLIENT_DIRECT_CHANNEL: - return "CLIENT_DIRECT_CHANNEL"; - case GRPC_NUM_CHANNEL_STACK_TYPES: - break; - } - GPR_UNREACHABLE_CODE(return "UNKNOWN"); -} - bool grpc_channel_init_create_stack(grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder, grpc_channel_stack_type type) { GPR_ASSERT(g_finalized); - grpc_channel_stack_builder_set_name(builder, name_for_type(type)); + grpc_channel_stack_builder_set_name(builder, + grpc_channel_stack_type_string(type)); for (size_t i = 0; i < g_slots[type].num_slots; i++) { const stage_slot *slot = &g_slots[type].slots[i]; diff --git a/src/core/lib/surface/channel_stack_type.c b/src/core/lib/surface/channel_stack_type.c index c35d603ca3..ed3b53fb36 100644 --- a/src/core/lib/surface/channel_stack_type.c +++ b/src/core/lib/surface/channel_stack_type.c @@ -52,3 +52,21 @@ bool grpc_channel_stack_type_is_client(grpc_channel_stack_type type) { } GPR_UNREACHABLE_CODE(return true;); } + +const char *grpc_channel_stack_type_string(grpc_channel_stack_type type) { + switch (type) { + case GRPC_CLIENT_CHANNEL: + return "CLIENT_CHANNEL"; + case GRPC_CLIENT_SUBCHANNEL: + return "CLIENT_SUBCHANNEL"; + case GRPC_SERVER_CHANNEL: + return "SERVER_CHANNEL"; + case GRPC_CLIENT_LAME_CHANNEL: + return "CLIENT_LAME_CHANNEL"; + case GRPC_CLIENT_DIRECT_CHANNEL: + return "CLIENT_DIRECT_CHANNEL"; + case GRPC_NUM_CHANNEL_STACK_TYPES: + break; + } + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} diff --git a/src/core/lib/surface/channel_stack_type.h b/src/core/lib/surface/channel_stack_type.h index 4eea4f1b01..ccf4e53d27 100644 --- a/src/core/lib/surface/channel_stack_type.h +++ b/src/core/lib/surface/channel_stack_type.h @@ -55,4 +55,6 @@ typedef enum { bool grpc_channel_stack_type_is_client(grpc_channel_stack_type type); +const char *grpc_channel_stack_type_string(grpc_channel_stack_type type); + #endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_STACK_TYPE_H */ diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index b4594817e4..df5b70205c 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,9 +50,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/event_string.h" -int grpc_trace_operation_failures; +grpc_tracer_flag grpc_trace_operation_failures = GRPC_TRACER_INITIALIZER(false); #ifndef NDEBUG -int grpc_trace_pending_tags; +grpc_tracer_flag grpc_trace_pending_tags = GRPC_TRACER_INITIALIZER(false); #endif typedef struct { @@ -60,10 +60,156 @@ typedef struct { void *tag; } plucker; +typedef struct { + bool can_get_pollset; + bool can_listen; + size_t (*size)(void); + void (*init)(grpc_pollset *pollset, gpr_mu **mu); + grpc_error *(*kick)(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker); + grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker, gpr_timespec now, + gpr_timespec deadline); + void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure); + void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); +} cq_poller_vtable; + +typedef struct non_polling_worker { + gpr_cv cv; + bool kicked; + struct non_polling_worker *next; + struct non_polling_worker *prev; +} non_polling_worker; + +typedef struct { + gpr_mu mu; + non_polling_worker *root; + grpc_closure *shutdown; +} non_polling_poller; + +static size_t non_polling_poller_size(void) { + return sizeof(non_polling_poller); +} + +static void non_polling_poller_init(grpc_pollset *pollset, gpr_mu **mu) { + non_polling_poller *npp = (non_polling_poller *)pollset; + gpr_mu_init(&npp->mu); + *mu = &npp->mu; +} + +static void non_polling_poller_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + non_polling_poller *npp = (non_polling_poller *)pollset; + gpr_mu_destroy(&npp->mu); +} + +static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker **worker, + gpr_timespec now, + gpr_timespec deadline) { + non_polling_poller *npp = (non_polling_poller *)pollset; + if (npp->shutdown) return GRPC_ERROR_NONE; + non_polling_worker w; + gpr_cv_init(&w.cv); + if (worker != NULL) *worker = (grpc_pollset_worker *)&w; + if (npp->root == NULL) { + npp->root = w.next = w.prev = &w; + } else { + w.next = npp->root; + w.prev = w.next->prev; + w.next->prev = w.prev->next = &w; + } + w.kicked = false; + while (!npp->shutdown && !w.kicked && !gpr_cv_wait(&w.cv, &npp->mu, deadline)) + ; + if (&w == npp->root) { + npp->root = w.next; + if (&w == npp->root) { + if (npp->shutdown) { + grpc_closure_sched(exec_ctx, npp->shutdown, GRPC_ERROR_NONE); + } + npp->root = NULL; + } + } + w.next->prev = w.prev; + w.prev->next = w.next; + gpr_cv_destroy(&w.cv); + if (worker != NULL) *worker = NULL; + return GRPC_ERROR_NONE; +} + +static grpc_error *non_polling_poller_kick( + grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + non_polling_poller *p = (non_polling_poller *)pollset; + if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; + if (specific_worker != NULL) { + non_polling_worker *w = (non_polling_worker *)specific_worker; + if (!w->kicked) { + w->kicked = true; + gpr_cv_signal(&w->cv); + } + } + return GRPC_ERROR_NONE; +} + +static void non_polling_poller_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_closure *closure) { + non_polling_poller *p = (non_polling_poller *)pollset; + GPR_ASSERT(closure != NULL); + p->shutdown = closure; + if (p->root == NULL) { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + } else { + non_polling_worker *w = p->root; + do { + gpr_cv_signal(&w->cv); + w = w->next; + } while (w != p->root); + } +} + +static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { + /* GRPC_CQ_DEFAULT_POLLING */ + {.can_get_pollset = true, + .can_listen = true, + .size = grpc_pollset_size, + .init = grpc_pollset_init, + .kick = grpc_pollset_kick, + .work = grpc_pollset_work, + .shutdown = grpc_pollset_shutdown, + .destroy = grpc_pollset_destroy}, + /* GRPC_CQ_NON_LISTENING */ + {.can_get_pollset = true, + .can_listen = false, + .size = grpc_pollset_size, + .init = grpc_pollset_init, + .kick = grpc_pollset_kick, + .work = grpc_pollset_work, + .shutdown = grpc_pollset_shutdown, + .destroy = grpc_pollset_destroy}, + /* GRPC_CQ_NON_POLLING */ + {.can_get_pollset = false, + .can_listen = false, + .size = non_polling_poller_size, + .init = non_polling_poller_init, + .kick = non_polling_poller_kick, + .work = non_polling_poller_work, + .shutdown = non_polling_poller_shutdown, + .destroy = non_polling_poller_destroy}, +}; + /* Completion queue structure */ struct grpc_completion_queue { /** owned by pollset */ gpr_mu *mu; + + grpc_cq_completion_type completion_type; + + const cq_poller_vtable *poller_vtable; + /** completed events */ grpc_cq_completion completed_head; grpc_cq_completion *completed_tail; @@ -79,6 +225,7 @@ struct grpc_completion_queue { int shutdown_called; int is_server_cq; /** Can the server cq accept incoming channels */ + /* TODO: sreek - This will no longer be needed. Use polling_type set */ int is_non_listening_server_cq; int num_pluckers; plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS]; @@ -96,35 +243,46 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) #define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1) -int grpc_cq_pluck_trace; -int grpc_cq_event_timeout_trace; +grpc_tracer_flag grpc_cq_pluck_trace = GRPC_TRACER_INITIALIZER(true); +grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true); -#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ - if (grpc_api_trace && \ - (grpc_cq_pluck_trace || (event)->type != GRPC_QUEUE_TIMEOUT)) { \ - char *_ev = grpc_event_string(event); \ - gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ - gpr_free(_ev); \ +#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ + if (GRPC_TRACER_ON(grpc_api_trace) && \ + (GRPC_TRACER_ON(grpc_cq_pluck_trace) || \ + (event)->type != GRPC_QUEUE_TIMEOUT)) { \ + char *_ev = grpc_event_string(event); \ + gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ + gpr_free(_ev); \ } static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, grpc_error *error); -grpc_completion_queue *grpc_completion_queue_create(void *reserved) { +grpc_completion_queue *grpc_completion_queue_create_internal( + grpc_cq_completion_type completion_type, + grpc_cq_polling_type polling_type) { grpc_completion_queue *cc; - GPR_ASSERT(!reserved); - GPR_TIMER_BEGIN("grpc_completion_queue_create", 0); + GPR_TIMER_BEGIN("grpc_completion_queue_create_internal", 0); - GRPC_API_TRACE("grpc_completion_queue_create(reserved=%p)", 1, (reserved)); + GRPC_API_TRACE( + "grpc_completion_queue_create_internal(completion_type=%d, " + "polling_type=%d)", + 2, (completion_type, polling_type)); + + const cq_poller_vtable *poller_vtable = + &g_poller_vtable_by_poller_type[polling_type]; - cc = gpr_zalloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); - grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); + cc = gpr_zalloc(sizeof(grpc_completion_queue) + poller_vtable->size()); + poller_vtable->init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG cc->outstanding_tags = NULL; cc->outstanding_tag_capacity = 0; #endif + cc->completion_type = completion_type; + cc->poller_vtable = poller_vtable; + /* Initial ref is dropped by grpc_completion_queue_shutdown */ gpr_ref_init(&cc->pending_events, 1); /* One for destroy(), one for pollset_shutdown */ @@ -143,11 +301,15 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) { grpc_closure_init(&cc->pollset_shutdown_done, on_pollset_shutdown_done, cc, grpc_schedule_on_exec_ctx); - GPR_TIMER_END("grpc_completion_queue_create", 0); + GPR_TIMER_END("grpc_completion_queue_create_internal", 0); return cc; } +grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc) { + return cc->completion_type; +} + #ifdef GRPC_CQ_REF_COUNT_DEBUG void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason, const char *file, int line) { @@ -162,20 +324,21 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) { static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_completion_queue *cc = arg; - GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, cc, "pollset_destroy"); } #ifdef GRPC_CQ_REF_COUNT_DEBUG -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "CQ:%p unref %d -> %d %s", cc, (int)cc->owning_refs.count, (int)cc->owning_refs.count - 1, reason); #else -void grpc_cq_internal_unref(grpc_completion_queue *cc) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, + grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_destroy(POLLSET_FROM_CQ(cc)); + cc->poller_vtable->destroy(exec_ctx, POLLSET_FROM_CQ(cc)); #ifndef NDEBUG gpr_free(cc->outstanding_tags); #endif @@ -215,14 +378,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, #endif GPR_TIMER_BEGIN("grpc_cq_end_op", 0); - if (grpc_api_trace || - (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) { + if (GRPC_TRACER_ON(grpc_api_trace) || + (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE)) { const char *errmsg = grpc_error_string(error); GRPC_API_TRACE( "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, " "done_arg=%p, storage=%p)", 7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage)); - if (grpc_trace_operation_failures && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg); } } @@ -260,7 +425,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, } } grpc_error *kick_error = - grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); + cc->poller_vtable->kick(POLLSET_FROM_CQ(cc), pluck_worker); gpr_mu_unlock(cc->mu); if (kick_error != GRPC_ERROR_NONE) { const char *msg = grpc_error_string(kick_error); @@ -275,8 +440,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; - grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); + cc->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); gpr_mu_unlock(cc->mu); } @@ -321,7 +486,7 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) { #ifndef NDEBUG static void dump_pending_tags(grpc_completion_queue *cc) { - if (!grpc_trace_pending_tags) return; + if (!GRPC_TRACER_ON(grpc_trace_pending_tags)) return; gpr_strvec v; gpr_strvec_init(&v); @@ -345,9 +510,15 @@ static void dump_pending_tags(grpc_completion_queue *cc) {} grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec deadline, void *reserved) { grpc_event ret; - grpc_pollset_worker *worker = NULL; gpr_timespec now; + if (cc->completion_type != GRPC_CQ_NEXT) { + gpr_log(GPR_ERROR, + "grpc_completion_queue_next() cannot be called on this completion " + "queue since its completion type is not GRPC_CQ_NEXT"); + abort(); + } + GPR_TIMER_BEGIN("grpc_completion_queue_next", 0); GRPC_API_TRACE( @@ -414,36 +585,23 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + NULL, now, deadline); + if (err != GRPC_ERROR_NONE) { gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - continue; - } else { - grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), - &worker, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "next"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "next"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -517,7 +675,14 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0); - if (grpc_cq_pluck_trace) { + if (cc->completion_type != GRPC_CQ_PLUCK) { + gpr_log(GPR_ERROR, + "grpc_completion_queue_pluck() cannot be called on this completion " + "queue since its completion type is not GRPC_CQ_PLUCK"); + abort(); + } + + if (GRPC_TRACER_ON(grpc_cq_pluck_trace)) { GRPC_API_TRACE( "grpc_completion_queue_pluck(" "cc=%p, tag=%p, " @@ -600,38 +765,26 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + &worker, now, deadline); + if (err != GRPC_ERROR_NONE) { + del_plucker(cc, tag, &worker); gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - } else { - grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), - &worker, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - del_plucker(cc, tag, &worker); - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; del_plucker(cc, tag, &worker); } done: GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "pluck"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "pluck"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -656,8 +809,8 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { if (gpr_unref(&cc->pending_events)) { GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); + cc->poller_vtable->shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); } gpr_mu_unlock(cc->mu); grpc_exec_ctx_finish(&exec_ctx); @@ -668,12 +821,14 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { GRPC_API_TRACE("grpc_completion_queue_destroy(cc=%p)", 1, (cc)); GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0); grpc_completion_queue_shutdown(cc); - GRPC_CQ_INTERNAL_UNREF(cc, "destroy"); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "destroy"); + grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_destroy", 0); } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { - return POLLSET_FROM_CQ(cc); + return cc->poller_vtable->can_get_pollset ? POLLSET_FROM_CQ(cc) : NULL; } grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) { @@ -681,13 +836,23 @@ grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) { } void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) { + /* TODO: sreek - use cc->polling_type field here and add a validation check + (i.e grpc_cq_mark_non_listening_server_cq can only be called on a cc whose + polling_type is set to GRPC_CQ_NON_LISTENING */ cc->is_non_listening_server_cq = 1; } bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) { + /* TODO (sreek) - return (cc->polling_type == GRPC_CQ_NON_LISTENING) */ return (cc->is_non_listening_server_cq == 1); } void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; } -int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; } +bool grpc_cq_is_server_cq(grpc_completion_queue *cc) { + return cc->is_server_cq; +} + +bool grpc_cq_can_listen(grpc_completion_queue *cc) { + return cc->poller_vtable->can_listen; +} diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 5d73dd7216..8d9ce2ec02 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -37,15 +37,16 @@ /* Internal API for completion queues */ #include <grpc/grpc.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/pollset.h" /* These trace flags default to 1. The corresponding lines are only traced if grpc_api_trace is also truthy */ -extern int grpc_cq_pluck_trace; -extern int grpc_cq_event_timeout_trace; -extern int grpc_trace_operation_failures; +extern grpc_tracer_flag grpc_cq_pluck_trace; +extern grpc_tracer_flag grpc_cq_event_timeout_trace; +extern grpc_tracer_flag grpc_trace_operation_failures; #ifndef NDEBUG -extern int grpc_trace_pending_tags; +extern grpc_tracer_flag grpc_trace_pending_tags; #endif typedef struct grpc_cq_completion { @@ -65,17 +66,17 @@ typedef struct grpc_cq_completion { #ifdef GRPC_CQ_REF_COUNT_DEBUG void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason, const char *file, int line); -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line); #define GRPC_CQ_INTERNAL_REF(cc, reason) \ grpc_cq_internal_ref(cc, reason, __FILE__, __LINE__) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) \ - grpc_cq_internal_unref(cc, reason, __FILE__, __LINE__) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) \ + grpc_cq_internal_unref(ec, cc, reason, __FILE__, __LINE__) #else void grpc_cq_internal_ref(grpc_completion_queue *cc); -void grpc_cq_internal_unref(grpc_completion_queue *cc); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc); #define GRPC_CQ_INTERNAL_REF(cc, reason) grpc_cq_internal_ref(cc) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) grpc_cq_internal_unref(cc) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) grpc_cq_internal_unref(ec, cc) #endif /* Flag that an operation is beginning: the completion channel will not finish @@ -94,9 +95,13 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc); grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps); -void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc); -bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc); void grpc_cq_mark_server_cq(grpc_completion_queue *cc); -int grpc_cq_is_server_cq(grpc_completion_queue *cc); +bool grpc_cq_is_server_cq(grpc_completion_queue *cc); +bool grpc_cq_can_listen(grpc_completion_queue *cc); + +grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc); + +grpc_completion_queue *grpc_completion_queue_create_internal( + grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type); #endif /* GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H */ diff --git a/src/core/lib/surface/completion_queue_factory.c b/src/core/lib/surface/completion_queue_factory.c index db67a5192b..d68b84eddd 100644 --- a/src/core/lib/surface/completion_queue_factory.c +++ b/src/core/lib/surface/completion_queue_factory.c @@ -36,12 +36,15 @@ #include <grpc/support/log.h> -/* TODO (sreek) - Currently this does not use the attributes arg. This will be - added in a future PR */ +/* + * == Default completion queue factory implementation == + */ + static grpc_completion_queue* default_create( const grpc_completion_queue_factory* factory, - const grpc_completion_queue_attributes* attributes) { - return grpc_completion_queue_create(NULL); + const grpc_completion_queue_attributes* attr) { + return grpc_completion_queue_create_internal(attr->cq_completion_type, + attr->cq_polling_type); } static grpc_completion_queue_factory_vtable default_vtable = {default_create}; @@ -49,19 +52,24 @@ static grpc_completion_queue_factory_vtable default_vtable = {default_create}; static const grpc_completion_queue_factory g_default_cq_factory = { "Default Factory", NULL, &default_vtable}; +/* + * == Completion queue factory APIs + */ + const grpc_completion_queue_factory* grpc_completion_queue_factory_lookup( const grpc_completion_queue_attributes* attributes) { - /* As we add more fields to grpc_completion_queue_attributes, we may have to - change this assert to: - GPR_ASSERT (attributes->version >= 1 && - attributes->version <= GRPC_CQ_CURRENT_VERSION) */ - GPR_ASSERT(attributes->version == 1); + GPR_ASSERT(attributes->version >= 1 && + attributes->version <= GRPC_CQ_CURRENT_VERSION); /* The default factory can handle version 1 of the attributes structure. We may have to change this as more fields are added to the structure */ return &g_default_cq_factory; } +/* + * == Completion queue creation APIs == + */ + grpc_completion_queue* grpc_completion_queue_create_for_next(void* reserved) { GPR_ASSERT(!reserved); grpc_completion_queue_attributes attr = {1, GRPC_CQ_NEXT, @@ -75,3 +83,10 @@ grpc_completion_queue* grpc_completion_queue_create_for_pluck(void* reserved) { GRPC_CQ_DEFAULT_POLLING}; return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr); } + +grpc_completion_queue* grpc_completion_queue_create( + const grpc_completion_queue_factory* factory, + const grpc_completion_queue_attributes* attr, void* reserved) { + GPR_ASSERT(!reserved); + return factory->vtable->create(factory, attr); +} diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index b46ecac18d..6163776152 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -41,14 +41,8 @@ #include <grpc/support/log.h> #include <grpc/support/time.h> #include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/channel/compress_filter.h" #include "src/core/lib/channel/connected_channel.h" -#include "src/core/lib/channel/deadline_filter.h" #include "src/core/lib/channel/handshaker_registry.h" -#include "src/core/lib/channel/http_client_filter.h" -#include "src/core/lib/channel/http_server_filter.h" -#include "src/core/lib/channel/max_age_filter.h" -#include "src/core/lib/channel/message_size_filter.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/combiner.h" @@ -96,60 +90,13 @@ static bool prepend_filter(grpc_exec_ctx *exec_ctx, builder, (const grpc_channel_filter *)arg, NULL, NULL); } -static bool maybe_add_http_filter(grpc_exec_ctx *exec_ctx, - grpc_channel_stack_builder *builder, - void *arg) { - grpc_transport *t = grpc_channel_stack_builder_get_transport(builder); - if (t && strstr(t->vtable->name, "http")) { - return grpc_channel_stack_builder_prepend_filter( - builder, (const grpc_channel_filter *)arg, NULL, NULL); - } - return true; -} - static void register_builtin_channel_init() { - grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - prepend_filter, (void *)&grpc_client_deadline_filter); - grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, prepend_filter, - (void *)&grpc_server_deadline_filter); - grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, prepend_filter, - (void *)&grpc_max_age_filter); - grpc_channel_init_register_stage( - GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - prepend_filter, (void *)&grpc_message_size_filter); - grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - prepend_filter, (void *)&grpc_message_size_filter); - grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, prepend_filter, - (void *)&grpc_message_size_filter); - grpc_channel_init_register_stage( - GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, prepend_filter, - (void *)&grpc_compress_filter); - grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - prepend_filter, (void *)&grpc_compress_filter); - grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, prepend_filter, - (void *)&grpc_compress_filter); - grpc_channel_init_register_stage( - GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_http_filter, (void *)&grpc_http_client_filter); grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, grpc_add_connected_filter, NULL); - grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_http_filter, (void *)&grpc_http_client_filter); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, grpc_add_connected_filter, NULL); - grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_http_filter, (void *)&grpc_http_server_filter); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, grpc_add_connected_filter, NULL); @@ -193,16 +140,13 @@ void grpc_init(void) { grpc_register_tracer("channel_stack_builder", &grpc_trace_channel_stack_builder); grpc_register_tracer("http1", &grpc_http1_trace); - grpc_register_tracer("compression", &grpc_compression_trace); grpc_register_tracer("queue_pluck", &grpc_cq_pluck_trace); grpc_register_tracer("combiner", &grpc_combiner_trace); grpc_register_tracer("server_channel", &grpc_server_channel_trace); grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace); // Default pluck trace to 1 - grpc_cq_pluck_trace = 1; grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace); // Default timeout trace to 1 - grpc_cq_event_timeout_trace = 1; grpc_register_tracer("op_failure", &grpc_trace_operation_failures); grpc_register_tracer("resource_quota", &grpc_resource_quota_trace); grpc_register_tracer("call_error", &grpc_call_error_trace); @@ -227,6 +171,7 @@ void grpc_init(void) { grpc_tracer_init("GRPC_TRACE"); /* no more changes to channel init pipelines */ grpc_channel_init_finalize(); + grpc_iomgr_start(); } gpr_mu_unlock(&g_init_mu); GRPC_API_TRACE("grpc_init(void)", 0, ()); diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c index 921ef87e36..746134676f 100644 --- a/src/core/lib/surface/init_secure.c +++ b/src/core/lib/surface/init_secure.c @@ -31,6 +31,8 @@ * */ +#include <grpc/support/port_platform.h> + #include "src/core/lib/surface/init.h" #include <limits.h> diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.cc index 0c408aa288..88f4eaac08 100644 --- a/src/core/lib/surface/lame_client.c +++ b/src/core/lib/surface/lame_client.cc @@ -31,39 +31,50 @@ * */ -#include "src/core/lib/surface/lame_client.h" - #include <grpc/grpc.h> #include <string.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> + +#include "src/core/lib/support/atomic.h" + +extern "C" { #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/lame_client.h" #include "src/core/lib/transport/static_metadata.h" +} -typedef struct { +namespace grpc_core { + +namespace { + +struct CallData { grpc_linked_mdelem status; grpc_linked_mdelem details; - gpr_atm filled_metadata; -} call_data; + grpc_core::atomic<bool> filled_metadata; +}; -typedef struct { +struct ChannelData { grpc_status_code error_code; const char *error_message; -} channel_data; +}; static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_metadata_batch *mdb) { - call_data *calld = elem->call_data; - if (!gpr_atm_no_barrier_cas(&calld->filled_metadata, 0, 1)) { + CallData *calld = static_cast<CallData *>(elem->call_data); + bool expected = false; + if (!calld->filled_metadata.compare_exchange_strong( + expected, true, grpc_core::memory_order_relaxed, + grpc_core::memory_order_relaxed)) { return; } - channel_data *chand = elem->channel_data; + ChannelData *chand = static_cast<ChannelData *>(elem->channel_data); char tmp[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(chand->error_code, tmp); calld->status.md = grpc_mdelem_from_slices( @@ -80,16 +91,17 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, mdb->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); } -static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - if (op->recv_initial_metadata != NULL) { - fill_metadata(exec_ctx, elem, op->recv_initial_metadata); - } else if (op->recv_trailing_metadata != NULL) { - fill_metadata(exec_ctx, elem, op->recv_trailing_metadata); +static void lame_start_transport_stream_op_batch( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_transport_stream_op_batch *op) { + if (op->recv_initial_metadata) { + fill_metadata(exec_ctx, elem, + op->payload->recv_initial_metadata.recv_initial_metadata); + } else if (op->recv_trailing_metadata) { + fill_metadata(exec_ctx, elem, + op->payload->recv_trailing_metadata.recv_trailing_metadata); } - grpc_transport_stream_op_finish_with_failure( + grpc_transport_stream_op_batch_finish_with_failure( exec_ctx, op, GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel")); } @@ -125,8 +137,6 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { - call_data *calld = elem->call_data; - gpr_atm_no_barrier_store(&calld->filled_metadata, 0); return GRPC_ERROR_NONE; } @@ -147,18 +157,22 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem) {} -const grpc_channel_filter grpc_lame_filter = { - lame_start_transport_stream_op, - lame_start_transport_op, - sizeof(call_data), - init_call_elem, +} // namespace + +} // namespace grpc_core + +extern "C" const grpc_channel_filter grpc_lame_filter = { + grpc_core::lame_start_transport_stream_op_batch, + grpc_core::lame_start_transport_op, + sizeof(grpc_core::CallData), + grpc_core::init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - lame_get_peer, - lame_get_channel_info, + grpc_core::destroy_call_elem, + sizeof(grpc_core::ChannelData), + grpc_core::init_channel_elem, + grpc_core::destroy_channel_elem, + grpc_core::lame_get_peer, + grpc_core::lame_get_channel_info, "lame-client", }; @@ -169,7 +183,6 @@ grpc_channel *grpc_lame_client_channel_create(const char *target, const char *error_message) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_channel_element *elem; - channel_data *chand; grpc_channel *channel = grpc_channel_create(&exec_ctx, target, NULL, GRPC_CLIENT_LAME_CHANNEL, NULL); elem = grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0); @@ -178,7 +191,7 @@ grpc_channel *grpc_lame_client_channel_create(const char *target, "error_message=%s)", 3, (target, (int)error_code, error_message)); GPR_ASSERT(elem->filter == &grpc_lame_filter); - chand = (channel_data *)elem->channel_data; + auto chand = static_cast<grpc_core::ChannelData *>(elem->channel_data); chand->error_code = error_code; chand->error_message = error_message; grpc_exec_ctx_finish(&exec_ctx); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index a123c9ca43..560229e892 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -44,6 +44,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/stack_lockfree.h" @@ -72,7 +73,7 @@ typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; -int grpc_server_channel_trace = 0; +grpc_tracer_flag grpc_server_channel_trace = GRPC_TRACER_INITIALIZER(false); typedef struct requested_call { requested_call_type type; @@ -154,8 +155,7 @@ struct call_data { grpc_completion_queue *cq_new; grpc_metadata_batch *recv_initial_metadata; - bool recv_idempotent_request; - bool recv_cacheable_request; + uint32_t recv_initial_metadata_flags; grpc_metadata_array initial_metadata; request_matcher *request_matcher; @@ -212,6 +212,11 @@ struct grpc_server { gpr_mu mu_global; /* mutex for server and channel state */ gpr_mu mu_call; /* mutex for call-specific state */ + /* startup synchronization: flag is protected by mu_global, signals whether + we are doing the listener start routine or not */ + bool starting; + gpr_cv starting_cv; + registered_method *registered_methods; /** one request matcher for unregistered methods */ request_matcher unregistered_request_matcher; @@ -340,7 +345,7 @@ static void request_matcher_destroy(request_matcher *rm) { static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, grpc_error *error) { - grpc_call_destroy(grpc_call_from_top_element(elem)); + grpc_call_unref(grpc_call_from_top_element(elem)); } static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, @@ -389,6 +394,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { grpc_channel_args_destroy(exec_ctx, server->channel_args); gpr_mu_destroy(&server->mu_global); gpr_mu_destroy(&server->mu_call); + gpr_cv_destroy(&server->starting_cv); while ((rm = server->registered_methods) != NULL) { server->registered_methods = rm->next; if (server->started) { @@ -402,7 +408,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { request_matcher_destroy(&server->unregistered_request_matcher); } for (i = 0; i < server->cq_count; i++) { - GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, server->cqs[i], "server"); if (server->started) { gpr_stack_lockfree_destroy(server->request_freelist_per_cq[i]); gpr_free(server->requested_calls_per_cq[i]); @@ -450,7 +456,7 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand, grpc_closure_init(&chand->finish_destroy_channel_closure, finish_destroy_channel, chand, grpc_schedule_on_exec_ctx); - if (grpc_server_channel_trace && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_server_channel_trace) && error != GRPC_ERROR_NONE) { const char *msg = grpc_error_string(error); gpr_log(GPR_INFO, "Disconnected client: %s", msg); } @@ -498,13 +504,7 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server, rc->data.batch.details->host = grpc_slice_ref_internal(calld->host); rc->data.batch.details->method = grpc_slice_ref_internal(calld->path); rc->data.batch.details->deadline = calld->deadline; - rc->data.batch.details->flags = - (calld->recv_idempotent_request - ? GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST - : 0) | - (calld->recv_cacheable_request - ? GRPC_INITIAL_METADATA_CACHEABLE_REQUEST - : 0); + rc->data.batch.details->flags = calld->recv_initial_metadata_flags; break; case REGISTERED_CALL: *rc->data.registered.deadline = calld->deadline; @@ -632,7 +632,8 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { if (!grpc_slice_eq(rm->host, calld->host)) continue; if (!grpc_slice_eq(rm->method, calld->path)) continue; if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) && - !calld->recv_idempotent_request) { + 0 == (calld->recv_initial_metadata_flags & + GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST)) { continue; } finish_start_new_rpc(exec_ctx, server, elem, @@ -649,7 +650,8 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { if (rm->has_host) continue; if (!grpc_slice_eq(rm->method, calld->path)) continue; if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) && - !calld->recv_idempotent_request) { + 0 == (calld->recv_initial_metadata_flags & + GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST)) { continue; } finish_start_new_rpc(exec_ctx, server, elem, @@ -781,22 +783,25 @@ static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr, } static void server_mutate_op(grpc_call_element *elem, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { call_data *calld = elem->call_data; - if (op->recv_initial_metadata != NULL) { - GPR_ASSERT(op->recv_idempotent_request == NULL); - calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready; - op->recv_initial_metadata_ready = &calld->server_on_recv_initial_metadata; - op->recv_idempotent_request = &calld->recv_idempotent_request; - op->recv_cacheable_request = &calld->recv_cacheable_request; + if (op->recv_initial_metadata) { + GPR_ASSERT(op->payload->recv_initial_metadata.recv_flags == NULL); + calld->recv_initial_metadata = + op->payload->recv_initial_metadata.recv_initial_metadata; + calld->on_done_recv_initial_metadata = + op->payload->recv_initial_metadata.recv_initial_metadata_ready; + op->payload->recv_initial_metadata.recv_initial_metadata_ready = + &calld->server_on_recv_initial_metadata; + op->payload->recv_initial_metadata.recv_flags = + &calld->recv_initial_metadata_flags; } } -static void server_start_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { +static void server_start_transport_stream_op_batch( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_transport_stream_op_batch *op) { GRPC_CALL_LOG_OP(GPR_INFO, elem, op); server_mutate_op(elem, op); grpc_call_next_op(exec_ctx, elem, op); @@ -960,7 +965,7 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } const grpc_channel_filter grpc_server_top_filter = { - server_start_transport_stream_op, + server_start_transport_stream_op_batch, grpc_channel_next_op, sizeof(call_data), init_call_elem, @@ -976,7 +981,7 @@ const grpc_channel_filter grpc_server_top_filter = { static void register_completion_queue(grpc_server *server, grpc_completion_queue *cq, - bool is_non_listening, void *reserved) { + void *reserved) { size_t i, n; GPR_ASSERT(!reserved); for (i = 0; i < server->cq_count; i++) { @@ -985,10 +990,6 @@ static void register_completion_queue(grpc_server *server, grpc_cq_mark_server_cq(cq); - if (is_non_listening) { - grpc_cq_mark_non_listening_server_cq(cq); - } - GRPC_CQ_INTERNAL_REF(cq, "server"); n = server->cq_count++; server->cqs = gpr_realloc(server->cqs, @@ -1002,16 +1003,16 @@ void grpc_server_register_completion_queue(grpc_server *server, GRPC_API_TRACE( "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3, (server, cq, reserved)); - register_completion_queue(server, cq, false, reserved); -} -void grpc_server_register_non_listening_completion_queue( - grpc_server *server, grpc_completion_queue *cq, void *reserved) { - GRPC_API_TRACE( - "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, " - "reserved=%p)", - 3, (server, cq, reserved)); - register_completion_queue(server, cq, true, reserved); + if (grpc_get_cq_completion_type(cq) != GRPC_CQ_NEXT) { + gpr_log(GPR_INFO, + "Completion queue which is not of type GRPC_CQ_NEXT is being " + "registered as a server-completion-queue"); + /* Ideally we should log an error and abort but ruby-wrapped-language API + calls grpc_completion_queue_pluck() on server completion queues */ + } + + register_completion_queue(server, cq, reserved); } grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) { @@ -1019,10 +1020,9 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) { grpc_server *server = gpr_zalloc(sizeof(grpc_server)); - GPR_ASSERT(grpc_is_initialized() && "call grpc_init()"); - gpr_mu_init(&server->mu_global); gpr_mu_init(&server->mu_call); + gpr_cv_init(&server->starting_cv); /* decremented by grpc_server_destroy */ gpr_ref_init(&server->internal_refcount, 1); @@ -1079,8 +1079,22 @@ void *grpc_server_register_method( return m; } +static void start_listeners(grpc_exec_ctx *exec_ctx, void *s, + grpc_error *error) { + grpc_server *server = s; + for (listener *l = server->listeners; l; l = l->next) { + l->start(exec_ctx, server, l->arg, server->pollsets, server->pollset_count); + } + + gpr_mu_lock(&server->mu_global); + server->starting = false; + gpr_cv_signal(&server->starting_cv); + gpr_mu_unlock(&server->mu_global); + + server_unref(exec_ctx, server); +} + void grpc_server_start(grpc_server *server) { - listener *l; size_t i; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -1094,7 +1108,7 @@ void grpc_server_start(grpc_server *server) { server->requested_calls_per_cq = gpr_malloc(sizeof(*server->requested_calls_per_cq) * server->cq_count); for (i = 0; i < server->cq_count; i++) { - if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) { + if (grpc_cq_can_listen(server->cqs[i])) { server->pollsets[server->pollset_count++] = grpc_cq_pollset(server->cqs[i]); } @@ -1114,10 +1128,11 @@ void grpc_server_start(grpc_server *server) { (size_t)server->max_requested_calls_per_cq, server); } - for (l = server->listeners; l; l = l->next) { - l->start(&exec_ctx, server, l->arg, server->pollsets, - server->pollset_count); - } + server_ref(server); + server->starting = true; + grpc_closure_sched(&exec_ctx, grpc_closure_create(start_listeners, server, + grpc_executor_scheduler), + GRPC_ERROR_NONE); grpc_exec_ctx_finish(&exec_ctx); } @@ -1251,8 +1266,14 @@ void grpc_server_shutdown_and_notify(grpc_server *server, GRPC_API_TRACE("grpc_server_shutdown_and_notify(server=%p, cq=%p, tag=%p)", 3, (server, cq, tag)); - /* lock, and gather up some stuff to do */ + /* wait for startup to be finished: locks mu_global */ gpr_mu_lock(&server->mu_global); + while (server->starting) { + gpr_cv_wait(&server->starting_cv, &server->mu_global, + gpr_inf_future(GPR_CLOCK_REALTIME)); + } + + /* stay locked, and gather up some stuff to do */ grpc_cq_begin_op(cq, tag); if (server->shutdown_published) { grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown, diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index a85d9f4964..cd2fca0fe0 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -36,12 +36,13 @@ #include <grpc/grpc.h> #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/transport.h" extern const grpc_channel_filter grpc_server_top_filter; /** Lightweight tracing of server channel state */ -extern int grpc_server_channel_trace; +extern grpc_tracer_flag grpc_server_channel_trace; /* Add a listener to the server: when the server starts, it will call start, and when it shuts down, it will call destroy */ diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index ba80bd801e..cddc595e4c 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,6 +36,6 @@ #include <grpc/grpc.h> -const char *grpc_version_string(void) { return "3.0.0-dev"; } +const char *grpc_version_string(void) { return "4.0.0-dev"; } -const char *grpc_g_stands_for(void) { return "gentle"; } +const char *grpc_g_stands_for(void) { return "gregarious"; } diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c index e1483677fd..da8019d98b 100644 --- a/src/core/lib/transport/bdp_estimator.c +++ b/src/core/lib/transport/bdp_estimator.c @@ -38,12 +38,13 @@ #include <grpc/support/log.h> #include <grpc/support/useful.h> -int grpc_bdp_estimator_trace = 0; +grpc_tracer_flag grpc_bdp_estimator_trace = GRPC_TRACER_INITIALIZER(false); void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { estimator->estimate = 65536; estimator->ping_state = GRPC_BDP_PING_UNSCHEDULED; estimator->name = name; + estimator->bw_est = 0; } bool grpc_bdp_estimator_get_estimate(grpc_bdp_estimator *estimator, @@ -67,7 +68,7 @@ bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator, } void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } @@ -77,24 +78,34 @@ void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { } void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_SCHEDULED); estimator->ping_state = GRPC_BDP_PING_STARTED; estimator->accumulator = 0; + estimator->ping_start_time = gpr_now(GPR_CLOCK_MONOTONIC); } void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { - gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64, - estimator->name, estimator->accumulator, estimator->estimate); + gpr_timespec dt_ts = + gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), estimator->ping_start_time); + double dt = (double)dt_ts.tv_sec + 1e-9 * (double)dt_ts.tv_nsec; + double bw = dt > 0 ? ((double)estimator->accumulator / dt) : 0; + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { + gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64 + " dt=%lf bw=%lfMbs bw_est=%lfMbs", + estimator->name, estimator->accumulator, estimator->estimate, dt, + bw / 125000.0, estimator->bw_est / 125000.0); } GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_STARTED); - if (estimator->accumulator > 2 * estimator->estimate / 3) { - estimator->estimate *= 2; - if (grpc_bdp_estimator_trace) { + if (estimator->accumulator > 2 * estimator->estimate / 3 && + bw > estimator->bw_est) { + estimator->estimate = + GPR_MAX(estimator->accumulator, estimator->estimate * 2); + estimator->bw_est = bw; + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]: estimate increased to %" PRId64, estimator->name, estimator->estimate); } diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h index df8d1f6fc0..135376edd7 100644 --- a/src/core/lib/transport/bdp_estimator.h +++ b/src/core/lib/transport/bdp_estimator.h @@ -34,13 +34,15 @@ #ifndef GRPC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H #define GRPC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H +#include <grpc/support/time.h> #include <stdbool.h> #include <stdint.h> +#include "src/core/lib/debug/trace.h" #define GRPC_BDP_SAMPLES 16 #define GRPC_BDP_MIN_SAMPLES_FOR_ESTIMATE 3 -extern int grpc_bdp_estimator_trace; +extern grpc_tracer_flag grpc_bdp_estimator_trace; typedef enum { GRPC_BDP_PING_UNSCHEDULED, @@ -52,6 +54,8 @@ typedef struct grpc_bdp_estimator { grpc_bdp_estimator_ping_state ping_state; int64_t accumulator; int64_t estimate; + gpr_timespec ping_start_time; + double bw_est; const char *name; } grpc_bdp_estimator; diff --git a/src/core/lib/transport/byte_stream.c b/src/core/lib/transport/byte_stream.c index 4d4206189e..5800c70ef4 100644 --- a/src/core/lib/transport/byte_stream.c +++ b/src/core/lib/transport/byte_stream.c @@ -40,10 +40,15 @@ #include "src/core/lib/slice/slice_internal.h" int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, grpc_slice *slice, - size_t max_size_hint, grpc_closure *on_complete) { - return byte_stream->next(exec_ctx, byte_stream, slice, max_size_hint, - on_complete); + grpc_byte_stream *byte_stream, size_t max_size_hint, + grpc_closure *on_complete) { + return byte_stream->next(exec_ctx, byte_stream, max_size_hint, on_complete); +} + +grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice) { + return byte_stream->pull(exec_ctx, byte_stream, slice); } void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -53,16 +58,24 @@ void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, /* slice_buffer_stream */ -static int slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, - grpc_slice *slice, size_t max_size_hint, - grpc_closure *on_complete) { +static bool slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + size_t max_size_hint, + grpc_closure *on_complete) { + grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; + GPR_ASSERT(stream->cursor < stream->backing_buffer->count); + return true; +} + +static grpc_error *slice_buffer_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice) { grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; GPR_ASSERT(stream->cursor < stream->backing_buffer->count); *slice = grpc_slice_ref_internal(stream->backing_buffer->slices[stream->cursor]); stream->cursor++; - return 1; + return GRPC_ERROR_NONE; } static void slice_buffer_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -75,6 +88,7 @@ void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, stream->base.length = (uint32_t)slice_buffer->length; stream->base.flags = flags; stream->base.next = slice_buffer_stream_next; + stream->base.pull = slice_buffer_stream_pull; stream->base.destroy = slice_buffer_stream_destroy; stream->backing_buffer = slice_buffer; stream->cursor = 0; diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index 1fdd5b4d77..381c65fb04 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -49,9 +49,10 @@ typedef struct grpc_byte_stream grpc_byte_stream; struct grpc_byte_stream { uint32_t length; uint32_t flags; - int (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, - grpc_slice *slice, size_t max_size_hint, - grpc_closure *on_complete); + bool (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, + size_t max_size_hint, grpc_closure *on_complete); + grpc_error *(*pull)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, + grpc_slice *slice); void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream); }; @@ -61,12 +62,20 @@ struct grpc_byte_stream { * * max_size_hint can be set as a hint as to the maximum number * of bytes that would be acceptable to read. + */ +int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, size_t max_size_hint, + grpc_closure *on_complete); + +/* returns the next slice in the byte stream when it is ready (indicated by + * either grpc_byte_stream_next returning 1 or on_complete passed to + * grpc_byte_stream_next is called). * * once a slice is returned into *slice, it is owned by the caller. */ -int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, grpc_slice *slice, - size_t max_size_hint, grpc_closure *on_complete); +grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice); void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream); diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 3757b25267..e30cd523fa 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -39,7 +39,7 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -int grpc_connectivity_state_trace = 0; +grpc_tracer_flag grpc_connectivity_state_trace = GRPC_TRACER_INITIALIZER(false); const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { @@ -94,7 +94,7 @@ grpc_connectivity_state grpc_connectivity_state_check( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -106,7 +106,7 @@ grpc_connectivity_state grpc_connectivity_state_get( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -127,7 +127,7 @@ bool grpc_connectivity_state_notify_on_state_change( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { if (current == NULL) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker, tracker->name, notify); @@ -180,7 +180,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); grpc_connectivity_state_watcher *w; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { const char *error_string = grpc_error_string(error); gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker, tracker->name, grpc_connectivity_state_name(cur), @@ -208,7 +208,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, while ((w = tracker->watchers) != NULL) { *w->current = state; tracker->watchers = w->next; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name, w->notify); } diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h index c9604c34dd..cdc2930c11 100644 --- a/src/core/lib/transport/connectivity_state.h +++ b/src/core/lib/transport/connectivity_state.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H #include <grpc/grpc.h> +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" typedef struct grpc_connectivity_state_watcher { @@ -57,7 +58,7 @@ typedef struct { char *name; } grpc_connectivity_state_tracker; -extern int grpc_connectivity_state_trace; +extern grpc_tracer_flag grpc_connectivity_state_trace; /** enum --> string conversion */ const char *grpc_connectivity_state_name(grpc_connectivity_state state); diff --git a/src/core/lib/transport/pid_controller.c b/src/core/lib/transport/pid_controller.c index 19cb1c0b36..c851564201 100644 --- a/src/core/lib/transport/pid_controller.c +++ b/src/core/lib/transport/pid_controller.c @@ -49,6 +49,7 @@ void grpc_pid_controller_reset(grpc_pid_controller *pid_controller) { double grpc_pid_controller_update(grpc_pid_controller *pid_controller, double error, double dt) { + if (dt == 0) return pid_controller->last_control_value; /* integrate error using the trapezoid rule */ pid_controller->error_integral += dt * (pid_controller->last_error + error) * 0.5; diff --git a/src/core/lib/transport/service_config.c b/src/core/lib/transport/service_config.c index 1195f75044..6aecb7fa93 100644 --- a/src/core/lib/transport/service_config.c +++ b/src/core/lib/transport/service_config.c @@ -162,7 +162,6 @@ static char* parse_json_method_name(grpc_json* json) { static bool parse_json_method_config( grpc_exec_ctx* exec_ctx, grpc_json* json, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable, grpc_slice_hash_table_entry* entries, size_t* idx) { // Construct value. void* method_config = create_value(json); @@ -185,13 +184,11 @@ static bool parse_json_method_config( // Add entry for each path. for (size_t i = 0; i < paths.count; ++i) { entries[*idx].key = grpc_slice_from_copied_string(paths.strs[i]); - entries[*idx].value = vtable->copy_value(method_config); - entries[*idx].vtable = vtable; + entries[*idx].value = method_config; ++*idx; } success = true; done: - vtable->destroy_value(exec_ctx, method_config); gpr_strvec_destroy(&paths); return success; } @@ -199,7 +196,7 @@ done: grpc_slice_hash_table* grpc_service_config_create_method_config_table( grpc_exec_ctx* exec_ctx, const grpc_service_config* service_config, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable) { + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)) { const grpc_json* json = service_config->json_tree; // Traverse parsed JSON tree. if (json->type != GRPC_JSON_OBJECT || json->key != NULL) return NULL; @@ -220,8 +217,8 @@ grpc_slice_hash_table* grpc_service_config_create_method_config_table( size_t idx = 0; for (grpc_json* method = field->child; method != NULL; method = method->next) { - if (!parse_json_method_config(exec_ctx, method, create_value, vtable, - entries, &idx)) { + if (!parse_json_method_config(exec_ctx, method, create_value, entries, + &idx)) { return NULL; } } @@ -231,12 +228,8 @@ grpc_slice_hash_table* grpc_service_config_create_method_config_table( // Instantiate method config table. grpc_slice_hash_table* method_config_table = NULL; if (entries != NULL) { - method_config_table = grpc_slice_hash_table_create(num_entries, entries); - // Clean up. - for (size_t i = 0; i < num_entries; ++i) { - grpc_slice_unref_internal(exec_ctx, entries[i].key); - vtable->destroy_value(exec_ctx, entries[i].value); - } + method_config_table = + grpc_slice_hash_table_create(num_entries, entries, destroy_value); gpr_free(entries); } return method_config_table; diff --git a/src/core/lib/transport/service_config.h b/src/core/lib/transport/service_config.h index ebfc59b534..e0548b9c3f 100644 --- a/src/core/lib/transport/service_config.h +++ b/src/core/lib/transport/service_config.h @@ -57,12 +57,12 @@ const char* grpc_service_config_get_lb_policy_name( /// Creates a method config table based on the data in \a json. /// The table's keys are request paths. The table's value type is /// returned by \a create_value(), based on data parsed from the JSON tree. -/// \a vtable provides methods used to manage the values. +/// \a destroy_value is used to clean up values. /// Returns NULL on error. grpc_slice_hash_table* grpc_service_config_create_method_config_table( grpc_exec_ctx* exec_ctx, const grpc_service_config* service_config, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable); + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)); /// A helper function for looking up values in the table returned by /// \a grpc_service_config_create_method_config_table(). diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index c13ba230b3..862cdaa8e0 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -51,67 +51,69 @@ static uint8_t g_bytes[] = { 115, 103, 114, 112, 99, 45, 112, 97, 121, 108, 111, 97, 100, 45, 98, 105, 110, 103, 114, 112, 99, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 114, 112, 99, 45, 97, 99, 99, 101, 112, 116, 45, 101, 110, 99, - 111, 100, 105, 110, 103, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, - 112, 101, 103, 114, 112, 99, 45, 105, 110, 116, 101, 114, 110, 97, 108, - 45, 101, 110, 99, 111, 100, 105, 110, 103, 45, 114, 101, 113, 117, 101, - 115, 116, 117, 115, 101, 114, 45, 97, 103, 101, 110, 116, 104, 111, 115, - 116, 108, 98, 45, 116, 111, 107, 101, 110, 103, 114, 112, 99, 45, 116, - 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 45, 116, 114, 97, 99, - 105, 110, 103, 45, 98, 105, 110, 103, 114, 112, 99, 45, 115, 116, 97, - 116, 115, 45, 98, 105, 110, 103, 114, 112, 99, 46, 119, 97, 105, 116, - 95, 102, 111, 114, 95, 114, 101, 97, 100, 121, 103, 114, 112, 99, 46, - 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, 109, 97, 120, - 95, 114, 101, 113, 117, 101, 115, 116, 95, 109, 101, 115, 115, 97, 103, - 101, 95, 98, 121, 116, 101, 115, 103, 114, 112, 99, 46, 109, 97, 120, - 95, 114, 101, 115, 112, 111, 110, 115, 101, 95, 109, 101, 115, 115, 97, - 103, 101, 95, 98, 121, 116, 101, 115, 47, 103, 114, 112, 99, 46, 108, - 98, 46, 118, 49, 46, 76, 111, 97, 100, 66, 97, 108, 97, 110, 99, - 101, 114, 47, 66, 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 48, - 49, 50, 105, 100, 101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, - 101, 102, 108, 97, 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 97, - 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 103, 114, 112, 99, - 80, 79, 83, 84, 50, 48, 48, 52, 48, 52, 104, 116, 116, 112, 104, - 116, 116, 112, 115, 103, 114, 112, 99, 71, 69, 84, 80, 85, 84, 47, - 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, 108, 50, 48, 52, 50, - 48, 54, 51, 48, 52, 52, 48, 48, 53, 48, 48, 97, 99, 99, 101, - 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 97, 99, 99, 101, 112, - 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 122, 105, 112, 44, - 32, 100, 101, 102, 108, 97, 116, 101, 97, 99, 99, 101, 112, 116, 45, - 108, 97, 110, 103, 117, 97, 103, 101, 97, 99, 99, 101, 112, 116, 45, - 114, 97, 110, 103, 101, 115, 97, 99, 99, 101, 112, 116, 97, 99, 99, - 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, - 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, - 111, 119, 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, - 99, 97, 99, 104, 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, - 110, 116, 101, 110, 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, - 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, - 105, 110, 103, 99, 111, 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, - 117, 97, 103, 101, 99, 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, - 103, 116, 104, 99, 111, 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, - 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, - 103, 101, 99, 111, 111, 107, 105, 101, 100, 97, 116, 101, 101, 116, 97, - 103, 101, 120, 112, 101, 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, - 114, 111, 109, 105, 102, 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, - 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, - 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, - 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, - 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, - 100, 105, 102, 105, 101, 100, 108, 105, 110, 107, 108, 111, 99, 97, 116, - 105, 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, - 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, - 97, 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, - 105, 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, - 101, 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, - 121, 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, - 116, 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, - 116, 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, - 105, 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, - 111, 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, - 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, - 101, 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, - 100, 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, - 108, 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, - 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; + 111, 100, 105, 110, 103, 103, 114, 112, 99, 45, 115, 101, 114, 118, 101, + 114, 45, 115, 116, 97, 116, 115, 45, 98, 105, 110, 103, 114, 112, 99, + 45, 116, 97, 103, 115, 45, 98, 105, 110, 103, 114, 112, 99, 45, 116, + 114, 97, 99, 101, 45, 98, 105, 110, 99, 111, 110, 116, 101, 110, 116, + 45, 116, 121, 112, 101, 103, 114, 112, 99, 45, 105, 110, 116, 101, 114, + 110, 97, 108, 45, 101, 110, 99, 111, 100, 105, 110, 103, 45, 114, 101, + 113, 117, 101, 115, 116, 117, 115, 101, 114, 45, 97, 103, 101, 110, 116, + 104, 111, 115, 116, 108, 98, 45, 116, 111, 107, 101, 110, 103, 114, 112, + 99, 45, 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, 119, + 97, 105, 116, 95, 102, 111, 114, 95, 114, 101, 97, 100, 121, 103, 114, + 112, 99, 46, 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, + 109, 97, 120, 95, 114, 101, 113, 117, 101, 115, 116, 95, 109, 101, 115, + 115, 97, 103, 101, 95, 98, 121, 116, 101, 115, 103, 114, 112, 99, 46, + 109, 97, 120, 95, 114, 101, 115, 112, 111, 110, 115, 101, 95, 109, 101, + 115, 115, 97, 103, 101, 95, 98, 121, 116, 101, 115, 47, 103, 114, 112, + 99, 46, 108, 98, 46, 118, 49, 46, 76, 111, 97, 100, 66, 97, 108, + 97, 110, 99, 101, 114, 47, 66, 97, 108, 97, 110, 99, 101, 76, 111, + 97, 100, 48, 49, 50, 105, 100, 101, 110, 116, 105, 116, 121, 103, 122, + 105, 112, 100, 101, 102, 108, 97, 116, 101, 116, 114, 97, 105, 108, 101, + 114, 115, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 103, + 114, 112, 99, 80, 79, 83, 84, 50, 48, 48, 52, 48, 52, 104, 116, + 116, 112, 104, 116, 116, 112, 115, 103, 114, 112, 99, 71, 69, 84, 80, + 85, 84, 47, 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, 108, 50, + 48, 52, 50, 48, 54, 51, 48, 52, 52, 48, 48, 53, 48, 48, 97, + 99, 99, 101, 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 97, 99, + 99, 101, 112, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 122, + 105, 112, 44, 32, 100, 101, 102, 108, 97, 116, 101, 97, 99, 99, 101, + 112, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 97, 99, 99, 101, + 112, 116, 45, 114, 97, 110, 103, 101, 115, 97, 99, 99, 101, 112, 116, + 97, 99, 99, 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, 45, + 97, 108, 108, 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, 101, + 97, 108, 108, 111, 119, 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, + 105, 111, 110, 99, 97, 99, 104, 101, 45, 99, 111, 110, 116, 114, 111, + 108, 99, 111, 110, 116, 101, 110, 116, 45, 100, 105, 115, 112, 111, 115, + 105, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 101, 110, + 99, 111, 100, 105, 110, 103, 99, 111, 110, 116, 101, 110, 116, 45, 108, + 97, 110, 103, 117, 97, 103, 101, 99, 111, 110, 116, 101, 110, 116, 45, + 108, 101, 110, 103, 116, 104, 99, 111, 110, 116, 101, 110, 116, 45, 108, + 111, 99, 97, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, + 114, 97, 110, 103, 101, 99, 111, 111, 107, 105, 101, 100, 97, 116, 101, + 101, 116, 97, 103, 101, 120, 112, 101, 99, 116, 101, 120, 112, 105, 114, + 101, 115, 102, 114, 111, 109, 105, 102, 45, 109, 97, 116, 99, 104, 105, + 102, 45, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, + 101, 105, 102, 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, + 102, 45, 114, 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, + 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, + 45, 109, 111, 100, 105, 102, 105, 101, 100, 108, 98, 45, 99, 111, 115, + 116, 45, 98, 105, 110, 108, 105, 110, 107, 108, 111, 99, 97, 116, 105, + 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, 112, + 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, + 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, 105, + 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, 101, + 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121, + 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116, + 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, 116, + 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, 105, + 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, + 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, 45, + 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, 101, + 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, + 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, 108, + 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, + 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; static void static_ref(void *unused) {} static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} @@ -220,6 +222,8 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { @@ -246,196 +250,201 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[11], - .data.refcounted = {g_bytes + 110, 12}}, + .data.refcounted = {g_bytes + 110, 21}}, {.refcount = &grpc_static_metadata_refcounts[12], - .data.refcounted = {g_bytes + 122, 30}}, + .data.refcounted = {g_bytes + 131, 13}}, {.refcount = &grpc_static_metadata_refcounts[13], - .data.refcounted = {g_bytes + 152, 10}}, + .data.refcounted = {g_bytes + 144, 14}}, {.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 162, 4}}, + .data.refcounted = {g_bytes + 158, 12}}, {.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 166, 8}}, + .data.refcounted = {g_bytes + 170, 30}}, {.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 174, 12}}, + .data.refcounted = {g_bytes + 200, 10}}, {.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 186, 16}}, + .data.refcounted = {g_bytes + 210, 4}}, {.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 202, 14}}, + .data.refcounted = {g_bytes + 214, 8}}, {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}, + .data.refcounted = {g_bytes + 222, 12}}, {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 216, 19}}, + .data.refcounted = {g_bytes + 234, 0}}, {.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 235, 12}}, + .data.refcounted = {g_bytes + 234, 19}}, {.refcount = &grpc_static_metadata_refcounts[22], - .data.refcounted = {g_bytes + 247, 30}}, + .data.refcounted = {g_bytes + 253, 12}}, {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 277, 31}}, + .data.refcounted = {g_bytes + 265, 30}}, {.refcount = &grpc_static_metadata_refcounts[24], - .data.refcounted = {g_bytes + 308, 36}}, + .data.refcounted = {g_bytes + 295, 31}}, {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 344, 1}}, + .data.refcounted = {g_bytes + 326, 36}}, {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 345, 1}}, + .data.refcounted = {g_bytes + 362, 1}}, {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 346, 1}}, + .data.refcounted = {g_bytes + 363, 1}}, {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 347, 8}}, + .data.refcounted = {g_bytes + 364, 1}}, {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 355, 4}}, + .data.refcounted = {g_bytes + 365, 8}}, {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 359, 7}}, + .data.refcounted = {g_bytes + 373, 4}}, {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 366, 8}}, + .data.refcounted = {g_bytes + 377, 7}}, {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 374, 16}}, + .data.refcounted = {g_bytes + 384, 8}}, {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 390, 4}}, + .data.refcounted = {g_bytes + 392, 16}}, {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 394, 3}}, + .data.refcounted = {g_bytes + 408, 4}}, {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 397, 3}}, + .data.refcounted = {g_bytes + 412, 3}}, {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 400, 4}}, + .data.refcounted = {g_bytes + 415, 3}}, {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 404, 5}}, + .data.refcounted = {g_bytes + 418, 4}}, {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 409, 4}}, + .data.refcounted = {g_bytes + 422, 5}}, {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 413, 3}}, + .data.refcounted = {g_bytes + 427, 4}}, {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 416, 3}}, + .data.refcounted = {g_bytes + 431, 3}}, {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 419, 1}}, + .data.refcounted = {g_bytes + 434, 3}}, {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 420, 11}}, + .data.refcounted = {g_bytes + 437, 1}}, {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 431, 3}}, + .data.refcounted = {g_bytes + 438, 11}}, {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 434, 3}}, + .data.refcounted = {g_bytes + 449, 3}}, {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 437, 3}}, + .data.refcounted = {g_bytes + 452, 3}}, {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 440, 3}}, + .data.refcounted = {g_bytes + 455, 3}}, {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 443, 3}}, + .data.refcounted = {g_bytes + 458, 3}}, {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 446, 14}}, + .data.refcounted = {g_bytes + 461, 3}}, {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 460, 15}}, + .data.refcounted = {g_bytes + 464, 14}}, {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 475, 13}}, + .data.refcounted = {g_bytes + 478, 15}}, {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 488, 15}}, + .data.refcounted = {g_bytes + 493, 13}}, {.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 503, 13}}, + .data.refcounted = {g_bytes + 506, 15}}, {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 516, 6}}, + .data.refcounted = {g_bytes + 521, 13}}, {.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 522, 27}}, + .data.refcounted = {g_bytes + 534, 6}}, {.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 549, 3}}, + .data.refcounted = {g_bytes + 540, 27}}, {.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 552, 5}}, + .data.refcounted = {g_bytes + 567, 3}}, {.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 557, 13}}, + .data.refcounted = {g_bytes + 570, 5}}, {.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 570, 13}}, + .data.refcounted = {g_bytes + 575, 13}}, {.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 583, 19}}, + .data.refcounted = {g_bytes + 588, 13}}, {.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 602, 16}}, + .data.refcounted = {g_bytes + 601, 19}}, {.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 618, 16}}, + .data.refcounted = {g_bytes + 620, 16}}, {.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 634, 14}}, + .data.refcounted = {g_bytes + 636, 16}}, {.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 648, 16}}, + .data.refcounted = {g_bytes + 652, 14}}, {.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 664, 13}}, + .data.refcounted = {g_bytes + 666, 16}}, {.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 677, 6}}, + .data.refcounted = {g_bytes + 682, 13}}, {.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 683, 4}}, + .data.refcounted = {g_bytes + 695, 6}}, {.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 687, 4}}, + .data.refcounted = {g_bytes + 701, 4}}, {.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 691, 6}}, + .data.refcounted = {g_bytes + 705, 4}}, {.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 697, 7}}, + .data.refcounted = {g_bytes + 709, 6}}, {.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 704, 4}}, + .data.refcounted = {g_bytes + 715, 7}}, {.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 708, 8}}, + .data.refcounted = {g_bytes + 722, 4}}, {.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 716, 17}}, + .data.refcounted = {g_bytes + 726, 8}}, {.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 733, 13}}, + .data.refcounted = {g_bytes + 734, 17}}, {.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 746, 8}}, + .data.refcounted = {g_bytes + 751, 13}}, {.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 754, 19}}, + .data.refcounted = {g_bytes + 764, 8}}, {.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 773, 13}}, + .data.refcounted = {g_bytes + 772, 19}}, {.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 786, 4}}, + .data.refcounted = {g_bytes + 791, 13}}, {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 790, 8}}, + .data.refcounted = {g_bytes + 804, 11}}, {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 798, 12}}, + .data.refcounted = {g_bytes + 815, 4}}, {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 810, 18}}, + .data.refcounted = {g_bytes + 819, 8}}, {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 828, 19}}, + .data.refcounted = {g_bytes + 827, 12}}, {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 847, 5}}, + .data.refcounted = {g_bytes + 839, 18}}, {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 852, 7}}, + .data.refcounted = {g_bytes + 857, 19}}, {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 859, 7}}, + .data.refcounted = {g_bytes + 876, 5}}, {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 866, 11}}, + .data.refcounted = {g_bytes + 881, 7}}, {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 877, 6}}, + .data.refcounted = {g_bytes + 888, 7}}, {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 883, 10}}, + .data.refcounted = {g_bytes + 895, 11}}, {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 893, 25}}, + .data.refcounted = {g_bytes + 906, 6}}, {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 918, 17}}, + .data.refcounted = {g_bytes + 912, 10}}, {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 935, 4}}, + .data.refcounted = {g_bytes + 922, 25}}, {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 939, 3}}, + .data.refcounted = {g_bytes + 947, 17}}, {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 942, 16}}, + .data.refcounted = {g_bytes + 964, 4}}, {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 958, 16}}, + .data.refcounted = {g_bytes + 968, 3}}, {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 974, 13}}, + .data.refcounted = {g_bytes + 971, 16}}, {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 987, 12}}, + .data.refcounted = {g_bytes + 987, 16}}, {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 999, 21}}, + .data.refcounted = {g_bytes + 1003, 13}}, + {.refcount = &grpc_static_metadata_refcounts[97], + .data.refcounted = {g_bytes + 1016, 12}}, + {.refcount = &grpc_static_metadata_refcounts[98], + .data.refcounted = {g_bytes + 1028, 21}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; static const int8_t elems_r[] = { - 10, 8, -3, 0, 9, 21, -75, 22, 0, 10, -7, 20, 0, 19, 18, 17, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - -48, -49, 16, -51, -52, -53, -54, -54, -55, -56, -57, 0, 37, 36, 35, 34, - 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, - 17, 16, 15, 14, 13, 12, 11, 14, 13, 12, 11, 10, 9, 8, 0}; + 10, 8, -3, 0, 9, 21, -77, 22, 0, 10, -7, 0, 0, 0, + 14, 0, 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, -50, -51, 16, -53, -54, -55, -56, + -56, -57, -58, -59, 0, 37, 36, 35, 34, 33, 32, 31, 30, 29, + 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, + 14, 13, 12, 11, 10, 13, 12, 11, 10, 9, 8, 7, 0}; static uint32_t elems_phash(uint32_t i) { - i -= 41; - uint32_t x = i % 95; - uint32_t y = i / 95; + i -= 42; + uint32_t x = i % 97; + uint32_t y = i / 97; uint32_t h = x; if (y < GPR_ARRAY_SIZE(elems_r)) { uint32_t delta = (uint32_t)elems_r[y]; @@ -445,29 +454,30 @@ static uint32_t elems_phash(uint32_t i) { } static const uint16_t elem_keys[] = { - 998, 999, 1000, 237, 238, 239, 240, 241, 136, 137, 41, 42, - 424, 425, 426, 901, 902, 903, 704, 705, 1086, 516, 706, 1280, - 1377, 1474, 4675, 4772, 4803, 4966, 5063, 5160, 5257, 1099, 5354, 5451, - 5548, 5645, 5742, 5839, 5936, 6033, 6130, 6227, 6324, 6421, 6518, 6615, - 6712, 6809, 6906, 7003, 7100, 7197, 7294, 7391, 7488, 7585, 7682, 7779, - 7876, 7973, 8070, 8167, 8264, 1063, 1064, 1065, 1066, 8361, 8458, 8555, - 8652, 8749, 8846, 8943, 310, 0, 0, 0, 0, 0, 0, 0, + 1019, 1020, 1021, 242, 243, 244, 245, 246, 139, 140, 42, 43, + 433, 434, 435, 920, 921, 922, 719, 720, 1406, 527, 721, 1604, + 1703, 1802, 4871, 4970, 5001, 5168, 5267, 5366, 5465, 1419, 5564, 5663, + 5762, 5861, 5960, 6059, 6158, 6257, 6356, 6455, 6554, 6653, 6752, 6851, + 6950, 7049, 7148, 7247, 7346, 7445, 7544, 7643, 7742, 7841, 7940, 8039, + 8138, 8237, 8336, 8435, 8534, 8633, 1085, 1086, 1087, 1088, 8732, 8831, + 8930, 9029, 9128, 9227, 9326, 0, 317, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 130, 228, 229, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 133, 233, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0}; static const uint8_t elem_idxs[] = { - 73, 76, 74, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, - 3, 4, 5, 0, 1, 41, 6, 2, 69, 48, 55, 24, 25, 26, 27, + 74, 77, 75, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, + 3, 4, 5, 0, 1, 41, 6, 2, 70, 48, 55, 24, 25, 26, 27, 28, 29, 30, 7, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 75, 77, 78, 79, 65, 66, 67, 68, 70, 71, - 72, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; + 60, 61, 62, 63, 64, 65, 76, 78, 79, 80, 66, 67, 68, 69, 71, + 72, 73, 255, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { if (a == -1 || b == -1) return GRPC_MDNULL; - uint32_t k = (uint32_t)(a * 97 + b); + uint32_t k = (uint32_t)(a * 99 + b); uint32_t h = elems_phash(k); return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k ? GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[elem_idxs[h]], @@ -478,324 +488,328 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 344, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 345, 1}}}, + .data.refcounted = {g_bytes + 362, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 346, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, + .data.refcounted = {g_bytes + 363, 1}}}, + {{.refcount = &grpc_static_metadata_refcounts[7], + .data.refcounted = {g_bytes + 50, 11}}, {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 347, 8}}}, + .data.refcounted = {g_bytes + 364, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 355, 4}}}, + .data.refcounted = {g_bytes + 365, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 359, 7}}}, + .data.refcounted = {g_bytes + 373, 4}}}, + {{.refcount = &grpc_static_metadata_refcounts[9], + .data.refcounted = {g_bytes + 77, 13}}, + {.refcount = &grpc_static_metadata_refcounts[31], + .data.refcounted = {g_bytes + 377, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[5], .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 366, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[11], - .data.refcounted = {g_bytes + 110, 12}}, {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 374, 16}}}, + .data.refcounted = {g_bytes + 384, 8}}}, + {{.refcount = &grpc_static_metadata_refcounts[14], + .data.refcounted = {g_bytes + 158, 12}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 392, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 390, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 394, 3}}}, + .data.refcounted = {g_bytes + 408, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 397, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, + .data.refcounted = {g_bytes + 412, 3}}}, + {{.refcount = &grpc_static_metadata_refcounts[2], + .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 400, 4}}}, + .data.refcounted = {g_bytes + 415, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 404, 5}}}, + .data.refcounted = {g_bytes + 418, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 409, 4}}}, + .data.refcounted = {g_bytes + 422, 5}}}, + {{.refcount = &grpc_static_metadata_refcounts[4], + .data.refcounted = {g_bytes + 29, 7}}, + {.refcount = &grpc_static_metadata_refcounts[39], + .data.refcounted = {g_bytes + 427, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[3], .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 413, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[40], + .data.refcounted = {g_bytes + 431, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 416, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 419, 1}}}, + .data.refcounted = {g_bytes + 434, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[0], .data.refcounted = {g_bytes + 0, 5}}, {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 420, 11}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, + .data.refcounted = {g_bytes + 437, 1}}}, + {{.refcount = &grpc_static_metadata_refcounts[0], + .data.refcounted = {g_bytes + 0, 5}}, {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 431, 3}}}, + .data.refcounted = {g_bytes + 438, 11}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 434, 3}}}, + .data.refcounted = {g_bytes + 449, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 437, 3}}}, + .data.refcounted = {g_bytes + 452, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 440, 3}}}, + .data.refcounted = {g_bytes + 455, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 443, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 446, 14}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 460, 15}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 458, 3}}}, + {{.refcount = &grpc_static_metadata_refcounts[2], + .data.refcounted = {g_bytes + 12, 7}}, + {.refcount = &grpc_static_metadata_refcounts[48], + .data.refcounted = {g_bytes + 461, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 460, 15}}, - {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 475, 13}}}, - {{.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 488, 15}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 464, 14}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[50], + .data.refcounted = {g_bytes + 478, 15}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[50], + .data.refcounted = {g_bytes + 478, 15}}, + {.refcount = &grpc_static_metadata_refcounts[51], + .data.refcounted = {g_bytes + 493, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 503, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 506, 15}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 516, 6}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 521, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 522, 27}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 534, 6}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 549, 3}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 540, 27}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 552, 5}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 567, 3}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 557, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 570, 5}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 570, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 575, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 583, 19}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 588, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 602, 16}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 601, 19}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 618, 16}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 620, 16}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 634, 14}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 636, 16}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 648, 16}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 652, 14}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 664, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[11], - .data.refcounted = {g_bytes + 110, 12}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 666, 16}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 677, 6}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 682, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[14], + .data.refcounted = {g_bytes + 158, 12}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 683, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 695, 6}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 687, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 701, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 691, 6}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 705, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 697, 7}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 709, 6}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 704, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 162, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 715, 7}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 708, 8}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 722, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[17], + .data.refcounted = {g_bytes + 210, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 716, 17}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 726, 8}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 733, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 734, 17}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 746, 8}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 751, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 754, 19}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 764, 8}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 773, 13}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 166, 8}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 772, 19}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 786, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 791, 13}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[18], + .data.refcounted = {g_bytes + 214, 8}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 790, 8}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 804, 11}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 798, 12}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 815, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 810, 18}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 819, 8}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 828, 19}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 827, 12}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 847, 5}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 839, 18}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 852, 7}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 857, 19}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 859, 7}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 876, 5}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 866, 11}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 881, 7}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 877, 6}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 888, 7}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 883, 10}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 895, 11}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 893, 25}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 906, 6}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 918, 17}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[13], - .data.refcounted = {g_bytes + 152, 10}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 912, 10}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 935, 4}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 922, 25}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 939, 3}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 947, 17}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 200, 10}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 942, 16}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 216, 0}}}, + .data.refcounted = {g_bytes + 964, 4}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[93], + .data.refcounted = {g_bytes + 968, 3}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[94], + .data.refcounted = {g_bytes + 971, 16}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 347, 8}}}, + {.refcount = &grpc_static_metadata_refcounts[29], + .data.refcounted = {g_bytes + 365, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 359, 7}}}, + {.refcount = &grpc_static_metadata_refcounts[31], + .data.refcounted = {g_bytes + 377, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 958, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[95], + .data.refcounted = {g_bytes + 987, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 355, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[30], + .data.refcounted = {g_bytes + 373, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 974, 13}}}, + {.refcount = &grpc_static_metadata_refcounts[96], + .data.refcounted = {g_bytes + 1003, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 987, 12}}}, + {.refcount = &grpc_static_metadata_refcounts[97], + .data.refcounted = {g_bytes + 1016, 12}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 999, 21}}}, + {.refcount = &grpc_static_metadata_refcounts[98], + .data.refcounted = {g_bytes + 1028, 21}}}, }; -const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 73, 74, 75, - 76, 77, 78, 79}; +const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 74, 75, 76, + 77, 78, 79, 80}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index f9600ee2e4..84fb316fd6 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -44,7 +44,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 97 +#define GRPC_STATIC_MDSTR_COUNT 99 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* ":path" */ #define GRPC_MDSTR_PATH (grpc_static_slice_table[0]) @@ -68,182 +68,186 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; #define GRPC_MDSTR_GRPC_ENCODING (grpc_static_slice_table[9]) /* "grpc-accept-encoding" */ #define GRPC_MDSTR_GRPC_ACCEPT_ENCODING (grpc_static_slice_table[10]) +/* "grpc-server-stats-bin" */ +#define GRPC_MDSTR_GRPC_SERVER_STATS_BIN (grpc_static_slice_table[11]) +/* "grpc-tags-bin" */ +#define GRPC_MDSTR_GRPC_TAGS_BIN (grpc_static_slice_table[12]) +/* "grpc-trace-bin" */ +#define GRPC_MDSTR_GRPC_TRACE_BIN (grpc_static_slice_table[13]) /* "content-type" */ -#define GRPC_MDSTR_CONTENT_TYPE (grpc_static_slice_table[11]) +#define GRPC_MDSTR_CONTENT_TYPE (grpc_static_slice_table[14]) /* "grpc-internal-encoding-request" */ -#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[12]) +#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[15]) /* "user-agent" */ -#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[13]) +#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[16]) /* "host" */ -#define GRPC_MDSTR_HOST (grpc_static_slice_table[14]) +#define GRPC_MDSTR_HOST (grpc_static_slice_table[17]) /* "lb-token" */ -#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[15]) +#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[18]) /* "grpc-timeout" */ -#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[16]) -/* "grpc-tracing-bin" */ -#define GRPC_MDSTR_GRPC_TRACING_BIN (grpc_static_slice_table[17]) -/* "grpc-stats-bin" */ -#define GRPC_MDSTR_GRPC_STATS_BIN (grpc_static_slice_table[18]) +#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[19]) /* "" */ -#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[19]) +#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[20]) /* "grpc.wait_for_ready" */ -#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[20]) +#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[21]) /* "grpc.timeout" */ -#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[21]) +#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[22]) /* "grpc.max_request_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_REQUEST_MESSAGE_BYTES \ - (grpc_static_slice_table[22]) + (grpc_static_slice_table[23]) /* "grpc.max_response_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_RESPONSE_MESSAGE_BYTES \ - (grpc_static_slice_table[23]) + (grpc_static_slice_table[24]) /* "/grpc.lb.v1.LoadBalancer/BalanceLoad" */ #define GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD \ - (grpc_static_slice_table[24]) + (grpc_static_slice_table[25]) /* "0" */ -#define GRPC_MDSTR_0 (grpc_static_slice_table[25]) +#define GRPC_MDSTR_0 (grpc_static_slice_table[26]) /* "1" */ -#define GRPC_MDSTR_1 (grpc_static_slice_table[26]) +#define GRPC_MDSTR_1 (grpc_static_slice_table[27]) /* "2" */ -#define GRPC_MDSTR_2 (grpc_static_slice_table[27]) +#define GRPC_MDSTR_2 (grpc_static_slice_table[28]) /* "identity" */ -#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[28]) +#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[29]) /* "gzip" */ -#define GRPC_MDSTR_GZIP (grpc_static_slice_table[29]) +#define GRPC_MDSTR_GZIP (grpc_static_slice_table[30]) /* "deflate" */ -#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[30]) +#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[31]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[31]) +#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[32]) /* "application/grpc" */ -#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[32]) +#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[33]) /* "POST" */ -#define GRPC_MDSTR_POST (grpc_static_slice_table[33]) +#define GRPC_MDSTR_POST (grpc_static_slice_table[34]) /* "200" */ -#define GRPC_MDSTR_200 (grpc_static_slice_table[34]) +#define GRPC_MDSTR_200 (grpc_static_slice_table[35]) /* "404" */ -#define GRPC_MDSTR_404 (grpc_static_slice_table[35]) +#define GRPC_MDSTR_404 (grpc_static_slice_table[36]) /* "http" */ -#define GRPC_MDSTR_HTTP (grpc_static_slice_table[36]) +#define GRPC_MDSTR_HTTP (grpc_static_slice_table[37]) /* "https" */ -#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[37]) +#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[38]) /* "grpc" */ -#define GRPC_MDSTR_GRPC (grpc_static_slice_table[38]) +#define GRPC_MDSTR_GRPC (grpc_static_slice_table[39]) /* "GET" */ -#define GRPC_MDSTR_GET (grpc_static_slice_table[39]) +#define GRPC_MDSTR_GET (grpc_static_slice_table[40]) /* "PUT" */ -#define GRPC_MDSTR_PUT (grpc_static_slice_table[40]) +#define GRPC_MDSTR_PUT (grpc_static_slice_table[41]) /* "/" */ -#define GRPC_MDSTR_SLASH (grpc_static_slice_table[41]) +#define GRPC_MDSTR_SLASH (grpc_static_slice_table[42]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[42]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[43]) /* "204" */ -#define GRPC_MDSTR_204 (grpc_static_slice_table[43]) +#define GRPC_MDSTR_204 (grpc_static_slice_table[44]) /* "206" */ -#define GRPC_MDSTR_206 (grpc_static_slice_table[44]) +#define GRPC_MDSTR_206 (grpc_static_slice_table[45]) /* "304" */ -#define GRPC_MDSTR_304 (grpc_static_slice_table[45]) +#define GRPC_MDSTR_304 (grpc_static_slice_table[46]) /* "400" */ -#define GRPC_MDSTR_400 (grpc_static_slice_table[46]) +#define GRPC_MDSTR_400 (grpc_static_slice_table[47]) /* "500" */ -#define GRPC_MDSTR_500 (grpc_static_slice_table[47]) +#define GRPC_MDSTR_500 (grpc_static_slice_table[48]) /* "accept-charset" */ -#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[48]) +#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[49]) /* "accept-encoding" */ -#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[49]) +#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[50]) /* "gzip, deflate" */ -#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[50]) +#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[51]) /* "accept-language" */ -#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[51]) +#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[52]) /* "accept-ranges" */ -#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[52]) +#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[53]) /* "accept" */ -#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[53]) +#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[54]) /* "access-control-allow-origin" */ -#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[54]) +#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[55]) /* "age" */ -#define GRPC_MDSTR_AGE (grpc_static_slice_table[55]) +#define GRPC_MDSTR_AGE (grpc_static_slice_table[56]) /* "allow" */ -#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[56]) +#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[57]) /* "authorization" */ -#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[57]) +#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[58]) /* "cache-control" */ -#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[58]) +#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[59]) /* "content-disposition" */ -#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[59]) +#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[60]) /* "content-encoding" */ -#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[60]) +#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[61]) /* "content-language" */ -#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[61]) +#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[62]) /* "content-length" */ -#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[62]) +#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[63]) /* "content-location" */ -#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[63]) +#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[64]) /* "content-range" */ -#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[64]) +#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[65]) /* "cookie" */ -#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[65]) +#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[66]) /* "date" */ -#define GRPC_MDSTR_DATE (grpc_static_slice_table[66]) +#define GRPC_MDSTR_DATE (grpc_static_slice_table[67]) /* "etag" */ -#define GRPC_MDSTR_ETAG (grpc_static_slice_table[67]) +#define GRPC_MDSTR_ETAG (grpc_static_slice_table[68]) /* "expect" */ -#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[68]) +#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[69]) /* "expires" */ -#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[69]) +#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[70]) /* "from" */ -#define GRPC_MDSTR_FROM (grpc_static_slice_table[70]) +#define GRPC_MDSTR_FROM (grpc_static_slice_table[71]) /* "if-match" */ -#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[71]) +#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[72]) /* "if-modified-since" */ -#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[72]) +#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[73]) /* "if-none-match" */ -#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[73]) +#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[74]) /* "if-range" */ -#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[74]) +#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[75]) /* "if-unmodified-since" */ -#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[75]) +#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[76]) /* "last-modified" */ -#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[76]) +#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[77]) +/* "lb-cost-bin" */ +#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[78]) /* "link" */ -#define GRPC_MDSTR_LINK (grpc_static_slice_table[77]) +#define GRPC_MDSTR_LINK (grpc_static_slice_table[79]) /* "location" */ -#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[78]) +#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[80]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[79]) +#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[81]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[80]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[82]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[81]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[83]) /* "range" */ -#define GRPC_MDSTR_RANGE (grpc_static_slice_table[82]) +#define GRPC_MDSTR_RANGE (grpc_static_slice_table[84]) /* "referer" */ -#define GRPC_MDSTR_REFERER (grpc_static_slice_table[83]) +#define GRPC_MDSTR_REFERER (grpc_static_slice_table[85]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[84]) +#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[86]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[85]) +#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[87]) /* "server" */ -#define GRPC_MDSTR_SERVER (grpc_static_slice_table[86]) +#define GRPC_MDSTR_SERVER (grpc_static_slice_table[88]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[87]) +#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[89]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[88]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[90]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[89]) +#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[91]) /* "vary" */ -#define GRPC_MDSTR_VARY (grpc_static_slice_table[90]) +#define GRPC_MDSTR_VARY (grpc_static_slice_table[92]) /* "via" */ -#define GRPC_MDSTR_VIA (grpc_static_slice_table[91]) +#define GRPC_MDSTR_VIA (grpc_static_slice_table[93]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[92]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[94]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[93]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[95]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[94]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[96]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[95]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[97]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (grpc_static_slice_table[96]) + (grpc_static_slice_table[98]) extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable; extern grpc_slice_refcount @@ -255,7 +259,7 @@ extern grpc_slice_refcount #define GRPC_STATIC_METADATA_INDEX(static_slice) \ ((int)((static_slice).refcount - grpc_static_metadata_refcounts)) -#define GRPC_STATIC_MDELEM_COUNT 80 +#define GRPC_STATIC_MDELEM_COUNT 81 extern grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "grpc-status": "0" */ @@ -426,78 +430,81 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "lb-token": "" */ #define GRPC_MDELEM_LB_TOKEN_EMPTY \ (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC)) +/* "lb-cost-bin": "" */ +#define GRPC_MDELEM_LB_COST_BIN_EMPTY \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) /* "link": "" */ #define GRPC_MDELEM_LINK_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) /* "location": "" */ #define GRPC_MDELEM_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) /* "max-forwards": "" */ #define GRPC_MDELEM_MAX_FORWARDS_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authenticate": "" */ #define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authorization": "" */ #define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) /* "range": "" */ #define GRPC_MDELEM_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) /* "referer": "" */ #define GRPC_MDELEM_REFERER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) /* "refresh": "" */ #define GRPC_MDELEM_REFRESH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) /* "retry-after": "" */ #define GRPC_MDELEM_RETRY_AFTER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) /* "server": "" */ #define GRPC_MDELEM_SERVER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) /* "set-cookie": "" */ #define GRPC_MDELEM_SET_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) /* "strict-transport-security": "" */ #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) /* "transfer-encoding": "" */ #define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) /* "user-agent": "" */ #define GRPC_MDELEM_USER_AGENT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) /* "vary": "" */ #define GRPC_MDELEM_VARY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) /* "via": "" */ #define GRPC_MDELEM_VIA_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) /* "www-authenticate": "" */ #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC)) grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b); typedef enum { @@ -512,6 +519,9 @@ typedef enum { GRPC_BATCH_GRPC_PAYLOAD_BIN, GRPC_BATCH_GRPC_ENCODING, GRPC_BATCH_GRPC_ACCEPT_ENCODING, + GRPC_BATCH_GRPC_SERVER_STATS_BIN, + GRPC_BATCH_GRPC_TAGS_BIN, + GRPC_BATCH_GRPC_TRACE_BIN, GRPC_BATCH_CONTENT_TYPE, GRPC_BATCH_GRPC_INTERNAL_ENCODING_REQUEST, GRPC_BATCH_USER_AGENT, @@ -534,6 +544,9 @@ typedef union { struct grpc_linked_mdelem *grpc_payload_bin; struct grpc_linked_mdelem *grpc_encoding; struct grpc_linked_mdelem *grpc_accept_encoding; + struct grpc_linked_mdelem *grpc_server_stats_bin; + struct grpc_linked_mdelem *grpc_tags_bin; + struct grpc_linked_mdelem *grpc_trace_bin; struct grpc_linked_mdelem *content_type; struct grpc_linked_mdelem *grpc_internal_encoding_request; struct grpc_linked_mdelem *user_agent; diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index d56cb31ee0..82c4e004b7 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -170,7 +170,7 @@ int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, void grpc_transport_perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, - grpc_transport_stream_op *op) { + grpc_transport_stream_op_batch *op) { transport->vtable->perform_stream_op(exec_ctx, transport, stream, op); } @@ -213,14 +213,23 @@ grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, return transport->vtable->get_endpoint(exec_ctx, transport); } -void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx, - grpc_transport_stream_op *op, - grpc_error *error) { - grpc_closure_sched(exec_ctx, op->recv_message_ready, GRPC_ERROR_REF(error)); - grpc_closure_sched(exec_ctx, op->recv_initial_metadata_ready, - GRPC_ERROR_REF(error)); +void grpc_transport_stream_op_batch_finish_with_failure( + grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op, + grpc_error *error) { + if (op->recv_message) { + grpc_closure_sched(exec_ctx, op->payload->recv_message.recv_message_ready, + GRPC_ERROR_REF(error)); + } + if (op->recv_initial_metadata) { + grpc_closure_sched( + exec_ctx, + op->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_REF(error)); + } grpc_closure_sched(exec_ctx, op->on_complete, error); - GRPC_ERROR_UNREF(op->cancel_error); + if (op->cancel_stream) { + GRPC_ERROR_UNREF(op->payload->cancel_stream.cancel_error); + } } typedef struct { @@ -249,7 +258,8 @@ grpc_transport_op *grpc_make_transport_op(grpc_closure *on_complete) { typedef struct { grpc_closure outer_on_complete; grpc_closure *inner_on_complete; - grpc_transport_stream_op op; + grpc_transport_stream_op_batch op; + grpc_transport_stream_op_batch_payload payload; } made_transport_stream_op; static void destroy_made_transport_stream_op(grpc_exec_ctx *exec_ctx, void *arg, @@ -260,13 +270,13 @@ static void destroy_made_transport_stream_op(grpc_exec_ctx *exec_ctx, void *arg, grpc_closure_run(exec_ctx, c, GRPC_ERROR_REF(error)); } -grpc_transport_stream_op *grpc_make_transport_stream_op( +grpc_transport_stream_op_batch *grpc_make_transport_stream_op( grpc_closure *on_complete) { - made_transport_stream_op *op = gpr_malloc(sizeof(*op)); + made_transport_stream_op *op = gpr_zalloc(sizeof(*op)); + op->op.payload = &op->payload; grpc_closure_init(&op->outer_on_complete, destroy_made_transport_stream_op, op, grpc_schedule_on_exec_ctx); op->inner_on_complete = on_complete; - memset(&op->op, 0, sizeof(op->op)); op->op.on_complete = &op->outer_on_complete; return &op->op; } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 950b18aeda..93369cc689 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -109,55 +109,98 @@ void grpc_transport_move_stats(grpc_transport_stream_stats *from, grpc_transport_stream_stats *to); typedef struct { + void *extra_arg; grpc_closure closure; - void *args[2]; -} grpc_transport_private_op_data; +} grpc_handler_private_op_data; + +typedef struct grpc_transport_stream_op_batch_payload + grpc_transport_stream_op_batch_payload; /* Transport stream op: a set of operations to perform on a transport against a single stream */ -typedef struct grpc_transport_stream_op { +typedef struct grpc_transport_stream_op_batch { /** Should be enqueued when all requested operations (excluding recv_message and recv_initial_metadata which have their own closures) in a given batch have been completed. */ grpc_closure *on_complete; + /** Values for the stream op (fields set are determined by flags above) */ + grpc_transport_stream_op_batch_payload *payload; + /** Is the completion of this op covered by a poller (if false: the op should complete independently of some pollset being polled) */ - bool covered_by_poller; + bool covered_by_poller : 1; - /** Send initial metadata to the peer, from the provided metadata batch. - idempotent_request MUST be set if this is non-null */ - grpc_metadata_batch *send_initial_metadata; - /** Iff send_initial_metadata != NULL, flags associated with - send_initial_metadata: a bitfield of GRPC_INITIAL_METADATA_xxx */ - uint32_t send_initial_metadata_flags; + /** Send initial metadata to the peer, from the provided metadata batch. */ + bool send_initial_metadata : 1; /** Send trailing metadata to the peer, from the provided metadata batch. */ - grpc_metadata_batch *send_trailing_metadata; + bool send_trailing_metadata : 1; /** Send message data to the peer, from the provided byte stream. */ - grpc_byte_stream *send_message; + bool send_message : 1; /** Receive initial metadata from the stream, into provided metadata batch. */ - grpc_metadata_batch *recv_initial_metadata; - bool *recv_idempotent_request; - bool *recv_cacheable_request; - /** Should be enqueued when initial metadata is ready to be processed. */ - grpc_closure *recv_initial_metadata_ready; + bool recv_initial_metadata : 1; /** Receive message data from the stream, into provided byte stream. */ - grpc_byte_stream **recv_message; - /** Should be enqueued when one message is ready to be processed. */ - grpc_closure *recv_message_ready; + bool recv_message : 1; /** Receive trailing metadata from the stream, into provided metadata batch. */ - grpc_metadata_batch *recv_trailing_metadata; + bool recv_trailing_metadata : 1; /** Collect any stats into provided buffer, zero internal stat counters */ - grpc_transport_stream_stats *collect_stats; + bool collect_stats : 1; + + /** Cancel this stream with the provided error */ + bool cancel_stream : 1; + + /*************************************************************************** + * remaining fields are initialized and used at the discretion of the + * current handler of the op */ - /** If != GRPC_ERROR_NONE, forcefully close this stream. + grpc_handler_private_op_data handler_private; +} grpc_transport_stream_op_batch; + +struct grpc_transport_stream_op_batch_payload { + struct { + grpc_metadata_batch *send_initial_metadata; + /** Iff send_initial_metadata != NULL, flags associated with + send_initial_metadata: a bitfield of GRPC_INITIAL_METADATA_xxx */ + uint32_t send_initial_metadata_flags; + } send_initial_metadata; + + struct { + grpc_metadata_batch *send_trailing_metadata; + } send_trailing_metadata; + + struct { + grpc_byte_stream *send_message; + } send_message; + + struct { + grpc_metadata_batch *recv_initial_metadata; + uint32_t *recv_flags; + /** Should be enqueued when initial metadata is ready to be processed. */ + grpc_closure *recv_initial_metadata_ready; + } recv_initial_metadata; + + struct { + grpc_byte_stream **recv_message; + /** Should be enqueued when one message is ready to be processed. */ + grpc_closure *recv_message_ready; + } recv_message; + + struct { + grpc_metadata_batch *recv_trailing_metadata; + } recv_trailing_metadata; + + struct { + grpc_transport_stream_stats *collect_stats; + } collect_stats; + + /** Forcefully close this stream. The HTTP2 semantics should be: - server side: if cancel_error has GRPC_ERROR_INT_GRPC_STATUS, and trailing metadata has not been sent, send trailing metadata with status @@ -167,17 +210,13 @@ typedef struct grpc_transport_stream_op { convert to a HTTP2 error code using grpc_chttp2_grpc_status_to_http2_error. Send a RST_STREAM with this error. */ - grpc_error *cancel_error; + struct { + grpc_error *cancel_error; + } cancel_stream; /* Indexes correspond to grpc_context_index enum values */ grpc_call_context_element *context; - - /*************************************************************************** - * remaining fields are initialized and used at the discretion of the - * current handler of the op */ - - grpc_transport_private_op_data handler_private; -} grpc_transport_stream_op; +}; /** Transport op: a set of operations to perform on a transport as a whole */ typedef struct grpc_transport_op { @@ -210,7 +249,7 @@ typedef struct grpc_transport_op { * remaining fields are initialized and used at the discretion of the * transport implementation */ - grpc_transport_private_op_data transport_private; + grpc_handler_private_op_data handler_private; } grpc_transport_op; /* Returns the amount of memory required to store a grpc_stream for this @@ -250,11 +289,11 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, grpc_stream *stream, grpc_closure *then_schedule_closure); -void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx, - grpc_transport_stream_op *op, - grpc_error *error); +void grpc_transport_stream_op_batch_finish_with_failure( + grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op, + grpc_error *error); -char *grpc_transport_stream_op_string(grpc_transport_stream_op *op); +char *grpc_transport_stream_op_batch_string(grpc_transport_stream_op_batch *op); char *grpc_transport_op_string(grpc_transport_op *op); /* Send a batch of operations on a transport @@ -265,11 +304,12 @@ char *grpc_transport_op_string(grpc_transport_op *op); transport - the transport on which to initiate the stream stream - the stream on which to send the operations. This must be non-NULL and previously initialized by the same transport. - op - a grpc_transport_stream_op specifying the op to perform */ + op - a grpc_transport_stream_op_batch specifying the op to perform + */ void grpc_transport_perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, - grpc_transport_stream_op *op); + grpc_transport_stream_op_batch *op); void grpc_transport_perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *transport, @@ -301,9 +341,10 @@ grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, /* Allocate a grpc_transport_op, and preconfigure the on_consumed closure to \a on_consumed and then delete the returned transport op */ grpc_transport_op *grpc_make_transport_op(grpc_closure *on_consumed); -/* Allocate a grpc_transport_stream_op, and preconfigure the on_consumed closure +/* Allocate a grpc_transport_stream_op_batch, and preconfigure the on_consumed + closure to \a on_consumed and then delete the returned transport op */ -grpc_transport_stream_op *grpc_make_transport_stream_op( +grpc_transport_stream_op_batch *grpc_make_transport_stream_op( grpc_closure *on_consumed); #ifdef __cplusplus diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index 6f688bf8d2..bbb19a34bd 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -59,7 +59,8 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_perform_stream_op */ void (*perform_stream_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self, - grpc_stream *stream, grpc_transport_stream_op *op); + grpc_stream *stream, + grpc_transport_stream_op_batch *op); /* implementation of grpc_transport_perform_op */ void (*perform_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self, diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 28360e3784..3a2a793e01 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -71,7 +71,8 @@ static void put_metadata_list(gpr_strvec *b, grpc_metadata_batch md) { } } -char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) { +char *grpc_transport_stream_op_batch_string( + grpc_transport_stream_op_batch *op) { char *tmp; char *out; @@ -81,45 +82,49 @@ char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) { gpr_strvec_add( &b, gpr_strdup(op->covered_by_poller ? "[COVERED]" : "[UNCOVERED]")); - if (op->send_initial_metadata != NULL) { + if (op->send_initial_metadata) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_strvec_add(&b, gpr_strdup("SEND_INITIAL_METADATA{")); - put_metadata_list(&b, *op->send_initial_metadata); + put_metadata_list( + &b, *op->payload->send_initial_metadata.send_initial_metadata); gpr_strvec_add(&b, gpr_strdup("}")); } - if (op->send_message != NULL) { + if (op->send_message) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_asprintf(&tmp, "SEND_MESSAGE:flags=0x%08x:len=%d", - op->send_message->flags, op->send_message->length); + op->payload->send_message.send_message->flags, + op->payload->send_message.send_message->length); gpr_strvec_add(&b, tmp); } - if (op->send_trailing_metadata != NULL) { + if (op->send_trailing_metadata) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_strvec_add(&b, gpr_strdup("SEND_TRAILING_METADATA{")); - put_metadata_list(&b, *op->send_trailing_metadata); + put_metadata_list( + &b, *op->payload->send_trailing_metadata.send_trailing_metadata); gpr_strvec_add(&b, gpr_strdup("}")); } - if (op->recv_initial_metadata != NULL) { + if (op->recv_initial_metadata) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_strvec_add(&b, gpr_strdup("RECV_INITIAL_METADATA")); } - if (op->recv_message != NULL) { + if (op->recv_message) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_strvec_add(&b, gpr_strdup("RECV_MESSAGE")); } - if (op->recv_trailing_metadata != NULL) { + if (op->recv_trailing_metadata) { gpr_strvec_add(&b, gpr_strdup(" ")); gpr_strvec_add(&b, gpr_strdup("RECV_TRAILING_METADATA")); } - if (op->cancel_error != GRPC_ERROR_NONE) { + if (op->cancel_stream) { gpr_strvec_add(&b, gpr_strdup(" ")); - const char *msg = grpc_error_string(op->cancel_error); + const char *msg = + grpc_error_string(op->payload->cancel_stream.cancel_error); gpr_asprintf(&tmp, "CANCEL:%s", msg); gpr_strvec_add(&b, tmp); @@ -204,8 +209,9 @@ char *grpc_transport_op_string(grpc_transport_op *op) { } void grpc_call_log_op(char *file, int line, gpr_log_severity severity, - grpc_call_element *elem, grpc_transport_stream_op *op) { - char *str = grpc_transport_stream_op_string(op); + grpc_call_element *elem, + grpc_transport_stream_op_batch *op) { + char *str = grpc_transport_stream_op_batch_string(op); gpr_log(file, line, severity, "OP[%s:%p]: %s", elem->filter->name, elem, str); gpr_free(str); } |