diff options
Diffstat (limited to 'src/core/lib')
27 files changed, 1740 insertions, 848 deletions
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 8fdef0bc64..02db798b5c 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -221,6 +221,21 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( return GRPC_COMPRESS_NONE; } +grpc_stream_compression_algorithm +grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) { + size_t i; + if (a == NULL) return 0; + for (i = 0; i < a->num_args; ++i) { + if (a->args[i].type == GRPC_ARG_INTEGER && + !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, + a->args[i].key)) { + return (grpc_stream_compression_algorithm)a->args[i].value.integer; + break; + } + } + return GRPC_STREAM_COMPRESS_NONE; +} + grpc_channel_args *grpc_channel_args_set_compression_algorithm( grpc_channel_args *a, grpc_compression_algorithm algorithm) { GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); @@ -231,6 +246,16 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm( return grpc_channel_args_copy_and_add(a, &tmp, 1); } +grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm( + grpc_channel_args *a, grpc_stream_compression_algorithm algorithm) { + GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); + grpc_arg tmp; + tmp.type = GRPC_ARG_INTEGER; + tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.value.integer = algorithm; + return grpc_channel_args_copy_and_add(a, &tmp, 1); +} + /** Returns 1 if the argument for compression algorithm's enabled states bitset * was found in \a a, returning the arg's value in \a states. Otherwise, returns * 0. */ @@ -251,6 +276,26 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args *a, return 0; /* GPR_FALSE */ } +/** Returns 1 if the argument for compression algorithm's enabled states bitset + * was found in \a a, returning the arg's value in \a states. Otherwise, returns + * 0. */ +static int find_stream_compression_algorithm_states_bitset( + const grpc_channel_args *a, int **states_arg) { + if (a != NULL) { + size_t i; + for (i = 0; i < a->num_args; ++i) { + if (a->args[i].type == GRPC_ARG_INTEGER && + !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET, + a->args[i].key)) { + *states_arg = &a->args[i].value.integer; + **states_arg |= 0x1; /* forcefully enable support for no compression */ + return 1; + } + } + } + return 0; /* GPR_FALSE */ +} + grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( grpc_exec_ctx *exec_ctx, grpc_channel_args **a, grpc_compression_algorithm algorithm, int state) { @@ -292,6 +337,48 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( return result; } +grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( + grpc_exec_ctx *exec_ctx, grpc_channel_args **a, + grpc_stream_compression_algorithm algorithm, int state) { + int *states_arg = NULL; + grpc_channel_args *result = *a; + const int states_arg_found = + find_stream_compression_algorithm_states_bitset(*a, &states_arg); + + if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm && + state == 0) { + char *algo_name = NULL; + GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) != + 0); + gpr_log(GPR_ERROR, + "Tried to disable default stream compression algorithm '%s'. The " + "operation has been ignored.", + algo_name); + } else if (states_arg_found) { + if (state != 0) { + GPR_BITSET((unsigned *)states_arg, algorithm); + } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) { + GPR_BITCLEAR((unsigned *)states_arg, algorithm); + } + } else { + /* create a new arg */ + grpc_arg tmp; + tmp.type = GRPC_ARG_INTEGER; + tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + /* all enabled by default */ + tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; + if (state != 0) { + GPR_BITSET((unsigned *)&tmp.value.integer, algorithm); + } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) { + GPR_BITCLEAR((unsigned *)&tmp.value.integer, algorithm); + } + result = grpc_channel_args_copy_and_add(*a, &tmp, 1); + grpc_channel_args_destroy(exec_ctx, *a); + *a = result; + } + return result; +} + uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args *a) { int *states_arg; @@ -302,6 +389,17 @@ uint32_t grpc_channel_args_compression_algorithm_get_states( } } +uint32_t grpc_channel_args_stream_compression_algorithm_get_states( + const grpc_channel_args *a) { + int *states_arg; + if (find_stream_compression_algorithm_states_bitset(a, &states_arg)) { + return (uint32_t)*states_arg; + } else { + return (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - + 1; /* All algs. enabled */ + } +} + grpc_channel_args *grpc_channel_args_set_socket_mutator( grpc_channel_args *a, grpc_socket_mutator *mutator) { grpc_arg tmp = grpc_socket_mutator_to_arg(mutator); diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index f649a8d9ec..0599e189c3 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -59,12 +59,24 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a); grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args *a); +/** Returns the stream compression algorithm set in \a a. */ +grpc_stream_compression_algorithm +grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a); + /** Returns a channel arg instance with compression enabled. If \a a is * non-NULL, its args are copied. N.B. GRPC_COMPRESS_NONE disables compression * for the channel. */ grpc_channel_args *grpc_channel_args_set_compression_algorithm( grpc_channel_args *a, grpc_compression_algorithm algorithm); +/** Returns a channel arg instance with stream compression enabled. If \a a is + * non-NULL, its args are copied. N.B. GRPC_STREAM_COMPRESS_NONE disables + * stream compression for the channel. If a value other than + * GRPC_STREAM_COMPRESS_NONE is set, it takes precedence over message-wise + * compression algorithms. */ +grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm( + grpc_channel_args *a, grpc_stream_compression_algorithm algorithm); + /** Sets the support for the given compression algorithm. By default, all * compression algorithms are enabled. It's an error to disable an algorithm set * by grpc_channel_args_set_compression_algorithm. @@ -76,6 +88,17 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( grpc_exec_ctx *exec_ctx, grpc_channel_args **a, grpc_compression_algorithm algorithm, int enabled); +/** Sets the support for the given stream compression algorithm. By default, all + * stream compression algorithms are enabled. It's an error to disable an + * algorithm set by grpc_channel_args_set_stream_compression_algorithm. + * + * Returns an instance with the updated algorithm states. The \a a pointer is + * modified to point to the returned instance (which may be different from the + * input value of \a a). */ +grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( + grpc_exec_ctx *exec_ctx, grpc_channel_args **a, + grpc_stream_compression_algorithm algorithm, int enabled); + /** Returns the bitset representing the support state (true for enabled, false * for disabled) for compression algorithms. * @@ -84,6 +107,14 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args *a); +/** Returns the bitset representing the support state (true for enabled, false + * for disabled) for stream compression algorithms. + * + * The i-th bit of the returned bitset corresponds to the i-th entry in the + * grpc_stream_compression_algorithm enum. */ +uint32_t grpc_channel_args_stream_compression_algorithm_get_states( + const grpc_channel_args *a); + int grpc_channel_args_compare(const grpc_channel_args *a, const grpc_channel_args *b); diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 0f8e33c4be..775c8bc667 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -233,15 +233,10 @@ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack, void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { grpc_call_element *next_elem = elem + 1; + GRPC_CALL_LOG_OP(GPR_INFO, next_elem, op); next_elem->filter->start_transport_stream_op_batch(exec_ctx, next_elem, op); } -char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - grpc_call_element *next_elem = elem + 1; - return next_elem->filter->get_peer(exec_ctx, next_elem); -} - void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, const grpc_channel_info *channel_info) { @@ -265,12 +260,3 @@ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem) { return (grpc_call_stack *)((char *)(elem)-ROUND_UP_TO_ALIGNMENT_SIZE( sizeof(grpc_call_stack))); } - -void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_error *error) { - grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op(NULL); - op->cancel_stream = true; - op->payload->cancel_stream.cancel_error = error; - elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op); -} diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index a80f8aa826..ae1cac31f7 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -40,6 +40,7 @@ #include <grpc/support/time.h> #include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/support/arena.h" #include "src/core/lib/transport/transport.h" @@ -71,6 +72,7 @@ typedef struct { gpr_timespec start_time; gpr_timespec deadline; gpr_arena *arena; + grpc_call_combiner *call_combiner; } grpc_call_element_args; typedef struct { @@ -150,9 +152,6 @@ typedef struct { void (*destroy_channel_elem)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem); - /* Implement grpc_call_get_peer() */ - char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem); - /* Implement grpc_channel_get_info() */ void (*get_channel_info)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, const grpc_channel_info *channel_info); @@ -271,8 +270,6 @@ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, stack */ void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op); -/* Pass through a request to get_peer to the next child element */ -char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem); /* Pass through a request to get_channel_info() to the next child element */ void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, @@ -288,10 +285,6 @@ void grpc_call_log_op(char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op); -void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, - grpc_call_element *cur_elem, - grpc_error *error); - extern grpc_tracer_flag grpc_trace_channel; #define GRPC_CALL_LOG_OP(sev, elem, op) \ diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index af06ca802e..8285226fc4 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -36,7 +36,57 @@ typedef struct connected_channel_channel_data { grpc_transport *transport; } channel_data; -typedef struct connected_channel_call_data { void *unused; } call_data; +typedef struct { + grpc_closure closure; + grpc_closure *original_closure; + grpc_call_combiner *call_combiner; + const char *reason; +} callback_state; + +typedef struct connected_channel_call_data { + grpc_call_combiner *call_combiner; + // Closures used for returning results on the call combiner. + callback_state on_complete[6]; // Max number of pending batches. + callback_state recv_initial_metadata_ready; + callback_state recv_message_ready; +} call_data; + +static void run_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + callback_state *state = (callback_state *)arg; + GRPC_CALL_COMBINER_START(exec_ctx, state->call_combiner, + state->original_closure, GRPC_ERROR_REF(error), + state->reason); +} + +static void run_cancel_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + run_in_call_combiner(exec_ctx, arg, error); + gpr_free(arg); +} + +static void intercept_callback(call_data *calld, callback_state *state, + bool free_when_done, const char *reason, + grpc_closure **original_closure) { + state->original_closure = *original_closure; + state->call_combiner = calld->call_combiner; + state->reason = reason; + *original_closure = GRPC_CLOSURE_INIT( + &state->closure, + free_when_done ? run_cancel_in_call_combiner : run_in_call_combiner, + state, grpc_schedule_on_exec_ctx); +} + +static callback_state *get_state_for_batch( + call_data *calld, grpc_transport_stream_op_batch *batch) { + if (batch->send_initial_metadata) return &calld->on_complete[0]; + if (batch->send_message) return &calld->on_complete[1]; + if (batch->send_trailing_metadata) return &calld->on_complete[2]; + if (batch->recv_initial_metadata) return &calld->on_complete[3]; + if (batch->recv_message) return &calld->on_complete[4]; + if (batch->recv_trailing_metadata) return &calld->on_complete[5]; + GPR_UNREACHABLE_CODE(return NULL); +} /* We perform a small hack to locate transport data alongside the connected channel data in call allocations, to allow everything to be pulled in minimal @@ -49,13 +99,38 @@ typedef struct connected_channel_call_data { void *unused; } call_data; into transport stream operations */ static void con_start_transport_stream_op_batch( grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - grpc_transport_stream_op_batch *op) { + grpc_transport_stream_op_batch *batch) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - + if (batch->recv_initial_metadata) { + callback_state *state = &calld->recv_initial_metadata_ready; + intercept_callback( + calld, state, false, "recv_initial_metadata_ready", + &batch->payload->recv_initial_metadata.recv_initial_metadata_ready); + } + if (batch->recv_message) { + callback_state *state = &calld->recv_message_ready; + intercept_callback(calld, state, false, "recv_message_ready", + &batch->payload->recv_message.recv_message_ready); + } + if (batch->cancel_stream) { + // There can be more than one cancellation batch in flight at any + // given time, so we can't just pick out a fixed index into + // calld->on_complete like we can for the other ops. However, + // cancellation isn't in the fast path, so we just allocate a new + // closure for each one. + callback_state *state = (callback_state *)gpr_malloc(sizeof(*state)); + intercept_callback(calld, state, true, "on_complete (cancel_stream)", + &batch->on_complete); + } else { + callback_state *state = get_state_for_batch(calld, batch); + intercept_callback(calld, state, false, "on_complete", &batch->on_complete); + } grpc_transport_perform_stream_op(exec_ctx, chand->transport, - TRANSPORT_STREAM_FROM_CALL_DATA(calld), op); + TRANSPORT_STREAM_FROM_CALL_DATA(calld), + batch); + GRPC_CALL_COMBINER_STOP(exec_ctx, calld->call_combiner, + "passed batch to transport"); } static void con_start_transport_op(grpc_exec_ctx *exec_ctx, @@ -71,6 +146,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, const grpc_call_element_args *args) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; + calld->call_combiner = args->call_combiner; int r = grpc_transport_init_stream( exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld), &args->call_stack->refcount, args->server_transport_data, args->arena); @@ -118,11 +194,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } } -static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - channel_data *chand = elem->channel_data; - return grpc_transport_get_peer(exec_ctx, chand->transport); -} - /* No-op. */ static void con_get_channel_info(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, @@ -138,7 +209,6 @@ const grpc_channel_filter grpc_connected_filter = { sizeof(channel_data), init_channel_elem, destroy_channel_elem, - con_get_peer, con_get_channel_info, "connected", }; diff --git a/src/core/lib/compression/algorithm_metadata.h b/src/core/lib/compression/algorithm_metadata.h index 4717af6e2b..08feafc1bb 100644 --- a/src/core/lib/compression/algorithm_metadata.h +++ b/src/core/lib/compression/algorithm_metadata.h @@ -26,13 +26,27 @@ grpc_slice grpc_compression_algorithm_slice( grpc_compression_algorithm algorithm); +/** Return stream compression algorithm based metadata value */ +grpc_slice grpc_stream_compression_algorithm_slice( + grpc_stream_compression_algorithm algorithm); + /** Return compression algorithm based metadata element (grpc-encoding: xxx) */ grpc_mdelem grpc_compression_encoding_mdelem( grpc_compression_algorithm algorithm); +/** Return stream compression algorithm based metadata element + * (content-encoding: xxx) */ +grpc_mdelem grpc_stream_compression_encoding_mdelem( + grpc_stream_compression_algorithm algorithm); + /** Find compression algorithm based on passed in mdstr - returns * GRPC_COMPRESS_ALGORITHM_COUNT on failure */ grpc_compression_algorithm grpc_compression_algorithm_from_slice( grpc_slice str); +/** Find stream compression algorithm based on passed in mdstr - returns + * GRPC_STREAM_COMPRESS_ALGORITHM_COUNT on failure */ +grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice( + grpc_slice str); + #endif /* GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H */ diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c index 8deae2798f..ec84c01811 100644 --- a/src/core/lib/compression/compression.c +++ b/src/core/lib/compression/compression.c @@ -46,6 +46,19 @@ int grpc_compression_algorithm_parse(grpc_slice name, } } +int grpc_stream_compression_algorithm_parse( + grpc_slice name, grpc_stream_compression_algorithm *algorithm) { + if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) { + *algorithm = GRPC_STREAM_COMPRESS_NONE; + return 1; + } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) { + *algorithm = GRPC_STREAM_COMPRESS_GZIP; + return 1; + } else { + return 0; + } +} + int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, char **name) { GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2, @@ -66,6 +79,24 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, return 0; } +int grpc_stream_compression_algorithm_name( + grpc_stream_compression_algorithm algorithm, char **name) { + GRPC_API_TRACE( + "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2, + ((int)algorithm, name)); + switch (algorithm) { + case GRPC_STREAM_COMPRESS_NONE: + *name = "identity"; + return 1; + case GRPC_STREAM_COMPRESS_GZIP: + *name = "gzip"; + return 1; + case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT: + return 0; + } + return 0; +} + grpc_compression_algorithm grpc_compression_algorithm_from_slice( grpc_slice str) { if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE; @@ -74,6 +105,13 @@ grpc_compression_algorithm grpc_compression_algorithm_from_slice( return GRPC_COMPRESS_ALGORITHMS_COUNT; } +grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice( + grpc_slice str) { + if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_STREAM_COMPRESS_NONE; + if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_STREAM_COMPRESS_GZIP; + return GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT; +} + grpc_slice grpc_compression_algorithm_slice( grpc_compression_algorithm algorithm) { switch (algorithm) { @@ -89,6 +127,19 @@ grpc_slice grpc_compression_algorithm_slice( return grpc_empty_slice(); } +grpc_slice grpc_stream_compression_algorithm_slice( + grpc_stream_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_STREAM_COMPRESS_NONE: + return GRPC_MDSTR_IDENTITY; + case GRPC_STREAM_COMPRESS_GZIP: + return GRPC_MDSTR_GZIP; + case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT: + return grpc_empty_slice(); + } + return grpc_empty_slice(); +} + grpc_mdelem grpc_compression_encoding_mdelem( grpc_compression_algorithm algorithm) { switch (algorithm) { @@ -104,10 +155,25 @@ grpc_mdelem grpc_compression_encoding_mdelem( return GRPC_MDNULL; } +grpc_mdelem grpc_stream_compression_encoding_mdelem( + grpc_stream_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_STREAM_COMPRESS_NONE: + return GRPC_MDELEM_CONTENT_ENCODING_IDENTITY; + case GRPC_STREAM_COMPRESS_GZIP: + return GRPC_MDELEM_CONTENT_ENCODING_GZIP; + default: + break; + } + return GRPC_MDNULL; +} + void grpc_compression_options_init(grpc_compression_options *opts) { memset(opts, 0, sizeof(*opts)); /* all enabled by default */ opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; + opts->enabled_stream_compression_algorithms_bitset = + (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; } void grpc_compression_options_enable_algorithm( @@ -126,6 +192,13 @@ int grpc_compression_options_is_algorithm_enabled( return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm); } +int grpc_compression_options_is_stream_compression_algorithm_enabled( + const grpc_compression_options *opts, + grpc_stream_compression_algorithm algorithm) { + return GPR_BITGET(opts->enabled_stream_compression_algorithms_bitset, + algorithm); +} + /* TODO(dgq): Add the ability to specify parameters to the individual * compression algorithms */ grpc_compression_algorithm grpc_compression_algorithm_for_level( @@ -181,3 +254,30 @@ grpc_compression_algorithm grpc_compression_algorithm_for_level( abort(); }; } + +GRPCAPI grpc_stream_compression_algorithm +grpc_stream_compression_algorithm_for_level( + grpc_stream_compression_level level, uint32_t accepted_stream_encodings) { + GRPC_API_TRACE("grpc_stream_compression_algorithm_for_level(level=%d)", 1, + ((int)level)); + if (level > GRPC_STREAM_COMPRESS_LEVEL_HIGH) { + gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level); + abort(); + } + + switch (level) { + case GRPC_STREAM_COMPRESS_LEVEL_NONE: + return GRPC_STREAM_COMPRESS_NONE; + case GRPC_STREAM_COMPRESS_LEVEL_LOW: + case GRPC_STREAM_COMPRESS_LEVEL_MED: + case GRPC_STREAM_COMPRESS_LEVEL_HIGH: + if (GPR_BITGET(accepted_stream_encodings, GRPC_STREAM_COMPRESS_GZIP) == + 1) { + return GRPC_STREAM_COMPRESS_GZIP; + } else { + return GRPC_STREAM_COMPRESS_NONE; + } + default: + abort(); + } +} diff --git a/src/core/lib/iomgr/call_combiner.c b/src/core/lib/iomgr/call_combiner.c new file mode 100644 index 0000000000..899f98552d --- /dev/null +++ b/src/core/lib/iomgr/call_combiner.c @@ -0,0 +1,180 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/call_combiner.h" + +#include <grpc/support/log.h> + +grpc_tracer_flag grpc_call_combiner_trace = + GRPC_TRACER_INITIALIZER(false, "call_combiner"); + +static grpc_error* decode_cancel_state_error(gpr_atm cancel_state) { + if (cancel_state & 1) { + return (grpc_error*)(cancel_state & ~(gpr_atm)1); + } + return GRPC_ERROR_NONE; +} + +static gpr_atm encode_cancel_state_error(grpc_error* error) { + return (gpr_atm)1 | (gpr_atm)error; +} + +void grpc_call_combiner_init(grpc_call_combiner* call_combiner) { + gpr_mpscq_init(&call_combiner->queue); +} + +void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) { + gpr_mpscq_destroy(&call_combiner->queue); + GRPC_ERROR_UNREF(decode_cancel_state_error(call_combiner->cancel_state)); +} + +#ifndef NDEBUG +#define DEBUG_ARGS , const char *file, int line +#define DEBUG_FMT_STR "%s:%d: " +#define DEBUG_FMT_ARGS , file, line +#else +#define DEBUG_ARGS +#define DEBUG_FMT_STR +#define DEBUG_FMT_ARGS +#endif + +void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_closure* closure, + grpc_error* error DEBUG_ARGS, + const char* reason) { + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, + "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR + "%s] error=%s", + call_combiner, closure DEBUG_FMT_ARGS, reason, + grpc_error_string(error)); + } + size_t prev_size = + (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)1); + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " size: %" PRIdPTR " -> %" PRIdPTR, prev_size, + prev_size + 1); + } + if (prev_size == 0) { + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " EXECUTING IMMEDIATELY"); + } + // Queue was empty, so execute this closure immediately. + GRPC_CLOSURE_SCHED(exec_ctx, closure, error); + } else { + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_INFO, " QUEUING"); + } + // Queue was not empty, so add closure to queue. + closure->error_data.error = error; + gpr_mpscq_push(&call_combiner->queue, (gpr_mpscq_node*)closure); + } +} + +void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner DEBUG_ARGS, + const char* reason) { + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, + "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]", + call_combiner DEBUG_FMT_ARGS, reason); + } + size_t prev_size = + (size_t)gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)-1); + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " size: %" PRIdPTR " -> %" PRIdPTR, prev_size, + prev_size - 1); + } + GPR_ASSERT(prev_size >= 1); + if (prev_size > 1) { + while (true) { + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " checking queue"); + } + bool empty; + grpc_closure* closure = (grpc_closure*)gpr_mpscq_pop_and_check_end( + &call_combiner->queue, &empty); + if (closure == NULL) { + // This can happen either due to a race condition within the mpscq + // code or because of a race with grpc_call_combiner_start(). + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " queue returned no result; checking again"); + } + continue; + } + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " EXECUTING FROM QUEUE: closure=%p error=%s", + closure, grpc_error_string(closure->error_data.error)); + } + GRPC_CLOSURE_SCHED(exec_ctx, closure, closure->error_data.error); + break; + } + } else if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, " queue empty"); + } +} + +void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_closure* closure) { + while (true) { + // Decode original state. + gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state); + grpc_error* original_error = decode_cancel_state_error(original_state); + // If error is set, invoke the cancellation closure immediately. + // Otherwise, store the new closure. + if (original_error != GRPC_ERROR_NONE) { + GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_REF(original_error)); + break; + } else { + if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state, + (gpr_atm)closure)) { + break; + } + } + // cas failed, try again. + } +} + +void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_error* error) { + while (true) { + gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state); + grpc_error* original_error = decode_cancel_state_error(original_state); + if (original_error != GRPC_ERROR_NONE) { + GRPC_ERROR_UNREF(error); + break; + } + if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state, + encode_cancel_state_error(error))) { + if (original_state != 0) { + grpc_closure* notify_on_cancel = (grpc_closure*)original_state; + if (GRPC_TRACER_ON(grpc_call_combiner_trace)) { + gpr_log(GPR_DEBUG, + "call_combiner=%p: scheduling notify_on_cancel callback=%p", + call_combiner, notify_on_cancel); + } + GRPC_CLOSURE_SCHED(exec_ctx, notify_on_cancel, GRPC_ERROR_REF(error)); + } + break; + } + // cas failed, try again. + } +} diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h new file mode 100644 index 0000000000..621e2c3669 --- /dev/null +++ b/src/core/lib/iomgr/call_combiner.h @@ -0,0 +1,104 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H +#define GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H + +#include <stddef.h> + +#include <grpc/support/atm.h> + +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/support/mpscq.h" + +// A simple, lock-free mechanism for serializing activity related to a +// single call. This is similar to a combiner but is more lightweight. +// +// It requires the callback (or, in the common case where the callback +// actually kicks off a chain of callbacks, the last callback in that +// chain) to explicitly indicate (by calling GRPC_CALL_COMBINER_STOP()) +// when it is done with the action that was kicked off by the original +// callback. + +extern grpc_tracer_flag grpc_call_combiner_trace; + +typedef struct { + gpr_atm size; // size_t, num closures in queue or currently executing + gpr_mpscq queue; + // Either 0 (if not cancelled and no cancellation closure set), + // a grpc_closure* (if the lowest bit is 0), + // or a grpc_error* (if the lowest bit is 1). + gpr_atm cancel_state; +} grpc_call_combiner; + +// Assumes memory was initialized to zero. +void grpc_call_combiner_init(grpc_call_combiner* call_combiner); + +void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner); + +#ifndef NDEBUG +#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error, \ + reason) \ + grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \ + __FILE__, __LINE__, (reason)) +#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason) \ + grpc_call_combiner_stop((exec_ctx), (call_combiner), __FILE__, __LINE__, \ + (reason)) +/// Starts processing \a closure on \a call_combiner. +void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_closure* closure, grpc_error* error, + const char* file, int line, const char* reason); +/// Yields the call combiner to the next closure in the queue, if any. +void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + const char* file, int line, const char* reason); +#else +#define GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, closure, error, \ + reason) \ + grpc_call_combiner_start((exec_ctx), (call_combiner), (closure), (error), \ + (reason)) +#define GRPC_CALL_COMBINER_STOP(exec_ctx, call_combiner, reason) \ + grpc_call_combiner_stop((exec_ctx), (call_combiner), (reason)) +/// Starts processing \a closure on \a call_combiner. +void grpc_call_combiner_start(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_closure* closure, grpc_error* error, + const char* reason); +/// Yields the call combiner to the next closure in the queue, if any. +void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + const char* reason); +#endif + +/// Tells \a call_combiner to invoke \a closure when +/// grpc_call_combiner_cancel() is called. If grpc_call_combiner_cancel() +/// was previously called, \a closure will be invoked immediately. +/// If \a closure is NULL, then no closure will be invoked on +/// cancellation; this effectively unregisters the previously set closure. +void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_closure* closure); + +/// Indicates that the call has been cancelled. +void grpc_call_combiner_cancel(grpc_exec_ctx* exec_ctx, + grpc_call_combiner* call_combiner, + grpc_error* error); + +#endif /* GRPC_CORE_LIB_IOMGR_CALL_COMBINER_H */ diff --git a/src/core/lib/profiling/timers.h b/src/core/lib/profiling/timers.h index 58e6659e6d..4d1437f606 100644 --- a/src/core/lib/profiling/timers.h +++ b/src/core/lib/profiling/timers.h @@ -37,7 +37,8 @@ void gpr_timers_set_log_filename(const char *filename); void gpr_timer_set_enabled(int enabled); -#if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER)) +#if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER) + \ + defined(GRPC_CUSTOM_PROFILER)) /* No profiling. No-op all the things. */ #define GPR_TIMER_MARK(tag, important) \ do { \ @@ -56,6 +57,12 @@ void gpr_timer_set_enabled(int enabled); #if defined(GRPC_STAP_PROFILER) && defined(GRPC_BASIC_PROFILER) #error "GRPC_STAP_PROFILER and GRPC_BASIC_PROFILER are mutually exclusive." #endif +#if defined(GRPC_STAP_PROFILER) && defined(GRPC_CUSTOM_PROFILER) +#error "GRPC_STAP_PROFILER and GRPC_CUSTOM_PROFILER are mutually exclusive." +#endif +#if defined(GRPC_CUSTOM_PROFILER) && defined(GRPC_BASIC_PROFILER) +#error "GRPC_CUSTOM_PROFILER and GRPC_BASIC_PROFILER are mutually exclusive." +#endif /* Generic profiling interface. */ #define GPR_TIMER_MARK(tag, important) \ @@ -80,22 +87,25 @@ void gpr_timer_set_enabled(int enabled); #ifdef __cplusplus } -#if (defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER)) +#if (defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER) + \ + defined(GRPC_CUSTOM_PROFILER)) namespace grpc { class ProfileScope { public: - ProfileScope(const char *desc, bool important) : desc_(desc) { - GPR_TIMER_BEGIN(desc_, important ? 1 : 0); + ProfileScope(const char *desc, bool important, const char *file, int line) + : desc_(desc) { + gpr_timer_begin((desc_, important ? 1 : 0, file, line); } - ~ProfileScope() { GPR_TIMER_END(desc_, 0); } + ~ProfileScope() { gpr_timer_end(desc_, 0, "n/a", 0); } private: const char *const desc_; }; -} +} // namespace grpc -#define GPR_TIMER_SCOPE(tag, important) \ - ::grpc::ProfileScope _profile_scope_##__LINE__((tag), (important)) +#define GPR_TIMER_SCOPE(tag, important) \ + ::grpc::ProfileScope _profile_scope_##__LINE__((tag), (important), __FILE__, \ + __LINE__) #else #define GPR_TIMER_SCOPE(tag, important) \ do { \ diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index 531a88434f..e3f0163a6c 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -39,6 +39,7 @@ /* We can have a per-call credentials. */ typedef struct { + grpc_call_combiner *call_combiner; grpc_call_credentials *creds; bool have_host; bool have_method; @@ -49,17 +50,11 @@ typedef struct { pollset_set so that work can progress when this call wants work to progress */ grpc_polling_entity *pollent; - gpr_atm security_context_set; - gpr_mu security_context_mu; grpc_credentials_mdelem_array md_array; grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT]; grpc_auth_metadata_context auth_md_context; - grpc_closure closure; - // Either 0 (no cancellation and no async operation in flight), - // a grpc_closure* (if the lowest bit is 0), - // or a grpc_error* (if the lowest bit is 1). - gpr_atm cancellation_state; - grpc_closure cancel_closure; + grpc_closure async_cancel_closure; + grpc_closure async_result_closure; } call_data; /* We can have a per-channel credentials. */ @@ -68,43 +63,6 @@ typedef struct { grpc_auth_context *auth_context; } channel_data; -static void decode_cancel_state(gpr_atm cancel_state, grpc_closure **func, - grpc_error **error) { - // If the lowest bit is 1, the value is a grpc_error*. - // Otherwise, if non-zdero, the value is a grpc_closure*. - if (cancel_state & 1) { - *error = (grpc_error *)(cancel_state & ~(gpr_atm)1); - } else if (cancel_state != 0) { - *func = (grpc_closure *)cancel_state; - } -} - -static gpr_atm encode_cancel_state_error(grpc_error *error) { - // Set the lowest bit to 1 to indicate that it's an error. - return (gpr_atm)1 | (gpr_atm)error; -} - -// Returns an error if the call has been cancelled. Otherwise, sets the -// cancellation function to be called upon cancellation. -static grpc_error *set_cancel_func(grpc_call_element *elem, - grpc_iomgr_cb_func func) { - call_data *calld = (call_data *)elem->call_data; - // Decode original state. - gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state); - grpc_error *original_error = GRPC_ERROR_NONE; - grpc_closure *original_func = NULL; - decode_cancel_state(original_state, &original_func, &original_error); - // If error is set, return it. - if (original_error != GRPC_ERROR_NONE) return GRPC_ERROR_REF(original_error); - // Otherwise, store func. - GRPC_CLOSURE_INIT(&calld->cancel_closure, func, elem, - grpc_schedule_on_exec_ctx); - GPR_ASSERT(((gpr_atm)&calld->cancel_closure & (gpr_atm)1) == 0); - gpr_atm_rel_store(&calld->cancellation_state, - (gpr_atm)&calld->cancel_closure); - return GRPC_ERROR_NONE; -} - static void reset_auth_metadata_context( grpc_auth_metadata_context *auth_md_context) { if (auth_md_context->service_url != NULL) { @@ -135,6 +93,7 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg, grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg; grpc_call_element *elem = batch->handler_private.extra_arg; call_data *calld = elem->call_data; + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL); reset_auth_metadata_context(&calld->auth_md_context); grpc_error *error = GRPC_ERROR_REF(input_error); if (error == GRPC_ERROR_NONE) { @@ -153,7 +112,8 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg, } else { error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED); - grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error); + grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error, + calld->call_combiner); } } @@ -223,7 +183,8 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Incompatible credentials set on channel and call."), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED)); + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED), + calld->call_combiner); return; } } else { @@ -234,22 +195,23 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx, build_auth_metadata_context(&chand->security_connector->base, chand->auth_context, calld); - grpc_error *cancel_error = set_cancel_func(elem, cancel_get_request_metadata); - if (cancel_error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, - cancel_error); - return; - } GPR_ASSERT(calld->pollent != NULL); - GRPC_CLOSURE_INIT(&calld->closure, on_credentials_metadata, batch, - grpc_schedule_on_exec_ctx); + + GRPC_CLOSURE_INIT(&calld->async_result_closure, on_credentials_metadata, + batch, grpc_schedule_on_exec_ctx); grpc_error *error = GRPC_ERROR_NONE; if (grpc_call_credentials_get_request_metadata( exec_ctx, calld->creds, calld->pollent, calld->auth_md_context, - &calld->md_array, &calld->closure, &error)) { + &calld->md_array, &calld->async_result_closure, &error)) { // Synchronous return; invoke on_credentials_metadata() directly. on_credentials_metadata(exec_ctx, batch, error); GRPC_ERROR_UNREF(error); + } else { + // Async return; register cancellation closure with call combiner. + GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_get_request_metadata, + elem, grpc_schedule_on_exec_ctx); + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, + &calld->async_cancel_closure); } } @@ -258,7 +220,7 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg, grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg; grpc_call_element *elem = batch->handler_private.extra_arg; call_data *calld = elem->call_data; - + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL); if (error == GRPC_ERROR_NONE) { send_security_metadata(exec_ctx, elem, batch); } else { @@ -271,7 +233,8 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg, exec_ctx, batch, grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg), GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNAUTHENTICATED)); + GRPC_STATUS_UNAUTHENTICATED), + calld->call_combiner); gpr_free(error_msg); } } @@ -282,7 +245,7 @@ static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg, call_data *calld = (call_data *)elem->call_data; channel_data *chand = (channel_data *)elem->channel_data; grpc_channel_security_connector_cancel_check_call_host( - exec_ctx, chand->security_connector, &calld->closure, + exec_ctx, chand->security_connector, &calld->async_result_closure, GRPC_ERROR_REF(error)); } @@ -295,52 +258,19 @@ static void auth_start_transport_stream_op_batch( call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - if (batch->cancel_stream) { - while (true) { - // Decode the original cancellation state. - gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state); - grpc_error *cancel_error = GRPC_ERROR_NONE; - grpc_closure *func = NULL; - decode_cancel_state(original_state, &func, &cancel_error); - // If we had already set a cancellation error, there's nothing - // more to do. - if (cancel_error != GRPC_ERROR_NONE) break; - // If there's a cancel func, call it. - // Note that even if the cancel func has been changed by some - // other thread between when we decoded it and now, it will just - // be a no-op. - cancel_error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); - if (func != NULL) { - GRPC_CLOSURE_SCHED(exec_ctx, func, GRPC_ERROR_REF(cancel_error)); - } - // Encode the new error into cancellation state. - if (gpr_atm_full_cas(&calld->cancellation_state, original_state, - encode_cancel_state_error(cancel_error))) { - break; // Success. - } - // The cas failed, so try again. - } - } else { - /* double checked lock over security context to ensure it's set once */ - if (gpr_atm_acq_load(&calld->security_context_set) == 0) { - gpr_mu_lock(&calld->security_context_mu); - if (gpr_atm_acq_load(&calld->security_context_set) == 0) { - GPR_ASSERT(batch->payload->context != NULL); - if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) { - batch->payload->context[GRPC_CONTEXT_SECURITY].value = - grpc_client_security_context_create(); - batch->payload->context[GRPC_CONTEXT_SECURITY].destroy = - grpc_client_security_context_destroy; - } - grpc_client_security_context *sec_ctx = - batch->payload->context[GRPC_CONTEXT_SECURITY].value; - GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter"); - sec_ctx->auth_context = - GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter"); - gpr_atm_rel_store(&calld->security_context_set, 1); - } - gpr_mu_unlock(&calld->security_context_mu); + if (!batch->cancel_stream) { + GPR_ASSERT(batch->payload->context != NULL); + if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) { + batch->payload->context[GRPC_CONTEXT_SECURITY].value = + grpc_client_security_context_create(); + batch->payload->context[GRPC_CONTEXT_SECURITY].destroy = + grpc_client_security_context_destroy; } + grpc_client_security_context *sec_ctx = + batch->payload->context[GRPC_CONTEXT_SECURITY].value; + GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter"); + sec_ctx->auth_context = + GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter"); } if (batch->send_initial_metadata) { @@ -365,26 +295,25 @@ static void auth_start_transport_stream_op_batch( } } if (calld->have_host) { - grpc_error *cancel_error = set_cancel_func(elem, cancel_check_call_host); - if (cancel_error != GRPC_ERROR_NONE) { - grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, - cancel_error); + batch->handler_private.extra_arg = elem; + GRPC_CLOSURE_INIT(&calld->async_result_closure, on_host_checked, batch, + grpc_schedule_on_exec_ctx); + char *call_host = grpc_slice_to_c_string(calld->host); + grpc_error *error = GRPC_ERROR_NONE; + if (grpc_channel_security_connector_check_call_host( + exec_ctx, chand->security_connector, call_host, + chand->auth_context, &calld->async_result_closure, &error)) { + // Synchronous return; invoke on_host_checked() directly. + on_host_checked(exec_ctx, batch, error); + GRPC_ERROR_UNREF(error); } else { - char *call_host = grpc_slice_to_c_string(calld->host); - batch->handler_private.extra_arg = elem; - grpc_error *error = GRPC_ERROR_NONE; - if (grpc_channel_security_connector_check_call_host( - exec_ctx, chand->security_connector, call_host, - chand->auth_context, - GRPC_CLOSURE_INIT(&calld->closure, on_host_checked, batch, - grpc_schedule_on_exec_ctx), - &error)) { - // Synchronous return; invoke on_host_checked() directly. - on_host_checked(exec_ctx, batch, error); - GRPC_ERROR_UNREF(error); - } - gpr_free(call_host); + // Async return; register cancellation closure with call combiner. + GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_check_call_host, + elem, grpc_schedule_on_exec_ctx); + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, + &calld->async_cancel_closure); } + gpr_free(call_host); GPR_TIMER_END("auth_start_transport_stream_op_batch", 0); return; /* early exit */ } @@ -400,8 +329,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { call_data *calld = elem->call_data; - memset(calld, 0, sizeof(*calld)); - gpr_mu_init(&calld->security_context_mu); + calld->call_combiner = args->call_combiner; return GRPC_ERROR_NONE; } @@ -426,12 +354,6 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_slice_unref_internal(exec_ctx, calld->method); } reset_auth_metadata_context(&calld->auth_md_context); - gpr_mu_destroy(&calld->security_context_mu); - gpr_atm cancel_state = gpr_atm_acq_load(&calld->cancellation_state); - grpc_error *cancel_error = GRPC_ERROR_NONE; - grpc_closure *cancel_func = NULL; - decode_cancel_state(cancel_state, &cancel_func, &cancel_error); - GRPC_ERROR_UNREF(cancel_error); } /* Constructor for channel_data */ @@ -490,6 +412,5 @@ const grpc_channel_filter grpc_client_auth_filter = { sizeof(channel_data), init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, grpc_channel_next_get_info, "client-auth"}; diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index 9bf3f0ca0f..b721ce4a22 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -26,7 +26,15 @@ #include "src/core/lib/security/transport/auth_filters.h" #include "src/core/lib/slice/slice_internal.h" +typedef enum { + STATE_INIT = 0, + STATE_DONE, + STATE_CANCELLED, +} async_state; + typedef struct call_data { + grpc_call_combiner *call_combiner; + grpc_call_stack *owning_call; grpc_transport_stream_op_batch *recv_initial_metadata_batch; grpc_closure *original_recv_initial_metadata_ready; grpc_closure recv_initial_metadata_ready; @@ -34,6 +42,8 @@ typedef struct call_data { const grpc_metadata *consumed_md; size_t num_consumed_md; grpc_auth_context *auth_context; + grpc_closure cancel_closure; + gpr_atm state; // async_state } call_data; typedef struct channel_data { @@ -78,54 +88,92 @@ static grpc_filtered_mdelem remove_consumed_md(grpc_exec_ctx *exec_ctx, return GRPC_FILTERED_MDELEM(md); } -/* called from application code */ -static void on_md_processing_done( - void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md, - const grpc_metadata *response_md, size_t num_response_md, - grpc_status_code status, const char *error_details) { - grpc_call_element *elem = user_data; +static void on_md_processing_done_inner(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + const grpc_metadata *consumed_md, + size_t num_consumed_md, + const grpc_metadata *response_md, + size_t num_response_md, + grpc_error *error) { call_data *calld = elem->call_data; grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, NULL); /* TODO(jboeuf): Implement support for response_md. */ if (response_md != NULL && num_response_md > 0) { gpr_log(GPR_INFO, "response_md in auth metadata processing not supported for now. " "Ignoring..."); } - grpc_error *error = GRPC_ERROR_NONE; - if (status == GRPC_STATUS_OK) { + if (error == GRPC_ERROR_NONE) { calld->consumed_md = consumed_md; calld->num_consumed_md = num_consumed_md; error = grpc_metadata_batch_filter( - &exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata, + exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata, remove_consumed_md, elem, "Response metadata filtering error"); - } else { - if (error_details == NULL) { - error_details = "Authentication metadata processing failed."; + } + GRPC_CLOSURE_SCHED(exec_ctx, calld->original_recv_initial_metadata_ready, + error); +} + +// Called from application code. +static void on_md_processing_done( + void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md, + const grpc_metadata *response_md, size_t num_response_md, + grpc_status_code status, const char *error_details) { + grpc_call_element *elem = user_data; + call_data *calld = elem->call_data; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + // If the call was not cancelled while we were in flight, process the result. + if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT, + (gpr_atm)STATE_DONE)) { + grpc_error *error = GRPC_ERROR_NONE; + if (status != GRPC_STATUS_OK) { + if (error_details == NULL) { + error_details = "Authentication metadata processing failed."; + } + error = grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details), + GRPC_ERROR_INT_GRPC_STATUS, status); } - error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details), - GRPC_ERROR_INT_GRPC_STATUS, status); + on_md_processing_done_inner(&exec_ctx, elem, consumed_md, num_consumed_md, + response_md, num_response_md, error); } + // Clean up. for (size_t i = 0; i < calld->md.count; i++) { grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key); grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value); } grpc_metadata_array_destroy(&calld->md); - GRPC_CLOSURE_SCHED(&exec_ctx, calld->original_recv_initial_metadata_ready, - error); + GRPC_CALL_STACK_UNREF(&exec_ctx, calld->owning_call, "server_auth_metadata"); grpc_exec_ctx_finish(&exec_ctx); } +static void cancel_call(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_call_element *elem = (grpc_call_element *)arg; + call_data *calld = elem->call_data; + // If the result was not already processed, invoke the callback now. + if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT, + (gpr_atm)STATE_CANCELLED)) { + on_md_processing_done_inner(exec_ctx, elem, NULL, 0, NULL, 0, + GRPC_ERROR_REF(error)); + } +} + static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - grpc_call_element *elem = arg; + grpc_call_element *elem = (grpc_call_element *)arg; channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch; if (error == GRPC_ERROR_NONE) { if (chand->creds != NULL && chand->creds->processor.process != NULL) { + // We're calling out to the application, so we need to make sure + // to drop the call combiner early if we get cancelled. + GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem, + grpc_schedule_on_exec_ctx); + grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner, + &calld->cancel_closure); + GRPC_CALL_STACK_REF(calld->owning_call, "server_auth_metadata"); calld->md = metadata_batch_to_md_array( batch->payload->recv_initial_metadata.recv_initial_metadata); chand->creds->processor.process( @@ -159,6 +207,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, const grpc_call_element_args *args) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; + calld->call_combiner = args->call_combiner; + calld->owning_call = args->call_stack; GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready, recv_initial_metadata_ready, elem, grpc_schedule_on_exec_ctx); @@ -218,6 +268,5 @@ const grpc_channel_filter grpc_server_auth_filter = { sizeof(channel_data), init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, grpc_channel_next_get_info, "server-auth"}; diff --git a/src/core/lib/support/block_annotate.h b/src/core/lib/support/block_annotate.h index 0a2cb45018..8e3ef7df65 100644 --- a/src/core/lib/support/block_annotate.h +++ b/src/core/lib/support/block_annotate.h @@ -19,15 +19,37 @@ #ifndef GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H #define GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H +#ifdef __cplusplus +extern "C" { +#endif + +void gpr_thd_start_blocking_region(); +void gpr_thd_end_blocking_region(); + +#ifdef __cplusplus +} +#endif + /* These annotations identify the beginning and end of regions where the code may block for reasons other than synchronization functions. These include poll, epoll, and getaddrinfo. */ +#ifdef GRPC_SCHEDULING_MARK_BLOCKING_REGION +#define GRPC_SCHEDULING_START_BLOCKING_REGION \ + do { \ + gpr_thd_start_blocking_region(); \ + } while (0) +#define GRPC_SCHEDULING_END_BLOCKING_REGION \ + do { \ + gpr_thd_end_blocking_region(); \ + } while (0) +#else #define GRPC_SCHEDULING_START_BLOCKING_REGION \ do { \ } while (0) #define GRPC_SCHEDULING_END_BLOCKING_REGION \ do { \ } while (0) +#endif #endif /* GRPC_CORE_LIB_SUPPORT_BLOCK_ANNOTATE_H */ diff --git a/src/core/lib/support/thd_internal.h b/src/core/lib/support/thd_internal.h deleted file mode 100644 index cc468c7846..0000000000 --- a/src/core/lib/support/thd_internal.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H -#define GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H - -/* Internal interfaces between modules within the gpr support library. */ - -#endif /* GRPC_CORE_LIB_SUPPORT_THD_INTERNAL_H */ diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 00ec9c7c9a..4a7152ea60 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -121,6 +121,7 @@ typedef struct batch_control { bool is_closure; } notify_tag; } completion_data; + grpc_closure start_batch; grpc_closure finish_batch; gpr_refcount steps_to_complete; @@ -147,6 +148,7 @@ typedef struct { struct grpc_call { gpr_refcount ext_ref; gpr_arena *arena; + grpc_call_combiner call_combiner; grpc_completion_queue *cq; grpc_polling_entity pollent; grpc_channel *channel; @@ -183,6 +185,11 @@ struct grpc_call { Element 0 is initial metadata, element 1 is trailing metadata. */ grpc_metadata_array *buffered_metadata[2]; + grpc_metadata compression_md; + + // A char* indicating the peer name. + gpr_atm peer_string; + /* Packed received call statuses from various sources */ gpr_atm status[STATUS_SOURCE_COUNT]; @@ -192,8 +199,12 @@ struct grpc_call { /* Compression algorithm for *incoming* data */ grpc_compression_algorithm incoming_compression_algorithm; + /* Stream compression algorithm for *incoming* data */ + grpc_stream_compression_algorithm incoming_stream_compression_algorithm; /* Supported encodings (compression algorithms), a bitset */ uint32_t encodings_accepted_by_peer; + /* Supported stream encodings (stream compression algorithms), a bitset */ + uint32_t stream_encodings_accepted_by_peer; /* Contexts for various subsystems (security, tracing, ...). */ grpc_call_context_element context[GRPC_CONTEXT_COUNT]; @@ -241,8 +252,9 @@ grpc_tracer_flag grpc_compression_trace = #define CALL_FROM_TOP_ELEM(top_elem) \ CALL_FROM_CALL_STACK(grpc_call_stack_from_top_element(top_elem)) -static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_transport_stream_op_batch *op); +static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call, + grpc_transport_stream_op_batch *op, + grpc_closure *start_batch_closure); static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_status_code status, const char *description); @@ -307,6 +319,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, sizeof(grpc_call) + channel_stack->call_stack_size); gpr_ref_init(&call->ext_ref, 1); call->arena = arena; + grpc_call_combiner_init(&call->call_combiner); *out_call = call; call->channel = args->channel; call->cq = args->cq; @@ -410,7 +423,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, .path = path, .start_time = call->start_time, .deadline = send_deadline, - .arena = call->arena}; + .arena = call->arena, + .call_combiner = &call->call_combiner}; add_init_error(&error, grpc_call_stack_init(exec_ctx, channel_stack, 1, destroy_call, call, &call_args)); if (error != GRPC_ERROR_NONE) { @@ -477,6 +491,8 @@ static void release_call(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { grpc_call *c = call; grpc_channel *channel = c->channel; + grpc_call_combiner_destroy(&c->call_combiner); + gpr_free((char *)c->peer_string); grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena)); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); } @@ -576,30 +592,37 @@ grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) { return GRPC_CALL_OK; } -static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_transport_stream_op_batch *op) { - grpc_call_element *elem; - - GPR_TIMER_BEGIN("execute_op", 0); - elem = CALL_ELEM_FROM_CALL(call, 0); - elem->filter->start_transport_stream_op_batch(exec_ctx, elem, op); - GPR_TIMER_END("execute_op", 0); +// This is called via the call combiner to start sending a batch down +// the filter stack. +static void execute_batch_in_call_combiner(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *ignored) { + grpc_transport_stream_op_batch *batch = arg; + grpc_call *call = batch->handler_private.extra_arg; + GPR_TIMER_BEGIN("execute_batch", 0); + grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0); + GRPC_CALL_LOG_OP(GPR_INFO, elem, batch); + elem->filter->start_transport_stream_op_batch(exec_ctx, elem, batch); + GPR_TIMER_END("execute_batch", 0); +} + +// start_batch_closure points to a caller-allocated closure to be used +// for entering the call combiner. +static void execute_batch(grpc_exec_ctx *exec_ctx, grpc_call *call, + grpc_transport_stream_op_batch *batch, + grpc_closure *start_batch_closure) { + batch->handler_private.extra_arg = call; + GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch, + grpc_schedule_on_exec_ctx); + GRPC_CALL_COMBINER_START(exec_ctx, &call->call_combiner, start_batch_closure, + GRPC_ERROR_NONE, "executing batch"); } char *grpc_call_get_peer(grpc_call *call) { - grpc_call_element *elem = CALL_ELEM_FROM_CALL(call, 0); - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - char *result; - GRPC_API_TRACE("grpc_call_get_peer(%p)", 1, (call)); - result = elem->filter->get_peer(&exec_ctx, elem); - if (result == NULL) { - result = grpc_channel_get_target(call->channel); - } - if (result == NULL) { - result = gpr_strdup("unknown"); - } - grpc_exec_ctx_finish(&exec_ctx); - return result; + char *peer_string = (char *)gpr_atm_acq_load(&call->peer_string); + if (peer_string != NULL) return gpr_strdup(peer_string); + peer_string = grpc_channel_get_target(call->channel); + if (peer_string != NULL) return peer_string; + return gpr_strdup("unknown"); } grpc_call *grpc_call_from_top_element(grpc_call_element *elem) { @@ -626,20 +649,41 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, return GRPC_CALL_OK; } -static void done_termination(grpc_exec_ctx *exec_ctx, void *call, +typedef struct { + grpc_call *call; + grpc_closure start_batch; + grpc_closure finish_batch; +} cancel_state; + +// The on_complete callback used when sending a cancel_stream batch down +// the filter stack. Yields the call combiner when the batch is done. +static void done_termination(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "termination"); + cancel_state *state = (cancel_state *)arg; + GRPC_CALL_COMBINER_STOP(exec_ctx, &state->call->call_combiner, + "on_complete for cancel_stream op"); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, state->call, "termination"); + gpr_free(state); } static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_error *error) { GRPC_CALL_INTERNAL_REF(c, "termination"); + // Inform the call combiner of the cancellation, so that it can cancel + // any in-flight asynchronous actions that may be holding the call + // combiner. This ensures that the cancel_stream batch can be sent + // down the filter stack in a timely manner. + grpc_call_combiner_cancel(exec_ctx, &c->call_combiner, GRPC_ERROR_REF(error)); set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error)); - grpc_transport_stream_op_batch *op = grpc_make_transport_stream_op( - GRPC_CLOSURE_CREATE(done_termination, c, grpc_schedule_on_exec_ctx)); + cancel_state *state = (cancel_state *)gpr_malloc(sizeof(*state)); + state->call = c; + GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state, + grpc_schedule_on_exec_ctx); + grpc_transport_stream_op_batch *op = + grpc_make_transport_stream_op(&state->finish_batch); op->cancel_stream = true; op->payload->cancel_stream.cancel_error = error; - execute_op(exec_ctx, c, op); + execute_batch(exec_ctx, c, op, &state->start_batch); } static grpc_error *error_from_status(grpc_status_code status, @@ -752,6 +796,12 @@ static void set_incoming_compression_algorithm( call->incoming_compression_algorithm = algo; } +static void set_incoming_stream_compression_algorithm( + grpc_call *call, grpc_stream_compression_algorithm algo) { + GPR_ASSERT(algo < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); + call->incoming_stream_compression_algorithm = algo; +} + grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( grpc_call *call) { grpc_compression_algorithm algorithm; @@ -765,6 +815,13 @@ static grpc_compression_algorithm compression_algorithm_for_level_locked( call->encodings_accepted_by_peer); } +static grpc_stream_compression_algorithm +stream_compression_algorithm_for_level_locked( + grpc_call *call, grpc_stream_compression_level level) { + return grpc_stream_compression_algorithm_for_level( + level, call->stream_encodings_accepted_by_peer); +} + uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) { uint32_t flags; flags = call->test_only_last_message_flags; @@ -819,12 +876,70 @@ static void set_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx, (void *)(((uintptr_t)call->encodings_accepted_by_peer) + 1)); } +static void set_stream_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx, + grpc_call *call, + grpc_mdelem mdel) { + size_t i; + grpc_stream_compression_algorithm algorithm; + grpc_slice_buffer accept_encoding_parts; + grpc_slice accept_encoding_slice; + void *accepted_user_data; + + accepted_user_data = + grpc_mdelem_get_user_data(mdel, destroy_encodings_accepted_by_peer); + if (accepted_user_data != NULL) { + call->stream_encodings_accepted_by_peer = + (uint32_t)(((uintptr_t)accepted_user_data) - 1); + return; + } + + accept_encoding_slice = GRPC_MDVALUE(mdel); + grpc_slice_buffer_init(&accept_encoding_parts); + grpc_slice_split(accept_encoding_slice, ",", &accept_encoding_parts); + + /* Always support no compression */ + GPR_BITSET(&call->stream_encodings_accepted_by_peer, + GRPC_STREAM_COMPRESS_NONE); + for (i = 0; i < accept_encoding_parts.count; i++) { + grpc_slice accept_encoding_entry_slice = accept_encoding_parts.slices[i]; + if (grpc_stream_compression_algorithm_parse(accept_encoding_entry_slice, + &algorithm)) { + GPR_BITSET(&call->stream_encodings_accepted_by_peer, algorithm); + } else { + char *accept_encoding_entry_str = + grpc_slice_to_c_string(accept_encoding_entry_slice); + gpr_log(GPR_ERROR, + "Invalid entry in accept encoding metadata: '%s'. Ignoring.", + accept_encoding_entry_str); + gpr_free(accept_encoding_entry_str); + } + } + + grpc_slice_buffer_destroy_internal(exec_ctx, &accept_encoding_parts); + + grpc_mdelem_set_user_data( + mdel, destroy_encodings_accepted_by_peer, + (void *)(((uintptr_t)call->stream_encodings_accepted_by_peer) + 1)); +} + uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) { uint32_t encodings_accepted_by_peer; encodings_accepted_by_peer = call->encodings_accepted_by_peer; return encodings_accepted_by_peer; } +uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer( + grpc_call *call) { + uint32_t stream_encodings_accepted_by_peer; + stream_encodings_accepted_by_peer = call->stream_encodings_accepted_by_peer; + return stream_encodings_accepted_by_peer; +} + +grpc_stream_compression_algorithm +grpc_call_test_only_get_incoming_stream_encodings(grpc_call *call) { + return call->incoming_stream_compression_algorithm; +} + static grpc_linked_mdelem *linked_from_md(const grpc_metadata *md) { return (grpc_linked_mdelem *)&md->internal_data; } @@ -936,6 +1051,22 @@ static grpc_compression_algorithm decode_compression(grpc_mdelem md) { return algorithm; } +static grpc_stream_compression_algorithm decode_stream_compression( + grpc_mdelem md) { + grpc_stream_compression_algorithm algorithm = + grpc_stream_compression_algorithm_from_slice(GRPC_MDVALUE(md)); + if (algorithm == GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) { + char *md_c_str = grpc_slice_to_c_string(GRPC_MDVALUE(md)); + gpr_log(GPR_ERROR, + "Invalid incoming stream compression algorithm: '%s'. Interpreting " + "incoming data as uncompressed.", + md_c_str); + gpr_free(md_c_str); + return GRPC_STREAM_COMPRESS_NONE; + } + return algorithm; +} + static void publish_app_metadata(grpc_call *call, grpc_metadata_batch *b, int is_trailing) { if (b->list.count == 0) return; @@ -960,7 +1091,19 @@ static void publish_app_metadata(grpc_call *call, grpc_metadata_batch *b, static void recv_initial_filter(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_metadata_batch *b) { - if (b->idx.named.grpc_encoding != NULL) { + if (b->idx.named.content_encoding != NULL) { + if (b->idx.named.grpc_encoding != NULL) { + gpr_log(GPR_ERROR, + "Received both content-encoding and grpc-encoding header. " + "Ignoring grpc-encoding."); + grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_encoding); + } + GPR_TIMER_BEGIN("incoming_stream_compression_algorithm", 0); + set_incoming_stream_compression_algorithm( + call, decode_stream_compression(b->idx.named.content_encoding->md)); + GPR_TIMER_END("incoming_stream_compression_algorithm", 0); + grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.content_encoding); + } else if (b->idx.named.grpc_encoding != NULL) { GPR_TIMER_BEGIN("incoming_compression_algorithm", 0); set_incoming_compression_algorithm( call, decode_compression(b->idx.named.grpc_encoding->md)); @@ -974,6 +1117,13 @@ static void recv_initial_filter(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_accept_encoding); GPR_TIMER_END("encodings_accepted_by_peer", 0); } + if (b->idx.named.accept_encoding != NULL) { + GPR_TIMER_BEGIN("stream_encodings_accepted_by_peer", 0); + set_stream_encodings_accepted_by_peer(exec_ctx, call, + b->idx.named.accept_encoding->md); + grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.accept_encoding); + GPR_TIMER_END("stream_encodings_accepted_by_peer", 0); + } publish_app_metadata(call, b, false); } @@ -1298,11 +1448,64 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } } +// The recv_message_ready callback used when sending a batch containing +// a recv_message op down the filter stack. Yields the call combiner +// before processing the received message. +static void receiving_stream_ready_in_call_combiner(grpc_exec_ctx *exec_ctx, + void *bctlp, + grpc_error *error) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "recv_message_ready"); + receiving_stream_ready(exec_ctx, bctlp, error); +} + static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, batch_control *bctl) { grpc_call *call = bctl->call; - /* validate call->incoming_compression_algorithm */ - if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) { + /* validate compression algorithms */ + if (call->incoming_stream_compression_algorithm != + GRPC_STREAM_COMPRESS_NONE) { + const grpc_stream_compression_algorithm algo = + call->incoming_stream_compression_algorithm; + char *error_msg = NULL; + const grpc_compression_options compression_options = + grpc_channel_compression_options(call->channel); + if (algo >= GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) { + gpr_asprintf(&error_msg, + "Invalid stream compression algorithm value '%d'.", algo); + gpr_log(GPR_ERROR, "%s", error_msg); + cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_STATUS_UNIMPLEMENTED, error_msg); + } else if (grpc_compression_options_is_stream_compression_algorithm_enabled( + &compression_options, algo) == 0) { + /* check if algorithm is supported by current channel config */ + char *algo_name = NULL; + grpc_stream_compression_algorithm_name(algo, &algo_name); + gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.", + algo_name); + gpr_log(GPR_ERROR, "%s", error_msg); + cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_STATUS_UNIMPLEMENTED, error_msg); + } + gpr_free(error_msg); + + GPR_ASSERT(call->stream_encodings_accepted_by_peer != 0); + if (!GPR_BITGET(call->stream_encodings_accepted_by_peer, + call->incoming_stream_compression_algorithm)) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { + char *algo_name = NULL; + grpc_stream_compression_algorithm_name( + call->incoming_stream_compression_algorithm, &algo_name); + gpr_log( + GPR_ERROR, + "Stream compression algorithm (content-encoding = '%s') not " + "present in the bitset of accepted encodings (accept-encodings: " + "'0x%x')", + algo_name, call->stream_encodings_accepted_by_peer); + } + } + } else if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) { const grpc_compression_algorithm algo = call->incoming_compression_algorithm; char *error_msg = NULL; @@ -1329,22 +1532,20 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, call->incoming_compression_algorithm = algo; } gpr_free(error_msg); - } - /* make sure the received grpc-encoding is amongst the ones listed in - * grpc-accept-encoding */ - GPR_ASSERT(call->encodings_accepted_by_peer != 0); - if (!GPR_BITGET(call->encodings_accepted_by_peer, - call->incoming_compression_algorithm)) { - if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; - grpc_compression_algorithm_name(call->incoming_compression_algorithm, - &algo_name); - gpr_log(GPR_ERROR, - "Compression algorithm (grpc-encoding = '%s') not present in " - "the bitset of accepted encodings (grpc-accept-encodings: " - "'0x%x')", - algo_name, call->encodings_accepted_by_peer); + GPR_ASSERT(call->encodings_accepted_by_peer != 0); + if (!GPR_BITGET(call->encodings_accepted_by_peer, + call->incoming_compression_algorithm)) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { + char *algo_name = NULL; + grpc_compression_algorithm_name(call->incoming_compression_algorithm, + &algo_name); + gpr_log(GPR_ERROR, + "Compression algorithm (grpc-encoding = '%s') not present in " + "the bitset of accepted encodings (grpc-accept-encodings: " + "'0x%x')", + algo_name, call->encodings_accepted_by_peer); + } } } } @@ -1365,6 +1566,9 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, batch_control *bctl = bctlp; grpc_call *call = bctl->call; + GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, + "recv_initial_metadata_ready"); + add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false); if (error == GRPC_ERROR_NONE) { grpc_metadata_batch *md = @@ -1399,7 +1603,8 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; - + grpc_call *call = bctl->call; + GRPC_CALL_COMBINER_STOP(exec_ctx, &call->call_combiner, "on_complete"); add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false); finish_batch_step(exec_ctx, bctl); } @@ -1419,9 +1624,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, int num_completion_callbacks_needed = 1; grpc_call_error error = GRPC_CALL_OK; - // sent_initial_metadata guards against variable reuse. - grpc_metadata compression_md; - GPR_TIMER_BEGIN("grpc_call_start_batch", 0); GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag); @@ -1469,31 +1671,60 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, goto done_with_error; } /* process compression level */ - memset(&compression_md, 0, sizeof(compression_md)); + memset(&call->compression_md, 0, sizeof(call->compression_md)); size_t additional_metadata_count = 0; - grpc_compression_level effective_compression_level; + grpc_compression_level effective_compression_level = + GRPC_COMPRESS_LEVEL_NONE; + grpc_stream_compression_level effective_stream_compression_level = + GRPC_STREAM_COMPRESS_LEVEL_NONE; bool level_set = false; - if (op->data.send_initial_metadata.maybe_compression_level.is_set) { + bool stream_compression = false; + if (op->data.send_initial_metadata.maybe_stream_compression_level + .is_set) { + effective_stream_compression_level = + op->data.send_initial_metadata.maybe_stream_compression_level + .level; + level_set = true; + stream_compression = true; + } else if (op->data.send_initial_metadata.maybe_compression_level + .is_set) { effective_compression_level = op->data.send_initial_metadata.maybe_compression_level.level; level_set = true; } else { const grpc_compression_options copts = grpc_channel_compression_options(call->channel); - level_set = copts.default_level.is_set; - if (level_set) { + if (copts.default_stream_compression_level.is_set) { + level_set = true; + effective_stream_compression_level = + copts.default_stream_compression_level.level; + stream_compression = true; + } else if (copts.default_level.is_set) { + level_set = true; effective_compression_level = copts.default_level.level; } } if (level_set && !call->is_client) { - const grpc_compression_algorithm calgo = - compression_algorithm_for_level_locked( - call, effective_compression_level); - // the following will be picked up by the compress filter and used as - // the call's compression algorithm. - compression_md.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST; - compression_md.value = grpc_compression_algorithm_slice(calgo); - additional_metadata_count++; + if (stream_compression) { + const grpc_stream_compression_algorithm calgo = + stream_compression_algorithm_for_level_locked( + call, effective_stream_compression_level); + call->compression_md.key = + GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST; + call->compression_md.value = + grpc_stream_compression_algorithm_slice(calgo); + } else { + const grpc_compression_algorithm calgo = + compression_algorithm_for_level_locked( + call, effective_compression_level); + /* the following will be picked up by the compress filter and used + * as the call's compression algorithm. */ + call->compression_md.key = + GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST; + call->compression_md.value = + grpc_compression_algorithm_slice(calgo); + additional_metadata_count++; + } } if (op->data.send_initial_metadata.count + additional_metadata_count > @@ -1506,7 +1737,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (!prepare_application_metadata( exec_ctx, call, (int)op->data.send_initial_metadata.count, op->data.send_initial_metadata.metadata, 0, call->is_client, - &compression_md, (int)additional_metadata_count)) { + &call->compression_md, (int)additional_metadata_count)) { error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } @@ -1518,6 +1749,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]; stream_op_payload->send_initial_metadata.send_initial_metadata_flags = op->flags; + if (call->is_client) { + stream_op_payload->send_initial_metadata.peer_string = + &call->peer_string; + } break; case GRPC_OP_SEND_MESSAGE: if (!are_write_flags_valid(op->flags)) { @@ -1650,6 +1885,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready = &call->receiving_initial_metadata_ready; + if (!call->is_client) { + stream_op_payload->recv_initial_metadata.peer_string = + &call->peer_string; + } num_completion_callbacks_needed++; break; case GRPC_OP_RECV_MESSAGE: @@ -1666,8 +1905,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op->recv_message = true; call->receiving_buffer = op->data.recv_message.recv_message; stream_op_payload->recv_message.recv_message = &call->receiving_stream; - GRPC_CLOSURE_INIT(&call->receiving_stream_ready, receiving_stream_ready, - bctl, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&call->receiving_stream_ready, + receiving_stream_ready_in_call_combiner, bctl, + grpc_schedule_on_exec_ctx); stream_op_payload->recv_message.recv_message_ready = &call->receiving_stream_ready; num_completion_callbacks_needed++; @@ -1737,7 +1977,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op->on_complete = &bctl->finish_batch; gpr_atm_rel_store(&call->any_ops_sent_atm, 1); - execute_op(exec_ctx, call, stream_op); + execute_batch(exec_ctx, call, stream_op, &bctl->start_batch); done: GPR_TIMER_END("grpc_call_start_batch", 0); diff --git a/src/core/lib/surface/call_test_only.h b/src/core/lib/surface/call_test_only.h index 2f1b80bfd7..a5a01b3679 100644 --- a/src/core/lib/surface/call_test_only.h +++ b/src/core/lib/surface/call_test_only.h @@ -42,6 +42,18 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call *call); * To be indexed by grpc_compression_algorithm enum values. */ uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call); +/** Returns a bitset for the stream encodings (stream compression algorithms) + * supported by \a call's peer. + * + * To be indexed by grpc_stream_compression_algorithm enum values. */ +uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer( + grpc_call *call); + +/** Returns the incoming stream compression algorithm (content-encoding header) + * received by a call. */ +grpc_stream_compression_algorithm +grpc_call_test_only_get_incoming_stream_encodings(grpc_call *call); + #ifdef __cplusplus } #endif diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 5780a18ce8..850fbe6a69 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -142,6 +142,16 @@ grpc_channel *grpc_channel_create_with_builder( GRPC_COMPRESS_LEVEL_NONE, GRPC_COMPRESS_LEVEL_COUNT - 1}); } else if (0 == strcmp(args->args[i].key, + GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) { + channel->compression_options.default_stream_compression_level.is_set = + true; + channel->compression_options.default_stream_compression_level.level = + (grpc_stream_compression_level)grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){GRPC_STREAM_COMPRESS_LEVEL_NONE, + GRPC_STREAM_COMPRESS_LEVEL_NONE, + GRPC_STREAM_COMPRESS_LEVEL_COUNT - 1}); + } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { channel->compression_options.default_algorithm.is_set = true; channel->compression_options.default_algorithm.algorithm = @@ -149,12 +159,31 @@ grpc_channel *grpc_channel_create_with_builder( &args->args[i], (grpc_integer_options){GRPC_COMPRESS_NONE, GRPC_COMPRESS_NONE, GRPC_COMPRESS_ALGORITHMS_COUNT - 1}); + } else if (0 == strcmp(args->args[i].key, + GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { + channel->compression_options.default_stream_compression_algorithm.is_set = + true; + channel->compression_options.default_stream_compression_algorithm + .algorithm = + (grpc_stream_compression_algorithm)grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){ + GRPC_STREAM_COMPRESS_NONE, GRPC_STREAM_COMPRESS_NONE, + GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT - 1}); } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { channel->compression_options.enabled_algorithms_bitset = (uint32_t)args->args[i].value.integer | 0x1; /* always support no compression */ + } else if (0 == + strcmp( + args->args[i].key, + GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { + channel->compression_options + .enabled_stream_compression_algorithms_bitset = + (uint32_t)args->args[i].value.integer | + 0x1; /* always support no compression */ } } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index c20cfbc740..10e4e5ab0c 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -976,7 +976,6 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, if (cqd->shutdown_called) { gpr_mu_unlock(cq->mu); GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down"); - GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } cqd->shutdown_called = true; @@ -1208,7 +1207,6 @@ static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx, if (cqd->shutdown_called) { gpr_mu_unlock(cq->mu); GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down (pluck cq)"); - GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } cqd->shutdown_called = true; diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index d199ac060e..75a13d28fc 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -30,6 +30,7 @@ #include "src/core/lib/channel/handshaker_registry.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/http/parser.h" +#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" @@ -127,6 +128,7 @@ void grpc_init(void) { grpc_register_tracer(&grpc_trace_channel_stack_builder); grpc_register_tracer(&grpc_http1_trace); grpc_register_tracer(&grpc_cq_pluck_trace); // default on + grpc_register_tracer(&grpc_call_combiner_trace); grpc_register_tracer(&grpc_combiner_trace); grpc_register_tracer(&grpc_server_channel_trace); grpc_register_tracer(&grpc_bdp_estimator_trace); diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index a0791080a9..6286f9159d 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -40,6 +40,7 @@ namespace grpc_core { namespace { struct CallData { + grpc_call_combiner *call_combiner; grpc_linked_mdelem status; grpc_linked_mdelem details; grpc_core::atomic<bool> filled_metadata; @@ -52,14 +53,14 @@ struct ChannelData { static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_metadata_batch *mdb) { - CallData *calld = static_cast<CallData *>(elem->call_data); + CallData *calld = reinterpret_cast<CallData *>(elem->call_data); bool expected = false; if (!calld->filled_metadata.compare_exchange_strong( expected, true, grpc_core::memory_order_relaxed, grpc_core::memory_order_relaxed)) { return; } - ChannelData *chand = static_cast<ChannelData *>(elem->channel_data); + ChannelData *chand = reinterpret_cast<ChannelData *>(elem->channel_data); char tmp[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(chand->error_code, tmp); calld->status.md = grpc_mdelem_from_slices( @@ -79,6 +80,7 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, static void lame_start_transport_stream_op_batch( grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { + CallData *calld = reinterpret_cast<CallData *>(elem->call_data); if (op->recv_initial_metadata) { fill_metadata(exec_ctx, elem, op->payload->recv_initial_metadata.recv_initial_metadata); @@ -87,12 +89,8 @@ static void lame_start_transport_stream_op_batch( op->payload->recv_trailing_metadata.recv_trailing_metadata); } grpc_transport_stream_op_batch_finish_with_failure( - exec_ctx, op, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel")); -} - -static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - return NULL; + exec_ctx, op, GRPC_ERROR_CREATE_FROM_STATIC_STRING("lame client channel"), + calld->call_combiner); } static void lame_get_channel_info(grpc_exec_ctx *exec_ctx, @@ -122,6 +120,8 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { + CallData *calld = reinterpret_cast<CallData *>(elem->call_data); + calld->call_combiner = args->call_combiner; return GRPC_ERROR_NONE; } @@ -156,7 +156,6 @@ extern "C" const grpc_channel_filter grpc_lame_filter = { sizeof(grpc_core::ChannelData), grpc_core::init_channel_elem, grpc_core::destroy_channel_elem, - grpc_core::lame_get_peer, grpc_core::lame_get_channel_info, "lame-client", }; @@ -176,7 +175,7 @@ grpc_channel *grpc_lame_client_channel_create(const char *target, "error_message=%s)", 3, (target, (int)error_code, error_message)); GPR_ASSERT(elem->filter == &grpc_lame_filter); - auto chand = static_cast<grpc_core::ChannelData *>(elem->channel_data); + auto chand = reinterpret_cast<grpc_core::ChannelData *>(elem->channel_data); chand->error_code = error_code; chand->error_message = error_message; grpc_exec_ctx_finish(&exec_ctx); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 66dcc299aa..8582d826ca 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -789,7 +789,6 @@ static void server_mutate_op(grpc_call_element *elem, static void server_start_transport_stream_op_batch( grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); server_mutate_op(elem, op); grpc_call_next_op(exec_ctx, elem, op); } @@ -962,7 +961,6 @@ const grpc_channel_filter grpc_server_top_filter = { sizeof(channel_data), init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, grpc_channel_next_get_info, "server", }; diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 2388f19f81..28f05d5c44 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -40,65 +40,68 @@ static uint8_t g_bytes[] = { 114, 45, 115, 116, 97, 116, 115, 45, 98, 105, 110, 103, 114, 112, 99, 45, 116, 97, 103, 115, 45, 98, 105, 110, 103, 114, 112, 99, 45, 116, 114, 97, 99, 101, 45, 98, 105, 110, 99, 111, 110, 116, 101, 110, 116, - 45, 116, 121, 112, 101, 103, 114, 112, 99, 45, 105, 110, 116, 101, 114, - 110, 97, 108, 45, 101, 110, 99, 111, 100, 105, 110, 103, 45, 114, 101, - 113, 117, 101, 115, 116, 117, 115, 101, 114, 45, 97, 103, 101, 110, 116, - 104, 111, 115, 116, 108, 98, 45, 116, 111, 107, 101, 110, 103, 114, 112, - 99, 45, 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, 119, - 97, 105, 116, 95, 102, 111, 114, 95, 114, 101, 97, 100, 121, 103, 114, - 112, 99, 46, 116, 105, 109, 101, 111, 117, 116, 103, 114, 112, 99, 46, - 109, 97, 120, 95, 114, 101, 113, 117, 101, 115, 116, 95, 109, 101, 115, - 115, 97, 103, 101, 95, 98, 121, 116, 101, 115, 103, 114, 112, 99, 46, - 109, 97, 120, 95, 114, 101, 115, 112, 111, 110, 115, 101, 95, 109, 101, - 115, 115, 97, 103, 101, 95, 98, 121, 116, 101, 115, 47, 103, 114, 112, - 99, 46, 108, 98, 46, 118, 49, 46, 76, 111, 97, 100, 66, 97, 108, - 97, 110, 99, 101, 114, 47, 66, 97, 108, 97, 110, 99, 101, 76, 111, - 97, 100, 48, 49, 50, 105, 100, 101, 110, 116, 105, 116, 121, 103, 122, - 105, 112, 100, 101, 102, 108, 97, 116, 101, 116, 114, 97, 105, 108, 101, - 114, 115, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 103, - 114, 112, 99, 80, 79, 83, 84, 50, 48, 48, 52, 48, 52, 104, 116, - 116, 112, 104, 116, 116, 112, 115, 103, 114, 112, 99, 71, 69, 84, 80, - 85, 84, 47, 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, 108, 50, - 48, 52, 50, 48, 54, 51, 48, 52, 52, 48, 48, 53, 48, 48, 97, - 99, 99, 101, 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 97, 99, - 99, 101, 112, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 122, - 105, 112, 44, 32, 100, 101, 102, 108, 97, 116, 101, 97, 99, 99, 101, - 112, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 97, 99, 99, 101, - 112, 116, 45, 114, 97, 110, 103, 101, 115, 97, 99, 99, 101, 112, 116, - 97, 99, 99, 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, 45, - 97, 108, 108, 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, 101, - 97, 108, 108, 111, 119, 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, - 105, 111, 110, 99, 97, 99, 104, 101, 45, 99, 111, 110, 116, 114, 111, - 108, 99, 111, 110, 116, 101, 110, 116, 45, 100, 105, 115, 112, 111, 115, - 105, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 101, 110, - 99, 111, 100, 105, 110, 103, 99, 111, 110, 116, 101, 110, 116, 45, 108, - 97, 110, 103, 117, 97, 103, 101, 99, 111, 110, 116, 101, 110, 116, 45, - 108, 101, 110, 103, 116, 104, 99, 111, 110, 116, 101, 110, 116, 45, 108, - 111, 99, 97, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, - 114, 97, 110, 103, 101, 99, 111, 111, 107, 105, 101, 100, 97, 116, 101, - 101, 116, 97, 103, 101, 120, 112, 101, 99, 116, 101, 120, 112, 105, 114, - 101, 115, 102, 114, 111, 109, 105, 102, 45, 109, 97, 116, 99, 104, 105, - 102, 45, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, - 101, 105, 102, 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, - 102, 45, 114, 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, - 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, - 45, 109, 111, 100, 105, 102, 105, 101, 100, 108, 98, 45, 99, 111, 115, - 116, 45, 98, 105, 110, 108, 105, 110, 107, 108, 111, 99, 97, 116, 105, - 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, 112, - 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, - 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, 105, - 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, 101, - 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121, - 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116, - 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, 116, - 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, 105, - 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, - 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, 45, - 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, 101, - 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, - 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, 108, - 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, - 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; + 45, 116, 121, 112, 101, 99, 111, 110, 116, 101, 110, 116, 45, 101, 110, + 99, 111, 100, 105, 110, 103, 97, 99, 99, 101, 112, 116, 45, 101, 110, + 99, 111, 100, 105, 110, 103, 103, 114, 112, 99, 45, 105, 110, 116, 101, + 114, 110, 97, 108, 45, 101, 110, 99, 111, 100, 105, 110, 103, 45, 114, + 101, 113, 117, 101, 115, 116, 103, 114, 112, 99, 45, 105, 110, 116, 101, + 114, 110, 97, 108, 45, 115, 116, 114, 101, 97, 109, 45, 101, 110, 99, + 111, 100, 105, 110, 103, 45, 114, 101, 113, 117, 101, 115, 116, 117, 115, + 101, 114, 45, 97, 103, 101, 110, 116, 104, 111, 115, 116, 108, 98, 45, + 116, 111, 107, 101, 110, 103, 114, 112, 99, 45, 116, 105, 109, 101, 111, + 117, 116, 103, 114, 112, 99, 46, 119, 97, 105, 116, 95, 102, 111, 114, + 95, 114, 101, 97, 100, 121, 103, 114, 112, 99, 46, 116, 105, 109, 101, + 111, 117, 116, 103, 114, 112, 99, 46, 109, 97, 120, 95, 114, 101, 113, + 117, 101, 115, 116, 95, 109, 101, 115, 115, 97, 103, 101, 95, 98, 121, + 116, 101, 115, 103, 114, 112, 99, 46, 109, 97, 120, 95, 114, 101, 115, + 112, 111, 110, 115, 101, 95, 109, 101, 115, 115, 97, 103, 101, 95, 98, + 121, 116, 101, 115, 47, 103, 114, 112, 99, 46, 108, 98, 46, 118, 49, + 46, 76, 111, 97, 100, 66, 97, 108, 97, 110, 99, 101, 114, 47, 66, + 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 48, 49, 50, 105, 100, + 101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, 101, 102, 108, 97, + 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 97, 112, 112, 108, 105, + 99, 97, 116, 105, 111, 110, 47, 103, 114, 112, 99, 80, 79, 83, 84, + 50, 48, 48, 52, 48, 52, 104, 116, 116, 112, 104, 116, 116, 112, 115, + 103, 114, 112, 99, 71, 69, 84, 80, 85, 84, 47, 47, 105, 110, 100, + 101, 120, 46, 104, 116, 109, 108, 50, 48, 52, 50, 48, 54, 51, 48, + 52, 52, 48, 48, 53, 48, 48, 97, 99, 99, 101, 112, 116, 45, 99, + 104, 97, 114, 115, 101, 116, 103, 122, 105, 112, 44, 32, 100, 101, 102, + 108, 97, 116, 101, 97, 99, 99, 101, 112, 116, 45, 108, 97, 110, 103, + 117, 97, 103, 101, 97, 99, 99, 101, 112, 116, 45, 114, 97, 110, 103, + 101, 115, 97, 99, 99, 101, 112, 116, 97, 99, 99, 101, 115, 115, 45, + 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, 111, 119, 45, 111, + 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, 111, 119, 97, 117, + 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 99, 97, 99, 104, + 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, 110, 116, 101, 110, + 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, 111, 110, 99, 111, + 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 99, + 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, 103, 116, 104, 99, 111, + 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, 116, 105, 111, 110, 99, + 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, 103, 101, 99, 111, 111, + 107, 105, 101, 100, 97, 116, 101, 101, 116, 97, 103, 101, 120, 112, 101, + 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, 114, 111, 109, 105, 102, + 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, 111, 100, 105, 102, 105, + 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, 45, 110, 111, 110, 101, + 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, 97, 110, 103, 101, 105, + 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, + 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, 100, 105, 102, 105, 101, + 100, 108, 98, 45, 99, 111, 115, 116, 45, 98, 105, 110, 108, 105, 110, + 107, 108, 111, 99, 97, 116, 105, 111, 110, 109, 97, 120, 45, 102, 111, + 114, 119, 97, 114, 100, 115, 112, 114, 111, 120, 121, 45, 97, 117, 116, + 104, 101, 110, 116, 105, 99, 97, 116, 101, 112, 114, 111, 120, 121, 45, + 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 114, 97, + 110, 103, 101, 114, 101, 102, 101, 114, 101, 114, 114, 101, 102, 114, 101, + 115, 104, 114, 101, 116, 114, 121, 45, 97, 102, 116, 101, 114, 115, 101, + 114, 118, 101, 114, 115, 101, 116, 45, 99, 111, 111, 107, 105, 101, 115, + 116, 114, 105, 99, 116, 45, 116, 114, 97, 110, 115, 112, 111, 114, 116, + 45, 115, 101, 99, 117, 114, 105, 116, 121, 116, 114, 97, 110, 115, 102, + 101, 114, 45, 101, 110, 99, 111, 100, 105, 110, 103, 118, 97, 114, 121, + 118, 105, 97, 119, 119, 119, 45, 97, 117, 116, 104, 101, 110, 116, 105, + 99, 97, 116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, 101, + 102, 108, 97, 116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 103, + 122, 105, 112, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112, + 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, + 101, 44, 103, 122, 105, 112}; static void static_ref(void *unused) {} static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} @@ -209,6 +212,7 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { @@ -243,193 +247,194 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {.refcount = &grpc_static_metadata_refcounts[14], .data.refcounted = {g_bytes + 158, 12}}, {.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 30}}, + .data.refcounted = {g_bytes + 170, 16}}, {.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 200, 10}}, + .data.refcounted = {g_bytes + 186, 15}}, {.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 210, 4}}, + .data.refcounted = {g_bytes + 201, 30}}, {.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 214, 8}}, + .data.refcounted = {g_bytes + 231, 37}}, {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 222, 12}}, + .data.refcounted = {g_bytes + 268, 10}}, {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}, + .data.refcounted = {g_bytes + 278, 4}}, {.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 234, 19}}, + .data.refcounted = {g_bytes + 282, 8}}, {.refcount = &grpc_static_metadata_refcounts[22], - .data.refcounted = {g_bytes + 253, 12}}, + .data.refcounted = {g_bytes + 290, 12}}, {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 265, 30}}, + .data.refcounted = {g_bytes + 302, 0}}, {.refcount = &grpc_static_metadata_refcounts[24], - .data.refcounted = {g_bytes + 295, 31}}, + .data.refcounted = {g_bytes + 302, 19}}, {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 326, 36}}, + .data.refcounted = {g_bytes + 321, 12}}, {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 362, 1}}, + .data.refcounted = {g_bytes + 333, 30}}, {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 363, 1}}, + .data.refcounted = {g_bytes + 363, 31}}, {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 364, 1}}, + .data.refcounted = {g_bytes + 394, 36}}, {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 365, 8}}, + .data.refcounted = {g_bytes + 430, 1}}, {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 373, 4}}, + .data.refcounted = {g_bytes + 431, 1}}, {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 377, 7}}, + .data.refcounted = {g_bytes + 432, 1}}, {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 384, 8}}, + .data.refcounted = {g_bytes + 433, 8}}, {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 392, 16}}, + .data.refcounted = {g_bytes + 441, 4}}, {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 408, 4}}, + .data.refcounted = {g_bytes + 445, 7}}, {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 412, 3}}, + .data.refcounted = {g_bytes + 452, 8}}, {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 415, 3}}, + .data.refcounted = {g_bytes + 460, 16}}, {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 418, 4}}, + .data.refcounted = {g_bytes + 476, 4}}, {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 422, 5}}, + .data.refcounted = {g_bytes + 480, 3}}, {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 427, 4}}, + .data.refcounted = {g_bytes + 483, 3}}, {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 431, 3}}, + .data.refcounted = {g_bytes + 486, 4}}, {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 434, 3}}, + .data.refcounted = {g_bytes + 490, 5}}, {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 437, 1}}, + .data.refcounted = {g_bytes + 495, 4}}, {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 438, 11}}, + .data.refcounted = {g_bytes + 499, 3}}, {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 449, 3}}, + .data.refcounted = {g_bytes + 502, 3}}, {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 452, 3}}, + .data.refcounted = {g_bytes + 505, 1}}, {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 455, 3}}, + .data.refcounted = {g_bytes + 506, 11}}, {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 458, 3}}, + .data.refcounted = {g_bytes + 517, 3}}, {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 461, 3}}, + .data.refcounted = {g_bytes + 520, 3}}, {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 464, 14}}, + .data.refcounted = {g_bytes + 523, 3}}, {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 478, 15}}, + .data.refcounted = {g_bytes + 526, 3}}, {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 493, 13}}, + .data.refcounted = {g_bytes + 529, 3}}, {.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 506, 15}}, + .data.refcounted = {g_bytes + 532, 14}}, {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 521, 13}}, + .data.refcounted = {g_bytes + 546, 13}}, {.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 534, 6}}, + .data.refcounted = {g_bytes + 559, 15}}, {.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 540, 27}}, + .data.refcounted = {g_bytes + 574, 13}}, {.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 567, 3}}, + .data.refcounted = {g_bytes + 587, 6}}, {.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 570, 5}}, + .data.refcounted = {g_bytes + 593, 27}}, {.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 575, 13}}, + .data.refcounted = {g_bytes + 620, 3}}, {.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 588, 13}}, + .data.refcounted = {g_bytes + 623, 5}}, {.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 601, 19}}, + .data.refcounted = {g_bytes + 628, 13}}, {.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 620, 16}}, + .data.refcounted = {g_bytes + 641, 13}}, {.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 636, 16}}, + .data.refcounted = {g_bytes + 654, 19}}, {.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 652, 14}}, + .data.refcounted = {g_bytes + 673, 16}}, {.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 666, 16}}, + .data.refcounted = {g_bytes + 689, 14}}, {.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 682, 13}}, + .data.refcounted = {g_bytes + 703, 16}}, {.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 695, 6}}, + .data.refcounted = {g_bytes + 719, 13}}, {.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 701, 4}}, + .data.refcounted = {g_bytes + 732, 6}}, {.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 705, 4}}, + .data.refcounted = {g_bytes + 738, 4}}, {.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 709, 6}}, + .data.refcounted = {g_bytes + 742, 4}}, {.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 715, 7}}, + .data.refcounted = {g_bytes + 746, 6}}, {.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 722, 4}}, + .data.refcounted = {g_bytes + 752, 7}}, {.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 726, 8}}, + .data.refcounted = {g_bytes + 759, 4}}, {.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 734, 17}}, + .data.refcounted = {g_bytes + 763, 8}}, {.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 751, 13}}, + .data.refcounted = {g_bytes + 771, 17}}, {.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 764, 8}}, + .data.refcounted = {g_bytes + 788, 13}}, {.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 772, 19}}, + .data.refcounted = {g_bytes + 801, 8}}, {.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 791, 13}}, + .data.refcounted = {g_bytes + 809, 19}}, {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 804, 11}}, + .data.refcounted = {g_bytes + 828, 13}}, {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 815, 4}}, + .data.refcounted = {g_bytes + 841, 11}}, {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 819, 8}}, + .data.refcounted = {g_bytes + 852, 4}}, {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 827, 12}}, + .data.refcounted = {g_bytes + 856, 8}}, {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 839, 18}}, + .data.refcounted = {g_bytes + 864, 12}}, {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 857, 19}}, + .data.refcounted = {g_bytes + 876, 18}}, {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 876, 5}}, + .data.refcounted = {g_bytes + 894, 19}}, {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 881, 7}}, + .data.refcounted = {g_bytes + 913, 5}}, {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 888, 7}}, + .data.refcounted = {g_bytes + 918, 7}}, {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 895, 11}}, + .data.refcounted = {g_bytes + 925, 7}}, {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 906, 6}}, + .data.refcounted = {g_bytes + 932, 11}}, {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 912, 10}}, + .data.refcounted = {g_bytes + 943, 6}}, {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 922, 25}}, + .data.refcounted = {g_bytes + 949, 10}}, {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 947, 17}}, + .data.refcounted = {g_bytes + 959, 25}}, {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 964, 4}}, + .data.refcounted = {g_bytes + 984, 17}}, {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 968, 3}}, + .data.refcounted = {g_bytes + 1001, 4}}, {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 971, 16}}, + .data.refcounted = {g_bytes + 1005, 3}}, {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 987, 16}}, + .data.refcounted = {g_bytes + 1008, 16}}, {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1003, 13}}, + .data.refcounted = {g_bytes + 1024, 16}}, {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1016, 12}}, + .data.refcounted = {g_bytes + 1040, 13}}, {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1028, 21}}, + .data.refcounted = {g_bytes + 1053, 12}}, + {.refcount = &grpc_static_metadata_refcounts[99], + .data.refcounted = {g_bytes + 1065, 21}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8, 2, 4, 4}; static const int8_t elems_r[] = { - 10, 8, -3, 0, 9, 21, -77, 22, 0, 10, -7, 0, 0, 0, - 14, 0, 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -50, -51, 16, -53, -54, -55, -56, - -56, -57, -58, -59, 0, 37, 36, 35, 34, 33, 32, 31, 30, 29, - 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, - 14, 13, 12, 11, 10, 13, 12, 11, 10, 9, 8, 7, 0}; + 11, 9, -3, 0, 10, 27, -74, 28, 0, 14, -7, 0, 0, 0, 18, 8, -2, + 0, 0, 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, -50, 0, -33, -55, -56, -57, -58, -57, 0, 40, 39, 38, 37, 36, 35, 34, + 33, 32, 31, 30, 29, 28, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 22, + 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 12, 11, 0}; static uint32_t elems_phash(uint32_t i) { - i -= 42; - uint32_t x = i % 97; - uint32_t y = i / 97; + i -= 45; + uint32_t x = i % 98; + uint32_t y = i / 98; uint32_t h = x; if (y < GPR_ARRAY_SIZE(elems_r)) { uint32_t delta = (uint32_t)elems_r[y]; @@ -439,30 +444,31 @@ static uint32_t elems_phash(uint32_t i) { } static const uint16_t elem_keys[] = { - 1019, 1020, 1021, 242, 243, 244, 245, 246, 139, 140, 42, 43, - 433, 434, 435, 920, 921, 922, 719, 720, 1406, 527, 721, 1604, - 1703, 1802, 4871, 4970, 5001, 5168, 5267, 5366, 5465, 1419, 5564, 5663, - 5762, 5861, 5960, 6059, 6158, 6257, 6356, 6455, 6554, 6653, 6752, 6851, - 6950, 7049, 7148, 7247, 7346, 7445, 7544, 7643, 7742, 7841, 7940, 8039, - 8138, 8237, 8336, 8435, 8534, 8633, 1085, 1086, 1087, 1088, 8732, 8831, - 8930, 9029, 9128, 9227, 9326, 0, 317, 0, 0, 0, 0, 0, + 1032, 1033, 1034, 247, 248, 249, 250, 251, 1623, 143, 144, 45, + 46, 440, 441, 442, 1523, 1632, 1633, 932, 933, 934, 729, 730, + 1423, 1532, 1533, 535, 731, 1923, 2023, 2123, 5223, 5523, 5623, 5723, + 5823, 1436, 1653, 5923, 6023, 6123, 6223, 6323, 6423, 6523, 6623, 6723, + 6823, 6923, 7023, 7123, 7223, 5423, 7323, 7423, 7523, 7623, 7723, 7823, + 7923, 8023, 8123, 8223, 1096, 1097, 1098, 1099, 8323, 8423, 8523, 8623, + 8723, 8823, 8923, 9023, 9123, 9223, 9323, 323, 9423, 9523, 1697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 133, 233, 234, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 137, 238, 239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0}; + 0, 0, 0, 0, 0}; static const uint8_t elem_idxs[] = { - 74, 77, 75, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, - 3, 4, 5, 0, 1, 41, 6, 2, 70, 48, 55, 24, 25, 26, 27, - 28, 29, 30, 7, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, - 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 76, 78, 79, 80, 66, 67, 68, 69, 71, - 72, 73, 255, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; + 76, 79, 77, 19, 20, 21, 22, 23, 25, 15, 16, 17, 18, 11, + 12, 13, 38, 83, 84, 3, 4, 5, 0, 1, 43, 36, 37, 6, + 2, 72, 50, 57, 24, 28, 29, 30, 31, 7, 26, 32, 33, 34, + 35, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, 27, 51, 52, + 53, 54, 55, 56, 58, 59, 60, 61, 78, 80, 81, 82, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 73, 14, 74, 75, 85, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 8, 9, 10}; grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { if (a == -1 || b == -1) return GRPC_MDNULL; - uint32_t k = (uint32_t)(a * 99 + b); + uint32_t k = (uint32_t)(a * 100 + b); uint32_t h = elems_phash(k); return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k && elem_idxs[h] != 255 @@ -474,328 +480,350 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 362, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[29], + .data.refcounted = {g_bytes + 430, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 363, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[30], + .data.refcounted = {g_bytes + 431, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[7], .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 364, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[31], + .data.refcounted = {g_bytes + 432, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 365, 8}}}, + {.refcount = &grpc_static_metadata_refcounts[32], + .data.refcounted = {g_bytes + 433, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 373, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 441, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[9], .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 377, 7}}}, + {.refcount = &grpc_static_metadata_refcounts[34], + .data.refcounted = {g_bytes + 445, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[5], .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 384, 8}}}, + {.refcount = &grpc_static_metadata_refcounts[35], + .data.refcounted = {g_bytes + 452, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[14], .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 392, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[36], + .data.refcounted = {g_bytes + 460, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 408, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[37], + .data.refcounted = {g_bytes + 476, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 412, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[38], + .data.refcounted = {g_bytes + 480, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 415, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[39], + .data.refcounted = {g_bytes + 483, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 418, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[40], + .data.refcounted = {g_bytes + 486, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 422, 5}}}, + {.refcount = &grpc_static_metadata_refcounts[41], + .data.refcounted = {g_bytes + 490, 5}}}, {{.refcount = &grpc_static_metadata_refcounts[4], .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 427, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[42], + .data.refcounted = {g_bytes + 495, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[3], .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 431, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[43], + .data.refcounted = {g_bytes + 499, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[1], .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 434, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[44], + .data.refcounted = {g_bytes + 502, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[0], .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 437, 1}}}, + {.refcount = &grpc_static_metadata_refcounts[45], + .data.refcounted = {g_bytes + 505, 1}}}, {{.refcount = &grpc_static_metadata_refcounts[0], .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 438, 11}}}, + {.refcount = &grpc_static_metadata_refcounts[46], + .data.refcounted = {g_bytes + 506, 11}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 449, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[47], + .data.refcounted = {g_bytes + 517, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 452, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[48], + .data.refcounted = {g_bytes + 520, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 455, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[49], + .data.refcounted = {g_bytes + 523, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 458, 3}}}, + {.refcount = &grpc_static_metadata_refcounts[50], + .data.refcounted = {g_bytes + 526, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[2], .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 461, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 464, 14}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 478, 15}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 478, 15}}, {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 493, 13}}}, + .data.refcounted = {g_bytes + 529, 3}}}, {{.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 506, 15}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 521, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 532, 14}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 186, 15}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 186, 15}}, + {.refcount = &grpc_static_metadata_refcounts[53], + .data.refcounted = {g_bytes + 546, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 534, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 559, 15}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 540, 27}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 574, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 567, 3}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 587, 6}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 570, 5}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 593, 27}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 575, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 620, 3}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 588, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 623, 5}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 601, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 628, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 620, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 641, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 636, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 654, 19}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[15], + .data.refcounted = {g_bytes + 170, 16}}, + {.refcount = &grpc_static_metadata_refcounts[32], + .data.refcounted = {g_bytes + 433, 8}}}, + {{.refcount = &grpc_static_metadata_refcounts[15], + .data.refcounted = {g_bytes + 170, 16}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 441, 4}}}, + {{.refcount = &grpc_static_metadata_refcounts[15], + .data.refcounted = {g_bytes + 170, 16}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 652, 14}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 673, 16}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 666, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 689, 14}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 682, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 703, 16}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[66], + .data.refcounted = {g_bytes + 719, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[14], .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 695, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 701, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 732, 6}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 705, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 738, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 709, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 742, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 715, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 746, 6}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 722, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 210, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 752, 7}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 726, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 759, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 278, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 734, 17}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 763, 8}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 751, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 771, 17}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 764, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 788, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 772, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 801, 8}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 791, 13}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 214, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 809, 19}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 804, 11}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 828, 13}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[21], + .data.refcounted = {g_bytes + 282, 8}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 815, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 841, 11}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 819, 8}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 852, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 827, 12}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 856, 8}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 839, 18}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 864, 12}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 857, 19}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 876, 18}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 876, 5}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 894, 19}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 881, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 913, 5}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 888, 7}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 918, 7}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 895, 11}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 925, 7}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 906, 6}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 932, 11}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 912, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 943, 6}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 922, 25}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 949, 10}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 947, 17}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 200, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 959, 25}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 964, 4}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 984, 17}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[19], + .data.refcounted = {g_bytes + 268, 10}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 968, 3}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, + .data.refcounted = {g_bytes + 1001, 4}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 971, 16}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 365, 8}}}, + .data.refcounted = {g_bytes + 1005, 3}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[95], + .data.refcounted = {g_bytes + 1008, 16}}, + {.refcount = &grpc_static_metadata_refcounts[23], + .data.refcounted = {g_bytes + 302, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 377, 7}}}, + {.refcount = &grpc_static_metadata_refcounts[32], + .data.refcounted = {g_bytes + 433, 8}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 987, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[34], + .data.refcounted = {g_bytes + 445, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 373, 4}}}, + {.refcount = &grpc_static_metadata_refcounts[96], + .data.refcounted = {g_bytes + 1024, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1003, 13}}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 441, 4}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1016, 12}}}, + .data.refcounted = {g_bytes + 1040, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1028, 21}}}, + .data.refcounted = {g_bytes + 1053, 12}}}, + {{.refcount = &grpc_static_metadata_refcounts[10], + .data.refcounted = {g_bytes + 90, 20}}, + {.refcount = &grpc_static_metadata_refcounts[99], + .data.refcounted = {g_bytes + 1065, 21}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 186, 15}}, + {.refcount = &grpc_static_metadata_refcounts[32], + .data.refcounted = {g_bytes + 433, 8}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 186, 15}}, + {.refcount = &grpc_static_metadata_refcounts[33], + .data.refcounted = {g_bytes + 441, 4}}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 186, 15}}, + {.refcount = &grpc_static_metadata_refcounts[97], + .data.refcounted = {g_bytes + 1040, 13}}}, }; -const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 74, 75, 76, - 77, 78, 79, 80}; +const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 76, 77, 78, + 79, 80, 81, 82}; + +const uint8_t grpc_static_accept_stream_encoding_metadata[4] = {0, 83, 84, 85}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index baa86de142..93ab90dff8 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -29,7 +29,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 99 +#define GRPC_STATIC_MDSTR_COUNT 100 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* ":path" */ #define GRPC_MDSTR_PATH (grpc_static_slice_table[0]) @@ -61,178 +61,181 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; #define GRPC_MDSTR_GRPC_TRACE_BIN (grpc_static_slice_table[13]) /* "content-type" */ #define GRPC_MDSTR_CONTENT_TYPE (grpc_static_slice_table[14]) +/* "content-encoding" */ +#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[15]) +/* "accept-encoding" */ +#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[16]) /* "grpc-internal-encoding-request" */ -#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[15]) +#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (grpc_static_slice_table[17]) +/* "grpc-internal-stream-encoding-request" */ +#define GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST \ + (grpc_static_slice_table[18]) /* "user-agent" */ -#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[16]) +#define GRPC_MDSTR_USER_AGENT (grpc_static_slice_table[19]) /* "host" */ -#define GRPC_MDSTR_HOST (grpc_static_slice_table[17]) +#define GRPC_MDSTR_HOST (grpc_static_slice_table[20]) /* "lb-token" */ -#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[18]) +#define GRPC_MDSTR_LB_TOKEN (grpc_static_slice_table[21]) /* "grpc-timeout" */ -#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[19]) +#define GRPC_MDSTR_GRPC_TIMEOUT (grpc_static_slice_table[22]) /* "" */ -#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[20]) +#define GRPC_MDSTR_EMPTY (grpc_static_slice_table[23]) /* "grpc.wait_for_ready" */ -#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[21]) +#define GRPC_MDSTR_GRPC_DOT_WAIT_FOR_READY (grpc_static_slice_table[24]) /* "grpc.timeout" */ -#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[22]) +#define GRPC_MDSTR_GRPC_DOT_TIMEOUT (grpc_static_slice_table[25]) /* "grpc.max_request_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_REQUEST_MESSAGE_BYTES \ - (grpc_static_slice_table[23]) + (grpc_static_slice_table[26]) /* "grpc.max_response_message_bytes" */ #define GRPC_MDSTR_GRPC_DOT_MAX_RESPONSE_MESSAGE_BYTES \ - (grpc_static_slice_table[24]) + (grpc_static_slice_table[27]) /* "/grpc.lb.v1.LoadBalancer/BalanceLoad" */ #define GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD \ - (grpc_static_slice_table[25]) + (grpc_static_slice_table[28]) /* "0" */ -#define GRPC_MDSTR_0 (grpc_static_slice_table[26]) +#define GRPC_MDSTR_0 (grpc_static_slice_table[29]) /* "1" */ -#define GRPC_MDSTR_1 (grpc_static_slice_table[27]) +#define GRPC_MDSTR_1 (grpc_static_slice_table[30]) /* "2" */ -#define GRPC_MDSTR_2 (grpc_static_slice_table[28]) +#define GRPC_MDSTR_2 (grpc_static_slice_table[31]) /* "identity" */ -#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[29]) +#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[32]) /* "gzip" */ -#define GRPC_MDSTR_GZIP (grpc_static_slice_table[30]) +#define GRPC_MDSTR_GZIP (grpc_static_slice_table[33]) /* "deflate" */ -#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[31]) +#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[34]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[32]) +#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[35]) /* "application/grpc" */ -#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[33]) +#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[36]) /* "POST" */ -#define GRPC_MDSTR_POST (grpc_static_slice_table[34]) +#define GRPC_MDSTR_POST (grpc_static_slice_table[37]) /* "200" */ -#define GRPC_MDSTR_200 (grpc_static_slice_table[35]) +#define GRPC_MDSTR_200 (grpc_static_slice_table[38]) /* "404" */ -#define GRPC_MDSTR_404 (grpc_static_slice_table[36]) +#define GRPC_MDSTR_404 (grpc_static_slice_table[39]) /* "http" */ -#define GRPC_MDSTR_HTTP (grpc_static_slice_table[37]) +#define GRPC_MDSTR_HTTP (grpc_static_slice_table[40]) /* "https" */ -#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[38]) +#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[41]) /* "grpc" */ -#define GRPC_MDSTR_GRPC (grpc_static_slice_table[39]) +#define GRPC_MDSTR_GRPC (grpc_static_slice_table[42]) /* "GET" */ -#define GRPC_MDSTR_GET (grpc_static_slice_table[40]) +#define GRPC_MDSTR_GET (grpc_static_slice_table[43]) /* "PUT" */ -#define GRPC_MDSTR_PUT (grpc_static_slice_table[41]) +#define GRPC_MDSTR_PUT (grpc_static_slice_table[44]) /* "/" */ -#define GRPC_MDSTR_SLASH (grpc_static_slice_table[42]) +#define GRPC_MDSTR_SLASH (grpc_static_slice_table[45]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[43]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[46]) /* "204" */ -#define GRPC_MDSTR_204 (grpc_static_slice_table[44]) +#define GRPC_MDSTR_204 (grpc_static_slice_table[47]) /* "206" */ -#define GRPC_MDSTR_206 (grpc_static_slice_table[45]) +#define GRPC_MDSTR_206 (grpc_static_slice_table[48]) /* "304" */ -#define GRPC_MDSTR_304 (grpc_static_slice_table[46]) +#define GRPC_MDSTR_304 (grpc_static_slice_table[49]) /* "400" */ -#define GRPC_MDSTR_400 (grpc_static_slice_table[47]) +#define GRPC_MDSTR_400 (grpc_static_slice_table[50]) /* "500" */ -#define GRPC_MDSTR_500 (grpc_static_slice_table[48]) +#define GRPC_MDSTR_500 (grpc_static_slice_table[51]) /* "accept-charset" */ -#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[49]) -/* "accept-encoding" */ -#define GRPC_MDSTR_ACCEPT_ENCODING (grpc_static_slice_table[50]) +#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[52]) /* "gzip, deflate" */ -#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[51]) +#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[53]) /* "accept-language" */ -#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[52]) +#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[54]) /* "accept-ranges" */ -#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[53]) +#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[55]) /* "accept" */ -#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[54]) +#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[56]) /* "access-control-allow-origin" */ -#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[55]) +#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[57]) /* "age" */ -#define GRPC_MDSTR_AGE (grpc_static_slice_table[56]) +#define GRPC_MDSTR_AGE (grpc_static_slice_table[58]) /* "allow" */ -#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[57]) +#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[59]) /* "authorization" */ -#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[58]) +#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[60]) /* "cache-control" */ -#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[59]) +#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[61]) /* "content-disposition" */ -#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[60]) -/* "content-encoding" */ -#define GRPC_MDSTR_CONTENT_ENCODING (grpc_static_slice_table[61]) +#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[62]) /* "content-language" */ -#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[62]) +#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[63]) /* "content-length" */ -#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[63]) +#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[64]) /* "content-location" */ -#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[64]) +#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[65]) /* "content-range" */ -#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[65]) +#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[66]) /* "cookie" */ -#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[66]) +#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[67]) /* "date" */ -#define GRPC_MDSTR_DATE (grpc_static_slice_table[67]) +#define GRPC_MDSTR_DATE (grpc_static_slice_table[68]) /* "etag" */ -#define GRPC_MDSTR_ETAG (grpc_static_slice_table[68]) +#define GRPC_MDSTR_ETAG (grpc_static_slice_table[69]) /* "expect" */ -#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[69]) +#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[70]) /* "expires" */ -#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[70]) +#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[71]) /* "from" */ -#define GRPC_MDSTR_FROM (grpc_static_slice_table[71]) +#define GRPC_MDSTR_FROM (grpc_static_slice_table[72]) /* "if-match" */ -#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[72]) +#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[73]) /* "if-modified-since" */ -#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[73]) +#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[74]) /* "if-none-match" */ -#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[74]) +#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[75]) /* "if-range" */ -#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[75]) +#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[76]) /* "if-unmodified-since" */ -#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[76]) +#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[77]) /* "last-modified" */ -#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[77]) +#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[78]) /* "lb-cost-bin" */ -#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[78]) +#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[79]) /* "link" */ -#define GRPC_MDSTR_LINK (grpc_static_slice_table[79]) +#define GRPC_MDSTR_LINK (grpc_static_slice_table[80]) /* "location" */ -#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[80]) +#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[81]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[81]) +#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[82]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[82]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[83]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[83]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[84]) /* "range" */ -#define GRPC_MDSTR_RANGE (grpc_static_slice_table[84]) +#define GRPC_MDSTR_RANGE (grpc_static_slice_table[85]) /* "referer" */ -#define GRPC_MDSTR_REFERER (grpc_static_slice_table[85]) +#define GRPC_MDSTR_REFERER (grpc_static_slice_table[86]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[86]) +#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[87]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[87]) +#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[88]) /* "server" */ -#define GRPC_MDSTR_SERVER (grpc_static_slice_table[88]) +#define GRPC_MDSTR_SERVER (grpc_static_slice_table[89]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[89]) +#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[90]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[90]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[91]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[91]) +#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[92]) /* "vary" */ -#define GRPC_MDSTR_VARY (grpc_static_slice_table[92]) +#define GRPC_MDSTR_VARY (grpc_static_slice_table[93]) /* "via" */ -#define GRPC_MDSTR_VIA (grpc_static_slice_table[93]) +#define GRPC_MDSTR_VIA (grpc_static_slice_table[94]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[94]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[95]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[95]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[96]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[96]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[97]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[97]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[98]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (grpc_static_slice_table[98]) + (grpc_static_slice_table[99]) extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable; extern grpc_slice_refcount @@ -244,7 +247,7 @@ extern grpc_slice_refcount #define GRPC_STATIC_METADATA_INDEX(static_slice) \ ((int)((static_slice).refcount - grpc_static_metadata_refcounts)) -#define GRPC_STATIC_MDELEM_COUNT 81 +#define GRPC_STATIC_MDELEM_COUNT 86 extern grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "grpc-status": "0" */ @@ -355,141 +358,156 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "content-disposition": "" */ #define GRPC_MDELEM_CONTENT_DISPOSITION_EMPTY \ (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[35], GRPC_MDELEM_STORAGE_STATIC)) +/* "content-encoding": "identity" */ +#define GRPC_MDELEM_CONTENT_ENCODING_IDENTITY \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[36], GRPC_MDELEM_STORAGE_STATIC)) +/* "content-encoding": "gzip" */ +#define GRPC_MDELEM_CONTENT_ENCODING_GZIP \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[37], GRPC_MDELEM_STORAGE_STATIC)) /* "content-encoding": "" */ #define GRPC_MDELEM_CONTENT_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[36], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[38], GRPC_MDELEM_STORAGE_STATIC)) /* "content-language": "" */ #define GRPC_MDELEM_CONTENT_LANGUAGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[37], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[39], GRPC_MDELEM_STORAGE_STATIC)) /* "content-length": "" */ #define GRPC_MDELEM_CONTENT_LENGTH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[38], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[40], GRPC_MDELEM_STORAGE_STATIC)) /* "content-location": "" */ #define GRPC_MDELEM_CONTENT_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[39], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[41], GRPC_MDELEM_STORAGE_STATIC)) /* "content-range": "" */ #define GRPC_MDELEM_CONTENT_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[40], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[42], GRPC_MDELEM_STORAGE_STATIC)) /* "content-type": "" */ #define GRPC_MDELEM_CONTENT_TYPE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[41], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[43], GRPC_MDELEM_STORAGE_STATIC)) /* "cookie": "" */ #define GRPC_MDELEM_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[42], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[44], GRPC_MDELEM_STORAGE_STATIC)) /* "date": "" */ #define GRPC_MDELEM_DATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[43], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[45], GRPC_MDELEM_STORAGE_STATIC)) /* "etag": "" */ #define GRPC_MDELEM_ETAG_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[44], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[46], GRPC_MDELEM_STORAGE_STATIC)) /* "expect": "" */ #define GRPC_MDELEM_EXPECT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[45], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[47], GRPC_MDELEM_STORAGE_STATIC)) /* "expires": "" */ #define GRPC_MDELEM_EXPIRES_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[46], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[48], GRPC_MDELEM_STORAGE_STATIC)) /* "from": "" */ #define GRPC_MDELEM_FROM_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[47], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[49], GRPC_MDELEM_STORAGE_STATIC)) /* "host": "" */ #define GRPC_MDELEM_HOST_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[48], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[50], GRPC_MDELEM_STORAGE_STATIC)) /* "if-match": "" */ #define GRPC_MDELEM_IF_MATCH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[49], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[51], GRPC_MDELEM_STORAGE_STATIC)) /* "if-modified-since": "" */ #define GRPC_MDELEM_IF_MODIFIED_SINCE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[50], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[52], GRPC_MDELEM_STORAGE_STATIC)) /* "if-none-match": "" */ #define GRPC_MDELEM_IF_NONE_MATCH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[51], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[53], GRPC_MDELEM_STORAGE_STATIC)) /* "if-range": "" */ #define GRPC_MDELEM_IF_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[52], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[54], GRPC_MDELEM_STORAGE_STATIC)) /* "if-unmodified-since": "" */ #define GRPC_MDELEM_IF_UNMODIFIED_SINCE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[53], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC)) /* "last-modified": "" */ #define GRPC_MDELEM_LAST_MODIFIED_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[54], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) /* "lb-token": "" */ #define GRPC_MDELEM_LB_TOKEN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) /* "lb-cost-bin": "" */ #define GRPC_MDELEM_LB_COST_BIN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) /* "link": "" */ #define GRPC_MDELEM_LINK_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) /* "location": "" */ #define GRPC_MDELEM_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) /* "max-forwards": "" */ #define GRPC_MDELEM_MAX_FORWARDS_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authenticate": "" */ #define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authorization": "" */ #define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) /* "range": "" */ #define GRPC_MDELEM_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) /* "referer": "" */ #define GRPC_MDELEM_REFERER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) /* "refresh": "" */ #define GRPC_MDELEM_REFRESH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) /* "retry-after": "" */ #define GRPC_MDELEM_RETRY_AFTER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) /* "server": "" */ #define GRPC_MDELEM_SERVER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) /* "set-cookie": "" */ #define GRPC_MDELEM_SET_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) /* "strict-transport-security": "" */ #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) /* "transfer-encoding": "" */ #define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) /* "user-agent": "" */ #define GRPC_MDELEM_USER_AGENT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) /* "vary": "" */ #define GRPC_MDELEM_VARY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) /* "via": "" */ #define GRPC_MDELEM_VIA_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) /* "www-authenticate": "" */ #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[81], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[82], GRPC_MDELEM_STORAGE_STATIC)) +/* "accept-encoding": "identity" */ +#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[83], GRPC_MDELEM_STORAGE_STATIC)) +/* "accept-encoding": "gzip" */ +#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[84], GRPC_MDELEM_STORAGE_STATIC)) +/* "accept-encoding": "identity,gzip" */ +#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[85], GRPC_MDELEM_STORAGE_STATIC)) grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b); typedef enum { @@ -508,7 +526,10 @@ typedef enum { GRPC_BATCH_GRPC_TAGS_BIN, GRPC_BATCH_GRPC_TRACE_BIN, GRPC_BATCH_CONTENT_TYPE, + GRPC_BATCH_CONTENT_ENCODING, + GRPC_BATCH_ACCEPT_ENCODING, GRPC_BATCH_GRPC_INTERNAL_ENCODING_REQUEST, + GRPC_BATCH_GRPC_INTERNAL_STREAM_ENCODING_REQUEST, GRPC_BATCH_USER_AGENT, GRPC_BATCH_HOST, GRPC_BATCH_LB_TOKEN, @@ -533,7 +554,10 @@ typedef union { struct grpc_linked_mdelem *grpc_tags_bin; struct grpc_linked_mdelem *grpc_trace_bin; struct grpc_linked_mdelem *content_type; + struct grpc_linked_mdelem *content_encoding; + struct grpc_linked_mdelem *accept_encoding; struct grpc_linked_mdelem *grpc_internal_encoding_request; + struct grpc_linked_mdelem *grpc_internal_stream_encoding_request; struct grpc_linked_mdelem *user_agent; struct grpc_linked_mdelem *host; struct grpc_linked_mdelem *lb_token; @@ -552,4 +576,10 @@ extern const uint8_t grpc_static_accept_encoding_metadata[8]; (GRPC_MAKE_MDELEM( \ &grpc_static_mdelem_table[grpc_static_accept_encoding_metadata[(algs)]], \ GRPC_MDELEM_STORAGE_STATIC)) + +extern const uint8_t grpc_static_accept_stream_encoding_metadata[4]; +#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table \ + [grpc_static_accept_stream_encoding_metadata[(algs)]], \ + GRPC_MDELEM_STORAGE_STATIC)) #endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */ diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 6c61f4b8d9..650b0559aa 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -197,11 +197,6 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, then_schedule_closure); } -char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, - grpc_transport *transport) { - return transport->vtable->get_peer(exec_ctx, transport); -} - grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *transport) { return transport->vtable->get_endpoint(exec_ctx, transport); @@ -214,24 +209,24 @@ grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, // is a function that must always unref cancel_error // though it lives in lib, it handles transport stream ops sure // it's grpc_transport_stream_op_batch_finish_with_failure - void grpc_transport_stream_op_batch_finish_with_failure( grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *batch, - grpc_error *error) { + grpc_error *error, grpc_call_combiner *call_combiner) { if (batch->send_message) { grpc_byte_stream_destroy(exec_ctx, batch->payload->send_message.send_message); } if (batch->recv_message) { - GRPC_CLOSURE_SCHED(exec_ctx, - batch->payload->recv_message.recv_message_ready, - GRPC_ERROR_REF(error)); + GRPC_CALL_COMBINER_START(exec_ctx, call_combiner, + batch->payload->recv_message.recv_message_ready, + GRPC_ERROR_REF(error), + "failing recv_message_ready"); } if (batch->recv_initial_metadata) { - GRPC_CLOSURE_SCHED( - exec_ctx, + GRPC_CALL_COMBINER_START( + exec_ctx, call_combiner, batch->payload->recv_initial_metadata.recv_initial_metadata_ready, - GRPC_ERROR_REF(error)); + GRPC_ERROR_REF(error), "failing recv_initial_metadata_ready"); } GRPC_CLOSURE_SCHED(exec_ctx, batch->on_complete, error); if (batch->cancel_stream) { diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 099138ea14..fbf5dcb8b5 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -22,6 +22,7 @@ #include <stddef.h> #include "src/core/lib/channel/context.h" +#include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/pollset.h" @@ -152,6 +153,9 @@ struct grpc_transport_stream_op_batch_payload { /** Iff send_initial_metadata != NULL, flags associated with send_initial_metadata: a bitfield of GRPC_INITIAL_METADATA_xxx */ uint32_t send_initial_metadata_flags; + // If non-NULL, will be set by the transport to the peer string + // (a char*, which the caller takes ownership of). + gpr_atm *peer_string; } send_initial_metadata; struct { @@ -176,6 +180,9 @@ struct grpc_transport_stream_op_batch_payload { // immediately available. This may be a signal that we received a // Trailers-Only response. bool *trailing_metadata_available; + // If non-NULL, will be set by the transport to the peer string + // (a char*, which the caller takes ownership of). + gpr_atm *peer_string; } recv_initial_metadata; struct { @@ -293,7 +300,7 @@ void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, void grpc_transport_stream_op_batch_finish_with_failure( grpc_exec_ctx *exec_ctx, grpc_transport_stream_op_batch *op, - grpc_error *error); + grpc_error *error, grpc_call_combiner *call_combiner); char *grpc_transport_stream_op_batch_string(grpc_transport_stream_op_batch *op); char *grpc_transport_op_string(grpc_transport_op *op); @@ -332,10 +339,6 @@ void grpc_transport_close(grpc_transport *transport); /* Destroy the transport */ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport); -/* Get the transports peer */ -char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, - grpc_transport *transport); - /* Get the endpoint used by \a transport */ grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *transport); diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index fc772c6dd1..bbae69c223 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -59,9 +59,6 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_destroy */ void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_transport *self); - /* implementation of grpc_transport_get_peer */ - char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_transport *self); - /* implementation of grpc_transport_get_endpoint */ grpc_endpoint *(*get_endpoint)(grpc_exec_ctx *exec_ctx, grpc_transport *self); } grpc_transport_vtable; diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 7b18229ba6..409a6c4103 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -112,6 +112,13 @@ char *grpc_transport_stream_op_batch_string( gpr_strvec_add(&b, tmp); } + if (op->collect_stats) { + gpr_strvec_add(&b, gpr_strdup(" ")); + gpr_asprintf(&tmp, "COLLECT_STATS:%p", + op->payload->collect_stats.collect_stats); + gpr_strvec_add(&b, tmp); + } + out = gpr_strvec_flatten(&b, NULL); gpr_strvec_destroy(&b); |