diff options
Diffstat (limited to 'src/core/lib')
41 files changed, 1285 insertions, 1150 deletions
diff --git a/src/core/lib/channel/channel_args.cc b/src/core/lib/channel/channel_args.cc index 634286d403..101a6fffbc 100644 --- a/src/core/lib/channel/channel_args.cc +++ b/src/core/lib/channel/channel_args.cc @@ -222,21 +222,6 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( return GRPC_COMPRESS_NONE; } -grpc_stream_compression_algorithm -grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args* a) { - size_t i; - if (a == nullptr) return GRPC_STREAM_COMPRESS_NONE; - for (i = 0; i < a->num_args; ++i) { - if (a->args[i].type == GRPC_ARG_INTEGER && - !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, - a->args[i].key)) { - return (grpc_stream_compression_algorithm)a->args[i].value.integer; - break; - } - } - return GRPC_STREAM_COMPRESS_NONE; -} - grpc_channel_args* grpc_channel_args_set_compression_algorithm( grpc_channel_args* a, grpc_compression_algorithm algorithm) { GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); @@ -247,16 +232,6 @@ grpc_channel_args* grpc_channel_args_set_compression_algorithm( return grpc_channel_args_copy_and_add(a, &tmp, 1); } -grpc_channel_args* grpc_channel_args_set_stream_compression_algorithm( - grpc_channel_args* a, grpc_stream_compression_algorithm algorithm) { - GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); - grpc_arg tmp; - tmp.type = GRPC_ARG_INTEGER; - tmp.key = (char*)GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; - tmp.value.integer = algorithm; - return grpc_channel_args_copy_and_add(a, &tmp, 1); -} - /** Returns 1 if the argument for compression algorithm's enabled states bitset * was found in \a a, returning the arg's value in \a states. Otherwise, returns * 0. */ @@ -277,26 +252,6 @@ static int find_compression_algorithm_states_bitset(const grpc_channel_args* a, return 0; /* GPR_FALSE */ } -/** Returns 1 if the argument for compression algorithm's enabled states bitset - * was found in \a a, returning the arg's value in \a states. Otherwise, returns - * 0. */ -static int find_stream_compression_algorithm_states_bitset( - const grpc_channel_args* a, int** states_arg) { - if (a != nullptr) { - size_t i; - for (i = 0; i < a->num_args; ++i) { - if (a->args[i].type == GRPC_ARG_INTEGER && - !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET, - a->args[i].key)) { - *states_arg = &a->args[i].value.integer; - **states_arg |= 0x1; /* forcefully enable support for no compression */ - return 1; - } - } - } - return 0; /* GPR_FALSE */ -} - grpc_channel_args* grpc_channel_args_compression_algorithm_set_state( grpc_channel_args** a, grpc_compression_algorithm algorithm, int state) { int* states_arg = nullptr; @@ -337,48 +292,6 @@ grpc_channel_args* grpc_channel_args_compression_algorithm_set_state( return result; } -grpc_channel_args* grpc_channel_args_stream_compression_algorithm_set_state( - grpc_channel_args** a, grpc_stream_compression_algorithm algorithm, - int state) { - int* states_arg = nullptr; - grpc_channel_args* result = *a; - const int states_arg_found = - find_stream_compression_algorithm_states_bitset(*a, &states_arg); - - if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm && - state == 0) { - const char* algo_name = nullptr; - GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) != - 0); - gpr_log(GPR_ERROR, - "Tried to disable default stream compression algorithm '%s'. The " - "operation has been ignored.", - algo_name); - } else if (states_arg_found) { - if (state != 0) { - GPR_BITSET((unsigned*)states_arg, algorithm); - } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) { - GPR_BITCLEAR((unsigned*)states_arg, algorithm); - } - } else { - /* create a new arg */ - grpc_arg tmp; - tmp.type = GRPC_ARG_INTEGER; - tmp.key = (char*)GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; - /* all enabled by default */ - tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; - if (state != 0) { - GPR_BITSET((unsigned*)&tmp.value.integer, algorithm); - } else if (algorithm != GRPC_STREAM_COMPRESS_NONE) { - GPR_BITCLEAR((unsigned*)&tmp.value.integer, algorithm); - } - result = grpc_channel_args_copy_and_add(*a, &tmp, 1); - grpc_channel_args_destroy(*a); - *a = result; - } - return result; -} - uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args* a) { int* states_arg; @@ -389,17 +302,6 @@ uint32_t grpc_channel_args_compression_algorithm_get_states( } } -uint32_t grpc_channel_args_stream_compression_algorithm_get_states( - const grpc_channel_args* a) { - int* states_arg; - if (find_stream_compression_algorithm_states_bitset(a, &states_arg)) { - return (uint32_t)*states_arg; - } else { - return (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - - 1; /* All algs. enabled */ - } -} - grpc_channel_args* grpc_channel_args_set_socket_mutator( grpc_channel_args* a, grpc_socket_mutator* mutator) { grpc_arg tmp = grpc_socket_mutator_to_arg(mutator); diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 9c7d06f34e..73e9122e75 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -59,24 +59,12 @@ void grpc_channel_args_destroy(grpc_channel_args* a); grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args* a); -/** Returns the stream compression algorithm set in \a a. */ -grpc_stream_compression_algorithm -grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args* a); - /** Returns a channel arg instance with compression enabled. If \a a is * non-NULL, its args are copied. N.B. GRPC_COMPRESS_NONE disables compression * for the channel. */ grpc_channel_args* grpc_channel_args_set_compression_algorithm( grpc_channel_args* a, grpc_compression_algorithm algorithm); -/** Returns a channel arg instance with stream compression enabled. If \a a is - * non-NULL, its args are copied. N.B. GRPC_STREAM_COMPRESS_NONE disables - * stream compression for the channel. If a value other than - * GRPC_STREAM_COMPRESS_NONE is set, it takes precedence over message-wise - * compression algorithms. */ -grpc_channel_args* grpc_channel_args_set_stream_compression_algorithm( - grpc_channel_args* a, grpc_stream_compression_algorithm algorithm); - /** Sets the support for the given compression algorithm. By default, all * compression algorithms are enabled. It's an error to disable an algorithm set * by grpc_channel_args_set_compression_algorithm. @@ -87,17 +75,6 @@ grpc_channel_args* grpc_channel_args_set_stream_compression_algorithm( grpc_channel_args* grpc_channel_args_compression_algorithm_set_state( grpc_channel_args** a, grpc_compression_algorithm algorithm, int enabled); -/** Sets the support for the given stream compression algorithm. By default, all - * stream compression algorithms are enabled. It's an error to disable an - * algorithm set by grpc_channel_args_set_stream_compression_algorithm. - * - * Returns an instance with the updated algorithm states. The \a a pointer is - * modified to point to the returned instance (which may be different from the - * input value of \a a). */ -grpc_channel_args* grpc_channel_args_stream_compression_algorithm_set_state( - grpc_channel_args** a, grpc_stream_compression_algorithm algorithm, - int enabled); - /** Returns the bitset representing the support state (true for enabled, false * for disabled) for compression algorithms. * @@ -106,14 +83,6 @@ grpc_channel_args* grpc_channel_args_stream_compression_algorithm_set_state( uint32_t grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args* a); -/** Returns the bitset representing the support state (true for enabled, false - * for disabled) for stream compression algorithms. - * - * The i-th bit of the returned bitset corresponds to the i-th entry in the - * grpc_stream_compression_algorithm enum. */ -uint32_t grpc_channel_args_stream_compression_algorithm_get_states( - const grpc_channel_args* a); - int grpc_channel_args_compare(const grpc_channel_args* a, const grpc_channel_args* b); diff --git a/src/core/lib/compression/algorithm_metadata.h b/src/core/lib/compression/algorithm_metadata.h index 08feafc1bb..7db771ea74 100644 --- a/src/core/lib/compression/algorithm_metadata.h +++ b/src/core/lib/compression/algorithm_metadata.h @@ -20,20 +20,27 @@ #define GRPC_CORE_LIB_COMPRESSION_ALGORITHM_METADATA_H #include <grpc/compression.h> +#include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/transport/metadata.h" /** Return compression algorithm based metadata value */ grpc_slice grpc_compression_algorithm_slice( grpc_compression_algorithm algorithm); -/** Return stream compression algorithm based metadata value */ -grpc_slice grpc_stream_compression_algorithm_slice( - grpc_stream_compression_algorithm algorithm); +/** Find compression algorithm based on passed in mdstr - returns + * GRPC_COMPRESS_ALGORITHM_COUNT on failure */ +grpc_compression_algorithm grpc_compression_algorithm_from_slice( + grpc_slice str); -/** Return compression algorithm based metadata element (grpc-encoding: xxx) */ +/** Return compression algorithm based metadata element */ grpc_mdelem grpc_compression_encoding_mdelem( grpc_compression_algorithm algorithm); +/** Return message compression algorithm based metadata element (grpc-encoding: + * xxx) */ +grpc_mdelem grpc_message_compression_encoding_mdelem( + grpc_message_compression_algorithm algorithm); + /** Return stream compression algorithm based metadata element * (content-encoding: xxx) */ grpc_mdelem grpc_stream_compression_encoding_mdelem( @@ -41,8 +48,8 @@ grpc_mdelem grpc_stream_compression_encoding_mdelem( /** Find compression algorithm based on passed in mdstr - returns * GRPC_COMPRESS_ALGORITHM_COUNT on failure */ -grpc_compression_algorithm grpc_compression_algorithm_from_slice( - grpc_slice str); +grpc_message_compression_algorithm +grpc_message_compression_algorithm_from_slice(grpc_slice str); /** Find stream compression algorithm based on passed in mdstr - returns * GRPC_STREAM_COMPRESS_ALGORITHM_COUNT on failure */ diff --git a/src/core/lib/compression/compression.cc b/src/core/lib/compression/compression.cc index a0d5bdcc78..99e6014b23 100644 --- a/src/core/lib/compression/compression.cc +++ b/src/core/lib/compression/compression.cc @@ -23,40 +23,40 @@ #include <grpc/support/useful.h> #include "src/core/lib/compression/algorithm_metadata.h" +#include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/transport/static_metadata.h" +int grpc_compression_algorithm_is_message( + grpc_compression_algorithm algorithm) { + return (algorithm >= GRPC_COMPRESS_MESSAGE_DEFLATE && + algorithm <= GRPC_COMPRESS_MESSAGE_GZIP) + ? 1 + : 0; +} + +int grpc_compression_algorithm_is_stream(grpc_compression_algorithm algorithm) { + return (algorithm == GRPC_COMPRESS_STREAM_GZIP) ? 1 : 0; +} + int grpc_compression_algorithm_parse(grpc_slice name, grpc_compression_algorithm* algorithm) { - /* we use strncmp not only because it's safer (even though in this case it - * doesn't matter, given that we are comparing against string literals, but - * because this way we needn't have "name" nil-terminated (useful for slice - * data, for example) */ if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) { *algorithm = GRPC_COMPRESS_NONE; return 1; - } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) { - *algorithm = GRPC_COMPRESS_GZIP; - return 1; - } else if (grpc_slice_eq(name, GRPC_MDSTR_DEFLATE)) { - *algorithm = GRPC_COMPRESS_DEFLATE; + } else if (grpc_slice_eq(name, GRPC_MDSTR_MESSAGE_SLASH_DEFLATE)) { + *algorithm = GRPC_COMPRESS_MESSAGE_DEFLATE; return 1; - } else { - return 0; - } -} - -int grpc_stream_compression_algorithm_parse( - grpc_slice name, grpc_stream_compression_algorithm* algorithm) { - if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) { - *algorithm = GRPC_STREAM_COMPRESS_NONE; + } else if (grpc_slice_eq(name, GRPC_MDSTR_MESSAGE_SLASH_GZIP)) { + *algorithm = GRPC_COMPRESS_MESSAGE_GZIP; return 1; - } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) { - *algorithm = GRPC_STREAM_COMPRESS_GZIP; + } else if (grpc_slice_eq(name, GRPC_MDSTR_STREAM_SLASH_GZIP)) { + *algorithm = GRPC_COMPRESS_STREAM_GZIP; return 1; } else { return 0; } + return 0; } int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, @@ -67,113 +67,49 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, case GRPC_COMPRESS_NONE: *name = "identity"; return 1; - case GRPC_COMPRESS_DEFLATE: - *name = "deflate"; - return 1; - case GRPC_COMPRESS_GZIP: - *name = "gzip"; + case GRPC_COMPRESS_MESSAGE_DEFLATE: + *name = "message/deflate"; return 1; - case GRPC_COMPRESS_ALGORITHMS_COUNT: - return 0; - } - return 0; -} - -int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, const char** name) { - GRPC_API_TRACE( - "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2, - ((int)algorithm, name)); - switch (algorithm) { - case GRPC_STREAM_COMPRESS_NONE: - *name = "identity"; + case GRPC_COMPRESS_MESSAGE_GZIP: + *name = "message/gzip"; return 1; - case GRPC_STREAM_COMPRESS_GZIP: - *name = "gzip"; + case GRPC_COMPRESS_STREAM_GZIP: + *name = "stream/gzip"; return 1; - case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT: + case GRPC_COMPRESS_ALGORITHMS_COUNT: return 0; } return 0; } -grpc_compression_algorithm grpc_compression_algorithm_from_slice( - grpc_slice str) { - if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE; - if (grpc_slice_eq(str, GRPC_MDSTR_DEFLATE)) return GRPC_COMPRESS_DEFLATE; - if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_COMPRESS_GZIP; - return GRPC_COMPRESS_ALGORITHMS_COUNT; -} - -grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice( - grpc_slice str) { - if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_STREAM_COMPRESS_NONE; - if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_STREAM_COMPRESS_GZIP; - return GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT; -} - -grpc_slice grpc_compression_algorithm_slice( - grpc_compression_algorithm algorithm) { - switch (algorithm) { - case GRPC_COMPRESS_NONE: - return GRPC_MDSTR_IDENTITY; - case GRPC_COMPRESS_DEFLATE: - return GRPC_MDSTR_DEFLATE; - case GRPC_COMPRESS_GZIP: - return GRPC_MDSTR_GZIP; - case GRPC_COMPRESS_ALGORITHMS_COUNT: - return grpc_empty_slice(); - } - return grpc_empty_slice(); -} - -grpc_slice grpc_stream_compression_algorithm_slice( - grpc_stream_compression_algorithm algorithm) { - switch (algorithm) { - case GRPC_STREAM_COMPRESS_NONE: - return GRPC_MDSTR_IDENTITY; - case GRPC_STREAM_COMPRESS_GZIP: - return GRPC_MDSTR_GZIP; - case GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT: - return grpc_empty_slice(); - } - return grpc_empty_slice(); -} - -grpc_mdelem grpc_compression_encoding_mdelem( - grpc_compression_algorithm algorithm) { - switch (algorithm) { - case GRPC_COMPRESS_NONE: - return GRPC_MDELEM_GRPC_ENCODING_IDENTITY; - case GRPC_COMPRESS_DEFLATE: - return GRPC_MDELEM_GRPC_ENCODING_DEFLATE; - case GRPC_COMPRESS_GZIP: - return GRPC_MDELEM_GRPC_ENCODING_GZIP; - default: - break; - } - return GRPC_MDNULL; -} - -grpc_mdelem grpc_stream_compression_encoding_mdelem( - grpc_stream_compression_algorithm algorithm) { - switch (algorithm) { - case GRPC_STREAM_COMPRESS_NONE: - return GRPC_MDELEM_CONTENT_ENCODING_IDENTITY; - case GRPC_STREAM_COMPRESS_GZIP: - return GRPC_MDELEM_CONTENT_ENCODING_GZIP; - default: - break; +grpc_compression_algorithm grpc_compression_algorithm_for_level( + grpc_compression_level level, uint32_t accepted_encodings) { + grpc_compression_algorithm algo; + if (level == GRPC_COMPRESS_LEVEL_NONE) { + return GRPC_COMPRESS_NONE; + } else if (level <= GRPC_COMPRESS_LEVEL_HIGH) { + // TODO(mxyan): Design algorithm to select from all algorithms, including + // stream compression algorithm + if (!grpc_compression_algorithm_from_message_stream_compression_algorithm( + &algo, + grpc_message_compression_algorithm_for_level( + level, + grpc_compression_bitset_to_message_bitset(accepted_encodings)), + static_cast<grpc_stream_compression_algorithm>(0))) { + gpr_log(GPR_ERROR, "Parse compression level error"); + return GRPC_COMPRESS_NONE; + } + return algo; + } else { + gpr_log(GPR_ERROR, "Unknown compression level: %d", level); + return GRPC_COMPRESS_NONE; } - return GRPC_MDNULL; } void grpc_compression_options_init(grpc_compression_options* opts) { memset(opts, 0, sizeof(*opts)); /* all enabled by default */ opts->enabled_algorithms_bitset = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; - opts->enabled_stream_compression_algorithms_bitset = - (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; } void grpc_compression_options_enable_algorithm( @@ -192,92 +128,48 @@ int grpc_compression_options_is_algorithm_enabled( return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm); } -int grpc_compression_options_is_stream_compression_algorithm_enabled( - const grpc_compression_options* opts, - grpc_stream_compression_algorithm algorithm) { - return GPR_BITGET(opts->enabled_stream_compression_algorithms_bitset, - algorithm); -} - -/* TODO(dgq): Add the ability to specify parameters to the individual - * compression algorithms */ -grpc_compression_algorithm grpc_compression_algorithm_for_level( - grpc_compression_level level, uint32_t accepted_encodings) { - GRPC_API_TRACE("grpc_compression_algorithm_for_level(level=%d)", 1, - ((int)level)); - if (level > GRPC_COMPRESS_LEVEL_HIGH) { - gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level); - abort(); - } - - const size_t num_supported = - GPR_BITCOUNT(accepted_encodings) - 1; /* discard NONE */ - if (level == GRPC_COMPRESS_LEVEL_NONE || num_supported == 0) { - return GRPC_COMPRESS_NONE; - } - - GPR_ASSERT(level > 0); - - /* Establish a "ranking" or compression algorithms in increasing order of - * compression. - * This is simplistic and we will probably want to introduce other dimensions - * in the future (cpu/memory cost, etc). */ - const grpc_compression_algorithm algos_ranking[] = {GRPC_COMPRESS_GZIP, - GRPC_COMPRESS_DEFLATE}; - - /* intersect algos_ranking with the supported ones keeping the ranked order */ - grpc_compression_algorithm - sorted_supported_algos[GRPC_COMPRESS_ALGORITHMS_COUNT]; - size_t algos_supported_idx = 0; - for (size_t i = 0; i < GPR_ARRAY_SIZE(algos_ranking); i++) { - const grpc_compression_algorithm alg = algos_ranking[i]; - for (size_t j = 0; j < num_supported; j++) { - if (GPR_BITGET(accepted_encodings, alg) == 1) { - /* if \a alg in supported */ - sorted_supported_algos[algos_supported_idx++] = alg; - break; - } - } - if (algos_supported_idx == num_supported) break; +grpc_slice grpc_compression_algorithm_slice( + grpc_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_COMPRESS_NONE: + return GRPC_MDSTR_IDENTITY; + case GRPC_COMPRESS_MESSAGE_DEFLATE: + return GRPC_MDSTR_MESSAGE_SLASH_DEFLATE; + case GRPC_COMPRESS_MESSAGE_GZIP: + return GRPC_MDSTR_MESSAGE_SLASH_GZIP; + case GRPC_COMPRESS_STREAM_GZIP: + return GRPC_MDSTR_STREAM_SLASH_GZIP; + case GRPC_COMPRESS_ALGORITHMS_COUNT: + return grpc_empty_slice(); } - - switch (level) { - case GRPC_COMPRESS_LEVEL_NONE: - abort(); /* should have been handled already */ - case GRPC_COMPRESS_LEVEL_LOW: - return sorted_supported_algos[0]; - case GRPC_COMPRESS_LEVEL_MED: - return sorted_supported_algos[num_supported / 2]; - case GRPC_COMPRESS_LEVEL_HIGH: - return sorted_supported_algos[num_supported - 1]; - default: - abort(); - }; + return grpc_empty_slice(); } -GRPCAPI grpc_stream_compression_algorithm -grpc_stream_compression_algorithm_for_level( - grpc_stream_compression_level level, uint32_t accepted_stream_encodings) { - GRPC_API_TRACE("grpc_stream_compression_algorithm_for_level(level=%d)", 1, - ((int)level)); - if (level > GRPC_STREAM_COMPRESS_LEVEL_HIGH) { - gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level); - abort(); - } +grpc_compression_algorithm grpc_compression_algorithm_from_slice( + grpc_slice str) { + if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE; + if (grpc_slice_eq(str, GRPC_MDSTR_MESSAGE_SLASH_DEFLATE)) + return GRPC_COMPRESS_MESSAGE_DEFLATE; + if (grpc_slice_eq(str, GRPC_MDSTR_MESSAGE_SLASH_GZIP)) + return GRPC_COMPRESS_MESSAGE_GZIP; + if (grpc_slice_eq(str, GRPC_MDSTR_STREAM_SLASH_GZIP)) + return GRPC_COMPRESS_STREAM_GZIP; + return GRPC_COMPRESS_ALGORITHMS_COUNT; +} - switch (level) { - case GRPC_STREAM_COMPRESS_LEVEL_NONE: - return GRPC_STREAM_COMPRESS_NONE; - case GRPC_STREAM_COMPRESS_LEVEL_LOW: - case GRPC_STREAM_COMPRESS_LEVEL_MED: - case GRPC_STREAM_COMPRESS_LEVEL_HIGH: - if (GPR_BITGET(accepted_stream_encodings, GRPC_STREAM_COMPRESS_GZIP) == - 1) { - return GRPC_STREAM_COMPRESS_GZIP; - } else { - return GRPC_STREAM_COMPRESS_NONE; - } +grpc_mdelem grpc_compression_encoding_mdelem( + grpc_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_COMPRESS_NONE: + return GRPC_MDELEM_GRPC_ENCODING_IDENTITY; + case GRPC_COMPRESS_MESSAGE_DEFLATE: + return GRPC_MDELEM_GRPC_ENCODING_DEFLATE; + case GRPC_COMPRESS_MESSAGE_GZIP: + return GRPC_MDELEM_GRPC_ENCODING_GZIP; + case GRPC_COMPRESS_STREAM_GZIP: + return GRPC_MDELEM_GRPC_ENCODING_GZIP; default: - abort(); + break; } + return GRPC_MDNULL; } diff --git a/src/core/lib/compression/compression_internal.cc b/src/core/lib/compression/compression_internal.cc new file mode 100644 index 0000000000..263cdf06eb --- /dev/null +++ b/src/core/lib/compression/compression_internal.cc @@ -0,0 +1,273 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <stdlib.h> +#include <string.h> + +#include <grpc/compression.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/compression/algorithm_metadata.h" +#include "src/core/lib/compression/compression_internal.h" +#include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/transport/static_metadata.h" + +/* Interfaces related to MD */ + +grpc_message_compression_algorithm +grpc_message_compression_algorithm_from_slice(grpc_slice str) { + if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) + return GRPC_MESSAGE_COMPRESS_NONE; + if (grpc_slice_eq(str, GRPC_MDSTR_DEFLATE)) + return GRPC_MESSAGE_COMPRESS_DEFLATE; + if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_MESSAGE_COMPRESS_GZIP; + return GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT; +} + +grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice( + grpc_slice str) { + if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_STREAM_COMPRESS_NONE; + if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_STREAM_COMPRESS_GZIP; + return GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT; +} + +grpc_mdelem grpc_message_compression_encoding_mdelem( + grpc_message_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_MESSAGE_COMPRESS_NONE: + return GRPC_MDELEM_GRPC_ENCODING_IDENTITY; + case GRPC_MESSAGE_COMPRESS_DEFLATE: + return GRPC_MDELEM_GRPC_ENCODING_DEFLATE; + case GRPC_MESSAGE_COMPRESS_GZIP: + return GRPC_MDELEM_GRPC_ENCODING_GZIP; + default: + break; + } + return GRPC_MDNULL; +} + +grpc_mdelem grpc_stream_compression_encoding_mdelem( + grpc_stream_compression_algorithm algorithm) { + switch (algorithm) { + case GRPC_STREAM_COMPRESS_NONE: + return GRPC_MDELEM_CONTENT_ENCODING_IDENTITY; + case GRPC_STREAM_COMPRESS_GZIP: + return GRPC_MDELEM_CONTENT_ENCODING_GZIP; + default: + break; + } + return GRPC_MDNULL; +} + +/* Interfaces performing transformation between compression algorithms and + * levels. */ +grpc_message_compression_algorithm +grpc_compression_algorithm_to_message_compression_algorithm( + grpc_compression_algorithm algo) { + switch (algo) { + case GRPC_COMPRESS_MESSAGE_DEFLATE: + return GRPC_MESSAGE_COMPRESS_DEFLATE; + case GRPC_COMPRESS_MESSAGE_GZIP: + return GRPC_MESSAGE_COMPRESS_GZIP; + default: + return GRPC_MESSAGE_COMPRESS_NONE; + } +} + +grpc_stream_compression_algorithm +grpc_compression_algorithm_to_stream_compression_algorithm( + grpc_compression_algorithm algo) { + switch (algo) { + case GRPC_COMPRESS_STREAM_GZIP: + return GRPC_STREAM_COMPRESS_GZIP; + default: + return GRPC_STREAM_COMPRESS_NONE; + } +} + +uint32_t grpc_compression_bitset_to_message_bitset(uint32_t bitset) { + return bitset & ((1u << GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT) - 1); +} + +uint32_t grpc_compression_bitset_to_stream_bitset(uint32_t bitset) { + uint32_t identity = (bitset & 1u); + uint32_t other_bits = + (bitset >> (GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT - 1)) & + ((1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 2); + return identity | other_bits; +} + +uint32_t grpc_compression_bitset_from_message_stream_compression_bitset( + uint32_t message_bitset, uint32_t stream_bitset) { + uint32_t offset_stream_bitset = + (stream_bitset & 1u) | + ((stream_bitset & (~1u)) << (GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT - 1)); + return message_bitset | offset_stream_bitset; +} + +int grpc_compression_algorithm_from_message_stream_compression_algorithm( + grpc_compression_algorithm* algorithm, + grpc_message_compression_algorithm message_algorithm, + grpc_stream_compression_algorithm stream_algorithm) { + if (message_algorithm != GRPC_MESSAGE_COMPRESS_NONE && + stream_algorithm != GRPC_STREAM_COMPRESS_NONE) { + *algorithm = GRPC_COMPRESS_NONE; + return 0; + } + if (message_algorithm == GRPC_MESSAGE_COMPRESS_NONE) { + switch (stream_algorithm) { + case GRPC_STREAM_COMPRESS_NONE: + *algorithm = GRPC_COMPRESS_NONE; + return 1; + case GRPC_STREAM_COMPRESS_GZIP: + *algorithm = GRPC_COMPRESS_STREAM_GZIP; + return 1; + default: + *algorithm = GRPC_COMPRESS_NONE; + return 0; + } + } else { + switch (message_algorithm) { + case GRPC_MESSAGE_COMPRESS_NONE: + *algorithm = GRPC_COMPRESS_NONE; + return 1; + case GRPC_MESSAGE_COMPRESS_DEFLATE: + *algorithm = GRPC_COMPRESS_MESSAGE_DEFLATE; + return 1; + case GRPC_MESSAGE_COMPRESS_GZIP: + *algorithm = GRPC_COMPRESS_MESSAGE_GZIP; + return 1; + default: + *algorithm = GRPC_COMPRESS_NONE; + return 0; + } + } + return 0; +} + +/* Interfaces for message compression. */ + +int grpc_message_compression_algorithm_name( + grpc_message_compression_algorithm algorithm, const char** name) { + GRPC_API_TRACE( + "grpc_message_compression_algorithm_parse(algorithm=%d, name=%p)", 2, + ((int)algorithm, name)); + switch (algorithm) { + case GRPC_MESSAGE_COMPRESS_NONE: + *name = "identity"; + return 1; + case GRPC_MESSAGE_COMPRESS_DEFLATE: + *name = "deflate"; + return 1; + case GRPC_MESSAGE_COMPRESS_GZIP: + *name = "gzip"; + return 1; + case GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT: + return 0; + } + return 0; +} + +/* TODO(dgq): Add the ability to specify parameters to the individual + * compression algorithms */ +grpc_message_compression_algorithm grpc_message_compression_algorithm_for_level( + grpc_compression_level level, uint32_t accepted_encodings) { + GRPC_API_TRACE("grpc_message_compression_algorithm_for_level(level=%d)", 1, + ((int)level)); + if (level > GRPC_COMPRESS_LEVEL_HIGH) { + gpr_log(GPR_ERROR, "Unknown message compression level %d.", (int)level); + abort(); + } + + const size_t num_supported = + GPR_BITCOUNT(accepted_encodings) - 1; /* discard NONE */ + if (level == GRPC_COMPRESS_LEVEL_NONE || num_supported == 0) { + return GRPC_MESSAGE_COMPRESS_NONE; + } + + GPR_ASSERT(level > 0); + + /* Establish a "ranking" or compression algorithms in increasing order of + * compression. + * This is simplistic and we will probably want to introduce other dimensions + * in the future (cpu/memory cost, etc). */ + const grpc_message_compression_algorithm algos_ranking[] = { + GRPC_MESSAGE_COMPRESS_GZIP, GRPC_MESSAGE_COMPRESS_DEFLATE}; + + /* intersect algos_ranking with the supported ones keeping the ranked order */ + grpc_message_compression_algorithm + sorted_supported_algos[GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT]; + size_t algos_supported_idx = 0; + for (size_t i = 0; i < GPR_ARRAY_SIZE(algos_ranking); i++) { + const grpc_message_compression_algorithm alg = algos_ranking[i]; + for (size_t j = 0; j < num_supported; j++) { + if (GPR_BITGET(accepted_encodings, alg) == 1) { + /* if \a alg in supported */ + sorted_supported_algos[algos_supported_idx++] = alg; + break; + } + } + if (algos_supported_idx == num_supported) break; + } + + switch (level) { + case GRPC_COMPRESS_LEVEL_NONE: + abort(); /* should have been handled already */ + case GRPC_COMPRESS_LEVEL_LOW: + return sorted_supported_algos[0]; + case GRPC_COMPRESS_LEVEL_MED: + return sorted_supported_algos[num_supported / 2]; + case GRPC_COMPRESS_LEVEL_HIGH: + return sorted_supported_algos[num_supported - 1]; + default: + abort(); + }; +} + +int grpc_message_compression_algorithm_parse( + grpc_slice value, grpc_message_compression_algorithm* algorithm) { + if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { + *algorithm = GRPC_MESSAGE_COMPRESS_NONE; + return 1; + } else if (grpc_slice_eq(value, GRPC_MDSTR_DEFLATE)) { + *algorithm = GRPC_MESSAGE_COMPRESS_DEFLATE; + return 1; + } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { + *algorithm = GRPC_MESSAGE_COMPRESS_GZIP; + return 1; + } else { + return 0; + } + return 0; +} + +/* Interfaces for stream compression. */ + +int grpc_stream_compression_algorithm_parse( + grpc_slice value, grpc_stream_compression_algorithm* algorithm) { + if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { + *algorithm = GRPC_STREAM_COMPRESS_NONE; + return 1; + } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { + *algorithm = GRPC_STREAM_COMPRESS_GZIP; + return 1; + } else { + return 0; + } + return 0; +} diff --git a/src/core/lib/compression/compression_internal.h b/src/core/lib/compression/compression_internal.h new file mode 100644 index 0000000000..72f01dd1b7 --- /dev/null +++ b/src/core/lib/compression/compression_internal.h @@ -0,0 +1,86 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H +#define GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H + +#include <grpc/impl/codegen/compression_types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + GRPC_MESSAGE_COMPRESS_NONE = 0, + GRPC_MESSAGE_COMPRESS_DEFLATE, + GRPC_MESSAGE_COMPRESS_GZIP, + /* TODO(ctiller): snappy */ + GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT +} grpc_message_compression_algorithm; + +/** Stream compresssion algorithms supported by gRPC */ +typedef enum { + GRPC_STREAM_COMPRESS_NONE = 0, + GRPC_STREAM_COMPRESS_GZIP, + GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT +} grpc_stream_compression_algorithm; + +/* Interfaces performing transformation between compression algorithms and + * levels. */ + +grpc_message_compression_algorithm +grpc_compression_algorithm_to_message_compression_algorithm( + grpc_compression_algorithm algo); + +grpc_stream_compression_algorithm +grpc_compression_algorithm_to_stream_compression_algorithm( + grpc_compression_algorithm algo); + +uint32_t grpc_compression_bitset_to_message_bitset(uint32_t bitset); + +uint32_t grpc_compression_bitset_to_stream_bitset(uint32_t bitset); + +uint32_t grpc_compression_bitset_from_message_stream_compression_bitset( + uint32_t message_bitset, uint32_t stream_bitset); + +int grpc_compression_algorithm_from_message_stream_compression_algorithm( + grpc_compression_algorithm* algorithm, + grpc_message_compression_algorithm message_algorithm, + grpc_stream_compression_algorithm stream_algorithm); + +/* Interfaces for message compression. */ + +int grpc_message_compression_algorithm_name( + grpc_message_compression_algorithm algorithm, const char** name); + +grpc_message_compression_algorithm grpc_message_compression_algorithm_for_level( + grpc_compression_level level, uint32_t accepted_encodings); + +int grpc_message_compression_algorithm_parse( + grpc_slice value, grpc_message_compression_algorithm* algorithm); + +/* Interfaces for stream compression. */ + +int grpc_stream_compression_algorithm_parse( + grpc_slice value, grpc_stream_compression_algorithm* algorithm); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H */ diff --git a/src/core/lib/compression/compression_ruby.cc b/src/core/lib/compression/compression_ruby.cc new file mode 100644 index 0000000000..293062f5ed --- /dev/null +++ b/src/core/lib/compression/compression_ruby.cc @@ -0,0 +1,66 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <grpc/compression_ruby.h> +#include <grpc/support/useful.h> + +#include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/transport/static_metadata.h" + +int grpc_compression_algorithm_parse_ruby( + grpc_slice name, grpc_compression_algorithm* algorithm) { + if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) { + *algorithm = GRPC_COMPRESS_NONE; + return 1; + } else if (grpc_slice_eq(name, GRPC_MDSTR_DEFLATE)) { + *algorithm = GRPC_COMPRESS_MESSAGE_DEFLATE; + return 1; + } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) { + *algorithm = GRPC_COMPRESS_MESSAGE_GZIP; + return 1; + } else if (grpc_slice_eq(name, GRPC_MDSTR_STREAM_SLASH_GZIP)) { + *algorithm = GRPC_COMPRESS_STREAM_GZIP; + return 1; + } else { + return 0; + } + return 0; +} + +int grpc_compression_algorithm_name_ruby(grpc_compression_algorithm algorithm, + const char** name) { + GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2, + ((int)algorithm, name)); + switch (algorithm) { + case GRPC_COMPRESS_NONE: + *name = "identity"; + return 1; + case GRPC_COMPRESS_MESSAGE_DEFLATE: + *name = "deflate"; + return 1; + case GRPC_COMPRESS_MESSAGE_GZIP: + *name = "gzip"; + return 1; + case GRPC_COMPRESS_STREAM_GZIP: + *name = "stream/gzip"; + return 1; + case GRPC_COMPRESS_ALGORITHMS_COUNT: + return 0; + } + return 0; +} diff --git a/src/core/lib/compression/message_compress.cc b/src/core/lib/compression/message_compress.cc index aa43a53f2b..1e9b0f10ee 100644 --- a/src/core/lib/compression/message_compress.cc +++ b/src/core/lib/compression/message_compress.cc @@ -141,25 +141,25 @@ static int copy(grpc_slice_buffer* input, grpc_slice_buffer* output) { return 1; } -static int compress_inner(grpc_compression_algorithm algorithm, +static int compress_inner(grpc_message_compression_algorithm algorithm, grpc_slice_buffer* input, grpc_slice_buffer* output) { switch (algorithm) { - case GRPC_COMPRESS_NONE: + case GRPC_MESSAGE_COMPRESS_NONE: /* the fallback path always needs to be send uncompressed: we simply rely on that here */ return 0; - case GRPC_COMPRESS_DEFLATE: + case GRPC_MESSAGE_COMPRESS_DEFLATE: return zlib_compress(input, output, 0); - case GRPC_COMPRESS_GZIP: + case GRPC_MESSAGE_COMPRESS_GZIP: return zlib_compress(input, output, 1); - case GRPC_COMPRESS_ALGORITHMS_COUNT: + case GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT: break; } gpr_log(GPR_ERROR, "invalid compression algorithm %d", algorithm); return 0; } -int grpc_msg_compress(grpc_compression_algorithm algorithm, +int grpc_msg_compress(grpc_message_compression_algorithm algorithm, grpc_slice_buffer* input, grpc_slice_buffer* output) { if (!compress_inner(algorithm, input, output)) { copy(input, output); @@ -168,16 +168,16 @@ int grpc_msg_compress(grpc_compression_algorithm algorithm, return 1; } -int grpc_msg_decompress(grpc_compression_algorithm algorithm, +int grpc_msg_decompress(grpc_message_compression_algorithm algorithm, grpc_slice_buffer* input, grpc_slice_buffer* output) { switch (algorithm) { - case GRPC_COMPRESS_NONE: + case GRPC_MESSAGE_COMPRESS_NONE: return copy(input, output); - case GRPC_COMPRESS_DEFLATE: + case GRPC_MESSAGE_COMPRESS_DEFLATE: return zlib_decompress(input, output, 0); - case GRPC_COMPRESS_GZIP: + case GRPC_MESSAGE_COMPRESS_GZIP: return zlib_decompress(input, output, 1); - case GRPC_COMPRESS_ALGORITHMS_COUNT: + case GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT: break; } gpr_log(GPR_ERROR, "invalid compression algorithm %d", algorithm); diff --git a/src/core/lib/compression/message_compress.h b/src/core/lib/compression/message_compress.h index c963fccc73..ed9e5bfa39 100644 --- a/src/core/lib/compression/message_compress.h +++ b/src/core/lib/compression/message_compress.h @@ -19,19 +19,20 @@ #ifndef GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H #define GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H -#include <grpc/compression.h> #include <grpc/slice_buffer.h> +#include "src/core/lib/compression/compression_internal.h" + /* compress 'input' to 'output' using 'algorithm'. On success, appends compressed slices to output and returns 1. On failure, appends uncompressed slices to output and returns 0. */ -int grpc_msg_compress(grpc_compression_algorithm algorithm, +int grpc_msg_compress(grpc_message_compression_algorithm algorithm, grpc_slice_buffer* input, grpc_slice_buffer* output); /* decompress 'input' to 'output' using 'algorithm'. On success, appends slices to output and returns 1. On failure, output is unchanged, and returns 0. */ -int grpc_msg_decompress(grpc_compression_algorithm algorithm, +int grpc_msg_decompress(grpc_message_compression_algorithm algorithm, grpc_slice_buffer* input, grpc_slice_buffer* output); #endif /* GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H */ diff --git a/src/core/lib/gpr/alloc.cc b/src/core/lib/gpr/alloc.cc index 518bdb99f7..b1d1550630 100644 --- a/src/core/lib/gpr/alloc.cc +++ b/src/core/lib/gpr/alloc.cc @@ -50,48 +50,44 @@ void gpr_set_allocation_functions(gpr_allocation_functions functions) { } void* gpr_malloc(size_t size) { + GPR_TIMER_SCOPE("gpr_malloc", 0); void* p; if (size == 0) return nullptr; - GPR_TIMER_BEGIN("gpr_malloc", 0); p = g_alloc_functions.malloc_fn(size); if (!p) { abort(); } - GPR_TIMER_END("gpr_malloc", 0); return p; } void* gpr_zalloc(size_t size) { + GPR_TIMER_SCOPE("gpr_zalloc", 0); void* p; if (size == 0) return nullptr; - GPR_TIMER_BEGIN("gpr_zalloc", 0); p = g_alloc_functions.zalloc_fn(size); if (!p) { abort(); } - GPR_TIMER_END("gpr_zalloc", 0); return p; } void gpr_free(void* p) { - GPR_TIMER_BEGIN("gpr_free", 0); + GPR_TIMER_SCOPE("gpr_free", 0); g_alloc_functions.free_fn(p); - GPR_TIMER_END("gpr_free", 0); } void* gpr_realloc(void* p, size_t size) { + GPR_TIMER_SCOPE("gpr_realloc", 0); if ((size == 0) && (p == nullptr)) return nullptr; - GPR_TIMER_BEGIN("gpr_realloc", 0); p = g_alloc_functions.realloc_fn(p, size); if (!p) { abort(); } - GPR_TIMER_END("gpr_realloc", 0); return p; } -void* gpr_malloc_aligned(size_t size, size_t alignment_log) { - size_t alignment = ((size_t)1) << alignment_log; +void* gpr_malloc_aligned(size_t size, size_t alignment) { + GPR_ASSERT(((alignment - 1) & alignment) == 0); // Must be power of 2. size_t extra = alignment - 1 + sizeof(void*); void* p = gpr_malloc(size + extra); void** ret = (void**)(((uintptr_t)p + extra) & ~(alignment - 1)); diff --git a/src/core/lib/gpr/arena.cc b/src/core/lib/gpr/arena.cc index 177c176732..687592a140 100644 --- a/src/core/lib/gpr/arena.cc +++ b/src/core/lib/gpr/arena.cc @@ -17,11 +17,19 @@ */ #include "src/core/lib/gpr/arena.h" + +#include <string.h> + #include <grpc/support/alloc.h> #include <grpc/support/atm.h> #include <grpc/support/log.h> #include <grpc/support/useful.h> +// TODO(roth): We currently assume that all callers need alignment of 16 +// bytes, which may be wrong in some cases. As part of converting the +// arena API to C++, we should consider replacing gpr_arena_alloc() with a +// template that takes the type of the value being allocated, which +// would allow us to use the alignment actually needed by the caller. #define ROUND_UP_TO_ALIGNMENT_SIZE(x) \ (((x) + GPR_MAX_ALIGNMENT - 1u) & ~(GPR_MAX_ALIGNMENT - 1u)) @@ -36,9 +44,16 @@ struct gpr_arena { zone initial_zone; }; +static void* zalloc_aligned(size_t size) { + void* ptr = gpr_malloc_aligned(size, GPR_MAX_ALIGNMENT); + memset(ptr, 0, size); + return ptr; +} + gpr_arena* gpr_arena_create(size_t initial_size) { initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); - gpr_arena* a = (gpr_arena*)gpr_zalloc(sizeof(gpr_arena) + initial_size); + gpr_arena* a = (gpr_arena*)zalloc_aligned( + ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + initial_size); a->initial_zone.size_end = initial_size; return a; } @@ -46,10 +61,10 @@ gpr_arena* gpr_arena_create(size_t initial_size) { size_t gpr_arena_destroy(gpr_arena* arena) { gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); zone* z = (zone*)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); - gpr_free(arena); + gpr_free_aligned(arena); while (z) { zone* next_z = (zone*)gpr_atm_no_barrier_load(&z->next_atm); - gpr_free(z); + gpr_free_aligned(z); z = next_z; } return (size_t)size; @@ -64,11 +79,12 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { zone* next_z = (zone*)gpr_atm_acq_load(&z->next_atm); if (next_z == nullptr) { size_t next_z_size = (size_t)gpr_atm_no_barrier_load(&arena->size_so_far); - next_z = (zone*)gpr_zalloc(sizeof(zone) + next_z_size); + next_z = (zone*)zalloc_aligned(ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)) + + next_z_size); next_z->size_begin = z->size_end; next_z->size_end = z->size_end + next_z_size; if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) { - gpr_free(next_z); + gpr_free_aligned(next_z); next_z = (zone*)gpr_atm_acq_load(&z->next_atm); } } @@ -79,5 +95,8 @@ void* gpr_arena_alloc(gpr_arena* arena, size_t size) { } GPR_ASSERT(start >= z->size_begin); GPR_ASSERT(start + size <= z->size_end); - return ((char*)(z + 1)) + start - z->size_begin; + char* ptr = (z == &arena->initial_zone) + ? (char*)arena + ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(gpr_arena)) + : (char*)z + ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(zone)); + return ptr + start - z->size_begin; } diff --git a/src/core/lib/gpr/fork.cc b/src/core/lib/gpr/fork.cc index c47e686378..92023f4350 100644 --- a/src/core/lib/gpr/fork.cc +++ b/src/core/lib/gpr/fork.cc @@ -38,18 +38,32 @@ void grpc_fork_support_init() { fork_support_enabled = 1; #else fork_support_enabled = 0; +#endif + bool env_var_set = false; char* env = gpr_getenv("GRPC_ENABLE_FORK_SUPPORT"); if (env != nullptr) { static const char* truthy[] = {"yes", "Yes", "YES", "true", "True", "TRUE", "1"}; + static const char* falsey[] = {"no", "No", "NO", "false", + "False", "FALSE", "0"}; for (size_t i = 0; i < GPR_ARRAY_SIZE(truthy); i++) { if (0 == strcmp(env, truthy[i])) { fork_support_enabled = 1; + env_var_set = true; + break; + } + } + if (!env_var_set) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(falsey); i++) { + if (0 == strcmp(env, falsey[i])) { + fork_support_enabled = 0; + env_var_set = true; + break; + } } } gpr_free(env); } -#endif if (override_fork_support_enabled != -1) { fork_support_enabled = override_fork_support_enabled; } diff --git a/src/core/lib/gpr/sync_posix.cc b/src/core/lib/gpr/sync_posix.cc index c3f6b10463..f23a8b2486 100644 --- a/src/core/lib/gpr/sync_posix.cc +++ b/src/core/lib/gpr/sync_posix.cc @@ -43,23 +43,19 @@ void gpr_mu_lock(gpr_mu* mu) { #ifdef GPR_LOW_LEVEL_COUNTERS GPR_ATM_INC_COUNTER(gpr_mu_locks); #endif - GPR_TIMER_BEGIN("gpr_mu_lock", 0); + GPR_TIMER_SCOPE("gpr_mu_lock", 0); GPR_ASSERT(pthread_mutex_lock(mu) == 0); - GPR_TIMER_END("gpr_mu_lock", 0); } void gpr_mu_unlock(gpr_mu* mu) { - GPR_TIMER_BEGIN("gpr_mu_unlock", 0); + GPR_TIMER_SCOPE("gpr_mu_unlock", 0); GPR_ASSERT(pthread_mutex_unlock(mu) == 0); - GPR_TIMER_END("gpr_mu_unlock", 0); } int gpr_mu_trylock(gpr_mu* mu) { - int err; - GPR_TIMER_BEGIN("gpr_mu_trylock", 0); - err = pthread_mutex_trylock(mu); + GPR_TIMER_SCOPE("gpr_mu_trylock", 0); + int err = pthread_mutex_trylock(mu); GPR_ASSERT(err == 0 || err == EBUSY); - GPR_TIMER_END("gpr_mu_trylock", 0); return err == 0; } diff --git a/src/core/lib/gprpp/inlined_vector.h b/src/core/lib/gprpp/inlined_vector.h index b78f85b893..2ced3d74b8 100644 --- a/src/core/lib/gprpp/inlined_vector.h +++ b/src/core/lib/gprpp/inlined_vector.h @@ -40,24 +40,13 @@ namespace grpc_core { // ANY METHOD ADDED HERE MUST COMPLY WITH THE INTERFACE IN THE absl // IMPLEMENTATION! // -// TODO(ctiller, nnoble, roth): Replace this with absl::InlinedVector -// once we integrate absl into the gRPC build system in a usable way. +// TODO(nnoble, roth): Replace this with absl::InlinedVector once we +// integrate absl into the gRPC build system in a usable way. template <typename T, size_t N> class InlinedVector { public: - InlinedVector() {} - ~InlinedVector() { - for (size_t i = 0; i < size_ && i < N; ++i) { - T& value = *reinterpret_cast<T*>(inline_ + i); - value.~T(); - } - if (size_ > N) { // Avoid subtracting two signed values. - for (size_t i = 0; i < size_ - N; ++i) { - dynamic_[i].~T(); - } - } - gpr_free(dynamic_); - } + InlinedVector() { init_data(); } + ~InlinedVector() { destroy_elements(); } // For now, we do not support copying. InlinedVector(const InlinedVector&) = delete; @@ -72,6 +61,15 @@ class InlinedVector { } } + const T& operator[](size_t offset) const { + assert(offset < size_); + if (offset < N) { + return *reinterpret_cast<const T*>(inline_ + offset); + } else { + return dynamic_[offset - N]; + } + } + template <typename... Args> void emplace_back(Args&&... args) { if (size_ < N) { @@ -100,11 +98,35 @@ class InlinedVector { size_t size() const { return size_; } + void clear() { + destroy_elements(); + init_data(); + } + private: + void init_data() { + dynamic_ = nullptr; + size_ = 0; + dynamic_capacity_ = 0; + } + + void destroy_elements() { + for (size_t i = 0; i < size_ && i < N; ++i) { + T& value = *reinterpret_cast<T*>(inline_ + i); + value.~T(); + } + if (size_ > N) { // Avoid subtracting two signed values. + for (size_t i = 0; i < size_ - N; ++i) { + dynamic_[i].~T(); + } + } + gpr_free(dynamic_); + } + typename std::aligned_storage<sizeof(T)>::type inline_[N]; - T* dynamic_ = nullptr; - size_t size_ = 0; - size_t dynamic_capacity_ = 0; + T* dynamic_; + size_t size_; + size_t dynamic_capacity_; }; } // namespace grpc_core diff --git a/src/core/lib/iomgr/call_combiner.cc b/src/core/lib/iomgr/call_combiner.cc index a9f48fb3c2..0e4c9cb3b3 100644 --- a/src/core/lib/iomgr/call_combiner.cc +++ b/src/core/lib/iomgr/call_combiner.cc @@ -60,7 +60,7 @@ void grpc_call_combiner_start(grpc_call_combiner* call_combiner, grpc_closure* closure, grpc_error* error DEBUG_ARGS, const char* reason) { - GPR_TIMER_BEGIN("call_combiner_start", 0); + GPR_TIMER_SCOPE("call_combiner_start", 0); if (grpc_call_combiner_trace.enabled()) { gpr_log(GPR_DEBUG, "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR @@ -92,12 +92,11 @@ void grpc_call_combiner_start(grpc_call_combiner* call_combiner, closure->error_data.error = error; gpr_mpscq_push(&call_combiner->queue, (gpr_mpscq_node*)closure); } - GPR_TIMER_END("call_combiner_start", 0); } void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS, const char* reason) { - GPR_TIMER_BEGIN("call_combiner_stop", 0); + GPR_TIMER_SCOPE("call_combiner_stop", 0); if (grpc_call_combiner_trace.enabled()) { gpr_log(GPR_DEBUG, "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]", @@ -136,7 +135,6 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS, } else if (grpc_call_combiner_trace.enabled()) { gpr_log(GPR_DEBUG, " queue empty"); } - GPR_TIMER_END("call_combiner_stop", 0); } void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner, diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 249fca6cd6..2bf2578c63 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -247,7 +247,7 @@ inline void grpc_closure_run(const char* file, int line, grpc_closure* c, #else inline void grpc_closure_run(grpc_closure* c, grpc_error* error) { #endif - GPR_TIMER_BEGIN("grpc_closure_run", 0); + GPR_TIMER_SCOPE("grpc_closure_run", 0); if (c != nullptr) { #ifndef NDEBUG c->file_initiated = file; @@ -259,7 +259,6 @@ inline void grpc_closure_run(grpc_closure* c, grpc_error* error) { } else { GRPC_ERROR_UNREF(error); } - GPR_TIMER_END("grpc_closure_run", 0); } /** Run a closure directly. Caller ensures that no locks are being held above. @@ -278,7 +277,7 @@ inline void grpc_closure_sched(const char* file, int line, grpc_closure* c, #else inline void grpc_closure_sched(grpc_closure* c, grpc_error* error) { #endif - GPR_TIMER_BEGIN("grpc_closure_sched", 0); + GPR_TIMER_SCOPE("grpc_closure_sched", 0); if (c != nullptr) { #ifndef NDEBUG if (c->scheduled) { @@ -299,7 +298,6 @@ inline void grpc_closure_sched(grpc_closure* c, grpc_error* error) { } else { GRPC_ERROR_UNREF(error); } - GPR_TIMER_END("grpc_closure_sched", 0); } /** Schedule a closure to be run. Does not need to be run from a safe point. */ diff --git a/src/core/lib/iomgr/combiner.cc b/src/core/lib/iomgr/combiner.cc index e4d7a6abd8..2965d807c1 100644 --- a/src/core/lib/iomgr/combiner.cc +++ b/src/core/lib/iomgr/combiner.cc @@ -153,8 +153,8 @@ static void push_first_on_exec_ctx(grpc_combiner* lock) { offsetof(grpc_combiner, scheduler_name))) static void combiner_exec(grpc_closure* cl, grpc_error* error) { + GPR_TIMER_SCOPE("combiner.execute", 0); GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_ITEMS(); - GPR_TIMER_BEGIN("combiner.execute", 0); grpc_combiner* lock = COMBINER_FROM_CLOSURE_SCHEDULER(cl, scheduler); gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT); GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, @@ -181,7 +181,6 @@ static void combiner_exec(grpc_closure* cl, grpc_error* error) { assert(cl->cb); cl->error_data.error = error; gpr_mpscq_push(&lock->queue, &cl->next_data.atm_next); - GPR_TIMER_END("combiner.execute", 0); } static void move_next() { @@ -207,11 +206,10 @@ static void queue_offload(grpc_combiner* lock) { } bool grpc_combiner_continue_exec_ctx() { - GPR_TIMER_BEGIN("combiner.continue_exec_ctx", 0); + GPR_TIMER_SCOPE("combiner.continue_exec_ctx", 0); grpc_combiner* lock = grpc_core::ExecCtx::Get()->combiner_data()->active_combiner; if (lock == nullptr) { - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return false; } @@ -233,7 +231,6 @@ bool grpc_combiner_continue_exec_ctx() { // this execution context wants to move on: schedule remaining work to be // picked up on the executor queue_offload(lock); - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return true; } @@ -249,10 +246,9 @@ bool grpc_combiner_continue_exec_ctx() { // go off and do something else for a while (and come back later) GPR_TIMER_MARK("delay_busy", 0); queue_offload(lock); - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return true; } - GPR_TIMER_BEGIN("combiner.exec1", 0); + GPR_TIMER_SCOPE("combiner.exec1", 0); grpc_closure* cl = (grpc_closure*)n; grpc_error* cl_err = cl->error_data.error; #ifndef NDEBUG @@ -260,14 +256,13 @@ bool grpc_combiner_continue_exec_ctx() { #endif cl->cb(cl->cb_arg, cl_err); GRPC_ERROR_UNREF(cl_err); - GPR_TIMER_END("combiner.exec1", 0); } else { grpc_closure* c = lock->final_list.head; GPR_ASSERT(c != nullptr); grpc_closure_list_init(&lock->final_list); int loops = 0; while (c != nullptr) { - GPR_TIMER_BEGIN("combiner.exec_1final", 0); + GPR_TIMER_SCOPE("combiner.exec_1final", 0); GRPC_COMBINER_TRACE( gpr_log(GPR_DEBUG, "C:%p execute_final[%d] c=%p", lock, loops, c)); grpc_closure* next = c->next_data.next; @@ -278,7 +273,6 @@ bool grpc_combiner_continue_exec_ctx() { c->cb(c->cb_arg, error); GRPC_ERROR_UNREF(error); c = next; - GPR_TIMER_END("combiner.exec_1final", 0); } } @@ -308,41 +302,36 @@ bool grpc_combiner_continue_exec_ctx() { break; case OLD_STATE_WAS(false, 1): // had one count, one unorphaned --> unlocked unorphaned - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return true; case OLD_STATE_WAS(true, 1): // and one count, one orphaned --> unlocked and orphaned really_destroy(lock); - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return true; case OLD_STATE_WAS(false, 0): case OLD_STATE_WAS(true, 0): // these values are illegal - representing an already unlocked or // deleted lock - GPR_TIMER_END("combiner.continue_exec_ctx", 0); GPR_UNREACHABLE_CODE(return true); } push_first_on_exec_ctx(lock); - GPR_TIMER_END("combiner.continue_exec_ctx", 0); return true; } static void enqueue_finally(void* closure, grpc_error* error); static void combiner_finally_exec(grpc_closure* closure, grpc_error* error) { + GPR_TIMER_SCOPE("combiner.execute_finally", 0); GRPC_STATS_INC_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS(); grpc_combiner* lock = COMBINER_FROM_CLOSURE_SCHEDULER(closure, finally_scheduler); GRPC_COMBINER_TRACE(gpr_log( GPR_DEBUG, "C:%p grpc_combiner_execute_finally c=%p; ac=%p", lock, closure, grpc_core::ExecCtx::Get()->combiner_data()->active_combiner)); - GPR_TIMER_BEGIN("combiner.execute_finally", 0); if (grpc_core::ExecCtx::Get()->combiner_data()->active_combiner != lock) { GPR_TIMER_MARK("slowpath", 0); GRPC_CLOSURE_SCHED(GRPC_CLOSURE_CREATE(enqueue_finally, closure, grpc_combiner_scheduler(lock)), error); - GPR_TIMER_END("combiner.execute_finally", 0); return; } @@ -350,7 +339,6 @@ static void combiner_finally_exec(grpc_closure* closure, grpc_error* error) { gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT); } grpc_closure_list_append(&lock->final_list, closure, error); - GPR_TIMER_END("combiner.execute_finally", 0); } static void enqueue_finally(void* closure, grpc_error* error) { diff --git a/src/core/lib/iomgr/error.cc b/src/core/lib/iomgr/error.cc index 67c3caf5ee..0e27ad427c 100644 --- a/src/core/lib/iomgr/error.cc +++ b/src/core/lib/iomgr/error.cc @@ -314,7 +314,7 @@ static void internal_add_error(grpc_error** err, grpc_error* new_err) { grpc_error* grpc_error_create(const char* file, int line, grpc_slice desc, grpc_error** referencing, size_t num_referencing) { - GPR_TIMER_BEGIN("grpc_error_create", 0); + GPR_TIMER_SCOPE("grpc_error_create", 0); uint8_t initial_arena_capacity = (uint8_t)( DEFAULT_ERROR_CAPACITY + (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY); @@ -355,7 +355,6 @@ grpc_error* grpc_error_create(const char* file, int line, grpc_slice desc, gpr_atm_no_barrier_store(&err->atomics.error_string, 0); gpr_ref_init(&err->atomics.refs, 1); - GPR_TIMER_END("grpc_error_create", 0); return err; } @@ -378,7 +377,7 @@ static void ref_errs(grpc_error* err) { } static grpc_error* copy_error_and_unref(grpc_error* in) { - GPR_TIMER_BEGIN("copy_error_and_unref", 0); + GPR_TIMER_SCOPE("copy_error_and_unref", 0); grpc_error* out; if (grpc_error_is_special(in)) { out = GRPC_ERROR_CREATE_FROM_STATIC_STRING("unknown"); @@ -422,16 +421,14 @@ static grpc_error* copy_error_and_unref(grpc_error* in) { ref_errs(out); GRPC_ERROR_UNREF(in); } - GPR_TIMER_END("copy_error_and_unref", 0); return out; } grpc_error* grpc_error_set_int(grpc_error* src, grpc_error_ints which, intptr_t value) { - GPR_TIMER_BEGIN("grpc_error_set_int", 0); + GPR_TIMER_SCOPE("grpc_error_set_int", 0); grpc_error* new_err = copy_error_and_unref(src); internal_set_int(&new_err, which, value); - GPR_TIMER_END("grpc_error_set_int", 0); return new_err; } @@ -447,36 +444,31 @@ static special_error_status_map error_status_map[] = { }; bool grpc_error_get_int(grpc_error* err, grpc_error_ints which, intptr_t* p) { - GPR_TIMER_BEGIN("grpc_error_get_int", 0); + GPR_TIMER_SCOPE("grpc_error_get_int", 0); if (grpc_error_is_special(err)) { if (which == GRPC_ERROR_INT_GRPC_STATUS) { for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { if (error_status_map[i].error == err) { if (p != nullptr) *p = error_status_map[i].code; - GPR_TIMER_END("grpc_error_get_int", 0); return true; } } } - GPR_TIMER_END("grpc_error_get_int", 0); return false; } uint8_t slot = err->ints[which]; if (slot != UINT8_MAX) { if (p != nullptr) *p = err->arena[slot]; - GPR_TIMER_END("grpc_error_get_int", 0); return true; } - GPR_TIMER_END("grpc_error_get_int", 0); return false; } grpc_error* grpc_error_set_str(grpc_error* src, grpc_error_strs which, grpc_slice str) { - GPR_TIMER_BEGIN("grpc_error_set_str", 0); + GPR_TIMER_SCOPE("grpc_error_set_str", 0); grpc_error* new_err = copy_error_and_unref(src); internal_set_str(&new_err, which, str); - GPR_TIMER_END("grpc_error_set_str", 0); return new_err; } @@ -503,10 +495,9 @@ bool grpc_error_get_str(grpc_error* err, grpc_error_strs which, } grpc_error* grpc_error_add_child(grpc_error* src, grpc_error* child) { - GPR_TIMER_BEGIN("grpc_error_add_child", 0); + GPR_TIMER_SCOPE("grpc_error_add_child", 0); grpc_error* new_err = copy_error_and_unref(src); internal_add_error(&new_err, child); - GPR_TIMER_END("grpc_error_add_child", 0); return new_err; } @@ -722,14 +713,13 @@ static char* finish_kvs(kv_pairs* kvs) { } const char* grpc_error_string(grpc_error* err) { - GPR_TIMER_BEGIN("grpc_error_string", 0); + GPR_TIMER_SCOPE("grpc_error_string", 0); if (err == GRPC_ERROR_NONE) return no_error_string; if (err == GRPC_ERROR_OOM) return oom_error_string; if (err == GRPC_ERROR_CANCELLED) return cancelled_error_string; void* p = (void*)gpr_atm_acq_load(&err->atomics.error_string); if (p != nullptr) { - GPR_TIMER_END("grpc_error_string", 0); return (const char*)p; } @@ -752,7 +742,6 @@ const char* grpc_error_string(grpc_error* err) { out = (char*)gpr_atm_acq_load(&err->atomics.error_string); } - GPR_TIMER_END("grpc_error_string", 0); return out; } diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 42d7cdd348..1ef70d2f80 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -519,7 +519,7 @@ static void pollset_destroy(grpc_pollset* pollset) { } static grpc_error* pollset_kick_all(grpc_pollset* pollset) { - GPR_TIMER_BEGIN("pollset_kick_all", 0); + GPR_TIMER_SCOPE("pollset_kick_all", 0); grpc_error* error = GRPC_ERROR_NONE; if (pollset->root_worker != nullptr) { grpc_pollset_worker* worker = pollset->root_worker; @@ -549,7 +549,6 @@ static grpc_error* pollset_kick_all(grpc_pollset* pollset) { } // TODO: sreek. Check if we need to set 'kicked_without_poller' to true here // in the else case - GPR_TIMER_END("pollset_kick_all", 0); return error; } @@ -563,14 +562,13 @@ static void pollset_maybe_finish_shutdown(grpc_pollset* pollset) { } static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { - GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_TIMER_SCOPE("pollset_shutdown", 0); GPR_ASSERT(pollset->shutdown_closure == nullptr); GPR_ASSERT(!pollset->shutting_down); pollset->shutdown_closure = closure; pollset->shutting_down = true; GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); pollset_maybe_finish_shutdown(pollset); - GPR_TIMER_END("pollset_shutdown", 0); } static int poll_deadline_to_millis_timeout(grpc_millis millis) { @@ -594,10 +592,10 @@ static int poll_deadline_to_millis_timeout(grpc_millis millis) { called by g_active_poller thread. So there is no need for synchronization when accessing fields in g_epoll_set */ static grpc_error* process_epoll_events(grpc_pollset* pollset) { + GPR_TIMER_SCOPE("process_epoll_events", 0); + static const char* err_desc = "process_events"; grpc_error* error = GRPC_ERROR_NONE; - - GPR_TIMER_BEGIN("process_epoll_events", 0); long num_events = gpr_atm_acq_load(&g_epoll_set.num_events); long cursor = gpr_atm_acq_load(&g_epoll_set.cursor); for (int idx = 0; @@ -626,7 +624,6 @@ static grpc_error* process_epoll_events(grpc_pollset* pollset) { } } gpr_atm_rel_store(&g_epoll_set.cursor, cursor); - GPR_TIMER_END("process_epoll_events", 0); return error; } @@ -638,7 +635,7 @@ static grpc_error* process_epoll_events(grpc_pollset* pollset) { (i.e the designated poller thread) will be calling this function. So there is no need for any synchronization when accesing fields in g_epoll_set */ static grpc_error* do_epoll_wait(grpc_pollset* ps, grpc_millis deadline) { - GPR_TIMER_BEGIN("do_epoll_wait", 0); + GPR_TIMER_SCOPE("do_epoll_wait", 0); int r; int timeout = poll_deadline_to_millis_timeout(deadline); @@ -665,14 +662,13 @@ static grpc_error* do_epoll_wait(grpc_pollset* ps, grpc_millis deadline) { gpr_atm_rel_store(&g_epoll_set.num_events, r); gpr_atm_rel_store(&g_epoll_set.cursor, 0); - GPR_TIMER_END("do_epoll_wait", 0); return GRPC_ERROR_NONE; } static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { - GPR_TIMER_BEGIN("begin_worker", 0); + GPR_TIMER_SCOPE("begin_worker", 0); if (worker_hdl != nullptr) *worker_hdl = worker; worker->initialized_cv = false; SET_KICK_STATE(worker, UNKICKED); @@ -680,7 +676,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, pollset->begin_refs++; if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, "PS:%p BEGIN_STARTS:%p", pollset, worker); + gpr_log(GPR_DEBUG, "PS:%p BEGIN_STARTS:%p", pollset, worker); } if (pollset->seen_inactive) { @@ -699,7 +695,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", + gpr_log(GPR_DEBUG, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", pollset, worker, kick_state_string(worker->state), is_reassigning); } @@ -751,7 +747,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, gpr_cv_init(&worker->cv); while (worker->state == UNKICKED && !pollset->shutting_down) { if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d", + gpr_log(GPR_DEBUG, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d", pollset, worker, kick_state_string(worker->state), pollset->shutting_down); } @@ -768,7 +764,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, } if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, + gpr_log(GPR_DEBUG, "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d " "kicked_without_poller: %d", pollset, worker, kick_state_string(worker->state), @@ -786,17 +782,15 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; - GPR_TIMER_END("begin_worker", 0); return false; } - GPR_TIMER_END("begin_worker", 0); return worker->state == DESIGNATED_POLLER && !pollset->shutting_down; } static bool check_neighborhood_for_available_poller( pollset_neighborhood* neighborhood) { - GPR_TIMER_BEGIN("check_neighborhood_for_available_poller", 0); + GPR_TIMER_SCOPE("check_neighborhood_for_available_poller", 0); bool found_worker = false; do { grpc_pollset* inspect = neighborhood->active_root; @@ -855,13 +849,12 @@ static bool check_neighborhood_for_available_poller( } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - GPR_TIMER_END("check_neighborhood_for_available_poller", 0); return found_worker; } static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl) { - GPR_TIMER_BEGIN("end_worker", 0); + GPR_TIMER_SCOPE("end_worker", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PS:%p END_WORKER:%p", pollset, worker); } @@ -931,7 +924,6 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, pollset_maybe_finish_shutdown(pollset); } GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); - GPR_TIMER_END("end_worker", 0); } /* pollset->po.mu lock must be held by the caller before calling this. @@ -941,13 +933,12 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, static grpc_error* pollset_work(grpc_pollset* ps, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { + GPR_TIMER_SCOPE("pollset_work", 0); grpc_pollset_worker worker; grpc_error* error = GRPC_ERROR_NONE; static const char* err_desc = "pollset_work"; - GPR_TIMER_BEGIN("pollset_work", 0); if (ps->kicked_without_poller) { ps->kicked_without_poller = false; - GPR_TIMER_END("pollset_work", 0); return GRPC_ERROR_NONE; } @@ -987,13 +978,12 @@ static grpc_error* pollset_work(grpc_pollset* ps, end_worker(ps, &worker, worker_hdl); gpr_tls_set(&g_current_thread_pollset, 0); - GPR_TIMER_END("pollset_work", 0); return error; } static grpc_error* pollset_kick(grpc_pollset* pollset, grpc_pollset_worker* specific_worker) { - GPR_TIMER_BEGIN("pollset_kick", 0); + GPR_TIMER_SCOPE("pollset_kick", 0); GRPC_STATS_INC_POLLSET_KICK(); grpc_error* ret_err = GRPC_ERROR_NONE; if (grpc_polling_trace.enabled()) { @@ -1019,7 +1009,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } tmp = gpr_strvec_flatten(&log, nullptr); gpr_strvec_destroy(&log); - gpr_log(GPR_ERROR, "%s", tmp); + gpr_log(GPR_DEBUG, "%s", tmp); gpr_free(tmp); } @@ -1030,7 +1020,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(); pollset->kicked_without_poller = true; if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kicked_without_poller"); + gpr_log(GPR_DEBUG, " .. kicked_without_poller"); } goto done; } @@ -1038,14 +1028,14 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, if (root_worker->state == KICKED) { GRPC_STATS_INC_POLLSET_KICKED_AGAIN(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); + gpr_log(GPR_DEBUG, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; } else if (next_worker->state == KICKED) { GRPC_STATS_INC_POLLSET_KICKED_AGAIN(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); + gpr_log(GPR_DEBUG, " .. already kicked %p", next_worker); } SET_KICK_STATE(next_worker, KICKED); goto done; @@ -1056,7 +1046,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, &g_active_poller)) { GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kicked %p", root_worker); + gpr_log(GPR_DEBUG, " .. kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); @@ -1064,7 +1054,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } else if (next_worker->state == UNKICKED) { GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kicked %p", next_worker); + gpr_log(GPR_DEBUG, " .. kicked %p", next_worker); } GPR_ASSERT(next_worker->initialized_cv); SET_KICK_STATE(next_worker, KICKED); @@ -1074,7 +1064,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, if (root_worker->state != DESIGNATED_POLLER) { if (grpc_polling_trace.enabled()) { gpr_log( - GPR_ERROR, + GPR_DEBUG, " .. kicked root non-poller %p (initialized_cv=%d) (poller=%p)", root_worker, root_worker->initialized_cv, next_worker); } @@ -1087,7 +1077,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } else { GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker, + gpr_log(GPR_DEBUG, " .. non-root poller %p (root=%p)", next_worker, root_worker); } SET_KICK_STATE(next_worker, KICKED); @@ -1103,7 +1093,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } else { GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kicked while waking up"); + gpr_log(GPR_DEBUG, " .. kicked while waking up"); } goto done; } @@ -1113,14 +1103,14 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, if (specific_worker->state == KICKED) { if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. specific worker already kicked"); + gpr_log(GPR_DEBUG, " .. specific worker already kicked"); } goto done; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker); + gpr_log(GPR_DEBUG, " .. mark %p kicked", specific_worker); } SET_KICK_STATE(specific_worker, KICKED); goto done; @@ -1128,7 +1118,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, (grpc_pollset_worker*)gpr_atm_no_barrier_load(&g_active_poller)) { GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kick active poller"); + gpr_log(GPR_DEBUG, " .. kick active poller"); } SET_KICK_STATE(specific_worker, KICKED); ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); @@ -1136,7 +1126,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } else if (specific_worker->initialized_cv) { GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kick waiting worker"); + gpr_log(GPR_DEBUG, " .. kick waiting worker"); } SET_KICK_STATE(specific_worker, KICKED); gpr_cv_signal(&specific_worker->cv); @@ -1144,13 +1134,12 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } else { GRPC_STATS_INC_POLLSET_KICKED_AGAIN(); if (grpc_polling_trace.enabled()) { - gpr_log(GPR_ERROR, " .. kick non-waiting worker"); + gpr_log(GPR_DEBUG, " .. kick non-waiting worker"); } SET_KICK_STATE(specific_worker, KICKED); goto done; } done: - GPR_TIMER_END("pollset_kick", 0); return ret_err; } diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 416e8384b4..178ebd8977 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -199,6 +199,7 @@ struct grpc_pollset { pollable* active_pollable; bool kicked_without_poller; grpc_closure* shutdown_closure; + bool already_shutdown; grpc_pollset_worker* root_worker; int containing_pollset_set_count; }; @@ -560,8 +561,10 @@ static void pollset_maybe_finish_shutdown(grpc_pollset* pollset) { } if (pollset->shutdown_closure != nullptr && pollset->root_worker == nullptr && pollset->containing_pollset_set_count == 0) { + GPR_TIMER_MARK("pollset_finish_shutdown", 0); GRPC_CLOSURE_SCHED(pollset->shutdown_closure, GRPC_ERROR_NONE); pollset->shutdown_closure = nullptr; + pollset->already_shutdown = true; } } @@ -569,6 +572,7 @@ static void pollset_maybe_finish_shutdown(grpc_pollset* pollset) { * pollset->active_pollable->mu & specific_worker->pollable_obj->mu must not be * held */ static grpc_error* kick_one_worker(grpc_pollset_worker* specific_worker) { + GPR_TIMER_SCOPE("kick_one_worker", 0); pollable* p = specific_worker->pollable_obj; grpc_core::mu_guard lock(&p->mu); GPR_ASSERT(specific_worker != nullptr); @@ -612,6 +616,7 @@ static grpc_error* kick_one_worker(grpc_pollset_worker* specific_worker) { static grpc_error* pollset_kick(grpc_pollset* pollset, grpc_pollset_worker* specific_worker) { + GPR_TIMER_SCOPE("pollset_kick", 0); GRPC_STATS_INC_POLLSET_KICK(); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, @@ -661,6 +666,7 @@ static grpc_error* pollset_kick(grpc_pollset* pollset, } static grpc_error* pollset_kick_all(grpc_pollset* pollset) { + GPR_TIMER_SCOPE("pollset_kick_all", 0); grpc_error* error = GRPC_ERROR_NONE; const char* err_desc = "pollset_kick_all"; grpc_pollset_worker* w = pollset->root_worker; @@ -677,6 +683,11 @@ static grpc_error* pollset_kick_all(grpc_pollset* pollset) { static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) { gpr_mu_init(&pollset->mu); pollset->active_pollable = POLLABLE_REF(g_empty_pollable, "pollset"); + pollset->kicked_without_poller = false; + pollset->shutdown_closure = nullptr; + pollset->already_shutdown = false; + pollset->root_worker = nullptr; + pollset->containing_pollset_set_count = 0; *mu = &pollset->mu; } @@ -733,6 +744,7 @@ static grpc_error* fd_get_or_become_pollable(grpc_fd* fd, pollable** p) { /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { + GPR_TIMER_SCOPE("pollset_shutdown", 0); GPR_ASSERT(pollset->shutdown_closure == nullptr); pollset->shutdown_closure = closure; GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); @@ -741,6 +753,7 @@ static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { static grpc_error* pollable_process_events(grpc_pollset* pollset, pollable* pollable_obj, bool drain) { + GPR_TIMER_SCOPE("pollable_process_events", 0); static const char* err_desc = "pollset_process_events"; grpc_error* error = GRPC_ERROR_NONE; for (int i = 0; (drain || i < MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) && @@ -787,6 +800,7 @@ static void pollset_destroy(grpc_pollset* pollset) { } static grpc_error* pollable_epoll(pollable* p, grpc_millis deadline) { + GPR_TIMER_SCOPE("pollable_epoll", 0); int timeout = poll_deadline_to_millis_timeout(deadline); if (grpc_polling_trace.enabled()) { @@ -862,7 +876,9 @@ static worker_remove_result worker_remove(grpc_pollset_worker** root_worker, static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { - bool do_poll = (pollset->shutdown_closure == nullptr); + GPR_TIMER_SCOPE("begin_worker", 0); + bool do_poll = + (pollset->shutdown_closure == nullptr && !pollset->already_shutdown); if (worker_hdl != nullptr) *worker_hdl = worker; worker->initialized_cv = false; worker->kicked = false; @@ -913,6 +929,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl) { + GPR_TIMER_SCOPE("end_worker", 0); gpr_mu_lock(&pollset->mu); gpr_mu_lock(&worker->pollable_obj->mu); switch (worker_remove(&worker->pollable_obj->root_worker, worker, @@ -955,6 +972,7 @@ static long gettid(void) { return syscall(__NR_gettid); } static grpc_error* pollset_work(grpc_pollset* pollset, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { + GPR_TIMER_SCOPE("pollset_work", 0); #ifdef GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP grpc_pollset_worker* worker = (grpc_pollset_worker*)gpr_malloc(sizeof(*worker)); @@ -1092,6 +1110,16 @@ static grpc_error* pollset_as_multipollable_locked(grpc_pollset* pollset, case PO_EMPTY: POLLABLE_UNREF(pollset->active_pollable, "pollset"); error = pollable_create(PO_MULTI, &pollset->active_pollable); + /* Any workers currently polling on this pollset must now be woked up so + * that they can pick up the new active_pollable */ + if (grpc_polling_trace.enabled()) { + gpr_log(GPR_DEBUG, + "PS:%p active pollable transition from empty to multi", + pollset); + } + static const char* err_desc = + "pollset_as_multipollable_locked: empty -> multi"; + append_error(&error, pollset_kick_all(pollset), err_desc); break; case PO_FD: gpr_mu_lock(&po_at_start->owner_fd->orphan_mu); @@ -1120,6 +1148,7 @@ static grpc_error* pollset_as_multipollable_locked(grpc_pollset* pollset, } static void pollset_add_fd(grpc_pollset* pollset, grpc_fd* fd) { + GPR_TIMER_SCOPE("pollset_add_fd", 0); gpr_mu_lock(&pollset->mu); grpc_error* error = pollset_add_fd_locked(pollset, fd); gpr_mu_unlock(&pollset->mu); @@ -1168,6 +1197,7 @@ static void pollset_set_unref(grpc_pollset_set* pss) { } static void pollset_set_add_fd(grpc_pollset_set* pss, grpc_fd* fd) { + GPR_TIMER_SCOPE("pollset_set_add_fd", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PSS:%p: add fd %p (%d)", pss, fd, fd->fd); } @@ -1191,6 +1221,7 @@ static void pollset_set_add_fd(grpc_pollset_set* pss, grpc_fd* fd) { } static void pollset_set_del_fd(grpc_pollset_set* pss, grpc_fd* fd) { + GPR_TIMER_SCOPE("pollset_set_del_fd", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PSS:%p: del fd %p", pss, fd); } @@ -1211,6 +1242,7 @@ static void pollset_set_del_fd(grpc_pollset_set* pss, grpc_fd* fd) { } static void pollset_set_del_pollset(grpc_pollset_set* pss, grpc_pollset* ps) { + GPR_TIMER_SCOPE("pollset_set_del_pollset", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PSS:%p: del pollset %p", pss, ps); } @@ -1241,6 +1273,7 @@ static grpc_error* add_fds_to_pollsets(grpc_fd** fds, size_t fd_count, size_t pollset_count, const char* err_desc, grpc_fd** out_fds, size_t* out_fd_count) { + GPR_TIMER_SCOPE("add_fds_to_pollsets", 0); grpc_error* error = GRPC_ERROR_NONE; for (size_t i = 0; i < fd_count; i++) { gpr_mu_lock(&fds[i]->orphan_mu); @@ -1261,6 +1294,7 @@ static grpc_error* add_fds_to_pollsets(grpc_fd** fds, size_t fd_count, } static void pollset_set_add_pollset(grpc_pollset_set* pss, grpc_pollset* ps) { + GPR_TIMER_SCOPE("pollset_set_add_pollset", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PSS:%p: add pollset %p", pss, ps); } @@ -1297,6 +1331,7 @@ static void pollset_set_add_pollset(grpc_pollset_set* pss, grpc_pollset* ps) { static void pollset_set_add_pollset_set(grpc_pollset_set* a, grpc_pollset_set* b) { + GPR_TIMER_SCOPE("pollset_set_add_pollset_set", 0); if (grpc_polling_trace.enabled()) { gpr_log(GPR_DEBUG, "PSS: merge (%p, %p)", a, b); } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc index 1518348992..c8954894ec 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.cc +++ b/src/core/lib/iomgr/ev_epollsig_linux.cc @@ -1024,7 +1024,7 @@ static void push_front_worker(grpc_pollset* p, grpc_pollset_worker* worker) { /* p->mu must be held before calling this function */ static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* specific_worker) { - GPR_TIMER_BEGIN("pollset_kick", 0); + GPR_TIMER_SCOPE("pollset_kick", 0); grpc_error* error = GRPC_ERROR_NONE; GRPC_STATS_INC_POLLSET_KICK(); const char* err_desc = "Kick Failure"; @@ -1032,14 +1032,13 @@ static grpc_error* pollset_kick(grpc_pollset* p, if (worker != nullptr) { if (worker == GRPC_POLLSET_KICK_BROADCAST) { if (pollset_has_workers(p)) { - GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + GPR_TIMER_SCOPE("pollset_kick.broadcast", 0); for (worker = p->root_worker.next; worker != &p->root_worker; worker = worker->next) { if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { append_error(&error, pollset_worker_kick(worker), err_desc); } } - GPR_TIMER_END("pollset_kick.broadcast", 0); } else { p->kicked_without_pollers = true; } @@ -1069,7 +1068,6 @@ static grpc_error* pollset_kick(grpc_pollset* p, } } - GPR_TIMER_END("pollset_kick", 0); GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); return error; } @@ -1136,7 +1134,7 @@ static void finish_shutdown_locked(grpc_pollset* pollset) { /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { - GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_TIMER_SCOPE("pollset_shutdown", 0); GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; @@ -1150,7 +1148,6 @@ static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); finish_shutdown_locked(pollset); } - GPR_TIMER_END("pollset_shutdown", 0); } /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other @@ -1166,13 +1163,13 @@ static void pollset_destroy(grpc_pollset* pollset) { static void pollset_work_and_unlock(grpc_pollset* pollset, grpc_pollset_worker* worker, int timeout_ms, sigset_t* sig_mask, grpc_error** error) { + GPR_TIMER_SCOPE("pollset_work_and_unlock", 0); struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int epoll_fd = -1; int ep_rv; polling_island* pi = nullptr; char* err_msg; const char* err_desc = "pollset_work_and_unlock"; - GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the latest polling island pointed by pollset->po.pi @@ -1187,7 +1184,6 @@ static void pollset_work_and_unlock(grpc_pollset* pollset, if (pollset->po.pi == nullptr) { pollset->po.pi = polling_island_create(nullptr, error); if (pollset->po.pi == nullptr) { - GPR_TIMER_END("pollset_work_and_unlock", 0); return; /* Fatal error. We cannot continue */ } @@ -1279,8 +1275,6 @@ static void pollset_work_and_unlock(grpc_pollset* pollset, pollset->po.pi pointer might get udpated in other parts of the code when there is an island merge while we are doing epoll_wait() above */ PI_UNREF(pi, "ps_work"); - - GPR_TIMER_END("pollset_work_and_unlock", 0); } /* pollset->po.mu lock must be held by the caller before calling this. @@ -1290,7 +1284,7 @@ static void pollset_work_and_unlock(grpc_pollset* pollset, static grpc_error* pollset_work(grpc_pollset* pollset, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { - GPR_TIMER_BEGIN("pollset_work", 0); + GPR_TIMER_SCOPE("pollset_work", 0); grpc_error* error = GRPC_ERROR_NONE; int timeout_ms = poll_deadline_to_millis_timeout(deadline); @@ -1384,15 +1378,13 @@ static grpc_error* pollset_work(grpc_pollset* pollset, gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); gpr_tls_set(&g_current_thread_worker, (intptr_t)0); - GPR_TIMER_END("pollset_work", 0); - GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; } static void add_poll_object(poll_obj* bag, poll_obj_type bag_type, poll_obj* item, poll_obj_type item_type) { - GPR_TIMER_BEGIN("add_poll_object", 0); + GPR_TIMER_SCOPE("add_poll_object", 0); #ifndef NDEBUG GPR_ASSERT(item->obj_type == item_type); @@ -1535,7 +1527,6 @@ retry: gpr_mu_unlock(&bag->mu); GRPC_LOG_IF_ERROR("add_poll_object", error); - GPR_TIMER_END("add_poll_object", 0); } static void pollset_add_fd(grpc_pollset* pollset, grpc_fd* fd) { diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index 3b79728055..ad92a93beb 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -706,14 +706,14 @@ static void kick_append_error(grpc_error** composite, grpc_error* error) { static grpc_error* pollset_kick_ext(grpc_pollset* p, grpc_pollset_worker* specific_worker, uint32_t flags) { - GPR_TIMER_BEGIN("pollset_kick_ext", 0); + GPR_TIMER_SCOPE("pollset_kick_ext", 0); grpc_error* error = GRPC_ERROR_NONE; GRPC_STATS_INC_POLLSET_KICK(); /* pollset->mu already held */ if (specific_worker != nullptr) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { - GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0); + GPR_TIMER_SCOPE("pollset_kick_ext.broadcast", 0); GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); for (specific_worker = p->root_worker.next; specific_worker != &p->root_worker; @@ -722,7 +722,6 @@ static grpc_error* pollset_kick_ext(grpc_pollset* p, &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); } p->kicked_without_pollers = true; - GPR_TIMER_END("pollset_kick_ext.broadcast", 0); } else if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)specific_worker) { GPR_TIMER_MARK("different_thread_worker", 0); @@ -769,7 +768,6 @@ static grpc_error* pollset_kick_ext(grpc_pollset* p, } } - GPR_TIMER_END("pollset_kick_ext", 0); GRPC_LOG_IF_ERROR("pollset_kick_ext", GRPC_ERROR_REF(error)); return error; } @@ -864,6 +862,8 @@ static void work_combine_error(grpc_error** composite, grpc_error* error) { static grpc_error* pollset_work(grpc_pollset* pollset, grpc_pollset_worker** worker_hdl, grpc_millis deadline) { + GPR_TIMER_SCOPE("pollset_work", 0); + grpc_pollset_worker worker; if (worker_hdl) *worker_hdl = &worker; grpc_error* error = GRPC_ERROR_NONE; @@ -878,7 +878,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset, int locked = 1; int queued_work = 0; int keep_polling = 0; - GPR_TIMER_BEGIN("pollset_work", 0); /* this must happen before we (potentially) drop pollset->mu */ worker.next = worker.prev = nullptr; worker.reevaluate_polling_on_wakeup = 0; @@ -921,7 +920,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, added_worker = 1; gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); } - GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); + GPR_TIMER_SCOPE("maybe_work_and_unlock", 0); #define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) #define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) @@ -1037,7 +1036,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset, gpr_free(pfds); } - GPR_TIMER_END("maybe_work_and_unlock", 0); locked = 0; } else { GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); @@ -1097,7 +1095,6 @@ static grpc_error* pollset_work(grpc_pollset* pollset, } } if (worker_hdl) *worker_hdl = nullptr; - GPR_TIMER_END("pollset_work", 0); GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; } diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 3a5714132d..8100bece8f 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -39,6 +39,18 @@ grpc_core::TraceFlag grpc_polling_trace(false, "polling"); /* Disabled by default */ grpc_core::DebugOnlyTraceFlag grpc_trace_fd_refcount(false, "fd_refcount"); +grpc_core::DebugOnlyTraceFlag grpc_polling_api_trace(false, "polling_api"); + +#ifndef NDEBUG + +// Polling API trace only enabled in debug builds +#define GRPC_POLLING_API_TRACE(format, ...) \ + if (grpc_polling_api_trace.enabled()) { \ + gpr_log(GPR_DEBUG, "(polling-api) " format, __VA_ARGS__); \ + } +#else +#define GRPC_POLLING_API_TRACE(...) +#endif /** Default poll() function - a pointer so that it can be overridden by some * tests */ @@ -177,6 +189,7 @@ void grpc_event_engine_shutdown(void) { } grpc_fd* grpc_fd_create(int fd, const char* name) { + GRPC_POLLING_API_TRACE("fd_create(%d, %s)", fd, name); return g_event_engine->fd_create(fd, name); } @@ -186,10 +199,14 @@ int grpc_fd_wrapped_fd(grpc_fd* fd) { void grpc_fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, bool already_closed, const char* reason) { + GRPC_POLLING_API_TRACE("fd_orphan(%d, %p, %p, %d, %s)", + grpc_fd_wrapped_fd(fd), on_done, release_fd, + already_closed, reason); g_event_engine->fd_orphan(fd, on_done, release_fd, already_closed, reason); } void grpc_fd_shutdown(grpc_fd* fd, grpc_error* why) { + GRPC_POLLING_API_TRACE("fd_shutdown(%d)", grpc_fd_wrapped_fd(fd)); g_event_engine->fd_shutdown(fd, why); } @@ -208,65 +225,89 @@ void grpc_fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) { size_t grpc_pollset_size(void) { return g_event_engine->pollset_size; } void grpc_pollset_init(grpc_pollset* pollset, gpr_mu** mu) { + GRPC_POLLING_API_TRACE("pollset_init(%p)", pollset); g_event_engine->pollset_init(pollset, mu); } void grpc_pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { + GRPC_POLLING_API_TRACE("pollset_shutdown(%p)", pollset); g_event_engine->pollset_shutdown(pollset, closure); } void grpc_pollset_destroy(grpc_pollset* pollset) { + GRPC_POLLING_API_TRACE("pollset_destroy(%p)", pollset); g_event_engine->pollset_destroy(pollset); } grpc_error* grpc_pollset_work(grpc_pollset* pollset, grpc_pollset_worker** worker, grpc_millis deadline) { - return g_event_engine->pollset_work(pollset, worker, deadline); + GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRIdPTR ") begin", pollset, + deadline); + grpc_error* err = g_event_engine->pollset_work(pollset, worker, deadline); + GRPC_POLLING_API_TRACE("pollset_work(%p, %" PRIdPTR ") end", pollset, + deadline); + return err; } grpc_error* grpc_pollset_kick(grpc_pollset* pollset, grpc_pollset_worker* specific_worker) { + GRPC_POLLING_API_TRACE("pollset_kick(%p, %p)", pollset, specific_worker); return g_event_engine->pollset_kick(pollset, specific_worker); } void grpc_pollset_add_fd(grpc_pollset* pollset, struct grpc_fd* fd) { + GRPC_POLLING_API_TRACE("pollset_add_fd(%p, %d)", pollset, + grpc_fd_wrapped_fd(fd)); g_event_engine->pollset_add_fd(pollset, fd); } grpc_pollset_set* grpc_pollset_set_create(void) { - return g_event_engine->pollset_set_create(); + grpc_pollset_set* pss = g_event_engine->pollset_set_create(); + GRPC_POLLING_API_TRACE("pollset_set_create(%p)", pss); + return pss; } void grpc_pollset_set_destroy(grpc_pollset_set* pollset_set) { + GRPC_POLLING_API_TRACE("pollset_set_destroy(%p)", pollset_set); g_event_engine->pollset_set_destroy(pollset_set); } void grpc_pollset_set_add_pollset(grpc_pollset_set* pollset_set, grpc_pollset* pollset) { + GRPC_POLLING_API_TRACE("pollset_set_add_pollset(%p, %p)", pollset_set, + pollset); g_event_engine->pollset_set_add_pollset(pollset_set, pollset); } void grpc_pollset_set_del_pollset(grpc_pollset_set* pollset_set, grpc_pollset* pollset) { + GRPC_POLLING_API_TRACE("pollset_set_del_pollset(%p, %p)", pollset_set, + pollset); g_event_engine->pollset_set_del_pollset(pollset_set, pollset); } void grpc_pollset_set_add_pollset_set(grpc_pollset_set* bag, grpc_pollset_set* item) { + GRPC_POLLING_API_TRACE("pollset_set_add_pollset_set(%p, %p)", bag, item); g_event_engine->pollset_set_add_pollset_set(bag, item); } void grpc_pollset_set_del_pollset_set(grpc_pollset_set* bag, grpc_pollset_set* item) { + GRPC_POLLING_API_TRACE("pollset_set_del_pollset_set(%p, %p)", bag, item); g_event_engine->pollset_set_del_pollset_set(bag, item); } void grpc_pollset_set_add_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { + GRPC_POLLING_API_TRACE("pollset_set_add_fd(%p, %d)", pollset_set, + grpc_fd_wrapped_fd(fd)); g_event_engine->pollset_set_add_fd(pollset_set, fd); } void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { + GRPC_POLLING_API_TRACE("pollset_set_del_fd(%p, %d)", pollset_set, + grpc_fd_wrapped_fd(fd)); g_event_engine->pollset_set_del_fd(pollset_set, fd); } diff --git a/src/core/lib/iomgr/exec_ctx.cc b/src/core/lib/iomgr/exec_ctx.cc index e005437e0a..03c833ff07 100644 --- a/src/core/lib/iomgr/exec_ctx.cc +++ b/src/core/lib/iomgr/exec_ctx.cc @@ -113,7 +113,7 @@ void ExecCtx::GlobalInit(void) { bool ExecCtx::Flush() { bool did_something = 0; - GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); + GPR_TIMER_SCOPE("grpc_exec_ctx_flush", 0); for (;;) { if (!grpc_closure_list_empty(closure_list_)) { grpc_closure* c = closure_list_.head; @@ -130,7 +130,6 @@ bool ExecCtx::Flush() { } } GPR_ASSERT(combiner_data_.active_combiner == nullptr); - GPR_TIMER_END("grpc_exec_ctx_flush", 0); return did_something; } diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 0ec5926227..07aed2d1fd 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -130,7 +130,7 @@ static void run_poller(void* bp, grpc_error* error_ignored) { gpr_log(GPR_DEBUG, "BACKUP_POLLER:%p run", p); } gpr_mu_lock(p->pollset_mu); - grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + 13 * GPR_MS_PER_SEC; + grpc_millis deadline = grpc_core::ExecCtx::Get()->Now() + 10 * GPR_MS_PER_SEC; GRPC_STATS_INC_TCP_BACKUP_POLLER_POLLS(); GRPC_LOG_IF_ERROR( "backup_poller:pollset_work", @@ -368,6 +368,7 @@ static void call_read_cb(grpc_tcp* tcp, grpc_error* error) { #define MAX_READ_IOVEC 4 static void tcp_do_read(grpc_tcp* tcp) { + GPR_TIMER_SCOPE("tcp_continue_read", 0); struct msghdr msg; struct iovec iov[MAX_READ_IOVEC]; ssize_t read_bytes; @@ -375,7 +376,6 @@ static void tcp_do_read(grpc_tcp* tcp) { GPR_ASSERT(!tcp->finished_edge); GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC); - GPR_TIMER_BEGIN("tcp_continue_read", 0); for (i = 0; i < tcp->incoming_buffer->count; i++) { iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); @@ -393,12 +393,11 @@ static void tcp_do_read(grpc_tcp* tcp) { GRPC_STATS_INC_TCP_READ_OFFER(tcp->incoming_buffer->length); GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(tcp->incoming_buffer->count); - GPR_TIMER_BEGIN("recvmsg", 0); do { + GPR_TIMER_SCOPE("recvmsg", 0); GRPC_STATS_INC_SYSCALL_READ(); read_bytes = recvmsg(tcp->fd, &msg, 0); } while (read_bytes < 0 && errno == EINTR); - GPR_TIMER_END("recvmsg", read_bytes >= 0); if (read_bytes < 0) { /* NB: After calling call_read_cb a parallel call of the read handler may @@ -434,8 +433,6 @@ static void tcp_do_read(grpc_tcp* tcp) { call_read_cb(tcp, GRPC_ERROR_NONE); TCP_UNREF(tcp, "read"); } - - GPR_TIMER_END("tcp_continue_read", 0); } static void tcp_read_allocation_done(void* tcpp, grpc_error* error) { @@ -552,13 +549,12 @@ static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) { GRPC_STATS_INC_TCP_WRITE_SIZE(sending_length); GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(iov_size); - GPR_TIMER_BEGIN("sendmsg", 1); + GPR_TIMER_SCOPE("sendmsg", 1); do { /* TODO(klempner): Cork if this is a partial write */ GRPC_STATS_INC_SYSCALL_WRITE(); sent_length = sendmsg(tcp->fd, &msg, SENDMSG_FLAGS); } while (sent_length < 0 && errno == EINTR); - GPR_TIMER_END("sendmsg", 0); if (sent_length < 0) { if (errno == EAGAIN) { @@ -637,6 +633,7 @@ static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error) { static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf, grpc_closure* cb) { + GPR_TIMER_SCOPE("tcp_write", 0); grpc_tcp* tcp = (grpc_tcp*)ep; grpc_error* error = GRPC_ERROR_NONE; @@ -651,11 +648,9 @@ static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf, } } - GPR_TIMER_BEGIN("tcp_write", 0); GPR_ASSERT(tcp->write_cb == nullptr); if (buf->length == 0) { - GPR_TIMER_END("tcp_write", 0); GRPC_CLOSURE_SCHED( cb, grpc_fd_is_shutdown(tcp->em_fd) ? tcp_annotate_error( @@ -680,8 +675,6 @@ static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf, } GRPC_CLOSURE_SCHED(cb, error); } - - GPR_TIMER_END("tcp_write", 0); } static void tcp_add_to_pollset(grpc_endpoint* ep, grpc_pollset* pollset) { diff --git a/src/core/lib/iomgr/wakeup_fd_eventfd.cc b/src/core/lib/iomgr/wakeup_fd_eventfd.cc index 81cb7ee280..421ac55b00 100644 --- a/src/core/lib/iomgr/wakeup_fd_eventfd.cc +++ b/src/core/lib/iomgr/wakeup_fd_eventfd.cc @@ -52,15 +52,14 @@ static grpc_error* eventfd_consume(grpc_wakeup_fd* fd_info) { } static grpc_error* eventfd_wakeup(grpc_wakeup_fd* fd_info) { + GPR_TIMER_SCOPE("eventfd_wakeup", 0); int err; - GPR_TIMER_BEGIN("eventfd_wakeup", 0); do { err = eventfd_write(fd_info->read_fd, 1); } while (err < 0 && errno == EINTR); if (err < 0) { return GRPC_OS_ERROR(errno, "eventfd_write"); } - GPR_TIMER_END("eventfd_wakeup", 0); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/profiling/timers.h b/src/core/lib/profiling/timers.h index 9f11f771e6..d0188b5054 100644 --- a/src/core/lib/profiling/timers.h +++ b/src/core/lib/profiling/timers.h @@ -40,14 +40,10 @@ void gpr_timer_set_enabled(int enabled); do { \ } while (0) -#define GPR_TIMER_BEGIN(tag, important) \ +#define GPR_TIMER_SCOPE(tag, important) \ do { \ } while (0) -#define GPR_TIMER_END(tag, important) \ - do { \ - } while (0) - #else /* at least one profiler requested... */ /* ... hopefully only one. */ #if defined(GRPC_STAP_PROFILER) && defined(GRPC_BASIC_PROFILER) @@ -64,12 +60,6 @@ void gpr_timer_set_enabled(int enabled); #define GPR_TIMER_MARK(tag, important) \ gpr_timer_add_mark(tag, important, __FILE__, __LINE__); -#define GPR_TIMER_BEGIN(tag, important) \ - gpr_timer_begin(tag, important, __FILE__, __LINE__); - -#define GPR_TIMER_END(tag, important) \ - gpr_timer_end(tag, important, __FILE__, __LINE__); - #ifdef GRPC_STAP_PROFILER /* Empty placeholder for now. */ #endif /* GRPC_STAP_PROFILER */ @@ -78,10 +68,6 @@ void gpr_timer_set_enabled(int enabled); /* Empty placeholder for now. */ #endif /* GRPC_BASIC_PROFILER */ -#endif /* at least one profiler requested. */ - -#if (defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER) + \ - defined(GRPC_CUSTOM_PROFILER)) namespace grpc { class ProfileScope { public: @@ -99,10 +85,7 @@ class ProfileScope { #define GPR_TIMER_SCOPE(tag, important) \ ::grpc::ProfileScope _profile_scope_##__LINE__((tag), (important), __FILE__, \ __LINE__) -#else -#define GPR_TIMER_SCOPE(tag, important) \ - do { \ - } while (false) -#endif + +#endif /* at least one profiler requested. */ #endif /* GRPC_CORE_LIB_PROFILING_TIMERS_H */ diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index 045cb3e239..16814d2598 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -266,7 +266,7 @@ static void cancel_check_call_host(void* arg, grpc_error* error) { static void auth_start_transport_stream_op_batch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { - GPR_TIMER_BEGIN("auth_start_transport_stream_op_batch", 0); + GPR_TIMER_SCOPE("auth_start_transport_stream_op_batch", 0); /* grab pointers to our data from the call element */ call_data* calld = (call_data*)elem->call_data; @@ -332,14 +332,12 @@ static void auth_start_transport_stream_op_batch( grpc_schedule_on_exec_ctx)); } gpr_free(call_host); - GPR_TIMER_END("auth_start_transport_stream_op_batch", 0); return; /* early exit */ } } /* pass control down the stack */ grpc_call_next_op(elem, batch); - GPR_TIMER_END("auth_start_transport_stream_op_batch", 0); } /* Constructor for call_data */ diff --git a/src/core/lib/security/transport/secure_endpoint.cc b/src/core/lib/security/transport/secure_endpoint.cc index bd8a6cd76a..9eaa299897 100644 --- a/src/core/lib/security/transport/secure_endpoint.cc +++ b/src/core/lib/security/transport/secure_endpoint.cc @@ -252,7 +252,7 @@ static void flush_write_staging_buffer(secure_endpoint* ep, uint8_t** cur, static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices, grpc_closure* cb) { - GPR_TIMER_BEGIN("secure_endpoint.endpoint_write", 0); + GPR_TIMER_SCOPE("secure_endpoint.endpoint_write", 0); unsigned i; tsi_result result = TSI_OK; @@ -336,12 +336,10 @@ static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices, GRPC_CLOSURE_SCHED( cb, grpc_set_tsi_error_result( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Wrap failed"), result)); - GPR_TIMER_END("secure_endpoint.endpoint_write", 0); return; } grpc_endpoint_write(ep->wrapped_ep, &ep->output_buffer, cb); - GPR_TIMER_END("secure_endpoint.endpoint_write", 0); } static void endpoint_shutdown(grpc_endpoint* secure_ep, grpc_error* why) { diff --git a/src/core/lib/slice/slice_intern.cc b/src/core/lib/slice/slice_intern.cc index fe1770b92c..cf471f30f8 100644 --- a/src/core/lib/slice/slice_intern.cc +++ b/src/core/lib/slice/slice_intern.cc @@ -122,13 +122,13 @@ static const grpc_slice_refcount_vtable interned_slice_sub_vtable = { grpc_slice_default_eq_impl, grpc_slice_default_hash_impl}; static void grow_shard(slice_shard* shard) { + GPR_TIMER_SCOPE("grow_strtab", 0); + size_t capacity = shard->capacity * 2; size_t i; interned_slice_refcount** strtab; interned_slice_refcount *s, *next; - GPR_TIMER_BEGIN("grow_strtab", 0); - strtab = (interned_slice_refcount**)gpr_zalloc( sizeof(interned_slice_refcount*) * capacity); @@ -140,12 +140,9 @@ static void grow_shard(slice_shard* shard) { strtab[idx] = s; } } - gpr_free(shard->strs); shard->strs = strtab; shard->capacity = capacity; - - GPR_TIMER_END("grow_strtab", 0); } static grpc_slice materialize(interned_slice_refcount* s) { @@ -200,9 +197,8 @@ bool grpc_slice_is_interned(grpc_slice slice) { } grpc_slice grpc_slice_intern(grpc_slice slice) { - GPR_TIMER_BEGIN("grpc_slice_intern", 0); + GPR_TIMER_SCOPE("grpc_slice_intern", 0); if (GRPC_IS_STATIC_METADATA_STRING(slice)) { - GPR_TIMER_END("grpc_slice_intern", 0); return slice; } @@ -212,7 +208,6 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)]; if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT && grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) { - GPR_TIMER_END("grpc_slice_intern", 0); return grpc_static_slice_table[ent.idx]; } } @@ -235,7 +230,6 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { /* and treat this as if we were never here... sshhh */ } else { gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } } @@ -264,7 +258,6 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } diff --git a/src/core/lib/slice/slice_string_helpers.cc b/src/core/lib/slice/slice_string_helpers.cc index be0db09252..4441a26d8e 100644 --- a/src/core/lib/slice/slice_string_helpers.cc +++ b/src/core/lib/slice/slice_string_helpers.cc @@ -56,24 +56,59 @@ static int slice_find_separator_offset(const grpc_slice str, const char* sep, return 0; } -void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst) { +static void skip_leading_trailing_spaces(const uint8_t* str_buffer, + size_t* begin, size_t* end) { + while (*begin < *end && str_buffer[*begin] == ' ') { + (*begin)++; + } + while (*begin < *end && str_buffer[*end - 1] == ' ') { + (*end)--; + } +} + +static void grpc_slice_split_inner(grpc_slice str, const char* sep, + grpc_slice_buffer* dst, bool no_space) { const size_t sep_len = strlen(sep); size_t begin, end; + const uint8_t* str_buffer = GRPC_SLICE_START_PTR(str); + size_t sep_pos; GPR_ASSERT(sep_len > 0); if (slice_find_separator_offset(str, sep, 0, &begin, &end) != 0) { do { + sep_pos = end; + if (no_space) { + skip_leading_trailing_spaces(str_buffer, &begin, &end); + } grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end)); - } while (slice_find_separator_offset(str, sep, end + sep_len, &begin, + } while (slice_find_separator_offset(str, sep, sep_pos + sep_len, &begin, &end) != 0); - grpc_slice_buffer_add_indexed( - dst, grpc_slice_sub(str, end + sep_len, GRPC_SLICE_LENGTH(str))); + begin = sep_pos + sep_len; + end = GRPC_SLICE_LENGTH(str); + if (no_space) { + skip_leading_trailing_spaces(str_buffer, &begin, &end); + } + grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end)); } else { /* no sep found, add whole input */ - grpc_slice_buffer_add_indexed(dst, grpc_slice_ref_internal(str)); + begin = 0; + end = GRPC_SLICE_LENGTH(str); + if (no_space) { + skip_leading_trailing_spaces(str_buffer, &begin, &end); + } + grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end)); } } +void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst) { + grpc_slice_split_inner(str, sep, dst, false); +} + +void grpc_slice_split_without_space(grpc_slice str, const char* sep, + grpc_slice_buffer* dst) { + grpc_slice_split_inner(str, sep, dst, true); +} + bool grpc_parse_slice_to_uint32(grpc_slice str, uint32_t* result) { return gpr_parse_bytes_to_uint32((const char*)GRPC_SLICE_START_PTR(str), GRPC_SLICE_LENGTH(str), result) != 0; diff --git a/src/core/lib/slice/slice_string_helpers.h b/src/core/lib/slice/slice_string_helpers.h index 109084be1f..429f9ff4b5 100644 --- a/src/core/lib/slice/slice_string_helpers.h +++ b/src/core/lib/slice/slice_string_helpers.h @@ -35,6 +35,12 @@ char* grpc_dump_slice(grpc_slice slice, uint32_t flags); * should be a properly initialized instance. */ void grpc_slice_split(grpc_slice str, const char* sep, grpc_slice_buffer* dst); +/** Split \a str by the separator \a sep and remove the leading and trailing + * spaces of each resulting token. Results are stored in \a dst, which should be + * a properly initialized instance. */ +void grpc_slice_split_without_space(grpc_slice str, const char* sep, + grpc_slice_buffer* dst); + bool grpc_parse_slice_to_uint32(grpc_slice str, uint32_t* result); #endif /* GRPC_CORE_LIB_SLICE_SLICE_STRING_HELPERS_H */ diff --git a/src/core/lib/surface/byte_buffer_reader.cc b/src/core/lib/surface/byte_buffer_reader.cc index 81a48e95fc..f7ea5161c7 100644 --- a/src/core/lib/surface/byte_buffer_reader.cc +++ b/src/core/lib/surface/byte_buffer_reader.cc @@ -49,9 +49,12 @@ int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, case GRPC_BB_RAW: grpc_slice_buffer_init(&decompressed_slices_buffer); if (is_compressed(reader->buffer_in)) { - if (grpc_msg_decompress(reader->buffer_in->data.raw.compression, - &reader->buffer_in->data.raw.slice_buffer, - &decompressed_slices_buffer) == 0) { + if (grpc_msg_decompress( + + grpc_compression_algorithm_to_message_compression_algorithm( + reader->buffer_in->data.raw.compression), + &reader->buffer_in->data.raw.slice_buffer, + &decompressed_slices_buffer) == 0) { gpr_log(GPR_ERROR, "Unexpected error decompressing data for algorithm with enum " "value '%d'.", diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 8a579cabe7..f2096d8937 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -201,7 +201,7 @@ struct grpc_call { grpc_call_final_info final_info; /* Compression algorithm for *incoming* data */ - grpc_compression_algorithm incoming_compression_algorithm; + grpc_message_compression_algorithm incoming_message_compression_algorithm; /* Stream compression algorithm for *incoming* data */ grpc_stream_compression_algorithm incoming_stream_compression_algorithm; /* Supported encodings (compression algorithms), a bitset */ @@ -321,12 +321,12 @@ static parent_call* get_parent_call(grpc_call* call) { grpc_error* grpc_call_create(const grpc_call_create_args* args, grpc_call** out_call) { + GPR_TIMER_SCOPE("grpc_call_create", 0); size_t i, j; grpc_error* error = GRPC_ERROR_NONE; grpc_channel_stack* channel_stack = grpc_channel_get_channel_stack(args->channel); grpc_call* call; - GPR_TIMER_BEGIN("grpc_call_create", 0); size_t initial_size = grpc_channel_get_call_size_estimate(args->channel); GRPC_STATS_INC_CALL_INITIAL_SIZE(initial_size); gpr_arena* arena = gpr_arena_create(initial_size); @@ -340,7 +340,7 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, call->cq = args->cq; call->start_time = gpr_now(GPR_CLOCK_MONOTONIC); /* Always support no compression */ - GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); + GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_MESSAGE_COMPRESS_NONE); call->is_client = args->server_transport_data == nullptr; if (call->is_client) { GRPC_STATS_INC_CLIENT_CALLS_CREATED(); @@ -449,9 +449,9 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, cancel_with_error(call, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); } if (args->cq != nullptr) { - GPR_ASSERT( - args->pollset_set_alternative == nullptr && - "Only one of 'cq' and 'pollset_set_alternative' should be non-NULL."); + GPR_ASSERT(args->pollset_set_alternative == nullptr && + "Only one of 'cq' and 'pollset_set_alternative' should be " + "non-nullptr."); GRPC_CQ_INTERNAL_REF(args->cq, "bind"); call->pollent = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args->cq)); @@ -467,7 +467,6 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, grpc_slice_unref_internal(path); - GPR_TIMER_END("grpc_call_create", 0); return error; } @@ -511,10 +510,10 @@ static void release_call(void* call, grpc_error* error) { static void set_status_value_directly(grpc_status_code status, void* dest); static void destroy_call(void* call, grpc_error* error) { + GPR_TIMER_SCOPE("destroy_call", 0); size_t i; int ii; grpc_call* c = (grpc_call*)call; - GPR_TIMER_BEGIN("destroy_call", 0); for (i = 0; i < 2; i++) { grpc_metadata_batch_destroy( &c->metadata_batch[1 /* is_receiving */][i /* is_initial */]); @@ -551,7 +550,6 @@ static void destroy_call(void* call, grpc_error* error) { grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); - GPR_TIMER_END("destroy_call", 0); } void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } @@ -559,10 +557,11 @@ void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call* c) { if (!gpr_unref(&c->ext_ref)) return; + GPR_TIMER_SCOPE("grpc_call_unref", 0); + child_call* cc = c->child; grpc_core::ExecCtx exec_ctx; - GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { @@ -594,8 +593,6 @@ void grpc_call_unref(grpc_call* c) { grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr); } GRPC_CALL_INTERNAL_UNREF(c, "destroy"); - - GPR_TIMER_END("grpc_call_unref", 0); } grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) { @@ -610,13 +607,12 @@ grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) { // This is called via the call combiner to start sending a batch down // the filter stack. static void execute_batch_in_call_combiner(void* arg, grpc_error* ignored) { + GPR_TIMER_SCOPE("execute_batch", 0); grpc_transport_stream_op_batch* batch = (grpc_transport_stream_op_batch*)arg; grpc_call* call = (grpc_call*)batch->handler_private.extra_arg; - GPR_TIMER_BEGIN("execute_batch", 0); grpc_call_element* elem = CALL_ELEM_FROM_CALL(call, 0); GRPC_CALL_LOG_OP(GPR_INFO, elem, batch); elem->filter->start_transport_stream_op_batch(elem, batch); - GPR_TIMER_END("execute_batch", 0); } // start_batch_closure points to a caller-allocated closure to be used @@ -801,10 +797,10 @@ static void set_status_from_error(grpc_call* call, status_source source, * COMPRESSION */ -static void set_incoming_compression_algorithm( - grpc_call* call, grpc_compression_algorithm algo) { - GPR_ASSERT(algo < GRPC_COMPRESS_ALGORITHMS_COUNT); - call->incoming_compression_algorithm = algo; +static void set_incoming_message_compression_algorithm( + grpc_call* call, grpc_message_compression_algorithm algo) { + GPR_ASSERT(algo < GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT); + call->incoming_message_compression_algorithm = algo; } static void set_incoming_stream_compression_algorithm( @@ -815,8 +811,10 @@ static void set_incoming_stream_compression_algorithm( grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( grpc_call* call) { - grpc_compression_algorithm algorithm; - algorithm = call->incoming_compression_algorithm; + grpc_compression_algorithm algorithm = GRPC_COMPRESS_NONE; + grpc_compression_algorithm_from_message_stream_compression_algorithm( + &algorithm, call->incoming_message_compression_algorithm, + call->incoming_stream_compression_algorithm); return algorithm; } @@ -826,13 +824,6 @@ static grpc_compression_algorithm compression_algorithm_for_level_locked( call->encodings_accepted_by_peer); } -static grpc_stream_compression_algorithm -stream_compression_algorithm_for_level_locked( - grpc_call* call, grpc_stream_compression_level level) { - return grpc_stream_compression_algorithm_for_level( - level, call->stream_encodings_accepted_by_peer); -} - uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) { uint32_t flags; flags = call->test_only_last_message_flags; @@ -841,9 +832,11 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) { static void destroy_encodings_accepted_by_peer(void* p) { return; } -static void set_encodings_accepted_by_peer(grpc_call* call, grpc_mdelem mdel) { +static void set_encodings_accepted_by_peer(grpc_call* call, grpc_mdelem mdel, + uint32_t* encodings_accepted_by_peer, + bool stream_encoding) { size_t i; - grpc_compression_algorithm algorithm; + uint32_t algorithm; grpc_slice_buffer accept_encoding_parts; grpc_slice accept_encoding_slice; void* accepted_user_data; @@ -851,69 +844,33 @@ static void set_encodings_accepted_by_peer(grpc_call* call, grpc_mdelem mdel) { accepted_user_data = grpc_mdelem_get_user_data(mdel, destroy_encodings_accepted_by_peer); if (accepted_user_data != nullptr) { - call->encodings_accepted_by_peer = + *encodings_accepted_by_peer = (uint32_t)(((uintptr_t)accepted_user_data) - 1); return; } + *encodings_accepted_by_peer = 0; + accept_encoding_slice = GRPC_MDVALUE(mdel); grpc_slice_buffer_init(&accept_encoding_parts); - grpc_slice_split(accept_encoding_slice, ",", &accept_encoding_parts); + grpc_slice_split_without_space(accept_encoding_slice, ",", + &accept_encoding_parts); - /* No need to zero call->encodings_accepted_by_peer: grpc_call_create already - * zeroes the whole grpc_call */ - /* Always support no compression */ - GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); + GPR_BITSET(encodings_accepted_by_peer, GRPC_COMPRESS_NONE); for (i = 0; i < accept_encoding_parts.count; i++) { + int r; grpc_slice accept_encoding_entry_slice = accept_encoding_parts.slices[i]; - if (grpc_compression_algorithm_parse(accept_encoding_entry_slice, - &algorithm)) { - GPR_BITSET(&call->encodings_accepted_by_peer, algorithm); + if (!stream_encoding) { + r = grpc_message_compression_algorithm_parse( + accept_encoding_entry_slice, + (grpc_message_compression_algorithm*)&algorithm); } else { - char* accept_encoding_entry_str = - grpc_slice_to_c_string(accept_encoding_entry_slice); - gpr_log(GPR_ERROR, - "Invalid entry in accept encoding metadata: '%s'. Ignoring.", - accept_encoding_entry_str); - gpr_free(accept_encoding_entry_str); + r = grpc_stream_compression_algorithm_parse( + accept_encoding_entry_slice, + (grpc_stream_compression_algorithm*)&algorithm); } - } - - grpc_slice_buffer_destroy_internal(&accept_encoding_parts); - - grpc_mdelem_set_user_data( - mdel, destroy_encodings_accepted_by_peer, - (void*)(((uintptr_t)call->encodings_accepted_by_peer) + 1)); -} - -static void set_stream_encodings_accepted_by_peer(grpc_call* call, - grpc_mdelem mdel) { - size_t i; - grpc_stream_compression_algorithm algorithm; - grpc_slice_buffer accept_encoding_parts; - grpc_slice accept_encoding_slice; - void* accepted_user_data; - - accepted_user_data = - grpc_mdelem_get_user_data(mdel, destroy_encodings_accepted_by_peer); - if (accepted_user_data != nullptr) { - call->stream_encodings_accepted_by_peer = - (uint32_t)(((uintptr_t)accepted_user_data) - 1); - return; - } - - accept_encoding_slice = GRPC_MDVALUE(mdel); - grpc_slice_buffer_init(&accept_encoding_parts); - grpc_slice_split(accept_encoding_slice, ",", &accept_encoding_parts); - - /* Always support no compression */ - GPR_BITSET(&call->stream_encodings_accepted_by_peer, - GRPC_STREAM_COMPRESS_NONE); - for (i = 0; i < accept_encoding_parts.count; i++) { - grpc_slice accept_encoding_entry_slice = accept_encoding_parts.slices[i]; - if (grpc_stream_compression_algorithm_parse(accept_encoding_entry_slice, - &algorithm)) { - GPR_BITSET(&call->stream_encodings_accepted_by_peer, algorithm); + if (r) { + GPR_BITSET(encodings_accepted_by_peer, algorithm); } else { char* accept_encoding_entry_str = grpc_slice_to_c_string(accept_encoding_entry_slice); @@ -928,7 +885,7 @@ static void set_stream_encodings_accepted_by_peer(grpc_call* call, grpc_mdelem_set_user_data( mdel, destroy_encodings_accepted_by_peer, - (void*)(((uintptr_t)call->stream_encodings_accepted_by_peer) + 1)); + (void*)(((uintptr_t)(*encodings_accepted_by_peer)) + 1)); } uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) { @@ -937,13 +894,6 @@ uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) { return encodings_accepted_by_peer; } -uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer( - grpc_call* call) { - uint32_t stream_encodings_accepted_by_peer; - stream_encodings_accepted_by_peer = call->stream_encodings_accepted_by_peer; - return stream_encodings_accepted_by_peer; -} - grpc_stream_compression_algorithm grpc_call_test_only_get_incoming_stream_encodings(grpc_call* call) { return call->incoming_stream_compression_algorithm; @@ -1047,17 +997,18 @@ static uint32_t decode_status(grpc_mdelem md) { return status; } -static grpc_compression_algorithm decode_compression(grpc_mdelem md) { - grpc_compression_algorithm algorithm = - grpc_compression_algorithm_from_slice(GRPC_MDVALUE(md)); - if (algorithm == GRPC_COMPRESS_ALGORITHMS_COUNT) { +static grpc_message_compression_algorithm decode_message_compression( + grpc_mdelem md) { + grpc_message_compression_algorithm algorithm = + grpc_message_compression_algorithm_from_slice(GRPC_MDVALUE(md)); + if (algorithm == GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT) { char* md_c_str = grpc_slice_to_c_string(GRPC_MDVALUE(md)); gpr_log(GPR_ERROR, - "Invalid incoming compression algorithm: '%s'. Interpreting " - "incoming data as uncompressed.", + "Invalid incoming message compression algorithm: '%s'. " + "Interpreting incoming data as uncompressed.", md_c_str); gpr_free(md_c_str); - return GRPC_COMPRESS_NONE; + return GRPC_MESSAGE_COMPRESS_NONE; } return algorithm; } @@ -1081,7 +1032,7 @@ static grpc_stream_compression_algorithm decode_stream_compression( static void publish_app_metadata(grpc_call* call, grpc_metadata_batch* b, int is_trailing) { if (b->list.count == 0) return; - GPR_TIMER_BEGIN("publish_app_metadata", 0); + GPR_TIMER_SCOPE("publish_app_metadata", 0); grpc_metadata_array* dest; grpc_metadata* mdusr; dest = call->buffered_metadata[is_trailing]; @@ -1097,42 +1048,39 @@ static void publish_app_metadata(grpc_call* call, grpc_metadata_batch* b, mdusr->key = GRPC_MDKEY(l->md); mdusr->value = GRPC_MDVALUE(l->md); } - GPR_TIMER_END("publish_app_metadata", 0); } static void recv_initial_filter(grpc_call* call, grpc_metadata_batch* b) { if (b->idx.named.content_encoding != nullptr) { - if (b->idx.named.grpc_encoding != nullptr) { - gpr_log(GPR_ERROR, - "Received both content-encoding and grpc-encoding header. " - "Ignoring grpc-encoding."); - grpc_metadata_batch_remove(b, b->idx.named.grpc_encoding); - } - GPR_TIMER_BEGIN("incoming_stream_compression_algorithm", 0); + GPR_TIMER_SCOPE("incoming_stream_compression_algorithm", 0); set_incoming_stream_compression_algorithm( call, decode_stream_compression(b->idx.named.content_encoding->md)); - GPR_TIMER_END("incoming_stream_compression_algorithm", 0); grpc_metadata_batch_remove(b, b->idx.named.content_encoding); - } else if (b->idx.named.grpc_encoding != nullptr) { - GPR_TIMER_BEGIN("incoming_compression_algorithm", 0); - set_incoming_compression_algorithm( - call, decode_compression(b->idx.named.grpc_encoding->md)); - GPR_TIMER_END("incoming_compression_algorithm", 0); + } + if (b->idx.named.grpc_encoding != nullptr) { + GPR_TIMER_SCOPE("incoming_message_compression_algorithm", 0); + set_incoming_message_compression_algorithm( + call, decode_message_compression(b->idx.named.grpc_encoding->md)); grpc_metadata_batch_remove(b, b->idx.named.grpc_encoding); } + uint32_t message_encodings_accepted_by_peer = 1u; + uint32_t stream_encodings_accepted_by_peer = 1u; if (b->idx.named.grpc_accept_encoding != nullptr) { - GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0); - set_encodings_accepted_by_peer(call, b->idx.named.grpc_accept_encoding->md); + GPR_TIMER_SCOPE("encodings_accepted_by_peer", 0); + set_encodings_accepted_by_peer(call, b->idx.named.grpc_accept_encoding->md, + &message_encodings_accepted_by_peer, false); grpc_metadata_batch_remove(b, b->idx.named.grpc_accept_encoding); - GPR_TIMER_END("encodings_accepted_by_peer", 0); } if (b->idx.named.accept_encoding != nullptr) { - GPR_TIMER_BEGIN("stream_encodings_accepted_by_peer", 0); - set_stream_encodings_accepted_by_peer(call, - b->idx.named.accept_encoding->md); + GPR_TIMER_SCOPE("stream_encodings_accepted_by_peer", 0); + set_encodings_accepted_by_peer(call, b->idx.named.accept_encoding->md, + &stream_encodings_accepted_by_peer, true); grpc_metadata_batch_remove(b, b->idx.named.accept_encoding); - GPR_TIMER_END("stream_encodings_accepted_by_peer", 0); } + call->encodings_accepted_by_peer = + grpc_compression_bitset_from_message_stream_compression_bitset( + message_encodings_accepted_by_peer, + stream_encodings_accepted_by_peer); publish_app_metadata(call, b, false); } @@ -1270,6 +1218,7 @@ static void post_batch_completion(batch_control* bctl) { if (bctl->op.send_initial_metadata) { grpc_metadata_batch_destroy( + &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]); } if (bctl->op.send_message) { @@ -1277,6 +1226,7 @@ static void post_batch_completion(batch_control* bctl) { } if (bctl->op.send_trailing_metadata) { grpc_metadata_batch_destroy( + &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]); } if (bctl->op.recv_trailing_metadata) { @@ -1425,9 +1375,15 @@ static void process_data_after_md(batch_control* bctl) { } else { call->test_only_last_message_flags = call->receiving_stream->flags; if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) && - (call->incoming_compression_algorithm > GRPC_COMPRESS_NONE)) { - *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create( - nullptr, 0, call->incoming_compression_algorithm); + (call->incoming_message_compression_algorithm > + GRPC_MESSAGE_COMPRESS_NONE)) { + grpc_compression_algorithm algo; + GPR_ASSERT( + grpc_compression_algorithm_from_message_stream_compression_algorithm( + &algo, call->incoming_message_compression_algorithm, + (grpc_stream_compression_algorithm)0)); + *call->receiving_buffer = + grpc_raw_compressed_byte_buffer_create(nullptr, 0, algo); } else { *call->receiving_buffer = grpc_raw_byte_buffer_create(nullptr, 0); } @@ -1469,88 +1425,66 @@ static void receiving_stream_ready_in_call_combiner(void* bctlp, } static void validate_filtered_metadata(batch_control* bctl) { + grpc_compression_algorithm compression_algorithm; grpc_call* call = bctl->call; - /* validate compression algorithms */ if (call->incoming_stream_compression_algorithm != - GRPC_STREAM_COMPRESS_NONE) { - const grpc_stream_compression_algorithm algo = - call->incoming_stream_compression_algorithm; + GRPC_STREAM_COMPRESS_NONE && + call->incoming_message_compression_algorithm != + GRPC_MESSAGE_COMPRESS_NONE) { char* error_msg = nullptr; - const grpc_compression_options compression_options = - grpc_channel_compression_options(call->channel); - if (algo >= GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) { - gpr_asprintf(&error_msg, - "Invalid stream compression algorithm value '%d'.", algo); - gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, - error_msg); - } else if (grpc_compression_options_is_stream_compression_algorithm_enabled( - &compression_options, algo) == 0) { - /* check if algorithm is supported by current channel config */ - const char* algo_name = nullptr; - grpc_stream_compression_algorithm_name(algo, &algo_name); - gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.", - algo_name); - gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, - error_msg); - } + gpr_asprintf(&error_msg, + "Incoming stream has both stream compression (%d) and message " + "compression (%d).", + call->incoming_stream_compression_algorithm, + call->incoming_message_compression_algorithm); + gpr_log(GPR_ERROR, "%s", error_msg); + cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_INTERNAL, + error_msg); gpr_free(error_msg); - - GPR_ASSERT(call->stream_encodings_accepted_by_peer != 0); - if (!GPR_BITGET(call->stream_encodings_accepted_by_peer, - call->incoming_stream_compression_algorithm)) { - if (grpc_compression_trace.enabled()) { - const char* algo_name = nullptr; - grpc_stream_compression_algorithm_name( - call->incoming_stream_compression_algorithm, &algo_name); - gpr_log( - GPR_ERROR, - "Stream compression algorithm (content-encoding = '%s') not " - "present in the bitset of accepted encodings (accept-encodings: " - "'0x%x')", - algo_name, call->stream_encodings_accepted_by_peer); - } - } - } else if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) { - const grpc_compression_algorithm algo = - call->incoming_compression_algorithm; + } else if ( + grpc_compression_algorithm_from_message_stream_compression_algorithm( + &compression_algorithm, call->incoming_message_compression_algorithm, + call->incoming_stream_compression_algorithm) == 0) { + char* error_msg = nullptr; + gpr_asprintf(&error_msg, + "Error in incoming message compression (%d) or stream " + "compression (%d).", + call->incoming_stream_compression_algorithm, + call->incoming_message_compression_algorithm); + cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_INTERNAL, + error_msg); + gpr_free(error_msg); + } else { char* error_msg = nullptr; const grpc_compression_options compression_options = grpc_channel_compression_options(call->channel); - /* check if algorithm is known */ - if (algo >= GRPC_COMPRESS_ALGORITHMS_COUNT) { + if (compression_algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) { gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.", - algo); + compression_algorithm); gpr_log(GPR_ERROR, "%s", error_msg); cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, error_msg); } else if (grpc_compression_options_is_algorithm_enabled( - &compression_options, algo) == 0) { + &compression_options, compression_algorithm) == 0) { /* check if algorithm is supported by current channel config */ const char* algo_name = nullptr; - grpc_compression_algorithm_name(algo, &algo_name); + grpc_compression_algorithm_name(compression_algorithm, &algo_name); gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); gpr_log(GPR_ERROR, "%s", error_msg); cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, error_msg); - } else { - call->incoming_compression_algorithm = algo; } gpr_free(error_msg); GPR_ASSERT(call->encodings_accepted_by_peer != 0); - if (!GPR_BITGET(call->encodings_accepted_by_peer, - call->incoming_compression_algorithm)) { + if (!GPR_BITGET(call->encodings_accepted_by_peer, compression_algorithm)) { if (grpc_compression_trace.enabled()) { const char* algo_name = nullptr; - grpc_compression_algorithm_name(call->incoming_compression_algorithm, - &algo_name); + grpc_compression_algorithm_name(compression_algorithm, &algo_name); gpr_log(GPR_ERROR, - "Compression algorithm (grpc-encoding = '%s') not present in " - "the bitset of accepted encodings (grpc-accept-encodings: " - "'0x%x')", + "Compression algorithm ('%s') not present in the bitset of " + "accepted encodings ('0x%x')", algo_name, call->encodings_accepted_by_peer); } } @@ -1580,9 +1514,8 @@ static void receiving_initial_metadata_ready(void* bctlp, grpc_error* error) { recv_initial_filter(call, md); /* TODO(ctiller): this could be moved into recv_initial_filter now */ - GPR_TIMER_BEGIN("validate_filtered_metadata", 0); + GPR_TIMER_SCOPE("validate_filtered_metadata", 0); validate_filtered_metadata(bctl); - GPR_TIMER_END("validate_filtered_metadata", 0); if (md->deadline != GRPC_MILLIS_INF_FUTURE && !call->is_client) { call->send_deadline = md->deadline; @@ -1635,6 +1568,8 @@ static void free_no_op_completion(void* p, grpc_cq_completion* completion) { static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, size_t nops, void* notify_tag, int is_notify_tag_closure) { + GPR_TIMER_SCOPE("grpc_call_start_batch", 0); + size_t i; const grpc_op* op; batch_control* bctl; @@ -1643,7 +1578,6 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, grpc_transport_stream_op_batch* stream_op; grpc_transport_stream_op_batch_payload* stream_op_payload; - GPR_TIMER_BEGIN("grpc_call_start_batch", 0); GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag); if (nops == 0) { @@ -1693,56 +1627,28 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, size_t additional_metadata_count = 0; grpc_compression_level effective_compression_level = GRPC_COMPRESS_LEVEL_NONE; - grpc_stream_compression_level effective_stream_compression_level = - GRPC_STREAM_COMPRESS_LEVEL_NONE; bool level_set = false; - bool stream_compression = false; - if (op->data.send_initial_metadata.maybe_stream_compression_level - .is_set) { - effective_stream_compression_level = - op->data.send_initial_metadata.maybe_stream_compression_level - .level; - level_set = true; - stream_compression = true; - } else if (op->data.send_initial_metadata.maybe_compression_level - .is_set) { + if (op->data.send_initial_metadata.maybe_compression_level.is_set) { effective_compression_level = op->data.send_initial_metadata.maybe_compression_level.level; level_set = true; } else { const grpc_compression_options copts = grpc_channel_compression_options(call->channel); - if (copts.default_stream_compression_level.is_set) { - level_set = true; - effective_stream_compression_level = - copts.default_stream_compression_level.level; - stream_compression = true; - } else if (copts.default_level.is_set) { + if (copts.default_level.is_set) { level_set = true; effective_compression_level = copts.default_level.level; } } if (level_set && !call->is_client) { - if (stream_compression) { - const grpc_stream_compression_algorithm calgo = - stream_compression_algorithm_for_level_locked( - call, effective_stream_compression_level); - call->compression_md.key = - GRPC_MDSTR_GRPC_INTERNAL_STREAM_ENCODING_REQUEST; - call->compression_md.value = - grpc_stream_compression_algorithm_slice(calgo); - } else { - const grpc_compression_algorithm calgo = - compression_algorithm_for_level_locked( - call, effective_compression_level); - /* the following will be picked up by the compress filter and used - * as the call's compression algorithm. */ - call->compression_md.key = - GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST; - call->compression_md.value = - grpc_compression_algorithm_slice(calgo); - additional_metadata_count++; - } + const grpc_compression_algorithm calgo = + compression_algorithm_for_level_locked( + call, effective_compression_level); + /* the following will be picked up by the compress filter and used + * as the call's compression algorithm. */ + call->compression_md.key = GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST; + call->compression_md.value = grpc_compression_algorithm_slice(calgo); + additional_metadata_count++; } if (op->data.send_initial_metadata.count + additional_metadata_count > @@ -2008,7 +1914,6 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, execute_batch(call, stream_op, &bctl->start_batch); done: - GPR_TIMER_END("grpc_call_start_batch", 0); return error; done_with_error: diff --git a/src/core/lib/surface/call_test_only.h b/src/core/lib/surface/call_test_only.h index 90444f85b6..9eb32f03fb 100644 --- a/src/core/lib/surface/call_test_only.h +++ b/src/core/lib/surface/call_test_only.h @@ -21,7 +21,7 @@ #include <grpc/grpc.h> -/** Return the compression algorithm from \a call. +/** Return the message compression algorithm from \a call. * * \warning This function should \b only be used in test code. */ grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( @@ -38,16 +38,4 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call* call); * To be indexed by grpc_compression_algorithm enum values. */ uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call); -/** Returns a bitset for the stream encodings (stream compression algorithms) - * supported by \a call's peer. - * - * To be indexed by grpc_stream_compression_algorithm enum values. */ -uint32_t grpc_call_test_only_get_stream_encodings_accepted_by_peer( - grpc_call* call); - -/** Returns the incoming stream compression algorithm (content-encoding header) - * received by a call. */ -grpc_stream_compression_algorithm -grpc_call_test_only_get_incoming_stream_encodings(grpc_call* call); - #endif /* GRPC_CORE_LIB_SURFACE_CALL_TEST_ONLY_H */ diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 62db82a1d9..679666bc5f 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -142,50 +142,23 @@ grpc_channel* grpc_channel_create_with_builder( GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) { channel->compression_options.default_level.is_set = true; channel->compression_options.default_level.level = - (grpc_compression_level)grpc_channel_arg_get_integer( + static_cast<grpc_compression_level>(grpc_channel_arg_get_integer( &args->args[i], {GRPC_COMPRESS_LEVEL_NONE, GRPC_COMPRESS_LEVEL_NONE, - GRPC_COMPRESS_LEVEL_COUNT - 1}); - } else if (0 == strcmp(args->args[i].key, - GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) { - channel->compression_options.default_stream_compression_level.is_set = - true; - channel->compression_options.default_stream_compression_level.level = - (grpc_stream_compression_level)grpc_channel_arg_get_integer( - &args->args[i], - {GRPC_STREAM_COMPRESS_LEVEL_NONE, GRPC_STREAM_COMPRESS_LEVEL_NONE, - GRPC_STREAM_COMPRESS_LEVEL_COUNT - 1}); + GRPC_COMPRESS_LEVEL_COUNT - 1})); } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { channel->compression_options.default_algorithm.is_set = true; channel->compression_options.default_algorithm.algorithm = - (grpc_compression_algorithm)grpc_channel_arg_get_integer( + static_cast<grpc_compression_algorithm>(grpc_channel_arg_get_integer( &args->args[i], {GRPC_COMPRESS_NONE, GRPC_COMPRESS_NONE, - GRPC_COMPRESS_ALGORITHMS_COUNT - 1}); - } else if (0 == strcmp(args->args[i].key, - GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) { - channel->compression_options.default_stream_compression_algorithm.is_set = - true; - channel->compression_options.default_stream_compression_algorithm - .algorithm = - (grpc_stream_compression_algorithm)grpc_channel_arg_get_integer( - &args->args[i], - {GRPC_STREAM_COMPRESS_NONE, GRPC_STREAM_COMPRESS_NONE, - GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT - 1}); + GRPC_COMPRESS_ALGORITHMS_COUNT - 1})); } else if (0 == strcmp(args->args[i].key, GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { channel->compression_options.enabled_algorithms_bitset = (uint32_t)args->args[i].value.integer | 0x1; /* always support no compression */ - } else if (0 == - strcmp( - args->args[i].key, - GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) { - channel->compression_options - .enabled_stream_compression_algorithms_bitset = - (uint32_t)args->args[i].value.integer | - 0x1; /* always support no compression */ } } diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc index e731e2139f..8362522fa1 100644 --- a/src/core/lib/surface/completion_queue.cc +++ b/src/core/lib/surface/completion_queue.cc @@ -421,9 +421,9 @@ static long cq_event_queue_num_items(grpc_cq_event_queue* q) { grpc_completion_queue* grpc_completion_queue_create_internal( grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type) { - grpc_completion_queue* cq; + GPR_TIMER_SCOPE("grpc_completion_queue_create_internal", 0); - GPR_TIMER_BEGIN("grpc_completion_queue_create_internal", 0); + grpc_completion_queue* cq; GRPC_API_TRACE( "grpc_completion_queue_create_internal(completion_type=%d, " @@ -452,9 +452,6 @@ grpc_completion_queue* grpc_completion_queue_create_internal( GRPC_CLOSURE_INIT(&cq->pollset_shutdown_done, on_pollset_shutdown_done, cq, grpc_schedule_on_exec_ctx); - - GPR_TIMER_END("grpc_completion_queue_create_internal", 0); - return cq; } @@ -622,7 +619,7 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag, void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg, grpc_cq_completion* storage) { - GPR_TIMER_BEGIN("cq_end_op_for_next", 0); + GPR_TIMER_SCOPE("cq_end_op_for_next", 0); if (grpc_api_trace.enabled() || (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) { @@ -691,8 +688,6 @@ static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag, } } - GPR_TIMER_END("cq_end_op_for_next", 0); - GRPC_ERROR_UNREF(error); } @@ -704,11 +699,11 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag, void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg, grpc_cq_completion* storage) { + GPR_TIMER_SCOPE("cq_end_op_for_pluck", 0); + cq_pluck_data* cqd = (cq_pluck_data*)DATA_FROM_CQ(cq); int is_success = (error == GRPC_ERROR_NONE); - GPR_TIMER_BEGIN("cq_end_op_for_pluck", 0); - if (grpc_api_trace.enabled() || (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) { const char* errmsg = grpc_error_string(error); @@ -760,8 +755,6 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag, } } - GPR_TIMER_END("cq_end_op_for_pluck", 0); - GRPC_ERROR_UNREF(error); } @@ -840,11 +833,11 @@ static void dump_pending_tags(grpc_completion_queue* cq) {} static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline, void* reserved) { + GPR_TIMER_SCOPE("grpc_completion_queue_next", 0); + grpc_event ret; cq_next_data* cqd = (cq_next_data*)DATA_FROM_CQ(cq); - GPR_TIMER_BEGIN("grpc_completion_queue_next", 0); - GRPC_API_TRACE( "grpc_completion_queue_next(" "cq=%p, " @@ -958,8 +951,6 @@ static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline, GPR_ASSERT(is_finished_arg.stolen_completion == nullptr); - GPR_TIMER_END("grpc_completion_queue_next", 0); - return ret; } @@ -1078,14 +1069,14 @@ class ExecCtxPluck : public grpc_core::ExecCtx { static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag, gpr_timespec deadline, void* reserved) { + GPR_TIMER_SCOPE("grpc_completion_queue_pluck", 0); + grpc_event ret; grpc_cq_completion* c; grpc_cq_completion* prev; grpc_pollset_worker* worker = nullptr; cq_pluck_data* cqd = (cq_pluck_data*)DATA_FROM_CQ(cq); - GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0); - if (grpc_cq_pluck_trace.enabled()) { GRPC_API_TRACE( "grpc_completion_queue_pluck(" @@ -1191,8 +1182,6 @@ done: GPR_ASSERT(is_finished_arg.stolen_completion == nullptr); - GPR_TIMER_END("grpc_completion_queue_pluck", 0); - return ret; } @@ -1240,23 +1229,19 @@ static void cq_shutdown_pluck(grpc_completion_queue* cq) { /* Shutdown simply drops a ref that we reserved at creation time; if we drop to zero here, then enter shutdown mode and wake up any waiters */ void grpc_completion_queue_shutdown(grpc_completion_queue* cq) { + GPR_TIMER_SCOPE("grpc_completion_queue_shutdown", 0); grpc_core::ExecCtx exec_ctx; - GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cq=%p)", 1, (cq)); cq->vtable->shutdown(cq); - - GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } void grpc_completion_queue_destroy(grpc_completion_queue* cq) { + GPR_TIMER_SCOPE("grpc_completion_queue_destroy", 0); GRPC_API_TRACE("grpc_completion_queue_destroy(cq=%p)", 1, (cq)); - GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0); grpc_completion_queue_shutdown(cq); grpc_core::ExecCtx exec_ctx; GRPC_CQ_INTERNAL_UNREF(cq, "destroy"); - - GPR_TIMER_END("grpc_completion_queue_destroy", 0); } grpc_pollset* grpc_cq_pollset(grpc_completion_queue* cq) { diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index 70329b09f4..b9eefda0ec 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -162,9 +162,9 @@ void grpc_shutdown(void) { { grpc_core::ExecCtx exec_ctx(0); { - grpc_executor_shutdown(); grpc_timer_manager_set_threading( false); // shutdown timer_manager thread + grpc_executor_shutdown(); for (i = g_number_of_plugins; i >= 0; i--) { if (g_all_of_the_plugins[i].destroy != nullptr) { g_all_of_the_plugins[i].destroy(); diff --git a/src/core/lib/transport/metadata.cc b/src/core/lib/transport/metadata.cc index 652222bd0b..cc1edd586f 100644 --- a/src/core/lib/transport/metadata.cc +++ b/src/core/lib/transport/metadata.cc @@ -166,12 +166,13 @@ static void ref_md_locked(mdtab_shard* shard, } static void gc_mdtab(mdtab_shard* shard) { + GPR_TIMER_SCOPE("gc_mdtab", 0); + size_t i; interned_metadata** prev_next; interned_metadata *md, *next; gpr_atm num_freed = 0; - GPR_TIMER_BEGIN("gc_mdtab", 0); for (i = 0; i < shard->capacity; i++) { prev_next = &shard->elems[i]; for (md = shard->elems[i]; md; md = next) { @@ -194,18 +195,17 @@ static void gc_mdtab(mdtab_shard* shard) { } } gpr_atm_no_barrier_fetch_add(&shard->free_estimate, -num_freed); - GPR_TIMER_END("gc_mdtab", 0); } static void grow_mdtab(mdtab_shard* shard) { + GPR_TIMER_SCOPE("grow_mdtab", 0); + size_t capacity = shard->capacity * 2; size_t i; interned_metadata** mdtab; interned_metadata *md, *next; uint32_t hash; - GPR_TIMER_BEGIN("grow_mdtab", 0); - mdtab = (interned_metadata**)gpr_zalloc(sizeof(interned_metadata*) * capacity); @@ -220,12 +220,9 @@ static void grow_mdtab(mdtab_shard* shard) { mdtab[idx] = md; } } - gpr_free(shard->elems); shard->elems = mdtab; shard->capacity = capacity; - - GPR_TIMER_END("grow_mdtab", 0); } static void rehash_mdtab(mdtab_shard* shard) { @@ -280,7 +277,7 @@ grpc_mdelem grpc_mdelem_create( mdtab_shard* shard = &g_shards[SHARD_IDX(hash)]; size_t idx; - GPR_TIMER_BEGIN("grpc_mdelem_from_metadata_strings", 0); + GPR_TIMER_SCOPE("grpc_mdelem_from_metadata_strings", 0); gpr_mu_lock(&shard->mu); @@ -290,7 +287,6 @@ grpc_mdelem grpc_mdelem_create( if (grpc_slice_eq(key, md->key) && grpc_slice_eq(value, md->value)) { REF_MD_LOCKED(shard, md); gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED); } } @@ -323,8 +319,6 @@ grpc_mdelem grpc_mdelem_create( gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); - return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED); } diff --git a/src/core/lib/transport/static_metadata.cc b/src/core/lib/transport/static_metadata.cc index 2213b30f56..82ba7ac51a 100644 --- a/src/core/lib/transport/static_metadata.cc +++ b/src/core/lib/transport/static_metadata.cc @@ -57,51 +57,53 @@ static uint8_t g_bytes[] = { 112, 111, 110, 115, 101, 95, 109, 101, 115, 115, 97, 103, 101, 95, 98, 121, 116, 101, 115, 47, 103, 114, 112, 99, 46, 108, 98, 46, 118, 49, 46, 76, 111, 97, 100, 66, 97, 108, 97, 110, 99, 101, 114, 47, 66, - 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 48, 49, 50, 105, 100, - 101, 110, 116, 105, 116, 121, 103, 122, 105, 112, 100, 101, 102, 108, 97, - 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 97, 112, 112, 108, 105, - 99, 97, 116, 105, 111, 110, 47, 103, 114, 112, 99, 80, 79, 83, 84, - 50, 48, 48, 52, 48, 52, 104, 116, 116, 112, 104, 116, 116, 112, 115, - 103, 114, 112, 99, 71, 69, 84, 80, 85, 84, 47, 47, 105, 110, 100, - 101, 120, 46, 104, 116, 109, 108, 50, 48, 52, 50, 48, 54, 51, 48, - 52, 52, 48, 48, 53, 48, 48, 97, 99, 99, 101, 112, 116, 45, 99, - 104, 97, 114, 115, 101, 116, 103, 122, 105, 112, 44, 32, 100, 101, 102, - 108, 97, 116, 101, 97, 99, 99, 101, 112, 116, 45, 108, 97, 110, 103, - 117, 97, 103, 101, 97, 99, 99, 101, 112, 116, 45, 114, 97, 110, 103, - 101, 115, 97, 99, 99, 101, 112, 116, 97, 99, 99, 101, 115, 115, 45, - 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, 111, 119, 45, 111, - 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, 111, 119, 97, 117, - 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 99, 97, 99, 104, - 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, 110, 116, 101, 110, - 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, 111, 110, 99, 111, - 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 99, - 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, 103, 116, 104, 99, 111, - 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, 116, 105, 111, 110, 99, - 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, 103, 101, 99, 111, 111, - 107, 105, 101, 100, 97, 116, 101, 101, 116, 97, 103, 101, 120, 112, 101, - 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, 114, 111, 109, 105, 102, - 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, 111, 100, 105, 102, 105, - 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, 45, 110, 111, 110, 101, - 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, 97, 110, 103, 101, 105, - 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, - 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, 100, 105, 102, 105, 101, - 100, 108, 98, 45, 99, 111, 115, 116, 45, 98, 105, 110, 108, 105, 110, - 107, 108, 111, 99, 97, 116, 105, 111, 110, 109, 97, 120, 45, 102, 111, - 114, 119, 97, 114, 100, 115, 112, 114, 111, 120, 121, 45, 97, 117, 116, - 104, 101, 110, 116, 105, 99, 97, 116, 101, 112, 114, 111, 120, 121, 45, - 97, 117, 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 114, 97, - 110, 103, 101, 114, 101, 102, 101, 114, 101, 114, 114, 101, 102, 114, 101, - 115, 104, 114, 101, 116, 114, 121, 45, 97, 102, 116, 101, 114, 115, 101, - 114, 118, 101, 114, 115, 101, 116, 45, 99, 111, 111, 107, 105, 101, 115, - 116, 114, 105, 99, 116, 45, 116, 114, 97, 110, 115, 112, 111, 114, 116, - 45, 115, 101, 99, 117, 114, 105, 116, 121, 116, 114, 97, 110, 115, 102, - 101, 114, 45, 101, 110, 99, 111, 100, 105, 110, 103, 118, 97, 114, 121, - 118, 105, 97, 119, 119, 119, 45, 97, 117, 116, 104, 101, 110, 116, 105, - 99, 97, 116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, 101, - 102, 108, 97, 116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 103, - 122, 105, 112, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112, - 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, - 101, 44, 103, 122, 105, 112}; + 97, 108, 97, 110, 99, 101, 76, 111, 97, 100, 109, 101, 115, 115, 97, + 103, 101, 47, 100, 101, 102, 108, 97, 116, 101, 109, 101, 115, 115, 97, + 103, 101, 47, 103, 122, 105, 112, 115, 116, 114, 101, 97, 109, 47, 103, + 122, 105, 112, 48, 49, 50, 105, 100, 101, 110, 116, 105, 116, 121, 103, + 122, 105, 112, 100, 101, 102, 108, 97, 116, 101, 116, 114, 97, 105, 108, + 101, 114, 115, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, + 103, 114, 112, 99, 80, 79, 83, 84, 50, 48, 48, 52, 48, 52, 104, + 116, 116, 112, 104, 116, 116, 112, 115, 103, 114, 112, 99, 71, 69, 84, + 80, 85, 84, 47, 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, 108, + 50, 48, 52, 50, 48, 54, 51, 48, 52, 52, 48, 48, 53, 48, 48, + 97, 99, 99, 101, 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 103, + 122, 105, 112, 44, 32, 100, 101, 102, 108, 97, 116, 101, 97, 99, 99, + 101, 112, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 97, 99, 99, + 101, 112, 116, 45, 114, 97, 110, 103, 101, 115, 97, 99, 99, 101, 112, + 116, 97, 99, 99, 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, + 45, 97, 108, 108, 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, + 101, 97, 108, 108, 111, 119, 97, 117, 116, 104, 111, 114, 105, 122, 97, + 116, 105, 111, 110, 99, 97, 99, 104, 101, 45, 99, 111, 110, 116, 114, + 111, 108, 99, 111, 110, 116, 101, 110, 116, 45, 100, 105, 115, 112, 111, + 115, 105, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, 108, + 97, 110, 103, 117, 97, 103, 101, 99, 111, 110, 116, 101, 110, 116, 45, + 108, 101, 110, 103, 116, 104, 99, 111, 110, 116, 101, 110, 116, 45, 108, + 111, 99, 97, 116, 105, 111, 110, 99, 111, 110, 116, 101, 110, 116, 45, + 114, 97, 110, 103, 101, 99, 111, 111, 107, 105, 101, 100, 97, 116, 101, + 101, 116, 97, 103, 101, 120, 112, 101, 99, 116, 101, 120, 112, 105, 114, + 101, 115, 102, 114, 111, 109, 105, 102, 45, 109, 97, 116, 99, 104, 105, + 102, 45, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, + 101, 105, 102, 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, + 102, 45, 114, 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, + 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, + 45, 109, 111, 100, 105, 102, 105, 101, 100, 108, 98, 45, 99, 111, 115, + 116, 45, 98, 105, 110, 108, 105, 110, 107, 108, 111, 99, 97, 116, 105, + 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, 112, + 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, + 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, 105, + 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, 101, + 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121, + 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116, + 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, 116, + 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, 105, + 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, + 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, 45, + 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, 101, + 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, + 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, 108, + 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, + 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; static void static_ref(void* unused) {} static void static_unref(void* unused) {} @@ -213,6 +215,9 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { @@ -245,77 +250,80 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_refcounts[26], {{g_bytes + 333, 30}}}, {&grpc_static_metadata_refcounts[27], {{g_bytes + 363, 31}}}, {&grpc_static_metadata_refcounts[28], {{g_bytes + 394, 36}}}, - {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}, - {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}, - {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}, - {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}, - {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}, - {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}, - {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}, - {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}, - {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}, - {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}, - {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}, - {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}, - {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}, - {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}, - {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}, - {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}, - {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}, - {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}, - {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}, - {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}, - {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}, - {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}, - {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}, - {&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, - {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}, - {&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, - {&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, - {&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, - {&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, - {&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, - {&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, - {&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, - {&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, - {&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, - {&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, - {&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, - {&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, - {&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, - {&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, - {&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, - {&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, - {&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, - {&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, - {&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, - {&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, - {&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, - {&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 15}}}, + {&grpc_static_metadata_refcounts[30], {{g_bytes + 445, 12}}}, + {&grpc_static_metadata_refcounts[31], {{g_bytes + 457, 11}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 468, 1}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 469, 1}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 470, 1}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 471, 8}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 479, 4}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 483, 7}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 490, 8}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 498, 16}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 514, 4}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 518, 3}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 521, 3}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 524, 4}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 528, 5}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 533, 4}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 537, 3}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 540, 3}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 543, 1}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 544, 11}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 555, 3}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 558, 3}}}, + {&grpc_static_metadata_refcounts[52], {{g_bytes + 561, 3}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 564, 3}}}, + {&grpc_static_metadata_refcounts[54], {{g_bytes + 567, 3}}}, + {&grpc_static_metadata_refcounts[55], {{g_bytes + 570, 14}}}, + {&grpc_static_metadata_refcounts[56], {{g_bytes + 584, 13}}}, + {&grpc_static_metadata_refcounts[57], {{g_bytes + 597, 15}}}, + {&grpc_static_metadata_refcounts[58], {{g_bytes + 612, 13}}}, + {&grpc_static_metadata_refcounts[59], {{g_bytes + 625, 6}}}, + {&grpc_static_metadata_refcounts[60], {{g_bytes + 631, 27}}}, + {&grpc_static_metadata_refcounts[61], {{g_bytes + 658, 3}}}, + {&grpc_static_metadata_refcounts[62], {{g_bytes + 661, 5}}}, + {&grpc_static_metadata_refcounts[63], {{g_bytes + 666, 13}}}, + {&grpc_static_metadata_refcounts[64], {{g_bytes + 679, 13}}}, + {&grpc_static_metadata_refcounts[65], {{g_bytes + 692, 19}}}, + {&grpc_static_metadata_refcounts[66], {{g_bytes + 711, 16}}}, + {&grpc_static_metadata_refcounts[67], {{g_bytes + 727, 14}}}, + {&grpc_static_metadata_refcounts[68], {{g_bytes + 741, 16}}}, + {&grpc_static_metadata_refcounts[69], {{g_bytes + 757, 13}}}, + {&grpc_static_metadata_refcounts[70], {{g_bytes + 770, 6}}}, + {&grpc_static_metadata_refcounts[71], {{g_bytes + 776, 4}}}, + {&grpc_static_metadata_refcounts[72], {{g_bytes + 780, 4}}}, + {&grpc_static_metadata_refcounts[73], {{g_bytes + 784, 6}}}, + {&grpc_static_metadata_refcounts[74], {{g_bytes + 790, 7}}}, + {&grpc_static_metadata_refcounts[75], {{g_bytes + 797, 4}}}, {&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, - {&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, - {&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, - {&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, - {&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, - {&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, - {&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, - {&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, - {&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, - {&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, - {&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, - {&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, - {&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, - {&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, - {&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, - {&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, - {&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, - {&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, - {&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, - {&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, - {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}, - {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}, - {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}, - {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}, + {&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 17}}}, + {&grpc_static_metadata_refcounts[78], {{g_bytes + 826, 13}}}, + {&grpc_static_metadata_refcounts[79], {{g_bytes + 839, 8}}}, + {&grpc_static_metadata_refcounts[80], {{g_bytes + 847, 19}}}, + {&grpc_static_metadata_refcounts[81], {{g_bytes + 866, 13}}}, + {&grpc_static_metadata_refcounts[82], {{g_bytes + 879, 11}}}, + {&grpc_static_metadata_refcounts[83], {{g_bytes + 890, 4}}}, + {&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 8}}}, + {&grpc_static_metadata_refcounts[85], {{g_bytes + 902, 12}}}, + {&grpc_static_metadata_refcounts[86], {{g_bytes + 914, 18}}}, + {&grpc_static_metadata_refcounts[87], {{g_bytes + 932, 19}}}, + {&grpc_static_metadata_refcounts[88], {{g_bytes + 951, 5}}}, + {&grpc_static_metadata_refcounts[89], {{g_bytes + 956, 7}}}, + {&grpc_static_metadata_refcounts[90], {{g_bytes + 963, 7}}}, + {&grpc_static_metadata_refcounts[91], {{g_bytes + 970, 11}}}, + {&grpc_static_metadata_refcounts[92], {{g_bytes + 981, 6}}}, + {&grpc_static_metadata_refcounts[93], {{g_bytes + 987, 10}}}, + {&grpc_static_metadata_refcounts[94], {{g_bytes + 997, 25}}}, + {&grpc_static_metadata_refcounts[95], {{g_bytes + 1022, 17}}}, + {&grpc_static_metadata_refcounts[96], {{g_bytes + 1039, 4}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1043, 3}}}, + {&grpc_static_metadata_refcounts[98], {{g_bytes + 1046, 16}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1062, 16}}}, + {&grpc_static_metadata_refcounts[100], {{g_bytes + 1078, 13}}}, + {&grpc_static_metadata_refcounts[101], {{g_bytes + 1091, 12}}}, + {&grpc_static_metadata_refcounts[102], {{g_bytes + 1103, 21}}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { @@ -325,16 +333,16 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8, 2, 4, 4}; static const int8_t elems_r[] = { - 11, 9, -3, 0, 10, 27, -74, 28, 0, 14, -7, 0, 0, 0, 18, 8, -2, - 0, 0, 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -50, 0, -33, -55, -56, -57, -58, -57, 0, 40, 39, 38, 37, 36, 35, 34, - 33, 32, 31, 30, 29, 28, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 22, - 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 12, 11, 0}; + 11, 9, -3, 0, 10, 25, -77, 26, 0, 11, -7, 0, 0, 0, 21, 14, 1, + 0, 0, 33, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, -56, 0, -36, -61, -60, -39, -63, -64, 0, 36, 35, 34, 33, + 34, 33, 32, 31, 31, 30, 29, 28, 27, 26, 26, 25, 25, 24, 23, 22, 21, + 20, 19, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 12, 11, 0}; static uint32_t elems_phash(uint32_t i) { - i -= 45; - uint32_t x = i % 98; - uint32_t y = i / 98; + i -= 48; + uint32_t x = i % 101; + uint32_t y = i / 101; uint32_t h = x; if (y < GPR_ARRAY_SIZE(elems_r)) { uint32_t delta = (uint32_t)elems_r[y]; @@ -344,31 +352,31 @@ static uint32_t elems_phash(uint32_t i) { } static const uint16_t elem_keys[] = { - 1032, 1033, 1034, 247, 248, 249, 250, 251, 1623, 143, 144, 45, - 46, 440, 441, 442, 1523, 1632, 1633, 932, 933, 934, 729, 730, - 1423, 1532, 1533, 535, 731, 1923, 2023, 2123, 5223, 5523, 5623, 5723, - 5823, 1436, 1653, 5923, 6023, 6123, 6223, 6323, 6423, 6523, 6623, 6723, - 6823, 6923, 7023, 7123, 7223, 5423, 7323, 7423, 7523, 7623, 7723, 7823, - 7923, 8023, 8123, 8223, 1096, 1097, 1098, 1099, 8323, 8423, 8523, 8623, - 8723, 8823, 8923, 9023, 9123, 9223, 9323, 323, 9423, 9523, 1697, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 137, 238, 239, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0}; + 1065, 1066, 1067, 256, 257, 258, 259, 260, 1671, 149, 150, 48, + 49, 455, 456, 457, 962, 963, 964, 1568, 1683, 1684, 753, 754, + 1465, 553, 755, 2083, 2186, 5688, 5997, 1580, 1581, 6100, 6306, 6409, + 6512, 6615, 6718, 6821, 1481, 1704, 6924, 7027, 7130, 7233, 1980, 7336, + 7439, 7542, 7645, 7748, 7851, 5894, 7954, 8057, 6203, 8160, 8263, 8366, + 8469, 8572, 8675, 8778, 1129, 1130, 1131, 1132, 8881, 8984, 9087, 9190, + 9293, 9396, 9499, 9602, 9705, 9808, 9911, 332, 10014, 10117, 0, 0, + 0, 1748, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 247, + 248, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0}; static const uint8_t elem_idxs[] = { 76, 79, 77, 19, 20, 21, 22, 23, 25, 15, 16, 17, 18, 11, - 12, 13, 38, 83, 84, 3, 4, 5, 0, 1, 43, 36, 37, 6, - 2, 72, 50, 57, 24, 28, 29, 30, 31, 7, 26, 32, 33, 34, - 35, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, 27, 51, 52, - 53, 54, 55, 56, 58, 59, 60, 61, 78, 80, 81, 82, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 73, 14, 74, 75, 85, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 8, 9, 10}; + 12, 13, 3, 4, 5, 38, 83, 84, 0, 1, 43, 6, 2, 50, + 57, 24, 28, 36, 37, 29, 31, 32, 33, 34, 35, 39, 7, 26, + 40, 41, 42, 44, 72, 45, 46, 47, 48, 49, 51, 27, 52, 53, + 30, 54, 55, 56, 58, 59, 60, 61, 78, 80, 81, 82, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, 73, 14, 74, 75, 255, 255, + 255, 85, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { if (a == -1 || b == -1) return GRPC_MDNULL; - uint32_t k = (uint32_t)(a * 100 + b); + uint32_t k = (uint32_t)(a * 103 + b); uint32_t h = elems_phash(k); return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k && elem_idxs[h] != 255 @@ -379,177 +387,177 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, - {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 468, 1}}}}, {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, - {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 469, 1}}}}, {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, - {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 470, 1}}}}, {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, - {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 471, 8}}}}, {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, - {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 479, 4}}}}, {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, - {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 483, 7}}}}, {{&grpc_static_metadata_refcounts[5], {{g_bytes + 36, 2}}}, - {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 490, 8}}}}, {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, - {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 498, 16}}}}, {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, - {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 514, 4}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 518, 3}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 521, 3}}}}, {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, - {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 524, 4}}}}, {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, - {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 528, 5}}}}, {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, - {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 533, 4}}}}, {{&grpc_static_metadata_refcounts[3], {{g_bytes + 19, 10}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, - {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 537, 3}}}}, {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, - {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 540, 3}}}}, {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, - {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 543, 1}}}}, {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, - {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 544, 11}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 555, 3}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 558, 3}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}}, + {&grpc_static_metadata_refcounts[52], {{g_bytes + 561, 3}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 564, 3}}}}, {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, - {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}}, - {{&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, + {&grpc_static_metadata_refcounts[54], {{g_bytes + 567, 3}}}}, + {{&grpc_static_metadata_refcounts[55], {{g_bytes + 570, 14}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, - {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}}, - {{&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, + {&grpc_static_metadata_refcounts[56], {{g_bytes + 584, 13}}}}, + {{&grpc_static_metadata_refcounts[57], {{g_bytes + 597, 15}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, + {{&grpc_static_metadata_refcounts[58], {{g_bytes + 612, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, + {{&grpc_static_metadata_refcounts[59], {{g_bytes + 625, 6}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, + {{&grpc_static_metadata_refcounts[60], {{g_bytes + 631, 27}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, + {{&grpc_static_metadata_refcounts[61], {{g_bytes + 658, 3}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, + {{&grpc_static_metadata_refcounts[62], {{g_bytes + 661, 5}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, + {{&grpc_static_metadata_refcounts[63], {{g_bytes + 666, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, + {{&grpc_static_metadata_refcounts[64], {{g_bytes + 679, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, + {{&grpc_static_metadata_refcounts[65], {{g_bytes + 692, 19}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, - {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 471, 8}}}}, {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, - {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 479, 4}}}}, {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, + {{&grpc_static_metadata_refcounts[66], {{g_bytes + 711, 16}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, + {{&grpc_static_metadata_refcounts[67], {{g_bytes + 727, 14}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, + {{&grpc_static_metadata_refcounts[68], {{g_bytes + 741, 16}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, + {{&grpc_static_metadata_refcounts[69], {{g_bytes + 757, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, + {{&grpc_static_metadata_refcounts[70], {{g_bytes + 770, 6}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, + {{&grpc_static_metadata_refcounts[71], {{g_bytes + 776, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, + {{&grpc_static_metadata_refcounts[72], {{g_bytes + 780, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, + {{&grpc_static_metadata_refcounts[73], {{g_bytes + 784, 6}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, + {{&grpc_static_metadata_refcounts[74], {{g_bytes + 790, 7}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, + {{&grpc_static_metadata_refcounts[75], {{g_bytes + 797, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[20], {{g_bytes + 278, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, + {{&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, + {{&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 17}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {{&grpc_static_metadata_refcounts[78], {{g_bytes + 826, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, + {{&grpc_static_metadata_refcounts[79], {{g_bytes + 839, 8}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, + {{&grpc_static_metadata_refcounts[80], {{g_bytes + 847, 19}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, + {{&grpc_static_metadata_refcounts[81], {{g_bytes + 866, 13}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[21], {{g_bytes + 282, 8}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, + {{&grpc_static_metadata_refcounts[82], {{g_bytes + 879, 11}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, + {{&grpc_static_metadata_refcounts[83], {{g_bytes + 890, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, + {{&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 8}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, + {{&grpc_static_metadata_refcounts[85], {{g_bytes + 902, 12}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, + {{&grpc_static_metadata_refcounts[86], {{g_bytes + 914, 18}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, + {{&grpc_static_metadata_refcounts[87], {{g_bytes + 932, 19}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, + {{&grpc_static_metadata_refcounts[88], {{g_bytes + 951, 5}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, + {{&grpc_static_metadata_refcounts[89], {{g_bytes + 956, 7}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, + {{&grpc_static_metadata_refcounts[90], {{g_bytes + 963, 7}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, + {{&grpc_static_metadata_refcounts[91], {{g_bytes + 970, 11}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, + {{&grpc_static_metadata_refcounts[92], {{g_bytes + 981, 6}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, + {{&grpc_static_metadata_refcounts[93], {{g_bytes + 987, 10}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, + {{&grpc_static_metadata_refcounts[94], {{g_bytes + 997, 25}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, + {{&grpc_static_metadata_refcounts[95], {{g_bytes + 1022, 17}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[19], {{g_bytes + 268, 10}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, + {{&grpc_static_metadata_refcounts[96], {{g_bytes + 1039, 4}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, + {{&grpc_static_metadata_refcounts[97], {{g_bytes + 1043, 3}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, - {{&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, + {{&grpc_static_metadata_refcounts[98], {{g_bytes + 1046, 16}}}, {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 471, 8}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 483, 7}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1062, 16}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 479, 4}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, + {&grpc_static_metadata_refcounts[100], {{g_bytes + 1078, 13}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}}, + {&grpc_static_metadata_refcounts[101], {{g_bytes + 1091, 12}}}}, {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, - {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}}, + {&grpc_static_metadata_refcounts[102], {{g_bytes + 1103, 21}}}}, {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, - {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 471, 8}}}}, {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, - {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 479, 4}}}}, {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, - {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, + {&grpc_static_metadata_refcounts[100], {{g_bytes + 1078, 13}}}}, }; bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = { true, // :path diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index ce3a11b009..21dc7a3d3f 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -29,7 +29,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 100 +#define GRPC_STATIC_MDSTR_COUNT 103 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* ":path" */ #define GRPC_MDSTR_PATH (grpc_static_slice_table[0]) @@ -93,149 +93,155 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* "/grpc.lb.v1.LoadBalancer/BalanceLoad" */ #define GRPC_MDSTR_SLASH_GRPC_DOT_LB_DOT_V1_DOT_LOADBALANCER_SLASH_BALANCELOAD \ (grpc_static_slice_table[28]) +/* "message/deflate" */ +#define GRPC_MDSTR_MESSAGE_SLASH_DEFLATE (grpc_static_slice_table[29]) +/* "message/gzip" */ +#define GRPC_MDSTR_MESSAGE_SLASH_GZIP (grpc_static_slice_table[30]) +/* "stream/gzip" */ +#define GRPC_MDSTR_STREAM_SLASH_GZIP (grpc_static_slice_table[31]) /* "0" */ -#define GRPC_MDSTR_0 (grpc_static_slice_table[29]) +#define GRPC_MDSTR_0 (grpc_static_slice_table[32]) /* "1" */ -#define GRPC_MDSTR_1 (grpc_static_slice_table[30]) +#define GRPC_MDSTR_1 (grpc_static_slice_table[33]) /* "2" */ -#define GRPC_MDSTR_2 (grpc_static_slice_table[31]) +#define GRPC_MDSTR_2 (grpc_static_slice_table[34]) /* "identity" */ -#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[32]) +#define GRPC_MDSTR_IDENTITY (grpc_static_slice_table[35]) /* "gzip" */ -#define GRPC_MDSTR_GZIP (grpc_static_slice_table[33]) +#define GRPC_MDSTR_GZIP (grpc_static_slice_table[36]) /* "deflate" */ -#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[34]) +#define GRPC_MDSTR_DEFLATE (grpc_static_slice_table[37]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[35]) +#define GRPC_MDSTR_TRAILERS (grpc_static_slice_table[38]) /* "application/grpc" */ -#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[36]) +#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (grpc_static_slice_table[39]) /* "POST" */ -#define GRPC_MDSTR_POST (grpc_static_slice_table[37]) +#define GRPC_MDSTR_POST (grpc_static_slice_table[40]) /* "200" */ -#define GRPC_MDSTR_200 (grpc_static_slice_table[38]) +#define GRPC_MDSTR_200 (grpc_static_slice_table[41]) /* "404" */ -#define GRPC_MDSTR_404 (grpc_static_slice_table[39]) +#define GRPC_MDSTR_404 (grpc_static_slice_table[42]) /* "http" */ -#define GRPC_MDSTR_HTTP (grpc_static_slice_table[40]) +#define GRPC_MDSTR_HTTP (grpc_static_slice_table[43]) /* "https" */ -#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[41]) +#define GRPC_MDSTR_HTTPS (grpc_static_slice_table[44]) /* "grpc" */ -#define GRPC_MDSTR_GRPC (grpc_static_slice_table[42]) +#define GRPC_MDSTR_GRPC (grpc_static_slice_table[45]) /* "GET" */ -#define GRPC_MDSTR_GET (grpc_static_slice_table[43]) +#define GRPC_MDSTR_GET (grpc_static_slice_table[46]) /* "PUT" */ -#define GRPC_MDSTR_PUT (grpc_static_slice_table[44]) +#define GRPC_MDSTR_PUT (grpc_static_slice_table[47]) /* "/" */ -#define GRPC_MDSTR_SLASH (grpc_static_slice_table[45]) +#define GRPC_MDSTR_SLASH (grpc_static_slice_table[48]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[46]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (grpc_static_slice_table[49]) /* "204" */ -#define GRPC_MDSTR_204 (grpc_static_slice_table[47]) +#define GRPC_MDSTR_204 (grpc_static_slice_table[50]) /* "206" */ -#define GRPC_MDSTR_206 (grpc_static_slice_table[48]) +#define GRPC_MDSTR_206 (grpc_static_slice_table[51]) /* "304" */ -#define GRPC_MDSTR_304 (grpc_static_slice_table[49]) +#define GRPC_MDSTR_304 (grpc_static_slice_table[52]) /* "400" */ -#define GRPC_MDSTR_400 (grpc_static_slice_table[50]) +#define GRPC_MDSTR_400 (grpc_static_slice_table[53]) /* "500" */ -#define GRPC_MDSTR_500 (grpc_static_slice_table[51]) +#define GRPC_MDSTR_500 (grpc_static_slice_table[54]) /* "accept-charset" */ -#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[52]) +#define GRPC_MDSTR_ACCEPT_CHARSET (grpc_static_slice_table[55]) /* "gzip, deflate" */ -#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[53]) +#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (grpc_static_slice_table[56]) /* "accept-language" */ -#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[54]) +#define GRPC_MDSTR_ACCEPT_LANGUAGE (grpc_static_slice_table[57]) /* "accept-ranges" */ -#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[55]) +#define GRPC_MDSTR_ACCEPT_RANGES (grpc_static_slice_table[58]) /* "accept" */ -#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[56]) +#define GRPC_MDSTR_ACCEPT (grpc_static_slice_table[59]) /* "access-control-allow-origin" */ -#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[57]) +#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (grpc_static_slice_table[60]) /* "age" */ -#define GRPC_MDSTR_AGE (grpc_static_slice_table[58]) +#define GRPC_MDSTR_AGE (grpc_static_slice_table[61]) /* "allow" */ -#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[59]) +#define GRPC_MDSTR_ALLOW (grpc_static_slice_table[62]) /* "authorization" */ -#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[60]) +#define GRPC_MDSTR_AUTHORIZATION (grpc_static_slice_table[63]) /* "cache-control" */ -#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[61]) +#define GRPC_MDSTR_CACHE_CONTROL (grpc_static_slice_table[64]) /* "content-disposition" */ -#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[62]) +#define GRPC_MDSTR_CONTENT_DISPOSITION (grpc_static_slice_table[65]) /* "content-language" */ -#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[63]) +#define GRPC_MDSTR_CONTENT_LANGUAGE (grpc_static_slice_table[66]) /* "content-length" */ -#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[64]) +#define GRPC_MDSTR_CONTENT_LENGTH (grpc_static_slice_table[67]) /* "content-location" */ -#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[65]) +#define GRPC_MDSTR_CONTENT_LOCATION (grpc_static_slice_table[68]) /* "content-range" */ -#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[66]) +#define GRPC_MDSTR_CONTENT_RANGE (grpc_static_slice_table[69]) /* "cookie" */ -#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[67]) +#define GRPC_MDSTR_COOKIE (grpc_static_slice_table[70]) /* "date" */ -#define GRPC_MDSTR_DATE (grpc_static_slice_table[68]) +#define GRPC_MDSTR_DATE (grpc_static_slice_table[71]) /* "etag" */ -#define GRPC_MDSTR_ETAG (grpc_static_slice_table[69]) +#define GRPC_MDSTR_ETAG (grpc_static_slice_table[72]) /* "expect" */ -#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[70]) +#define GRPC_MDSTR_EXPECT (grpc_static_slice_table[73]) /* "expires" */ -#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[71]) +#define GRPC_MDSTR_EXPIRES (grpc_static_slice_table[74]) /* "from" */ -#define GRPC_MDSTR_FROM (grpc_static_slice_table[72]) +#define GRPC_MDSTR_FROM (grpc_static_slice_table[75]) /* "if-match" */ -#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[73]) +#define GRPC_MDSTR_IF_MATCH (grpc_static_slice_table[76]) /* "if-modified-since" */ -#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[74]) +#define GRPC_MDSTR_IF_MODIFIED_SINCE (grpc_static_slice_table[77]) /* "if-none-match" */ -#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[75]) +#define GRPC_MDSTR_IF_NONE_MATCH (grpc_static_slice_table[78]) /* "if-range" */ -#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[76]) +#define GRPC_MDSTR_IF_RANGE (grpc_static_slice_table[79]) /* "if-unmodified-since" */ -#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[77]) +#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[80]) /* "last-modified" */ -#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[78]) +#define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[81]) /* "lb-cost-bin" */ -#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[79]) +#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[82]) /* "link" */ -#define GRPC_MDSTR_LINK (grpc_static_slice_table[80]) +#define GRPC_MDSTR_LINK (grpc_static_slice_table[83]) /* "location" */ -#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[81]) +#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[84]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[82]) +#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[85]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[83]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[86]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[84]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[87]) /* "range" */ -#define GRPC_MDSTR_RANGE (grpc_static_slice_table[85]) +#define GRPC_MDSTR_RANGE (grpc_static_slice_table[88]) /* "referer" */ -#define GRPC_MDSTR_REFERER (grpc_static_slice_table[86]) +#define GRPC_MDSTR_REFERER (grpc_static_slice_table[89]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[87]) +#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[90]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[88]) +#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[91]) /* "server" */ -#define GRPC_MDSTR_SERVER (grpc_static_slice_table[89]) +#define GRPC_MDSTR_SERVER (grpc_static_slice_table[92]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[90]) +#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[93]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[91]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[94]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[92]) +#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[95]) /* "vary" */ -#define GRPC_MDSTR_VARY (grpc_static_slice_table[93]) +#define GRPC_MDSTR_VARY (grpc_static_slice_table[96]) /* "via" */ -#define GRPC_MDSTR_VIA (grpc_static_slice_table[94]) +#define GRPC_MDSTR_VIA (grpc_static_slice_table[97]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[95]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[98]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[96]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[99]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[97]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[100]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[98]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[101]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (grpc_static_slice_table[99]) + (grpc_static_slice_table[102]) extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable; extern grpc_slice_refcount |