diff options
Diffstat (limited to 'src/core/lib')
86 files changed, 1392 insertions, 3087 deletions
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index cfc072c0b5..401a2ad4fe 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -298,6 +298,12 @@ uint32_t grpc_channel_args_compression_algorithm_get_states( } } +grpc_channel_args *grpc_channel_args_set_socket_mutator( + grpc_channel_args *a, grpc_socket_mutator *mutator) { + grpc_arg tmp = grpc_socket_mutator_to_arg(mutator); + return grpc_channel_args_copy_and_add(a, &tmp, 1); +} + int grpc_channel_args_compare(const grpc_channel_args *a, const grpc_channel_args *b) { int c = GPR_ICMP(a->num_args, b->num_args); diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 1e05303471..88fc0e37a3 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -36,6 +36,7 @@ #include <grpc/compression.h> #include <grpc/grpc.h> +#include "src/core/lib/iomgr/socket_mutator.h" // Channel args are intentionally immutable, to avoid the need for locking. @@ -100,6 +101,13 @@ uint32_t grpc_channel_args_compression_algorithm_get_states( int grpc_channel_args_compare(const grpc_channel_args *a, const grpc_channel_args *b); +/** Returns a channel arg instance with socket mutator added. The socket mutator + * will perform its mutate_fd method on all file descriptors used by the + * channel. + * If \a a is non-MULL, its args are copied. */ +grpc_channel_args *grpc_channel_args_set_socket_mutator( + grpc_channel_args *a, grpc_socket_mutator *mutator); + /** Returns the value of argument \a name from \a args, or NULL if not found. */ const grpc_arg *grpc_channel_args_find(const grpc_channel_args *args, const char *name); diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 2c5367901d..999ad5f507 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -162,7 +162,8 @@ grpc_error *grpc_call_stack_init( grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack, int initial_refs, grpc_iomgr_cb_func destroy, void *destroy_arg, grpc_call_context_element *context, const void *transport_server_data, - grpc_mdstr *path, gpr_timespec deadline, grpc_call_stack *call_stack) { + grpc_mdstr *path, gpr_timespec start_time, gpr_timespec deadline, + grpc_call_stack *call_stack) { grpc_channel_element *channel_elems = CHANNEL_ELEMS_FROM_STACK(channel_stack); grpc_call_element_args args; size_t count = channel_stack->count; @@ -179,7 +180,7 @@ grpc_error *grpc_call_stack_init( /* init per-filter data */ grpc_error *first_error = GRPC_ERROR_NONE; - args.start_time = gpr_now(GPR_CLOCK_MONOTONIC); + args.start_time = start_time; for (i = 0; i < count; i++) { args.call_stack = call_stack; args.server_transport_data = transport_server_data; @@ -255,6 +256,13 @@ char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, return next_elem->filter->get_peer(exec_ctx, next_elem); } +void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx, + grpc_channel_element *elem, + const grpc_channel_info *channel_info) { + grpc_channel_element *next_elem = elem + 1; + next_elem->filter->get_channel_info(exec_ctx, next_elem, channel_info); +} + void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { grpc_channel_element *next_elem = elem + 1; @@ -288,7 +296,7 @@ void grpc_call_element_send_cancel(grpc_exec_ctx *exec_ctx, void grpc_call_element_send_cancel_with_message(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_status_code status, - gpr_slice *optional_message) { + grpc_slice *optional_message) { grpc_transport_stream_op *op = gpr_malloc(sizeof(*op)); memset(op, 0, sizeof(*op)); op->on_complete = grpc_closure_create(destroy_op, op); @@ -300,7 +308,7 @@ void grpc_call_element_send_cancel_with_message(grpc_exec_ctx *exec_ctx, void grpc_call_element_send_close_with_message(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_status_code status, - gpr_slice *optional_message) { + grpc_slice *optional_message) { grpc_transport_stream_op *op = gpr_malloc(sizeof(*op)); memset(op, 0, sizeof(*op)); op->on_complete = grpc_closure_create(destroy_op, op); diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 27f3be7b29..004643d45f 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -156,6 +156,10 @@ typedef struct { /* Implement grpc_call_get_peer() */ char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem); + /* Implement grpc_channel_get_info() */ + void (*get_channel_info)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, + const grpc_channel_info *channel_info); + /* The name of this filter */ const char *name; } grpc_channel_filter; @@ -227,7 +231,8 @@ grpc_error *grpc_call_stack_init( grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack, int initial_refs, grpc_iomgr_cb_func destroy, void *destroy_arg, grpc_call_context_element *context, const void *transport_server_data, - grpc_mdstr *path, gpr_timespec deadline, grpc_call_stack *call_stack); + grpc_mdstr *path, gpr_timespec start_time, gpr_timespec deadline, + grpc_call_stack *call_stack); /* Set a pollset or a pollset_set for a call stack: must occur before the first * op is started */ void grpc_call_stack_set_pollset_or_pollset_set(grpc_exec_ctx *exec_ctx, @@ -273,6 +278,10 @@ void grpc_channel_next_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op); /* Pass through a request to get_peer to the next child element */ char *grpc_call_next_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem); +/* Pass through a request to get_channel_info() to the next child element */ +void grpc_channel_next_get_info(grpc_exec_ctx *exec_ctx, + grpc_channel_element *elem, + const grpc_channel_info *channel_info); /* Given the top element of a channel stack, get the channel stack itself */ grpc_channel_stack *grpc_channel_stack_from_top_element( @@ -289,12 +298,12 @@ void grpc_call_element_send_cancel(grpc_exec_ctx *exec_ctx, void grpc_call_element_send_cancel_with_message(grpc_exec_ctx *exec_ctx, grpc_call_element *cur_elem, grpc_status_code status, - gpr_slice *optional_message); + grpc_slice *optional_message); void grpc_call_element_send_close_with_message(grpc_exec_ctx *exec_ctx, grpc_call_element *cur_elem, grpc_status_code status, - gpr_slice *optional_message); + grpc_slice *optional_message); extern int grpc_trace_channel; diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index 0981d59f63..2874d63fc7 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -35,9 +35,9 @@ #include <string.h> #include <grpc/compression.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/compress_filter.h" @@ -50,7 +50,7 @@ int grpc_compression_trace = 0; typedef struct call_data { - gpr_slice_buffer slices; /**< Buffers up input slices to be compressed */ + grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */ grpc_linked_mdelem compression_algorithm_storage; grpc_linked_mdelem accept_encoding_storage; uint32_t remaining_slice_bytes; @@ -63,7 +63,7 @@ typedef struct call_data { grpc_transport_stream_op *send_op; uint32_t send_length; uint32_t send_flags; - gpr_slice incoming_slice; + grpc_slice incoming_slice; grpc_slice_buffer_stream replacement_stream; grpc_closure *post_send; grpc_closure send_done; @@ -111,9 +111,13 @@ static grpc_mdelem *compression_md_filter(void *user_data, grpc_mdelem *md) { return md; } -static int skip_compression(grpc_call_element *elem) { +static int skip_compression(grpc_call_element *elem, uint32_t flags) { call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; + + if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { + return 1; + } if (calld->has_compression_algorithm) { if (calld->compression_algorithm == GRPC_COMPRESS_NONE) { return 1; @@ -157,7 +161,7 @@ static void continue_send_message(grpc_exec_ctx *exec_ctx, static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; - gpr_slice_buffer_reset_and_unref(&calld->slices); + grpc_slice_buffer_reset_and_unref(&calld->slices); calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error); } @@ -165,8 +169,8 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { call_data *calld = elem->call_data; int did_compress; - gpr_slice_buffer tmp; - gpr_slice_buffer_init(&tmp); + grpc_slice_buffer tmp; + grpc_slice_buffer_init(&tmp); did_compress = grpc_msg_compress(calld->compression_algorithm, &calld->slices, &tmp); if (did_compress) { @@ -181,7 +185,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, " bytes (%.2f%% savings)", algo_name, before_size, after_size, 100 * savings_ratio); } - gpr_slice_buffer_swap(&calld->slices, &tmp); + grpc_slice_buffer_swap(&calld->slices, &tmp); calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { if (grpc_compression_trace) { @@ -195,7 +199,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, } } - gpr_slice_buffer_destroy(&tmp); + grpc_slice_buffer_destroy(&tmp); grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices, calld->send_flags); @@ -209,7 +213,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; - gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { finish_send_message(exec_ctx, elem); } else { @@ -223,7 +227,7 @@ static void continue_send_message(grpc_exec_ctx *exec_ctx, while (grpc_byte_stream_next(exec_ctx, calld->send_op->send_message, &calld->incoming_slice, ~(size_t)0, &calld->got_slice)) { - gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { finish_send_message(exec_ctx, elem); break; @@ -241,8 +245,8 @@ static void compress_start_transport_stream_op(grpc_exec_ctx *exec_ctx, if (op->send_initial_metadata) { process_send_initial_metadata(elem, op->send_initial_metadata); } - if (op->send_message != NULL && !skip_compression(elem) && - 0 == (op->send_message->flags & GRPC_WRITE_NO_COMPRESS)) { + if (op->send_message != NULL && + !skip_compression(elem, op->send_message->flags)) { calld->send_op = op; calld->send_length = op->send_message->length; calld->send_flags = op->send_message->flags; @@ -263,7 +267,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, call_data *calld = elem->call_data; /* initialize members */ - gpr_slice_buffer_init(&calld->slices); + grpc_slice_buffer_init(&calld->slices); calld->has_compression_algorithm = 0; grpc_closure_init(&calld->got_slice, got_slice, elem); grpc_closure_init(&calld->send_done, send_done, elem); @@ -277,7 +281,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, void *ignored) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; - gpr_slice_buffer_destroy(&calld->slices); + grpc_slice_buffer_destroy(&calld->slices); } /* Constructor for channel_data */ @@ -328,4 +332,5 @@ const grpc_channel_filter grpc_compress_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "compress"}; diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index 918379c845..038e819f72 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -38,9 +38,9 @@ #include <string.h> #include <grpc/byte_buffer.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/transport.h" @@ -134,6 +134,11 @@ static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { return grpc_transport_get_peer(exec_ctx, chand->transport); } +/* No-op. */ +static void con_get_channel_info(grpc_exec_ctx *exec_ctx, + grpc_channel_element *elem, + const grpc_channel_info *channel_info) {} + static const grpc_channel_filter connected_channel_filter = { con_start_transport_stream_op, con_start_transport_op, @@ -145,6 +150,7 @@ static const grpc_channel_filter connected_channel_filter = { init_channel_elem, destroy_channel_elem, con_get_peer, + con_get_channel_info, "connected", }; diff --git a/src/core/lib/channel/deadline_filter.c b/src/core/lib/channel/deadline_filter.c index d2ea5250f6..0e703d8d27 100644 --- a/src/core/lib/channel/deadline_filter.c +++ b/src/core/lib/channel/deadline_filter.c @@ -55,10 +55,10 @@ static void timer_callback(grpc_exec_ctx* exec_ctx, void* arg, deadline_state->timer_pending = false; gpr_mu_unlock(&deadline_state->timer_mu); if (error != GRPC_ERROR_CANCELLED) { - gpr_slice msg = gpr_slice_from_static_string("Deadline Exceeded"); + grpc_slice msg = grpc_slice_from_static_string("Deadline Exceeded"); grpc_call_element_send_cancel_with_message( exec_ctx, elem, GRPC_STATUS_DEADLINE_EXCEEDED, &msg); - gpr_slice_unref(msg); + grpc_slice_unref(msg); } GRPC_CALL_STACK_UNREF(exec_ctx, deadline_state->call_stack, "deadline_timer"); } @@ -316,6 +316,7 @@ const grpc_channel_filter grpc_client_deadline_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "deadline", }; @@ -330,5 +331,6 @@ const grpc_channel_filter grpc_server_deadline_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "deadline", }; diff --git a/src/core/lib/channel/handshaker.c b/src/core/lib/channel/handshaker.c index 0d759887bc..a45a39981c 100644 --- a/src/core/lib/channel/handshaker.c +++ b/src/core/lib/channel/handshaker.c @@ -62,7 +62,7 @@ void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, + grpc_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data) { @@ -146,8 +146,8 @@ void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx, static void call_next_handshaker(grpc_exec_ctx* exec_ctx, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, void* user_data, - grpc_error* error) { + grpc_slice_buffer* read_buffer, + void* user_data, grpc_error* error) { grpc_handshake_manager* mgr = user_data; GPR_ASSERT(mgr->state != NULL); GPR_ASSERT(mgr->state->index < mgr->count); @@ -183,8 +183,8 @@ void grpc_handshake_manager_do_handshake( gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data) { grpc_channel_args* args_copy = grpc_channel_args_copy(args); - gpr_slice_buffer* read_buffer = gpr_malloc(sizeof(*read_buffer)); - gpr_slice_buffer_init(read_buffer); + grpc_slice_buffer* read_buffer = gpr_malloc(sizeof(*read_buffer)); + grpc_slice_buffer_init(read_buffer); if (mgr->count == 0) { // No handshakers registered, so we just immediately call the done // callback with the passed-in endpoint. diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h index d574b46242..f8a36c6473 100644 --- a/src/core/lib/channel/handshaker.h +++ b/src/core/lib/channel/handshaker.h @@ -59,7 +59,7 @@ typedef struct grpc_handshaker grpc_handshaker; typedef void (*grpc_handshaker_done_cb)(grpc_exec_ctx* exec_ctx, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, + grpc_slice_buffer* read_buffer, void* user_data, grpc_error* error); struct grpc_handshaker_vtable { @@ -77,7 +77,7 @@ struct grpc_handshaker_vtable { /// \a acceptor will be NULL for client-side handshakers. void (*do_handshake)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, gpr_timespec deadline, + grpc_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data); }; @@ -103,7 +103,7 @@ void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker, grpc_endpoint* endpoint, grpc_channel_args* args, - gpr_slice_buffer* read_buffer, + grpc_slice_buffer* read_buffer, gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb, void* user_data); diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index 0875cc18c4..dbe0d25211 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -36,7 +36,7 @@ #include <grpc/support/string_util.h> #include <string.h> #include "src/core/lib/profiling/timers.h" -#include "src/core/lib/support/percent_encoding.h" +#include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" #include "src/core/lib/transport/transport_impl.h" @@ -64,9 +64,9 @@ typedef struct call_data { grpc_transport_stream_op send_op; uint32_t send_length; uint32_t send_flags; - gpr_slice incoming_slice; + grpc_slice incoming_slice; grpc_slice_buffer_stream replacement_stream; - gpr_slice_buffer slices; + grpc_slice_buffer slices; /* flag that indicates that all slices of send_messages aren't availble */ bool send_message_blocked; @@ -105,16 +105,16 @@ static grpc_mdelem *client_recv_filter(void *user_data, grpc_mdelem *md) { char *message_string; gpr_asprintf(&message_string, "Received http2 header with status: %s", grpc_mdstr_as_c_string(md->value)); - gpr_slice message = gpr_slice_from_copied_string(message_string); + grpc_slice message = grpc_slice_from_copied_string(message_string); gpr_free(message_string); grpc_call_element_send_close_with_message(a->exec_ctx, a->elem, GRPC_STATUS_CANCELLED, &message); return NULL; } else if (md->key == GRPC_MDSTR_GRPC_MESSAGE) { - gpr_slice pct_decoded_msg = - gpr_permissive_percent_decode_slice(md->value->slice); - if (gpr_slice_is_equivalent(pct_decoded_msg, md->value->slice)) { - gpr_slice_unref(pct_decoded_msg); + grpc_slice pct_decoded_msg = + grpc_permissive_percent_decode_slice(md->value->slice); + if (grpc_slice_is_equivalent(pct_decoded_msg, md->value->slice)) { + grpc_slice_unref(pct_decoded_msg); return md; } else { return grpc_mdelem_from_metadata_strings( @@ -183,7 +183,7 @@ static void hc_on_complete(grpc_exec_ctx *exec_ctx, void *user_data, static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; - gpr_slice_buffer_reset_and_unref(&calld->slices); + grpc_slice_buffer_reset_and_unref(&calld->slices); calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, error); } @@ -204,10 +204,10 @@ static void continue_send_message(grpc_exec_ctx *exec_ctx, while (grpc_byte_stream_next(exec_ctx, calld->send_op.send_message, &calld->incoming_slice, ~(size_t)0, &calld->got_slice)) { - memcpy(wrptr, GPR_SLICE_START_PTR(calld->incoming_slice), - GPR_SLICE_LENGTH(calld->incoming_slice)); - wrptr += GPR_SLICE_LENGTH(calld->incoming_slice); - gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + memcpy(wrptr, GRPC_SLICE_START_PTR(calld->incoming_slice), + GRPC_SLICE_LENGTH(calld->incoming_slice)); + wrptr += GRPC_SLICE_LENGTH(calld->incoming_slice); + grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { calld->send_message_blocked = false; break; @@ -219,7 +219,7 @@ static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; calld->send_message_blocked = false; - gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { /* Pass down the original send_message op that was blocked.*/ grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices, @@ -347,7 +347,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, calld->on_done_recv_trailing_metadata = NULL; calld->on_complete = NULL; calld->payload_bytes = NULL; - gpr_slice_buffer_init(&calld->slices); + grpc_slice_buffer_init(&calld->slices); grpc_closure_init(&calld->hc_on_recv_initial_metadata, hc_on_recv_initial_metadata, elem); grpc_closure_init(&calld->hc_on_recv_trailing_metadata, @@ -363,7 +363,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, void *ignored) { call_data *calld = elem->call_data; - gpr_slice_buffer_destroy(&calld->slices); + grpc_slice_buffer_destroy(&calld->slices); } static grpc_mdelem *scheme_from_args(const grpc_channel_args *args) { @@ -487,4 +487,5 @@ const grpc_channel_filter grpc_http_client_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "http-client"}; diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index eb335b6b5c..80f3fc3cc3 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -69,7 +69,7 @@ typedef struct call_data { grpc_closure *recv_message_ready; grpc_closure *on_complete; grpc_byte_stream **pp_recv_message; - gpr_slice_buffer read_slice_buffer; + grpc_slice_buffer read_slice_buffer; grpc_slice_buffer_stream read_stream; /** Receive closures are chained: we inject this closure as the on_done_recv @@ -180,9 +180,8 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) { /* Retrieve the payload from the value of the 'grpc-internal-payload-bin' header field */ calld->seen_payload_bin = 1; - gpr_slice_buffer_init(&calld->read_slice_buffer); - gpr_slice_buffer_add(&calld->read_slice_buffer, - gpr_slice_ref(md->value->slice)); + grpc_slice_buffer_add(&calld->read_slice_buffer, + grpc_slice_ref(md->value->slice)); grpc_slice_buffer_stream_init(&calld->read_stream, &calld->read_slice_buffer, 0); return NULL; @@ -338,13 +337,17 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem); grpc_closure_init(&calld->hs_on_complete, hs_on_complete, elem); grpc_closure_init(&calld->hs_recv_message_ready, hs_recv_message_ready, elem); + grpc_slice_buffer_init(&calld->read_slice_buffer); return GRPC_ERROR_NONE; } /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_final_info *final_info, - void *ignored) {} + void *ignored) { + call_data *calld = elem->call_data; + grpc_slice_buffer_destroy(&calld->read_slice_buffer); +} /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, @@ -368,4 +371,5 @@ const grpc_channel_filter grpc_http_server_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "http-server"}; diff --git a/src/core/lib/channel/message_size_filter.c b/src/core/lib/channel/message_size_filter.c index 7dc5ae0df1..1331fe1c65 100644 --- a/src/core/lib/channel/message_size_filter.c +++ b/src/core/lib/channel/message_size_filter.c @@ -141,7 +141,7 @@ static void start_transport_stream_op(grpc_exec_ctx* exec_ctx, char* message_string; gpr_asprintf(&message_string, "Sent message larger than max (%u vs. %d)", op->send_message->length, calld->max_send_size); - gpr_slice message = gpr_slice_from_copied_string(message_string); + grpc_slice message = grpc_slice_from_copied_string(message_string); gpr_free(message_string); grpc_call_element_send_close_with_message( exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT, &message); @@ -248,4 +248,5 @@ const grpc_channel_filter grpc_message_size_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "message_size"}; diff --git a/src/core/lib/compression/message_compress.c b/src/core/lib/compression/message_compress.c index cbe0b5a285..6c245acf61 100644 --- a/src/core/lib/compression/message_compress.c +++ b/src/core/lib/compression/message_compress.c @@ -42,31 +42,31 @@ #define OUTPUT_BLOCK_SIZE 1024 -static int zlib_body(z_stream* zs, gpr_slice_buffer* input, - gpr_slice_buffer* output, +static int zlib_body(z_stream* zs, grpc_slice_buffer* input, + grpc_slice_buffer* output, int (*flate)(z_stream* zs, int flush)) { int r; int flush; size_t i; - gpr_slice outbuf = gpr_slice_malloc(OUTPUT_BLOCK_SIZE); + grpc_slice outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); const uInt uint_max = ~(uInt)0; - GPR_ASSERT(GPR_SLICE_LENGTH(outbuf) <= uint_max); - zs->avail_out = (uInt)GPR_SLICE_LENGTH(outbuf); - zs->next_out = GPR_SLICE_START_PTR(outbuf); + GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); + zs->avail_out = (uInt)GRPC_SLICE_LENGTH(outbuf); + zs->next_out = GRPC_SLICE_START_PTR(outbuf); flush = Z_NO_FLUSH; for (i = 0; i < input->count; i++) { if (i == input->count - 1) flush = Z_FINISH; - GPR_ASSERT(GPR_SLICE_LENGTH(input->slices[i]) <= uint_max); - zs->avail_in = (uInt)GPR_SLICE_LENGTH(input->slices[i]); - zs->next_in = GPR_SLICE_START_PTR(input->slices[i]); + GPR_ASSERT(GRPC_SLICE_LENGTH(input->slices[i]) <= uint_max); + zs->avail_in = (uInt)GRPC_SLICE_LENGTH(input->slices[i]); + zs->next_in = GRPC_SLICE_START_PTR(input->slices[i]); do { if (zs->avail_out == 0) { - gpr_slice_buffer_add_indexed(output, outbuf); - outbuf = gpr_slice_malloc(OUTPUT_BLOCK_SIZE); - GPR_ASSERT(GPR_SLICE_LENGTH(outbuf) <= uint_max); - zs->avail_out = (uInt)GPR_SLICE_LENGTH(outbuf); - zs->next_out = GPR_SLICE_START_PTR(outbuf); + grpc_slice_buffer_add_indexed(output, outbuf); + outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); + GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); + zs->avail_out = (uInt)GRPC_SLICE_LENGTH(outbuf); + zs->next_out = GRPC_SLICE_START_PTR(outbuf); } r = flate(zs, flush); if (r < 0 && r != Z_BUF_ERROR /* not fatal */) { @@ -82,12 +82,12 @@ static int zlib_body(z_stream* zs, gpr_slice_buffer* input, GPR_ASSERT(outbuf.refcount); outbuf.data.refcounted.length -= zs->avail_out; - gpr_slice_buffer_add_indexed(output, outbuf); + grpc_slice_buffer_add_indexed(output, outbuf); return 1; error: - gpr_slice_unref(outbuf); + grpc_slice_unref(outbuf); return 0; } @@ -97,7 +97,7 @@ static void* zalloc_gpr(void* opaque, unsigned int items, unsigned int size) { static void zfree_gpr(void* opaque, void* address) { gpr_free(address); } -static int zlib_compress(gpr_slice_buffer* input, gpr_slice_buffer* output, +static int zlib_compress(grpc_slice_buffer* input, grpc_slice_buffer* output, int gzip) { z_stream zs; int r; @@ -113,7 +113,7 @@ static int zlib_compress(gpr_slice_buffer* input, gpr_slice_buffer* output, r = zlib_body(&zs, input, output, deflate) && output->length < input->length; if (!r) { for (i = count_before; i < output->count; i++) { - gpr_slice_unref(output->slices[i]); + grpc_slice_unref(output->slices[i]); } output->count = count_before; output->length = length_before; @@ -122,7 +122,7 @@ static int zlib_compress(gpr_slice_buffer* input, gpr_slice_buffer* output, return r; } -static int zlib_decompress(gpr_slice_buffer* input, gpr_slice_buffer* output, +static int zlib_decompress(grpc_slice_buffer* input, grpc_slice_buffer* output, int gzip) { z_stream zs; int r; @@ -137,7 +137,7 @@ static int zlib_decompress(gpr_slice_buffer* input, gpr_slice_buffer* output, r = zlib_body(&zs, input, output, inflate); if (!r) { for (i = count_before; i < output->count; i++) { - gpr_slice_unref(output->slices[i]); + grpc_slice_unref(output->slices[i]); } output->count = count_before; output->length = length_before; @@ -146,16 +146,16 @@ static int zlib_decompress(gpr_slice_buffer* input, gpr_slice_buffer* output, return r; } -static int copy(gpr_slice_buffer* input, gpr_slice_buffer* output) { +static int copy(grpc_slice_buffer* input, grpc_slice_buffer* output) { size_t i; for (i = 0; i < input->count; i++) { - gpr_slice_buffer_add(output, gpr_slice_ref(input->slices[i])); + grpc_slice_buffer_add(output, grpc_slice_ref(input->slices[i])); } return 1; } static int compress_inner(grpc_compression_algorithm algorithm, - gpr_slice_buffer* input, gpr_slice_buffer* output) { + grpc_slice_buffer* input, grpc_slice_buffer* output) { switch (algorithm) { case GRPC_COMPRESS_NONE: /* the fallback path always needs to be send uncompressed: we simply @@ -173,7 +173,7 @@ static int compress_inner(grpc_compression_algorithm algorithm, } int grpc_msg_compress(grpc_compression_algorithm algorithm, - gpr_slice_buffer* input, gpr_slice_buffer* output) { + grpc_slice_buffer* input, grpc_slice_buffer* output) { if (!compress_inner(algorithm, input, output)) { copy(input, output); return 0; @@ -182,7 +182,7 @@ int grpc_msg_compress(grpc_compression_algorithm algorithm, } int grpc_msg_decompress(grpc_compression_algorithm algorithm, - gpr_slice_buffer* input, gpr_slice_buffer* output) { + grpc_slice_buffer* input, grpc_slice_buffer* output) { switch (algorithm) { case GRPC_COMPRESS_NONE: return copy(input, output); diff --git a/src/core/lib/compression/message_compress.h b/src/core/lib/compression/message_compress.h index c69eaaf006..448d36a863 100644 --- a/src/core/lib/compression/message_compress.h +++ b/src/core/lib/compression/message_compress.h @@ -35,18 +35,18 @@ #define GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H #include <grpc/compression.h> -#include <grpc/support/slice_buffer.h> +#include <grpc/slice_buffer.h> /* compress 'input' to 'output' using 'algorithm'. On success, appends compressed slices to output and returns 1. On failure, appends uncompressed slices to output and returns 0. */ int grpc_msg_compress(grpc_compression_algorithm algorithm, - gpr_slice_buffer* input, gpr_slice_buffer* output); + grpc_slice_buffer* input, grpc_slice_buffer* output); /* decompress 'input' to 'output' using 'algorithm'. On success, appends slices to output and returns 1. On failure, output is unchanged, and returns 0. */ int grpc_msg_decompress(grpc_compression_algorithm algorithm, - gpr_slice_buffer* input, gpr_slice_buffer* output); + grpc_slice_buffer* input, grpc_slice_buffer* output); #endif /* GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H */ diff --git a/src/core/lib/http/format_request.c b/src/core/lib/http/format_request.c index e818b70113..024664b6ee 100644 --- a/src/core/lib/http/format_request.c +++ b/src/core/lib/http/format_request.c @@ -37,8 +37,8 @@ #include <stdio.h> #include <string.h> +#include <grpc/slice.h> #include <grpc/support/alloc.h> -#include <grpc/support/slice.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> #include "src/core/lib/support/string.h" @@ -65,7 +65,8 @@ static void fill_common_header(const grpc_httpcli_request *request, } } -gpr_slice grpc_httpcli_format_get_request(const grpc_httpcli_request *request) { +grpc_slice grpc_httpcli_format_get_request( + const grpc_httpcli_request *request) { gpr_strvec out; char *flat; size_t flat_len; @@ -78,12 +79,12 @@ gpr_slice grpc_httpcli_format_get_request(const grpc_httpcli_request *request) { flat = gpr_strvec_flatten(&out, &flat_len); gpr_strvec_destroy(&out); - return gpr_slice_new(flat, flat_len, gpr_free); + return grpc_slice_new(flat, flat_len, gpr_free); } -gpr_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request, - const char *body_bytes, - size_t body_size) { +grpc_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request, + const char *body_bytes, + size_t body_size) { gpr_strvec out; char *tmp; size_t out_len; @@ -117,10 +118,10 @@ gpr_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request, out_len += body_size; } - return gpr_slice_new(tmp, out_len, gpr_free); + return grpc_slice_new(tmp, out_len, gpr_free); } -gpr_slice grpc_httpcli_format_connect_request( +grpc_slice grpc_httpcli_format_connect_request( const grpc_httpcli_request *request) { gpr_strvec out; gpr_strvec_init(&out); @@ -130,5 +131,5 @@ gpr_slice grpc_httpcli_format_connect_request( size_t flat_len; char *flat = gpr_strvec_flatten(&out, &flat_len); gpr_strvec_destroy(&out); - return gpr_slice_new(flat, flat_len, gpr_free); + return grpc_slice_new(flat, flat_len, gpr_free); } diff --git a/src/core/lib/http/format_request.h b/src/core/lib/http/format_request.h index 7abd55f2f7..1c8e3f68c5 100644 --- a/src/core/lib/http/format_request.h +++ b/src/core/lib/http/format_request.h @@ -34,14 +34,14 @@ #ifndef GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H #define GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/lib/http/httpcli.h" -gpr_slice grpc_httpcli_format_get_request(const grpc_httpcli_request *request); -gpr_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request, - const char *body_bytes, - size_t body_size); -gpr_slice grpc_httpcli_format_connect_request( +grpc_slice grpc_httpcli_format_get_request(const grpc_httpcli_request *request); +grpc_slice grpc_httpcli_format_post_request(const grpc_httpcli_request *request, + const char *body_bytes, + size_t body_size); +grpc_slice grpc_httpcli_format_connect_request( const grpc_httpcli_request *request); #endif /* GRPC_CORE_LIB_HTTP_FORMAT_REQUEST_H */ diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 411e669b53..fdb8abaa2d 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -50,7 +50,7 @@ #include "src/core/lib/support/string.h" typedef struct { - gpr_slice request_text; + grpc_slice request_text; grpc_http_parser parser; grpc_resolved_addresses *addresses; size_t next_address; @@ -64,8 +64,8 @@ typedef struct { grpc_httpcli_context *context; grpc_polling_entity *pollent; grpc_iomgr_object iomgr_obj; - gpr_slice_buffer incoming; - gpr_slice_buffer outgoing; + grpc_slice_buffer incoming; + grpc_slice_buffer outgoing; grpc_closure on_read; grpc_closure done_write; grpc_closure connected; @@ -111,12 +111,12 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req, if (req->ep != NULL) { grpc_endpoint_destroy(exec_ctx, req->ep); } - gpr_slice_unref(req->request_text); + grpc_slice_unref(req->request_text); gpr_free(req->host); gpr_free(req->ssl_host_override); grpc_iomgr_unregister_object(&req->iomgr_obj); - gpr_slice_buffer_destroy(&req->incoming); - gpr_slice_buffer_destroy(&req->outgoing); + grpc_slice_buffer_destroy(&req->incoming); + grpc_slice_buffer_destroy(&req->outgoing); GRPC_ERROR_UNREF(req->overall_error); grpc_resource_quota_internal_unref(exec_ctx, req->resource_quota); gpr_free(req); @@ -144,7 +144,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, size_t i; for (i = 0; i < req->incoming.count; i++) { - if (GPR_SLICE_LENGTH(req->incoming.slices[i])) { + if (GRPC_SLICE_LENGTH(req->incoming.slices[i])) { req->have_read_byte = 1; grpc_error *err = grpc_http_parser_parse(&req->parser, req->incoming.slices[i], NULL); @@ -178,8 +178,8 @@ static void done_write(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } static void start_write(grpc_exec_ctx *exec_ctx, internal_request *req) { - gpr_slice_ref(req->request_text); - gpr_slice_buffer_add(&req->outgoing, req->request_text); + grpc_slice_ref(req->request_text); + grpc_slice_buffer_add(&req->outgoing, req->request_text); grpc_endpoint_write(exec_ctx, req->ep, &req->outgoing, &req->done_write); } @@ -253,7 +253,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, const grpc_httpcli_request *request, gpr_timespec deadline, grpc_closure *on_done, grpc_httpcli_response *response, - const char *name, gpr_slice request_text) { + const char *name, grpc_slice request_text) { internal_request *req = gpr_malloc(sizeof(internal_request)); memset(req, 0, sizeof(*req)); req->request_text = request_text; @@ -268,8 +268,8 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx, req->resource_quota = grpc_resource_quota_internal_ref(resource_quota); grpc_closure_init(&req->on_read, on_read, req); grpc_closure_init(&req->done_write, done_write, req); - gpr_slice_buffer_init(&req->incoming); - gpr_slice_buffer_init(&req->outgoing); + grpc_slice_buffer_init(&req->incoming); + grpc_slice_buffer_init(&req->outgoing); grpc_iomgr_register_object(&req->iomgr_obj, name); req->host = gpr_strdup(request->host); req->ssl_host_override = gpr_strdup(request->ssl_host_override); diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 0006e809a6..24d264c32a 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -61,7 +61,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) { static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index be9e9b6b63..2f84adc187 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -333,12 +333,12 @@ void grpc_http_response_destroy(grpc_http_response *response) { gpr_free(response->hdrs); } -grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice, +grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, grpc_slice slice, size_t *start_of_body) { - for (size_t i = 0; i < GPR_SLICE_LENGTH(slice); i++) { + for (size_t i = 0; i < GRPC_SLICE_LENGTH(slice); i++) { bool found_body_start = false; grpc_error *err = - addbyte(parser, GPR_SLICE_START_PTR(slice)[i], &found_body_start); + addbyte(parser, GRPC_SLICE_START_PTR(slice)[i], &found_body_start); if (err != GRPC_ERROR_NONE) return err; if (found_body_start && start_of_body != NULL) *start_of_body = i + 1; } diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h index fab42979cd..a68011dd43 100644 --- a/src/core/lib/http/parser.h +++ b/src/core/lib/http/parser.h @@ -34,8 +34,8 @@ #ifndef GRPC_CORE_LIB_HTTP_PARSER_H #define GRPC_CORE_LIB_HTTP_PARSER_H +#include <grpc/slice.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> #include "src/core/lib/iomgr/error.h" /* Maximum length of a header string of the form 'Key: Value\r\n' */ @@ -114,7 +114,7 @@ void grpc_http_parser_init(grpc_http_parser *parser, grpc_http_type type, void grpc_http_parser_destroy(grpc_http_parser *parser); /* Sets \a start_of_body to the offset in \a slice of the start of the body. */ -grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, gpr_slice slice, +grpc_error *grpc_http_parser_parse(grpc_http_parser *parser, grpc_slice slice, size_t *start_of_body); grpc_error *grpc_http_parser_eof(grpc_http_parser *parser); diff --git a/src/core/lib/iomgr/endpoint.c b/src/core/lib/iomgr/endpoint.c index 74fa9c45df..2d300f4560 100644 --- a/src/core/lib/iomgr/endpoint.c +++ b/src/core/lib/iomgr/endpoint.c @@ -34,12 +34,12 @@ #include "src/core/lib/iomgr/endpoint.h" void grpc_endpoint_read(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep, - gpr_slice_buffer* slices, grpc_closure* cb) { + grpc_slice_buffer* slices, grpc_closure* cb) { ep->vtable->read(exec_ctx, ep, slices, cb); } void grpc_endpoint_write(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep, - gpr_slice_buffer* slices, grpc_closure* cb) { + grpc_slice_buffer* slices, grpc_closure* cb) { ep->vtable->write(exec_ctx, ep, slices, cb); } @@ -66,6 +66,8 @@ char* grpc_endpoint_get_peer(grpc_endpoint* ep) { return ep->vtable->get_peer(ep); } +int grpc_endpoint_get_fd(grpc_endpoint* ep) { return ep->vtable->get_fd(ep); } + grpc_workqueue* grpc_endpoint_get_workqueue(grpc_endpoint* ep) { return ep->vtable->get_workqueue(ep); } diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h index 0ac5486ff5..1609b64f2b 100644 --- a/src/core/lib/iomgr/endpoint.h +++ b/src/core/lib/iomgr/endpoint.h @@ -34,8 +34,8 @@ #ifndef GRPC_CORE_LIB_IOMGR_ENDPOINT_H #define GRPC_CORE_LIB_IOMGR_ENDPOINT_H -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/time.h> #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" @@ -49,9 +49,9 @@ typedef struct grpc_endpoint_vtable grpc_endpoint_vtable; struct grpc_endpoint_vtable { void (*read)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *slices, grpc_closure *cb); + grpc_slice_buffer *slices, grpc_closure *cb); void (*write)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *slices, grpc_closure *cb); + grpc_slice_buffer *slices, grpc_closure *cb); grpc_workqueue *(*get_workqueue)(grpc_endpoint *ep); void (*add_to_pollset)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_pollset *pollset); @@ -61,6 +61,7 @@ struct grpc_endpoint_vtable { void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep); grpc_resource_user *(*get_resource_user)(grpc_endpoint *ep); char *(*get_peer)(grpc_endpoint *ep); + int (*get_fd)(grpc_endpoint *ep); }; /* When data is available on the connection, calls the callback with slices. @@ -69,10 +70,14 @@ struct grpc_endpoint_vtable { Valid slices may be placed into \a slices even when the callback is invoked with error != GRPC_ERROR_NONE. */ void grpc_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *slices, grpc_closure *cb); + grpc_slice_buffer *slices, grpc_closure *cb); char *grpc_endpoint_get_peer(grpc_endpoint *ep); +/* Get the file descriptor used by \a ep. Return -1 if \a ep is not using an fd. + */ +int grpc_endpoint_get_fd(grpc_endpoint *ep); + /* Retrieve a reference to the workqueue associated with this endpoint */ grpc_workqueue *grpc_endpoint_get_workqueue(grpc_endpoint *ep); @@ -87,7 +92,7 @@ grpc_workqueue *grpc_endpoint_get_workqueue(grpc_endpoint *ep); it is a valid slice buffer. */ void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *slices, grpc_closure *cb); + grpc_slice_buffer *slices, grpc_closure *cb); /* Causes any pending and future read/write callbacks to run immediately with success==0 */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index db51ec4939..91041a7c28 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -163,7 +163,7 @@ static void fd_global_shutdown(void); #define PI_ADD_REF(p, r) pi_add_ref((p)) #define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) -#endif /* !defined(GPRC_PI_REF_COUNT_DEBUG) */ +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ /* This is also used as grpc_workqueue (by directly casing it) */ typedef struct polling_island { diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c deleted file mode 100644 index bf51404203..0000000000 --- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c +++ /dev/null @@ -1,2076 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* This file will be removed shortly: it's here to keep refactoring - * steps simple and auditable. - * It's the combination of the old files: - * - fd_posix.{h,c} - * - pollset_posix.{h,c} - * - pullset_multipoller_with_{poll,epoll}.{h,c} - * The new version will be split into: - * - ev_poll_posix.{h,c} - * - ev_epoll_posix.{h,c} - */ - -#include "src/core/lib/iomgr/port.h" - -#ifdef GRPC_POSIX_SOCKET - -#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h" - -#include <assert.h> -#include <errno.h> -#include <poll.h> -#include <string.h> -#include <sys/socket.h> -#include <unistd.h> - -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/string_util.h> -#include <grpc/support/tls.h> -#include <grpc/support/useful.h> - -#include "src/core/lib/iomgr/iomgr_internal.h" -#include "src/core/lib/iomgr/wakeup_fd_posix.h" -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/support/block_annotate.h" - -/******************************************************************************* - * FD declarations - */ - -typedef struct grpc_fd_watcher { - struct grpc_fd_watcher *next; - struct grpc_fd_watcher *prev; - grpc_pollset *pollset; - grpc_pollset_worker *worker; - grpc_fd *fd; -} grpc_fd_watcher; - -struct grpc_fd { - int fd; - /* refst format: - bit0: 1=active/0=orphaned - bit1-n: refcount - meaning that mostly we ref by two to avoid altering the orphaned bit, - and just unref by 1 when we're ready to flag the object as orphaned */ - gpr_atm refst; - - gpr_mu mu; - int shutdown; - int closed; - int released; - - /* The watcher list. - - The following watcher related fields are protected by watcher_mu. - - An fd_watcher is an ephemeral object created when an fd wants to - begin polling, and destroyed after the poll. - - It denotes the fd's interest in whether to read poll or write poll - or both or neither on this fd. - - If a watcher is asked to poll for reads or writes, the read_watcher - or write_watcher fields are set respectively. A watcher may be asked - to poll for both, in which case both fields will be set. - - read_watcher and write_watcher may be NULL if no watcher has been - asked to poll for reads or writes. - - If an fd_watcher is not asked to poll for reads or writes, it's added - to a linked list of inactive watchers, rooted at inactive_watcher_root. - If at a later time there becomes need of a poller to poll, one of - the inactive pollers may be kicked out of their poll loops to take - that responsibility. */ - grpc_fd_watcher inactive_watcher_root; - grpc_fd_watcher *read_watcher; - grpc_fd_watcher *write_watcher; - - grpc_closure *read_closure; - grpc_closure *write_closure; - - struct grpc_fd *freelist_next; - - grpc_closure *on_done_closure; - - grpc_iomgr_object iomgr_object; - - /* The pollset that last noticed and notified that the fd is readable */ - grpc_pollset *read_notifier_pollset; -}; - -/* Begin polling on an fd. - Registers that the given pollset is interested in this fd - so that if read - or writability interest changes, the pollset can be kicked to pick up that - new interest. - Return value is: - (fd_needs_read? read_mask : 0) | (fd_needs_write? write_mask : 0) - i.e. a combination of read_mask and write_mask determined by the fd's current - interest in said events. - Polling strategies that do not need to alter their behavior depending on the - fd's current interest (such as epoll) do not need to call this function. - MUST NOT be called with a pollset lock taken */ -static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, - grpc_pollset_worker *worker, uint32_t read_mask, - uint32_t write_mask, grpc_fd_watcher *rec); -/* Complete polling previously started with fd_begin_poll - MUST NOT be called with a pollset lock taken - if got_read or got_write are 1, also does the become_{readable,writable} as - appropriate. */ -static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec, - int got_read, int got_write, - grpc_pollset *read_notifier_pollset); - -/* Return 1 if this fd is orphaned, 0 otherwise */ -static bool fd_is_orphaned(grpc_fd *fd); - -/* Reference counting for fds */ -/*#define GRPC_FD_REF_COUNT_DEBUG*/ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - -static void fd_global_init(void); -static void fd_global_shutdown(void); - -#define CLOSURE_NOT_READY ((grpc_closure *)0) -#define CLOSURE_READY ((grpc_closure *)1) - -/******************************************************************************* - * pollset declarations - */ - -typedef struct grpc_pollset_vtable grpc_pollset_vtable; - -typedef struct grpc_cached_wakeup_fd { - grpc_wakeup_fd fd; - struct grpc_cached_wakeup_fd *next; -} grpc_cached_wakeup_fd; - -struct grpc_pollset_worker { - grpc_cached_wakeup_fd *wakeup_fd; - int reevaluate_polling_on_wakeup; - int kicked_specifically; - struct grpc_pollset_worker *next; - struct grpc_pollset_worker *prev; -}; - -struct grpc_pollset { - /* pollsets under posix can mutate representation as fds are added and - removed. - For example, we may choose a poll() based implementation on linux for - few fds, and an epoll() based implementation for many fds */ - const grpc_pollset_vtable *vtable; - gpr_mu mu; - grpc_pollset_worker root_worker; - int in_flight_cbs; - int shutting_down; - int called_shutdown; - int kicked_without_pollers; - grpc_closure *shutdown_done; - grpc_closure_list idle_jobs; - union { - int fd; - void *ptr; - } data; - /* Local cache of eventfds for workers */ - grpc_cached_wakeup_fd *local_wakeup_cache; -}; - -struct grpc_pollset_vtable { - void (*add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - struct grpc_fd *fd, int and_unlock_pollset); - grpc_error *(*maybe_work_and_unlock)(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, - grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now); - void (*finish_shutdown)(grpc_pollset *pollset); - void (*destroy)(grpc_pollset *pollset); -}; - -/* Add an fd to a pollset */ -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - struct grpc_fd *fd); - -static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pollset_set, grpc_fd *fd); - -/* Convert a timespec to milliseconds: - - very small or negative poll times are clamped to zero to do a - non-blocking poll (which becomes spin polling) - - other small values are rounded up to one millisecond - - longer than a millisecond polls are rounded up to the next nearest - millisecond to avoid spinning - - infinite timeouts are converted to -1 */ -static int poll_deadline_to_millis_timeout(gpr_timespec deadline, - gpr_timespec now); - -/* Allow kick to wakeup the currently polling worker */ -#define GRPC_POLLSET_CAN_KICK_SELF 1 -/* Force the wakee to repoll when awoken */ -#define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 -/* As per pollset_kick, with an extended set of flags (defined above) - -- mostly for fd_posix's use. */ -static grpc_error *pollset_kick_ext(grpc_pollset *p, - grpc_pollset_worker *specific_worker, - uint32_t flags) GRPC_MUST_USE_RESULT; - -/* turn a pollset into a multipoller: platform specific */ -typedef void (*platform_become_multipoller_type)(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, - struct grpc_fd **fds, - size_t fd_count); -static platform_become_multipoller_type platform_become_multipoller; - -/* Return 1 if the pollset has active threads in pollset_work (pollset must - * be locked) */ -static int pollset_has_workers(grpc_pollset *pollset); - -static void remove_fd_from_all_epoll_sets(int fd); - -/******************************************************************************* - * pollset_set definitions - */ - -struct grpc_pollset_set { - gpr_mu mu; - - size_t pollset_count; - size_t pollset_capacity; - grpc_pollset **pollsets; - - size_t pollset_set_count; - size_t pollset_set_capacity; - struct grpc_pollset_set **pollset_sets; - - size_t fd_count; - size_t fd_capacity; - grpc_fd **fds; -}; - -/******************************************************************************* - * fd_posix.c - */ - -/* We need to keep a freelist not because of any concerns of malloc performance - * but instead so that implementations with multiple threads in (for example) - * epoll_wait deal with the race between pollset removal and incoming poll - * notifications. - * - * The problem is that the poller ultimately holds a reference to this - * object, so it is very difficult to know when is safe to free it, at least - * without some expensive synchronization. - * - * If we keep the object freelisted, in the worst case losing this race just - * becomes a spurious read notification on a reused fd. - */ -/* TODO(klempner): We could use some form of polling generation count to know - * when these are safe to free. */ -/* TODO(klempner): Consider disabling freelisting if we don't have multiple - * threads in poll on the same fd */ -/* TODO(klempner): Batch these allocations to reduce fragmentation */ -static grpc_fd *fd_freelist = NULL; -static gpr_mu fd_freelist_mu; - -static void freelist_fd(grpc_fd *fd) { - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - gpr_mu_unlock(&fd_freelist_mu); -} - -static grpc_fd *alloc_fd(int fd) { - grpc_fd *r = NULL; - gpr_mu_lock(&fd_freelist_mu); - if (fd_freelist != NULL) { - r = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - } - gpr_mu_unlock(&fd_freelist_mu); - if (r == NULL) { - r = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&r->mu); - } - - gpr_mu_lock(&r->mu); - gpr_atm_rel_store(&r->refst, 1); - r->shutdown = 0; - r->read_closure = CLOSURE_NOT_READY; - r->write_closure = CLOSURE_NOT_READY; - r->fd = fd; - r->inactive_watcher_root.next = r->inactive_watcher_root.prev = - &r->inactive_watcher_root; - r->freelist_next = NULL; - r->read_watcher = r->write_watcher = NULL; - r->on_done_closure = NULL; - r->closed = 0; - r->released = 0; - r->read_notifier_pollset = NULL; - gpr_mu_unlock(&r->mu); - return r; -} - -static void destroy(grpc_fd *fd) { - gpr_mu_destroy(&fd->mu); - gpr_free(fd); -} - -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, - gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} - -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %d -> %d [%s; %s:%d]", fd->fd, fd, n, - gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - freelist_fd(fd); - } else { - GPR_ASSERT(old > n); - } -} - -static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } - -static void fd_global_shutdown(void) { - gpr_mu_lock(&fd_freelist_mu); - gpr_mu_unlock(&fd_freelist_mu); - while (fd_freelist != NULL) { - grpc_fd *fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - destroy(fd); - } - gpr_mu_destroy(&fd_freelist_mu); -} - -static grpc_fd *fd_create(int fd, const char *name) { - grpc_fd *r = alloc_fd(fd); - char *name2; - gpr_asprintf(&name2, "%s fd=%d", name, fd); - grpc_iomgr_register_object(&r->iomgr_object, name2); - gpr_free(name2); -#ifdef GRPC_FD_REF_COUNT_DEBUG - gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, r, name); -#endif - return r; -} - -static bool fd_is_orphaned(grpc_fd *fd) { - return (gpr_atm_acq_load(&fd->refst) & 1) == 0; -} - -static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { - gpr_mu_lock(&watcher->pollset->mu); - GPR_ASSERT(watcher->worker); - grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); - gpr_mu_unlock(&watcher->pollset->mu); - return err; -} - -static void maybe_wake_one_watcher_locked(grpc_fd *fd) { - if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next); - } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); - } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher); - } -} - -static void wake_all_watchers_locked(grpc_fd *fd) { - grpc_fd_watcher *watcher; - for (watcher = fd->inactive_watcher_root.next; - watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher); - } - if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); - } - if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher); - } -} - -static int has_watchers(grpc_fd *fd) { - return fd->read_watcher != NULL || fd->write_watcher != NULL || - fd->inactive_watcher_root.next != &fd->inactive_watcher_root; -} - -static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - fd->closed = 1; - if (!fd->released) { - close(fd->fd); - } else { - remove_fd_from_all_epoll_sets(fd->fd); - } - grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL); -} - -static int fd_wrapped_fd(grpc_fd *fd) { - if (fd->released || fd->closed) { - return -1; - } else { - return fd->fd; - } -} - -static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *on_done, int *release_fd, - const char *reason) { - fd->on_done_closure = on_done; - fd->released = release_fd != NULL; - if (!fd->released) { - shutdown(fd->fd, SHUT_RDWR); - } else { - *release_fd = fd->fd; - } - gpr_mu_lock(&fd->mu); - REF_BY(fd, 1, reason); /* remove active status, but keep referenced */ - if (!has_watchers(fd)) { - close_fd_locked(exec_ctx, fd); - } else { - wake_all_watchers_locked(fd); - } - gpr_mu_unlock(&fd->mu); - UNREF_BY(fd, 2, reason); /* drop the reference */ -} - -/* increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } - -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - -static grpc_error *fd_shutdown_error(bool shutdown) { - if (!shutdown) { - return GRPC_ERROR_NONE; - } else { - return GRPC_ERROR_CREATE("FD shutdown"); - } -} - -static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure **st, grpc_closure *closure) { - if (fd->shutdown) { - grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"), - NULL); - } else if (*st == CLOSURE_NOT_READY) { - /* not ready ==> switch to a waiting state by setting the closure */ - *st = closure; - } else if (*st == CLOSURE_READY) { - /* already ready ==> queue the closure to run immediately */ - *st = CLOSURE_NOT_READY; - grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown), - NULL); - maybe_wake_one_watcher_locked(fd); - } else { - /* upcallptr was set to a different closure. This is an error! */ - gpr_log(GPR_ERROR, - "User called a notify_on function with a previous callback still " - "pending"); - abort(); - } -} - -/* returns 1 if state becomes not ready */ -static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure **st) { - if (*st == CLOSURE_READY) { - /* duplicate ready ==> ignore */ - return 0; - } else if (*st == CLOSURE_NOT_READY) { - /* not ready, and not waiting ==> flag ready */ - *st = CLOSURE_READY; - return 0; - } else { - /* waiting ==> queue closure */ - grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL); - *st = CLOSURE_NOT_READY; - return 1; - } -} - -static void set_read_notifier_pollset_locked( - grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) { - fd->read_notifier_pollset = read_notifier_pollset; -} - -static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - gpr_mu_lock(&fd->mu); - /* only shutdown once */ - if (!fd->shutdown) { - fd->shutdown = 1; - /* signal read/write closed to OS so that future operations fail */ - shutdown(fd->fd, SHUT_RDWR); - set_ready_locked(exec_ctx, fd, &fd->read_closure); - set_ready_locked(exec_ctx, fd, &fd->write_closure); - } - gpr_mu_unlock(&fd->mu); -} - -static bool fd_is_shutdown(grpc_fd *fd) { - gpr_mu_lock(&fd->mu); - bool r = fd->shutdown; - gpr_mu_unlock(&fd->mu); - return r; -} - -static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *closure) { - gpr_mu_lock(&fd->mu); - notify_on_locked(exec_ctx, fd, &fd->read_closure, closure); - gpr_mu_unlock(&fd->mu); -} - -static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_closure *closure) { - gpr_mu_lock(&fd->mu); - notify_on_locked(exec_ctx, fd, &fd->write_closure, closure); - gpr_mu_unlock(&fd->mu); -} - -/* Return the read-notifier pollset */ -static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, - grpc_fd *fd) { - grpc_pollset *notifier = NULL; - - gpr_mu_lock(&fd->mu); - notifier = fd->read_notifier_pollset; - gpr_mu_unlock(&fd->mu); - - return notifier; -} - -static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, - grpc_pollset_worker *worker, uint32_t read_mask, - uint32_t write_mask, grpc_fd_watcher *watcher) { - uint32_t mask = 0; - grpc_closure *cur; - int requested; - /* keep track of pollers that have requested our events, in case they change - */ - GRPC_FD_REF(fd, "poll"); - - gpr_mu_lock(&fd->mu); - - /* if we are shutdown, then don't add to the watcher set */ - if (fd->shutdown) { - watcher->fd = NULL; - watcher->pollset = NULL; - watcher->worker = NULL; - gpr_mu_unlock(&fd->mu); - GRPC_FD_UNREF(fd, "poll"); - return 0; - } - - /* if there is nobody polling for read, but we need to, then start doing so */ - cur = fd->read_closure; - requested = cur != CLOSURE_READY; - if (read_mask && fd->read_watcher == NULL && requested) { - fd->read_watcher = watcher; - mask |= read_mask; - } - /* if there is nobody polling for write, but we need to, then start doing so - */ - cur = fd->write_closure; - requested = cur != CLOSURE_READY; - if (write_mask && fd->write_watcher == NULL && requested) { - fd->write_watcher = watcher; - mask |= write_mask; - } - /* if not polling, remember this watcher in case we need someone to later */ - if (mask == 0 && worker != NULL) { - watcher->next = &fd->inactive_watcher_root; - watcher->prev = watcher->next->prev; - watcher->next->prev = watcher->prev->next = watcher; - } - watcher->pollset = pollset; - watcher->worker = worker; - watcher->fd = fd; - gpr_mu_unlock(&fd->mu); - - return mask; -} - -static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, - int got_read, int got_write, - grpc_pollset *read_notifier_pollset) { - int was_polling = 0; - int kick = 0; - grpc_fd *fd = watcher->fd; - - if (fd == NULL) { - return; - } - - gpr_mu_lock(&fd->mu); - - if (watcher == fd->read_watcher) { - /* remove read watcher, kick if we still need a read */ - was_polling = 1; - if (!got_read) { - kick = 1; - } - fd->read_watcher = NULL; - } - if (watcher == fd->write_watcher) { - /* remove write watcher, kick if we still need a write */ - was_polling = 1; - if (!got_write) { - kick = 1; - } - fd->write_watcher = NULL; - } - if (!was_polling && watcher->worker != NULL) { - /* remove from inactive list */ - watcher->next->prev = watcher->prev; - watcher->prev->next = watcher->next; - } - if (got_read) { - if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) { - kick = 1; - } - - if (read_notifier_pollset != NULL) { - set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset); - } - } - if (got_write) { - if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) { - kick = 1; - } - } - if (kick) { - maybe_wake_one_watcher_locked(fd); - } - if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { - close_fd_locked(exec_ctx, fd); - } - gpr_mu_unlock(&fd->mu); - - GRPC_FD_UNREF(fd, "poll"); -} - -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } - -/******************************************************************************* - * pollset_posix.c - */ - -GPR_TLS_DECL(g_current_thread_poller); -GPR_TLS_DECL(g_current_thread_worker); - -/** The alarm system needs to be able to wakeup 'some poller' sometimes - * (specifically when a new alarm needs to be triggered earlier than the next - * alarm 'epoch'). - * This wakeup_fd gives us something to alert on when such a case occurs. */ -grpc_wakeup_fd grpc_global_wakeup_fd; - -static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; -} - -static int pollset_has_workers(grpc_pollset *p) { - return p->root_worker.next != &p->root_worker; -} - -static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { - if (pollset_has_workers(p)) { - grpc_pollset_worker *w = p->root_worker.next; - remove_worker(p, w); - return w; - } else { - return NULL; - } -} - -static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->next = &p->root_worker; - worker->prev = worker->next->prev; - worker->prev->next = worker->next->prev = worker; -} - -static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev = &p->root_worker; - worker->next = worker->prev->next; - worker->prev->next = worker->next->prev = worker; -} - -static void kick_append_error(grpc_error **composite, grpc_error *error) { - if (error == GRPC_ERROR_NONE) return; - if (*composite == GRPC_ERROR_NONE) { - *composite = GRPC_ERROR_CREATE("Kick Failure"); - } - *composite = grpc_error_add_child(*composite, error); -} - -static grpc_error *pollset_kick_ext(grpc_pollset *p, - grpc_pollset_worker *specific_worker, - uint32_t flags) { - GPR_TIMER_BEGIN("pollset_kick_ext", 0); - grpc_error *error = GRPC_ERROR_NONE; - - /* pollset->mu already held */ - if (specific_worker != NULL) { - if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { - GPR_TIMER_BEGIN("pollset_kick_ext.broadcast", 0); - GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); - for (specific_worker = p->root_worker.next; - specific_worker != &p->root_worker; - specific_worker = specific_worker->next) { - kick_append_error( - &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); - } - p->kicked_without_pollers = true; - GPR_TIMER_END("pollset_kick_ext.broadcast", 0); - } else if (gpr_tls_get(&g_current_thread_worker) != - (intptr_t)specific_worker) { - GPR_TIMER_MARK("different_thread_worker", 0); - if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { - specific_worker->reevaluate_polling_on_wakeup = true; - } - specific_worker->kicked_specifically = true; - kick_append_error(&error, - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); - } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { - GPR_TIMER_MARK("kick_yoself", 0); - if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { - specific_worker->reevaluate_polling_on_wakeup = true; - } - specific_worker->kicked_specifically = true; - kick_append_error(&error, - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); - } - } else if (gpr_tls_get(&g_current_thread_poller) != (intptr_t)p) { - GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); - GPR_TIMER_MARK("kick_anonymous", 0); - specific_worker = pop_front_worker(p); - if (specific_worker != NULL) { - if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { - GPR_TIMER_MARK("kick_anonymous_not_self", 0); - push_back_worker(p, specific_worker); - specific_worker = pop_front_worker(p); - if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 && - gpr_tls_get(&g_current_thread_worker) == - (intptr_t)specific_worker) { - push_back_worker(p, specific_worker); - specific_worker = NULL; - } - } - if (specific_worker != NULL) { - GPR_TIMER_MARK("finally_kick", 0); - push_back_worker(p, specific_worker); - kick_append_error( - &error, grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd)); - } - } else { - GPR_TIMER_MARK("kicked_no_pollers", 0); - p->kicked_without_pollers = true; - } - } - - GPR_TIMER_END("pollset_kick_ext", 0); - return error; -} - -static grpc_error *pollset_kick(grpc_pollset *p, - grpc_pollset_worker *specific_worker) { - return pollset_kick_ext(p, specific_worker, 0); -} - -/* global state management */ - -static grpc_error *pollset_global_init(void) { - gpr_tls_init(&g_current_thread_poller); - gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&grpc_global_wakeup_fd); -} - -static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); - gpr_tls_destroy(&g_current_thread_poller); - gpr_tls_destroy(&g_current_thread_worker); -} - -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); -} - -/* main interface */ - -static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null); - -static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->mu); - *mu = &pollset->mu; - pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; - pollset->in_flight_cbs = 0; - pollset->shutting_down = 0; - pollset->called_shutdown = 0; - pollset->kicked_without_pollers = 0; - pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; - pollset->local_wakeup_cache = NULL; - pollset->kicked_without_pollers = 0; - become_basic_pollset(pollset, NULL); -} - -static void pollset_destroy(grpc_pollset *pollset) { - GPR_ASSERT(pollset->in_flight_cbs == 0); - GPR_ASSERT(!pollset_has_workers(pollset)); - GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); - pollset->vtable->destroy(pollset); - while (pollset->local_wakeup_cache) { - grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; - grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); - gpr_free(pollset->local_wakeup_cache); - pollset->local_wakeup_cache = next; - } - gpr_mu_destroy(&pollset->mu); -} - -static void pollset_reset(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT(pollset->in_flight_cbs == 0); - GPR_ASSERT(!pollset_has_workers(pollset)); - GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); - pollset->vtable->destroy(pollset); - pollset->shutting_down = 0; - pollset->called_shutdown = 0; - pollset->kicked_without_pollers = 0; - become_basic_pollset(pollset, NULL); -} - -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { - gpr_mu_lock(&pollset->mu); - pollset->vtable->add_fd(exec_ctx, pollset, fd, 1); -/* the following (enabled only in debug) will reacquire and then release - our lock - meaning that if the unlocking flag passed to add_fd above is - not respected, the code will deadlock (in a way that we have a chance of - debugging) */ -#ifndef NDEBUG - gpr_mu_lock(&pollset->mu); - gpr_mu_unlock(&pollset->mu); -#endif -} - -static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - GPR_ASSERT(grpc_closure_list_empty(pollset->idle_jobs)); - pollset->vtable->finish_shutdown(pollset); - grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL); -} - -static void work_combine_error(grpc_error **composite, grpc_error *error) { - if (error == GRPC_ERROR_NONE) return; - if (*composite == GRPC_ERROR_NONE) { - *composite = GRPC_ERROR_CREATE("pollset_work"); - } - *composite = grpc_error_add_child(*composite, error); -} - -static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker **worker_hdl, - gpr_timespec now, gpr_timespec deadline) { - grpc_pollset_worker worker; - *worker_hdl = &worker; - grpc_error *error = GRPC_ERROR_NONE; - - /* pollset->mu already held */ - int added_worker = 0; - int locked = 1; - int queued_work = 0; - int keep_polling = 0; - GPR_TIMER_BEGIN("pollset_work", 0); - /* this must happen before we (potentially) drop pollset->mu */ - worker.next = worker.prev = NULL; - worker.reevaluate_polling_on_wakeup = 0; - if (pollset->local_wakeup_cache != NULL) { - worker.wakeup_fd = pollset->local_wakeup_cache; - pollset->local_wakeup_cache = worker.wakeup_fd->next; - } else { - worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd)); - error = grpc_wakeup_fd_init(&worker.wakeup_fd->fd); - if (error != GRPC_ERROR_NONE) { - return error; - } - } - worker.kicked_specifically = 0; - /* If there's work waiting for the pollset to be idle, and the - pollset is idle, then do that work */ - if (!pollset_has_workers(pollset) && - !grpc_closure_list_empty(pollset->idle_jobs)) { - GPR_TIMER_MARK("pollset_work.idle_jobs", 0); - grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); - goto done; - } - /* If we're shutting down then we don't execute any extended work */ - if (pollset->shutting_down) { - GPR_TIMER_MARK("pollset_work.shutting_down", 0); - goto done; - } - /* Give do_promote priority so we don't starve it out */ - if (pollset->in_flight_cbs) { - GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0); - gpr_mu_unlock(&pollset->mu); - locked = 0; - goto done; - } - /* Start polling, and keep doing so while we're being asked to - re-evaluate our pollers (this allows poll() based pollers to - ensure they don't miss wakeups) */ - keep_polling = 1; - while (keep_polling) { - keep_polling = 0; - if (!pollset->kicked_without_pollers) { - if (!added_worker) { - push_front_worker(pollset, &worker); - added_worker = 1; - gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); - } - gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset); - GPR_TIMER_BEGIN("maybe_work_and_unlock", 0); - work_combine_error(&error, - pollset->vtable->maybe_work_and_unlock( - exec_ctx, pollset, &worker, deadline, now)); - GPR_TIMER_END("maybe_work_and_unlock", 0); - locked = 0; - gpr_tls_set(&g_current_thread_poller, 0); - } else { - GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); - pollset->kicked_without_pollers = 0; - } - /* Finished execution - start cleaning up. - Note that we may arrive here from outside the enclosing while() loop. - In that case we won't loop though as we haven't added worker to the - worker list, which means nobody could ask us to re-evaluate polling). */ - done: - if (!locked) { - queued_work |= grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); - locked = 1; - } - /* If we're forced to re-evaluate polling (via pollset_kick with - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force - a loop */ - if (worker.reevaluate_polling_on_wakeup) { - worker.reevaluate_polling_on_wakeup = 0; - pollset->kicked_without_pollers = 0; - if (queued_work || worker.kicked_specifically) { - /* If there's queued work on the list, then set the deadline to be - immediate so we get back out of the polling loop quickly */ - deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); - } - keep_polling = 1; - } - } - if (added_worker) { - remove_worker(pollset, &worker); - gpr_tls_set(&g_current_thread_worker, 0); - } - /* release wakeup fd to the local pool */ - worker.wakeup_fd->next = pollset->local_wakeup_cache; - pollset->local_wakeup_cache = worker.wakeup_fd; - /* check shutdown conditions */ - if (pollset->shutting_down) { - if (pollset_has_workers(pollset)) { - pollset_kick(pollset, NULL); - } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) { - pollset->called_shutdown = 1; - gpr_mu_unlock(&pollset->mu); - finish_shutdown(exec_ctx, pollset); - grpc_exec_ctx_flush(exec_ctx); - /* Continuing to access pollset here is safe -- it is the caller's - * responsibility to not destroy when it has outstanding calls to - * pollset_work. - * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ - gpr_mu_lock(&pollset->mu); - } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { - grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); - gpr_mu_unlock(&pollset->mu); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); - } - } - *worker_hdl = NULL; - GPR_TIMER_END("pollset_work", 0); - return error; -} - -static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_closure *closure) { - GPR_ASSERT(!pollset->shutting_down); - pollset->shutting_down = 1; - pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - if (!pollset_has_workers(pollset)) { - grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL); - } - if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && - !pollset_has_workers(pollset)) { - pollset->called_shutdown = 1; - finish_shutdown(exec_ctx, pollset); - } -} - -static int poll_deadline_to_millis_timeout(gpr_timespec deadline, - gpr_timespec now) { - gpr_timespec timeout; - static const int64_t max_spin_polling_us = 10; - if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { - return -1; - } - if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( - max_spin_polling_us, - GPR_TIMESPAN))) <= 0) { - return 0; - } - timeout = gpr_time_sub(deadline, now); - return gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); -} - -/* - * basic_pollset - a vtable that provides polling for zero or one file - * descriptor via poll() - */ - -typedef struct grpc_unary_promote_args { - const grpc_pollset_vtable *original_vtable; - grpc_pollset *pollset; - grpc_fd *fd; - grpc_closure promotion_closure; -} grpc_unary_promote_args; - -static void basic_do_promote(grpc_exec_ctx *exec_ctx, void *args, - grpc_error *error) { - grpc_unary_promote_args *up_args = args; - const grpc_pollset_vtable *original_vtable = up_args->original_vtable; - grpc_pollset *pollset = up_args->pollset; - grpc_fd *fd = up_args->fd; - - /* - * This is quite tricky. There are a number of cases to keep in mind here: - * 1. fd may have been orphaned - * 2. The pollset may no longer be a unary poller (and we can't let case #1 - * leak to other pollset types!) - * 3. pollset's fd (which may have changed) may have been orphaned - * 4. The pollset may be shutting down. - */ - - gpr_mu_lock(&pollset->mu); - /* First we need to ensure that nobody is polling concurrently */ - GPR_ASSERT(!pollset_has_workers(pollset)); - - gpr_free(up_args); - /* At this point the pollset may no longer be a unary poller. In that case - * we should just call the right add function and be done. */ - /* TODO(klempner): If we're not careful this could cause infinite recursion. - * That's not a problem for now because empty_pollset has a trivial poller - * and we don't have any mechanism to unbecome multipoller. */ - pollset->in_flight_cbs--; - if (pollset->shutting_down) { - /* We don't care about this pollset anymore. */ - if (pollset->in_flight_cbs == 0 && !pollset->called_shutdown) { - pollset->called_shutdown = 1; - finish_shutdown(exec_ctx, pollset); - } - } else if (fd_is_orphaned(fd)) { - /* Don't try to add it to anything, we'll drop our ref on it below */ - } else if (pollset->vtable != original_vtable) { - pollset->vtable->add_fd(exec_ctx, pollset, fd, 0); - } else if (fd != pollset->data.ptr) { - grpc_fd *fds[2]; - fds[0] = pollset->data.ptr; - fds[1] = fd; - - if (fds[0] && !fd_is_orphaned(fds[0])) { - platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds)); - GRPC_FD_UNREF(fds[0], "basicpoll"); - } else { - /* old fd is orphaned and we haven't cleaned it up until now, so remain a - * unary poller */ - /* Note that it is possible that fds[1] is also orphaned at this point. - * That's okay, we'll correct it at the next add or poll. */ - if (fds[0]) GRPC_FD_UNREF(fds[0], "basicpoll"); - pollset->data.ptr = fd; - GRPC_FD_REF(fd, "basicpoll"); - } - } - - gpr_mu_unlock(&pollset->mu); - - /* Matching ref in basic_pollset_add_fd */ - GRPC_FD_UNREF(fd, "basicpoll_add"); -} - -static void basic_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd, int and_unlock_pollset) { - grpc_unary_promote_args *up_args; - GPR_ASSERT(fd); - if (fd == pollset->data.ptr) goto exit; - - if (!pollset_has_workers(pollset)) { - /* Fast path -- no in flight cbs */ - /* TODO(klempner): Comment this out and fix any test failures or establish - * they are due to timing issues */ - grpc_fd *fds[2]; - fds[0] = pollset->data.ptr; - fds[1] = fd; - - if (fds[0] == NULL) { - pollset->data.ptr = fd; - GRPC_FD_REF(fd, "basicpoll"); - } else if (!fd_is_orphaned(fds[0])) { - platform_become_multipoller(exec_ctx, pollset, fds, GPR_ARRAY_SIZE(fds)); - GRPC_FD_UNREF(fds[0], "basicpoll"); - } else { - /* old fd is orphaned and we haven't cleaned it up until now, so remain a - * unary poller */ - GRPC_FD_UNREF(fds[0], "basicpoll"); - pollset->data.ptr = fd; - GRPC_FD_REF(fd, "basicpoll"); - } - goto exit; - } - - /* Now we need to promote. This needs to happen when we're not polling. Since - * this may be called from poll, the wait needs to happen asynchronously. */ - GRPC_FD_REF(fd, "basicpoll_add"); - pollset->in_flight_cbs++; - up_args = gpr_malloc(sizeof(*up_args)); - up_args->fd = fd; - up_args->original_vtable = pollset->vtable; - up_args->pollset = pollset; - up_args->promotion_closure.cb = basic_do_promote; - up_args->promotion_closure.cb_arg = up_args; - - grpc_closure_list_append(&pollset->idle_jobs, &up_args->promotion_closure, - GRPC_ERROR_NONE); - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - -exit: - if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); - } -} - -static grpc_error *basic_pollset_maybe_work_and_unlock( - grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now) { -#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) -#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) - - struct pollfd pfd[3]; - grpc_fd *fd; - grpc_fd_watcher fd_watcher; - int timeout; - int r; - nfds_t nfds; - grpc_error *error = GRPC_ERROR_NONE; - - fd = pollset->data.ptr; - if (fd && fd_is_orphaned(fd)) { - GRPC_FD_UNREF(fd, "basicpoll"); - fd = pollset->data.ptr = NULL; - } - timeout = poll_deadline_to_millis_timeout(deadline, now); - pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); - pfd[0].events = POLLIN; - pfd[0].revents = 0; - pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); - pfd[1].events = POLLIN; - pfd[1].revents = 0; - nfds = 2; - if (fd) { - pfd[2].fd = fd->fd; - pfd[2].revents = 0; - GRPC_FD_REF(fd, "basicpoll_begin"); - gpr_mu_unlock(&pollset->mu); - pfd[2].events = - (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, &fd_watcher); - if (pfd[2].events != 0) { - nfds++; - } - } else { - gpr_mu_unlock(&pollset->mu); - } - - /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid - even going into the blocking annotation if possible */ - /* poll fd count (argument 2) is shortened by one if we have no events - to poll on - such that it only includes the kicker */ - GPR_TIMER_BEGIN("poll", 0); - GRPC_SCHEDULING_START_BLOCKING_REGION; - r = grpc_poll_function(pfd, nfds, timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; - GPR_TIMER_END("poll", 0); - - if (r < 0) { - if (errno != EINTR) { - work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); - } - if (fd) { - fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); - } - } else if (r == 0) { - if (fd) { - fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); - } - } else { - if (pfd[0].revents & POLLIN_CHECK) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd)); - } - if (pfd[1].revents & POLLIN_CHECK) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); - } - if (nfds > 2) { - fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, - pfd[2].revents & POLLOUT_CHECK, pollset); - } else if (fd) { - fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL); - } - } - - if (fd) { - GRPC_FD_UNREF(fd, "basicpoll_begin"); - } - - return error; -} - -static void basic_pollset_destroy(grpc_pollset *pollset) { - if (pollset->data.ptr != NULL) { - GRPC_FD_UNREF(pollset->data.ptr, "basicpoll"); - pollset->data.ptr = NULL; - } -} - -static const grpc_pollset_vtable basic_pollset = { - basic_pollset_add_fd, basic_pollset_maybe_work_and_unlock, - basic_pollset_destroy, basic_pollset_destroy}; - -static void become_basic_pollset(grpc_pollset *pollset, grpc_fd *fd_or_null) { - pollset->vtable = &basic_pollset; - pollset->data.ptr = fd_or_null; - if (fd_or_null != NULL) { - GRPC_FD_REF(fd_or_null, "basicpoll"); - } -} - -/******************************************************************************* - * pollset_multipoller_with_poll_posix.c - */ - -#ifndef GRPC_LINUX_MULTIPOLL_WITH_EPOLL - -typedef struct { - /* all polled fds */ - size_t fd_count; - size_t fd_capacity; - grpc_fd **fds; - /* fds that have been removed from the pollset explicitly */ - size_t del_count; - size_t del_capacity; - grpc_fd **dels; -} poll_hdr; - -static void multipoll_with_poll_pollset_add_fd(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, - grpc_fd *fd, - int and_unlock_pollset) { - size_t i; - poll_hdr *h = pollset->data.ptr; - /* TODO(ctiller): this is O(num_fds^2); maybe switch to a hash set here */ - for (i = 0; i < h->fd_count; i++) { - if (h->fds[i] == fd) goto exit; - } - if (h->fd_count == h->fd_capacity) { - h->fd_capacity = GPR_MAX(h->fd_capacity + 8, h->fd_count * 3 / 2); - h->fds = gpr_realloc(h->fds, sizeof(grpc_fd *) * h->fd_capacity); - } - h->fds[h->fd_count++] = fd; - GRPC_FD_REF(fd, "multipoller"); -exit: - if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); - } -} - -static grpc_error *multipoll_with_poll_pollset_maybe_work_and_unlock( - grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now) { -#define POLLOUT_CHECK (POLLOUT | POLLHUP | POLLERR) -#define POLLIN_CHECK (POLLIN | POLLHUP | POLLERR) - - int timeout; - int r; - size_t i, j, fd_count; - nfds_t pfd_count; - poll_hdr *h; - /* TODO(ctiller): inline some elements to avoid an allocation */ - grpc_fd_watcher *watchers; - struct pollfd *pfds; - grpc_error *error = GRPC_ERROR_NONE; - - h = pollset->data.ptr; - timeout = poll_deadline_to_millis_timeout(deadline, now); - /* TODO(ctiller): perform just one malloc here if we exceed the inline case */ - pfds = gpr_malloc(sizeof(*pfds) * (h->fd_count + 2)); - watchers = gpr_malloc(sizeof(*watchers) * (h->fd_count + 2)); - fd_count = 0; - pfd_count = 2; - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); - pfds[0].events = POLLIN; - pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); - pfds[1].events = POLLIN; - pfds[1].revents = 0; - for (i = 0; i < h->fd_count; i++) { - int remove = fd_is_orphaned(h->fds[i]); - for (j = 0; !remove && j < h->del_count; j++) { - if (h->fds[i] == h->dels[j]) remove = 1; - } - if (remove) { - GRPC_FD_UNREF(h->fds[i], "multipoller"); - } else { - h->fds[fd_count++] = h->fds[i]; - watchers[pfd_count].fd = h->fds[i]; - GRPC_FD_REF(watchers[pfd_count].fd, "multipoller_start"); - pfds[pfd_count].fd = h->fds[i]->fd; - pfds[pfd_count].revents = 0; - pfd_count++; - } - } - for (j = 0; j < h->del_count; j++) { - GRPC_FD_UNREF(h->dels[j], "multipoller_del"); - } - h->del_count = 0; - h->fd_count = fd_count; - gpr_mu_unlock(&pollset->mu); - - for (i = 2; i < pfd_count; i++) { - grpc_fd *fd = watchers[i].fd; - pfds[i].events = (short)fd_begin_poll(fd, pollset, worker, POLLIN, POLLOUT, - &watchers[i]); - GRPC_FD_UNREF(fd, "multipoller_start"); - } - - /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid - even going into the blocking annotation if possible */ - GRPC_SCHEDULING_START_BLOCKING_REGION; - r = grpc_poll_function(pfds, pfd_count, timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; - - if (r < 0) { - if (errno != EINTR) { - work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); - } - for (i = 2; i < pfd_count; i++) { - fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); - } - } else if (r == 0) { - for (i = 2; i < pfd_count; i++) { - fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); - } - } else { - if (pfds[0].revents & POLLIN_CHECK) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd)); - } - if (pfds[1].revents & POLLIN_CHECK) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); - } - for (i = 2; i < pfd_count; i++) { - if (watchers[i].fd == NULL) { - fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); - continue; - } - fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK, - pfds[i].revents & POLLOUT_CHECK, pollset); - } - } - - gpr_free(pfds); - gpr_free(watchers); - - return error; -} - -static void multipoll_with_poll_pollset_finish_shutdown(grpc_pollset *pollset) { - size_t i; - poll_hdr *h = pollset->data.ptr; - for (i = 0; i < h->fd_count; i++) { - GRPC_FD_UNREF(h->fds[i], "multipoller"); - } - for (i = 0; i < h->del_count; i++) { - GRPC_FD_UNREF(h->dels[i], "multipoller_del"); - } - h->fd_count = 0; - h->del_count = 0; -} - -static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) { - poll_hdr *h = pollset->data.ptr; - multipoll_with_poll_pollset_finish_shutdown(pollset); - gpr_free(h->fds); - gpr_free(h->dels); - gpr_free(h); -} - -static const grpc_pollset_vtable multipoll_with_poll_pollset = { - multipoll_with_poll_pollset_add_fd, - multipoll_with_poll_pollset_maybe_work_and_unlock, - multipoll_with_poll_pollset_finish_shutdown, - multipoll_with_poll_pollset_destroy}; - -static void poll_become_multipoller(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, grpc_fd **fds, - size_t nfds) { - size_t i; - poll_hdr *h = gpr_malloc(sizeof(poll_hdr)); - pollset->vtable = &multipoll_with_poll_pollset; - pollset->data.ptr = h; - h->fd_count = nfds; - h->fd_capacity = nfds; - h->fds = gpr_malloc(nfds * sizeof(grpc_fd *)); - h->del_count = 0; - h->del_capacity = 0; - h->dels = NULL; - for (i = 0; i < nfds; i++) { - h->fds[i] = fds[i]; - GRPC_FD_REF(fds[i], "multipoller"); - } -} - -#endif /* !GRPC_LINUX_MULTIPOLL_WITH_EPOLL */ - -/******************************************************************************* - * pollset_multipoller_with_epoll_posix.c - */ - -#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL - -#include <errno.h> -#include <poll.h> -#include <string.h> -#include <sys/epoll.h> -#include <unistd.h> - -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/useful.h> - -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/profiling/timers.h" -#include "src/core/lib/support/block_annotate.h" - -static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st, - grpc_pollset *read_notifier_pollset) { - /* only one set_ready can be active at once (but there may be a racing - notify_on) */ - gpr_mu_lock(&fd->mu); - set_ready_locked(exec_ctx, fd, st); - - /* A non-NULL read_notifier_pollset means that the fd is readable. */ - if (read_notifier_pollset != NULL) { - /* Note: Since the fd might be a part of multiple pollsets, this might be - * called multiple times (for each time the fd becomes readable) and it is - * okay to set the fd's read-notifier pollset to anyone of these pollsets */ - set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset); - } - - gpr_mu_unlock(&fd->mu); -} - -static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_pollset *notifier_pollset) { - set_ready(exec_ctx, fd, &fd->read_closure, notifier_pollset); -} - -static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - set_ready(exec_ctx, fd, &fd->write_closure, NULL); -} - -struct epoll_fd_list { - int *epoll_fds; - size_t count; - size_t capacity; -}; - -static struct epoll_fd_list epoll_fd_global_list; -static gpr_once init_epoll_fd_list_mu = GPR_ONCE_INIT; -static gpr_mu epoll_fd_list_mu; - -static void init_mu(void) { gpr_mu_init(&epoll_fd_list_mu); } - -static void add_epoll_fd_to_global_list(int epoll_fd) { - gpr_once_init(&init_epoll_fd_list_mu, init_mu); - - gpr_mu_lock(&epoll_fd_list_mu); - if (epoll_fd_global_list.count == epoll_fd_global_list.capacity) { - epoll_fd_global_list.capacity = - GPR_MAX((size_t)8, epoll_fd_global_list.capacity * 2); - epoll_fd_global_list.epoll_fds = - gpr_realloc(epoll_fd_global_list.epoll_fds, - epoll_fd_global_list.capacity * sizeof(int)); - } - epoll_fd_global_list.epoll_fds[epoll_fd_global_list.count++] = epoll_fd; - gpr_mu_unlock(&epoll_fd_list_mu); -} - -static void remove_epoll_fd_from_global_list(int epoll_fd) { - gpr_mu_lock(&epoll_fd_list_mu); - GPR_ASSERT(epoll_fd_global_list.count > 0); - for (size_t i = 0; i < epoll_fd_global_list.count; i++) { - if (epoll_fd == epoll_fd_global_list.epoll_fds[i]) { - epoll_fd_global_list.epoll_fds[i] = - epoll_fd_global_list.epoll_fds[--(epoll_fd_global_list.count)]; - break; - } - } - gpr_mu_unlock(&epoll_fd_list_mu); -} - -static void remove_fd_from_all_epoll_sets(int fd) { - int err; - gpr_once_init(&init_epoll_fd_list_mu, init_mu); - gpr_mu_lock(&epoll_fd_list_mu); - if (epoll_fd_global_list.count == 0) { - gpr_mu_unlock(&epoll_fd_list_mu); - return; - } - for (size_t i = 0; i < epoll_fd_global_list.count; i++) { - err = epoll_ctl(epoll_fd_global_list.epoll_fds[i], EPOLL_CTL_DEL, fd, NULL); - if (err < 0 && errno != ENOENT) { - gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd, - strerror(errno)); - } - } - gpr_mu_unlock(&epoll_fd_list_mu); -} - -typedef struct { - grpc_pollset *pollset; - grpc_fd *fd; - grpc_closure closure; -} delayed_add; - -typedef struct { int epoll_fd; } epoll_hdr; - -static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { - epoll_hdr *h = pollset->data.ptr; - struct epoll_event ev; - int err; - grpc_fd_watcher watcher; - - /* We pretend to be polling whilst adding an fd to keep the fd from being - closed during the add. This may result in a spurious wakeup being assigned - to this pollset whilst adding, but that should be benign. */ - GPR_ASSERT(fd_begin_poll(fd, pollset, NULL, 0, 0, &watcher) == 0); - if (watcher.fd != NULL) { - ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); - ev.data.ptr = fd; - err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); - if (err < 0) { - /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ - if (errno != EEXIST) { - gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, - strerror(errno)); - } - } - } - fd_end_poll(exec_ctx, &watcher, 0, 0, NULL); -} - -static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - delayed_add *da = arg; - - if (!fd_is_orphaned(da->fd)) { - finally_add_fd(exec_ctx, da->pollset, da->fd); - } - - gpr_mu_lock(&da->pollset->mu); - da->pollset->in_flight_cbs--; - if (da->pollset->shutting_down) { - /* We don't care about this pollset anymore. */ - if (da->pollset->in_flight_cbs == 0 && !da->pollset->called_shutdown) { - da->pollset->called_shutdown = 1; - grpc_exec_ctx_sched(exec_ctx, da->pollset->shutdown_done, GRPC_ERROR_NONE, - NULL); - } - } - gpr_mu_unlock(&da->pollset->mu); - - GRPC_FD_UNREF(da->fd, "delayed_add"); - - gpr_free(da); -} - -static void multipoll_with_epoll_pollset_add_fd(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, - grpc_fd *fd, - int and_unlock_pollset) { - if (and_unlock_pollset) { - gpr_mu_unlock(&pollset->mu); - finally_add_fd(exec_ctx, pollset, fd); - } else { - delayed_add *da = gpr_malloc(sizeof(*da)); - da->pollset = pollset; - da->fd = fd; - GRPC_FD_REF(fd, "delayed_add"); - grpc_closure_init(&da->closure, perform_delayed_add, da); - pollset->in_flight_cbs++; - grpc_exec_ctx_sched(exec_ctx, &da->closure, GRPC_ERROR_NONE, NULL); - } -} - -/* TODO(klempner): We probably want to turn this down a bit */ -#define GRPC_EPOLL_MAX_EVENTS 1000 - -static grpc_error *multipoll_with_epoll_pollset_maybe_work_and_unlock( - grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, - gpr_timespec deadline, gpr_timespec now) { - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; - int ep_rv; - int poll_rv; - epoll_hdr *h = pollset->data.ptr; - int timeout_ms; - struct pollfd pfds[2]; - grpc_error *error = GRPC_ERROR_NONE; - - /* If you want to ignore epoll's ability to sanely handle parallel pollers, - * for a more apples-to-apples performance comparison with poll, add a - * if (pollset->counter != 0) { return 0; } - * here. - */ - - gpr_mu_unlock(&pollset->mu); - - timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); - pfds[0].events = POLLIN; - pfds[0].revents = 0; - pfds[1].fd = h->epoll_fd; - pfds[1].events = POLLIN; - pfds[1].revents = 0; - - /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid - even going into the blocking annotation if possible */ - GPR_TIMER_BEGIN("poll", 0); - GRPC_SCHEDULING_START_BLOCKING_REGION; - poll_rv = grpc_poll_function(pfds, 2, timeout_ms); - GRPC_SCHEDULING_END_BLOCKING_REGION; - GPR_TIMER_END("poll", 0); - - if (poll_rv < 0) { - if (errno != EINTR) { - work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); - } - } else if (poll_rv == 0) { - /* do nothing */ - } else { - if (pfds[0].revents) { - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd)); - } - if (pfds[1].revents) { - do { - /* The following epoll_wait never blocks; it has a timeout of 0 */ - ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - if (ep_rv < 0) { - if (errno != EINTR) { - work_combine_error(&error, GRPC_OS_ERROR(errno, "epoll_wait")); - } - } else { - int i; - for (i = 0; i < ep_rv; ++i) { - grpc_fd *fd = ep_ev[i].data.ptr; - /* TODO(klempner): We might want to consider making err and pri - * separate events */ - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (fd == NULL) { - work_combine_error(&error, grpc_wakeup_fd_consume_wakeup( - &grpc_global_wakeup_fd)); - } else { - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } - } - } while (ep_rv == GRPC_EPOLL_MAX_EVENTS); - } - } - return error; -} - -static void multipoll_with_epoll_pollset_finish_shutdown( - grpc_pollset *pollset) {} - -static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) { - epoll_hdr *h = pollset->data.ptr; - close(h->epoll_fd); - remove_epoll_fd_from_global_list(h->epoll_fd); - gpr_free(h); -} - -static const grpc_pollset_vtable multipoll_with_epoll_pollset = { - multipoll_with_epoll_pollset_add_fd, - multipoll_with_epoll_pollset_maybe_work_and_unlock, - multipoll_with_epoll_pollset_finish_shutdown, - multipoll_with_epoll_pollset_destroy}; - -static void epoll_become_multipoller(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, grpc_fd **fds, - size_t nfds) { - size_t i; - epoll_hdr *h = gpr_malloc(sizeof(epoll_hdr)); - struct epoll_event ev; - int err; - - pollset->vtable = &multipoll_with_epoll_pollset; - pollset->data.ptr = h; - h->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (h->epoll_fd < 0) { - /* TODO(klempner): Fall back to poll here, especially on ENOSYS */ - gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno)); - abort(); - } - add_epoll_fd_to_global_list(h->epoll_fd); - - ev.events = (uint32_t)(EPOLLIN | EPOLLET); - ev.data.ptr = NULL; - err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, - GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), &ev); - if (err < 0) { - gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", - GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd), - strerror(errno)); - } - - for (i = 0; i < nfds; i++) { - multipoll_with_epoll_pollset_add_fd(exec_ctx, pollset, fds[i], 0); - } -} - -#else /* GRPC_LINUX_MULTIPOLL_WITH_EPOLL */ - -static void remove_fd_from_all_epoll_sets(int fd) {} - -#endif /* GRPC_LINUX_MULTIPOLL_WITH_EPOLL */ - -/******************************************************************************* - * pollset_set_posix.c - */ - -static grpc_pollset_set *pollset_set_create(void) { - grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set)); - memset(pollset_set, 0, sizeof(*pollset_set)); - gpr_mu_init(&pollset_set->mu); - return pollset_set; -} - -static void pollset_set_destroy(grpc_pollset_set *pollset_set) { - size_t i; - gpr_mu_destroy(&pollset_set->mu); - for (i = 0; i < pollset_set->fd_count; i++) { - GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set"); - } - gpr_free(pollset_set->pollsets); - gpr_free(pollset_set->pollset_sets); - gpr_free(pollset_set->fds); - gpr_free(pollset_set); -} - -static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pollset_set, - grpc_pollset *pollset) { - size_t i, j; - gpr_mu_lock(&pollset_set->mu); - if (pollset_set->pollset_count == pollset_set->pollset_capacity) { - pollset_set->pollset_capacity = - GPR_MAX(8, 2 * pollset_set->pollset_capacity); - pollset_set->pollsets = - gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity * - sizeof(*pollset_set->pollsets)); - } - pollset_set->pollsets[pollset_set->pollset_count++] = pollset; - for (i = 0, j = 0; i < pollset_set->fd_count; i++) { - if (fd_is_orphaned(pollset_set->fds[i])) { - GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set"); - } else { - pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]); - pollset_set->fds[j++] = pollset_set->fds[i]; - } - } - pollset_set->fd_count = j; - gpr_mu_unlock(&pollset_set->mu); -} - -static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pollset_set, - grpc_pollset *pollset) { - size_t i; - gpr_mu_lock(&pollset_set->mu); - for (i = 0; i < pollset_set->pollset_count; i++) { - if (pollset_set->pollsets[i] == pollset) { - pollset_set->pollset_count--; - GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i], - pollset_set->pollsets[pollset_set->pollset_count]); - break; - } - } - gpr_mu_unlock(&pollset_set->mu); -} - -static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - size_t i, j; - gpr_mu_lock(&bag->mu); - if (bag->pollset_set_count == bag->pollset_set_capacity) { - bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity); - bag->pollset_sets = - gpr_realloc(bag->pollset_sets, - bag->pollset_set_capacity * sizeof(*bag->pollset_sets)); - } - bag->pollset_sets[bag->pollset_set_count++] = item; - for (i = 0, j = 0; i < bag->fd_count; i++) { - if (fd_is_orphaned(bag->fds[i])) { - GRPC_FD_UNREF(bag->fds[i], "pollset_set"); - } else { - pollset_set_add_fd(exec_ctx, item, bag->fds[i]); - bag->fds[j++] = bag->fds[i]; - } - } - bag->fd_count = j; - gpr_mu_unlock(&bag->mu); -} - -static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - size_t i; - gpr_mu_lock(&bag->mu); - for (i = 0; i < bag->pollset_set_count; i++) { - if (bag->pollset_sets[i] == item) { - bag->pollset_set_count--; - GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i], - bag->pollset_sets[bag->pollset_set_count]); - break; - } - } - gpr_mu_unlock(&bag->mu); -} - -static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pollset_set, grpc_fd *fd) { - size_t i; - gpr_mu_lock(&pollset_set->mu); - if (pollset_set->fd_count == pollset_set->fd_capacity) { - pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity); - pollset_set->fds = gpr_realloc( - pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds)); - } - GRPC_FD_REF(fd, "pollset_set"); - pollset_set->fds[pollset_set->fd_count++] = fd; - for (i = 0; i < pollset_set->pollset_count; i++) { - pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd); - } - for (i = 0; i < pollset_set->pollset_set_count; i++) { - pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd); - } - gpr_mu_unlock(&pollset_set->mu); -} - -static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pollset_set, grpc_fd *fd) { - size_t i; - gpr_mu_lock(&pollset_set->mu); - for (i = 0; i < pollset_set->fd_count; i++) { - if (pollset_set->fds[i] == fd) { - pollset_set->fd_count--; - GPR_SWAP(grpc_fd *, pollset_set->fds[i], - pollset_set->fds[pollset_set->fd_count]); - GRPC_FD_UNREF(fd, "pollset_set"); - break; - } - } - for (i = 0; i < pollset_set->pollset_set_count; i++) { - pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd); - } - gpr_mu_unlock(&pollset_set->mu); -} - -/******************************************************************************* - * workqueue stubs - */ - -#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, - const char *file, int line, - const char *reason) { - return workqueue; -} -static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - const char *file, int line, const char *reason) {} -#else -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { - return workqueue; -} -static void workqueue_unref(grpc_exec_ctx *exec_ctx, - grpc_workqueue *workqueue) {} -#endif - -static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, - grpc_workqueue *workqueue, grpc_closure *closure, - grpc_error *error) { - grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); -} - -/******************************************************************************* - * event engine binding - */ - -static void shutdown_engine(void) { - fd_global_shutdown(); - pollset_global_shutdown(); -} - -static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - .fd_get_workqueue = fd_get_workqueue, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_reset = pollset_reset, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .kick_poller = kick_poller, - - .workqueue_ref = workqueue_ref, - .workqueue_unref = workqueue_unref, - .workqueue_enqueue = workqueue_enqueue, - - .shutdown_engine = shutdown_engine, -}; - -const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void) { -#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL - platform_become_multipoller = epoll_become_multipoller; -#else - platform_become_multipoller = poll_become_multipoller; -#endif - fd_global_init(); - pollset_global_init(); - return &vtable; -} - -#endif diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index ef36ba89b2..ab139895fd 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,7 +45,6 @@ #include <grpc/support/useful.h> #include "src/core/lib/iomgr/ev_epoll_linux.h" -#include "src/core/lib/iomgr/ev_poll_and_epoll_posix.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -67,7 +66,6 @@ static const event_engine_factory g_factories[] = { {"epoll", grpc_init_epoll_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, - {"legacy", grpc_init_poll_and_epoll_posix}, }; static void add(const char *beg, const char *end, char ***ss, size_t *ns) { diff --git a/src/core/lib/iomgr/load_file.c b/src/core/lib/iomgr/load_file.c index b62ecbc534..217bc5da59 100644 --- a/src/core/lib/iomgr/load_file.c +++ b/src/core/lib/iomgr/load_file.c @@ -44,10 +44,10 @@ #include "src/core/lib/support/string.h" grpc_error *grpc_load_file(const char *filename, int add_null_terminator, - gpr_slice *output) { + grpc_slice *output) { unsigned char *contents = NULL; size_t contents_size = 0; - gpr_slice result = gpr_empty_slice(); + grpc_slice result = gpr_empty_slice(); FILE *file; size_t bytes_read = 0; grpc_error *error = GRPC_ERROR_NONE; @@ -72,7 +72,7 @@ grpc_error *grpc_load_file(const char *filename, int add_null_terminator, if (add_null_terminator) { contents[contents_size++] = 0; } - result = gpr_slice_new(contents, contents_size, gpr_free); + result = grpc_slice_new(contents, contents_size, gpr_free); end: *output = result; diff --git a/src/core/lib/iomgr/load_file.h b/src/core/lib/iomgr/load_file.h index 9aac2225d1..73ee8c3abf 100644 --- a/src/core/lib/iomgr/load_file.h +++ b/src/core/lib/iomgr/load_file.h @@ -36,7 +36,7 @@ #include <stdio.h> -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/lib/iomgr/error.h" @@ -47,7 +47,7 @@ extern "C" { /* Loads the content of a file into a slice. add_null_terminator will add a NULL terminator if non-zero. */ grpc_error *grpc_load_file(const char *filename, int add_null_terminator, - gpr_slice *slice); + grpc_slice *slice); #ifdef __cplusplus } diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 8a06443d58..051a30baa3 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,6 +44,81 @@ int grpc_resource_quota_trace = 0; +/* Internal linked list pointers for a resource user */ +typedef struct { + grpc_resource_user *next; + grpc_resource_user *prev; +} grpc_resource_user_link; + +/* Resource users are kept in (potentially) several intrusive linked lists + at once. These are the list names. */ +typedef enum { + /* Resource users that are waiting for an allocation */ + GRPC_RULIST_AWAITING_ALLOCATION, + /* Resource users that have free memory available for internal reclamation */ + GRPC_RULIST_NON_EMPTY_FREE_POOL, + /* Resource users that have published a benign reclamation is available */ + GRPC_RULIST_RECLAIMER_BENIGN, + /* Resource users that have published a destructive reclamation is + available */ + GRPC_RULIST_RECLAIMER_DESTRUCTIVE, + /* Number of lists: must be last */ + GRPC_RULIST_COUNT +} grpc_rulist; + +struct grpc_resource_user { + /* The quota this resource user consumes from */ + grpc_resource_quota *resource_quota; + + /* Closure to schedule an allocation under the resource quota combiner lock */ + grpc_closure allocate_closure; + /* Closure to publish a non empty free pool under the resource quota combiner + lock */ + grpc_closure add_to_free_pool_closure; + + /* one ref for each ref call (released by grpc_resource_user_unref), and one + ref for each byte allocated (released by grpc_resource_user_free) */ + gpr_atm refs; + /* is this resource user unlocked? starts at 0, increases for each shutdown + call */ + gpr_atm shutdown; + + gpr_mu mu; + /* The amount of memory (in bytes) this user has cached for its own use: to + avoid quota contention, each resource user can keep some memory in + addition to what it is immediately using (e.g., for caching), and the quota + can pull it back under memory pressure. + This value can become negative if more memory has been requested than + existed in the free pool, at which point the quota is consulted to bring + this value non-negative (asynchronously). */ + int64_t free_pool; + /* A list of closures to call once free_pool becomes non-negative - ie when + all outstanding allocations have been granted. */ + grpc_closure_list on_allocated; + /* True if we are currently trying to allocate from the quota, false if not */ + bool allocating; + /* True if we are currently trying to add ourselves to the non-free quota + list, false otherwise */ + bool added_to_free_pool; + + /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer + */ + grpc_closure *reclaimers[2]; + /* Trampoline closures to finish reclamation and re-enter the quota combiner + lock */ + grpc_closure post_reclaimer_closure[2]; + + /* Closure to execute under the quota combiner to de-register and shutdown the + resource user */ + grpc_closure destroy_closure; + + /* Links in the various grpc_rulist lists */ + grpc_resource_user_link links[GRPC_RULIST_COUNT]; + + /* The name of this resource user, for debugging/tracing */ + char *name; +}; + struct grpc_resource_quota { /* refcount */ gpr_refcount refs; @@ -272,7 +347,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx, */ typedef struct { - gpr_slice_refcount base; + grpc_slice_refcount base; gpr_refcount refs; grpc_resource_user *resource_user; size_t size; @@ -289,7 +364,7 @@ static void ru_slice_unref(void *p) { /* TODO(ctiller): this is dangerous, but I think safe for now: we have no guarantee here that we're at a safe point for creating an execution context, but we have no way of writing this code otherwise. - In the future: consider lifting gpr_slice to grpc, and offering an + In the future: consider lifting grpc_slice to grpc, and offering an internal_{ref,unref} pair that is execution context aware. Alternatively, make exec_ctx be thread local and 'do the right thing' (whatever that @@ -302,15 +377,15 @@ static void ru_slice_unref(void *p) { } } -static gpr_slice ru_slice_create(grpc_resource_user *resource_user, - size_t size) { +static grpc_slice ru_slice_create(grpc_resource_user *resource_user, + size_t size) { ru_slice_refcount *rc = gpr_malloc(sizeof(ru_slice_refcount) + size); rc->base.ref = ru_slice_ref; rc->base.unref = ru_slice_unref; gpr_ref_init(&rc->refs, 1); rc->resource_user = resource_user; rc->size = size; - gpr_slice slice; + grpc_slice slice; slice.refcount = &rc->base; slice.data.refcounted.bytes = (uint8_t *)(rc + 1); slice.data.refcounted.length = size; @@ -373,9 +448,19 @@ static void ru_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *ru, rulist_add_tail(resource_user, GRPC_RULIST_RECLAIMER_DESTRUCTIVE); } +static void ru_shutdown(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) { + grpc_resource_user *resource_user = ru; + grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[0], + GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[1], + GRPC_ERROR_CANCELLED, NULL); + resource_user->reclaimers[0] = NULL; + resource_user->reclaimers[1] = NULL; +} + static void ru_destroy(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) { grpc_resource_user *resource_user = ru; - GPR_ASSERT(resource_user->allocated == 0); + GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->refs) == 0); for (int i = 0; i < GRPC_RULIST_COUNT; i++) { rulist_remove(resource_user, (grpc_rulist)i); } @@ -383,13 +468,14 @@ static void ru_destroy(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) { GRPC_ERROR_CANCELLED, NULL); grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[1], GRPC_ERROR_CANCELLED, NULL); - grpc_exec_ctx_sched(exec_ctx, (grpc_closure *)gpr_atm_no_barrier_load( - &resource_user->on_done_destroy_closure), - GRPC_ERROR_NONE, NULL); if (resource_user->free_pool != 0) { resource_user->resource_quota->free_pool += resource_user->free_pool; rq_step_sched(exec_ctx, resource_user->resource_quota); } + grpc_resource_quota_internal_unref(exec_ctx, resource_user->resource_quota); + gpr_mu_destroy(&resource_user->mu); + gpr_free(resource_user->name); + gpr_free(resource_user); } static void ru_allocated_slices(grpc_exec_ctx *exec_ctx, void *arg, @@ -397,7 +483,7 @@ static void ru_allocated_slices(grpc_exec_ctx *exec_ctx, void *arg, grpc_resource_user_slice_allocator *slice_allocator = arg; if (error == GRPC_ERROR_NONE) { for (size_t i = 0; i < slice_allocator->count; i++) { - gpr_slice_buffer_add_indexed( + grpc_slice_buffer_add_indexed( slice_allocator->dest, ru_slice_create(slice_allocator->resource_user, slice_allocator->length)); } @@ -539,9 +625,9 @@ const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void) { * grpc_resource_user api */ -void grpc_resource_user_init(grpc_resource_user *resource_user, - grpc_resource_quota *resource_quota, - const char *name) { +grpc_resource_user *grpc_resource_user_create( + grpc_resource_quota *resource_quota, const char *name) { + grpc_resource_user *resource_user = gpr_malloc(sizeof(*resource_user)); resource_user->resource_quota = grpc_resource_quota_internal_ref(resource_quota); grpc_closure_init(&resource_user->allocate_closure, &ru_allocate, @@ -555,12 +641,12 @@ void grpc_resource_user_init(grpc_resource_user *resource_user, grpc_closure_init(&resource_user->destroy_closure, &ru_destroy, resource_user); gpr_mu_init(&resource_user->mu); - resource_user->allocated = 0; + gpr_atm_rel_store(&resource_user->refs, 1); + gpr_atm_rel_store(&resource_user->shutdown, 0); resource_user->free_pool = 0; grpc_closure_list_init(&resource_user->on_allocated); resource_user->allocating = false; resource_user->added_to_free_pool = false; - gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, 0); resource_user->reclaimers[0] = NULL; resource_user->reclaimers[1] = NULL; for (int i = 0; i < GRPC_RULIST_COUNT; i++) { @@ -572,56 +658,54 @@ void grpc_resource_user_init(grpc_resource_user *resource_user, gpr_asprintf(&resource_user->name, "anonymous_resource_user_%" PRIxPTR, (intptr_t)resource_user); } + return resource_user; } -void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user, - grpc_closure *on_done) { - gpr_mu_lock(&resource_user->mu); - GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->on_done_destroy_closure) == - 0); - gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, - (gpr_atm)on_done); - if (resource_user->allocated == 0) { +static void ru_ref_by(grpc_resource_user *resource_user, gpr_atm amount) { + GPR_ASSERT(amount > 0); + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&resource_user->refs, amount) != 0); +} + +static void ru_unref_by(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, gpr_atm amount) { + GPR_ASSERT(amount > 0); + gpr_atm old = gpr_atm_full_fetch_add(&resource_user->refs, -amount); + GPR_ASSERT(old >= amount); + if (old == amount) { grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, &resource_user->destroy_closure, GRPC_ERROR_NONE, false); } - gpr_mu_unlock(&resource_user->mu); } -void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user) { - grpc_resource_quota_internal_unref(exec_ctx, resource_user->resource_quota); - gpr_mu_destroy(&resource_user->mu); - gpr_free(resource_user->name); +void grpc_resource_user_ref(grpc_resource_user *resource_user) { + ru_ref_by(resource_user, 1); +} + +void grpc_resource_user_unref(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user) { + ru_unref_by(exec_ctx, resource_user, 1); +} + +void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user) { + if (gpr_atm_full_fetch_add(&resource_user->shutdown, 1) == 0) { + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + grpc_closure_create(ru_shutdown, resource_user), + GRPC_ERROR_NONE, false); + } } void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user, size_t size, grpc_closure *optional_on_done) { gpr_mu_lock(&resource_user->mu); - grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( - &resource_user->on_done_destroy_closure); - if (on_done_destroy != NULL) { - /* already shutdown */ - if (grpc_resource_quota_trace) { - gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR " after shutdown", - resource_user->resource_quota->name, resource_user->name, size); - } - grpc_exec_ctx_sched( - exec_ctx, optional_on_done, - GRPC_ERROR_CREATE("Buffer pool user is already shutdown"), NULL); - gpr_mu_unlock(&resource_user->mu); - return; - } - resource_user->allocated += (int64_t)size; + ru_ref_by(resource_user, (gpr_atm)size); resource_user->free_pool -= (int64_t)size; if (grpc_resource_quota_trace) { - gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; allocated -> %" PRId64 - ", free_pool -> %" PRId64, + gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, - resource_user->allocated, resource_user->free_pool); + resource_user->free_pool); } if (resource_user->free_pool < 0) { grpc_closure_list_append(&resource_user->on_allocated, optional_on_done, @@ -641,15 +725,12 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user, size_t size) { gpr_mu_lock(&resource_user->mu); - GPR_ASSERT(resource_user->allocated >= (int64_t)size); bool was_zero_or_negative = resource_user->free_pool <= 0; resource_user->free_pool += (int64_t)size; - resource_user->allocated -= (int64_t)size; if (grpc_resource_quota_trace) { - gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; allocated -> %" PRId64 - ", free_pool -> %" PRId64, + gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, - resource_user->allocated, resource_user->free_pool); + resource_user->free_pool); } bool is_bigger_than_zero = resource_user->free_pool > 0; if (is_bigger_than_zero && was_zero_or_negative && @@ -659,29 +740,23 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, &resource_user->add_to_free_pool_closure, GRPC_ERROR_NONE, false); } - grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( - &resource_user->on_done_destroy_closure); - if (on_done_destroy != NULL && resource_user->allocated == 0) { - grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, - &resource_user->destroy_closure, GRPC_ERROR_NONE, - false); - } gpr_mu_unlock(&resource_user->mu); + ru_unref_by(exec_ctx, resource_user, (gpr_atm)size); } void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user, bool destructive, grpc_closure *closure) { - if (gpr_atm_acq_load(&resource_user->on_done_destroy_closure) == 0) { - GPR_ASSERT(resource_user->reclaimers[destructive] == NULL); - resource_user->reclaimers[destructive] = closure; - grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, - &resource_user->post_reclaimer_closure[destructive], - GRPC_ERROR_NONE, false); - } else { + GPR_ASSERT(resource_user->reclaimers[destructive] == NULL); + if (gpr_atm_acq_load(&resource_user->shutdown) > 0) { grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL); + return; } + resource_user->reclaimers[destructive] = closure; + grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, + &resource_user->post_reclaimer_closure[destructive], + GRPC_ERROR_NONE, false); } void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, @@ -708,7 +783,7 @@ void grpc_resource_user_slice_allocator_init( void grpc_resource_user_alloc_slices( grpc_exec_ctx *exec_ctx, grpc_resource_user_slice_allocator *slice_allocator, size_t length, - size_t count, gpr_slice_buffer *dest) { + size_t count, grpc_slice_buffer *dest) { slice_allocator->length = length; slice_allocator->count = count; slice_allocator->dest = dest; @@ -716,9 +791,9 @@ void grpc_resource_user_alloc_slices( count * length, &slice_allocator->on_allocated); } -gpr_slice grpc_resource_user_slice_malloc(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user, - size_t size) { +grpc_slice grpc_resource_user_slice_malloc(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + size_t size) { grpc_resource_user_alloc(exec_ctx, resource_user, size, NULL); return ru_slice_create(resource_user, size); } diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index da68f21a2c..0181fd978b 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -84,91 +84,15 @@ void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx, grpc_resource_quota *grpc_resource_quota_from_channel_args( const grpc_channel_args *channel_args); -/* Resource users are kept in (potentially) several intrusive linked lists - at once. These are the list names. */ -typedef enum { - /* Resource users that are waiting for an allocation */ - GRPC_RULIST_AWAITING_ALLOCATION, - /* Resource users that have free memory available for internal reclamation */ - GRPC_RULIST_NON_EMPTY_FREE_POOL, - /* Resource users that have published a benign reclamation is available */ - GRPC_RULIST_RECLAIMER_BENIGN, - /* Resource users that have published a destructive reclamation is - available */ - GRPC_RULIST_RECLAIMER_DESTRUCTIVE, - /* Number of lists: must be last */ - GRPC_RULIST_COUNT -} grpc_rulist; - typedef struct grpc_resource_user grpc_resource_user; -/* Internal linked list pointers for a resource user */ -typedef struct { - grpc_resource_user *next; - grpc_resource_user *prev; -} grpc_resource_user_link; - -struct grpc_resource_user { - /* The quota this resource user consumes from */ - grpc_resource_quota *resource_quota; - - /* Closure to schedule an allocation under the resource quota combiner lock */ - grpc_closure allocate_closure; - /* Closure to publish a non empty free pool under the resource quota combiner - lock */ - grpc_closure add_to_free_pool_closure; - - gpr_mu mu; - /* Total allocated memory outstanding by this resource user in bytes; - always positive */ - int64_t allocated; - /* The amount of memory (in bytes) this user has cached for its own use: to - avoid quota contention, each resource user can keep some memory in - addition to what it is immediately using (e.g., for caching), and the quota - can pull it back under memory pressure. - This value can become negative if more memory has been requested than - existed in the free pool, at which point the quota is consulted to bring - this value non-negative (asynchronously). */ - int64_t free_pool; - /* A list of closures to call once free_pool becomes non-negative - ie when - all outstanding allocations have been granted. */ - grpc_closure_list on_allocated; - /* True if we are currently trying to allocate from the quota, false if not */ - bool allocating; - /* True if we are currently trying to add ourselves to the non-free quota - list, false otherwise */ - bool added_to_free_pool; - - /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer - */ - grpc_closure *reclaimers[2]; - /* Trampoline closures to finish reclamation and re-enter the quota combiner - lock */ - grpc_closure post_reclaimer_closure[2]; - - /* Closure to execute under the quota combiner to de-register and shutdown the - resource user */ - grpc_closure destroy_closure; - /* User supplied closure to call once the user has finished shutting down AND - all outstanding allocations have been freed. Real type is grpc_closure*, - but it's stored as an atomic to avoid a mutex on some fast paths. */ - gpr_atm on_done_destroy_closure; - - /* Links in the various grpc_rulist lists */ - grpc_resource_user_link links[GRPC_RULIST_COUNT]; - - /* The name of this resource user, for debugging/tracing */ - char *name; -}; - -void grpc_resource_user_init(grpc_resource_user *resource_user, - grpc_resource_quota *resource_quota, - const char *name); +grpc_resource_user *grpc_resource_user_create( + grpc_resource_quota *resource_quota, const char *name); +void grpc_resource_user_ref(grpc_resource_user *resource_user); +void grpc_resource_user_unref(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user); void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user, - grpc_closure *on_done); -void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user); + grpc_resource_user *resource_user); /* Allocate from the resource user (and its quota). If optional_on_done is NULL, then allocate immediately. This may push the @@ -203,7 +127,7 @@ typedef struct grpc_resource_user_slice_allocator { /* Number of slices to allocate on the current request */ size_t count; /* Destination for slices to allocate on the current request */ - gpr_slice_buffer *dest; + grpc_slice_buffer *dest; /* Parent resource user */ grpc_resource_user *resource_user; } grpc_resource_user_slice_allocator; @@ -219,11 +143,11 @@ void grpc_resource_user_slice_allocator_init( void grpc_resource_user_alloc_slices( grpc_exec_ctx *exec_ctx, grpc_resource_user_slice_allocator *slice_allocator, size_t length, - size_t count, gpr_slice_buffer *dest); + size_t count, grpc_slice_buffer *dest); /* Allocate one slice of length \a size synchronously. */ -gpr_slice grpc_resource_user_slice_malloc(grpc_exec_ctx *exec_ctx, - grpc_resource_user *resource_user, - size_t size); +grpc_slice grpc_resource_user_slice_malloc(grpc_exec_ctx *exec_ctx, + grpc_resource_user *resource_user, + size_t size); #endif /* GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H */ diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c new file mode 100644 index 0000000000..8b1efb6bab --- /dev/null +++ b/src/core/lib/iomgr/socket_mutator.c @@ -0,0 +1,98 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/socket_mutator.h" + +#include <grpc/impl/codegen/grpc_types.h> +#include <grpc/support/sync.h> +#include <grpc/support/useful.h> + +void grpc_socket_mutator_init(grpc_socket_mutator *mutator, + const grpc_socket_mutator_vtable *vtable) { + mutator->vtable = vtable; + gpr_ref_init(&mutator->refcount, 1); +} + +grpc_socket_mutator *grpc_socket_mutator_ref(grpc_socket_mutator *mutator) { + gpr_ref(&mutator->refcount); + return mutator; +} + +bool grpc_socket_mutator_mutate_fd(grpc_socket_mutator *mutator, int fd) { + return mutator->vtable->mutate_fd(fd, mutator); +} + +int grpc_socket_mutator_compare(grpc_socket_mutator *a, + grpc_socket_mutator *b) { + int c = GPR_ICMP(a, b); + if (c != 0) { + grpc_socket_mutator *sma = a; + grpc_socket_mutator *smb = b; + c = GPR_ICMP(sma->vtable, smb->vtable); + if (c == 0) { + c = sma->vtable->compare(sma, smb); + } + } + return c; +} + +void grpc_socket_mutator_unref(grpc_socket_mutator *mutator) { + if (gpr_unref(&mutator->refcount)) { + mutator->vtable->destory(mutator); + } +} + +static void *socket_mutator_arg_copy(void *p) { + return grpc_socket_mutator_ref(p); +} + +static void socket_mutator_arg_destroy(void *p) { + grpc_socket_mutator_unref(p); +} + +static int socket_mutator_cmp(void *a, void *b) { + return grpc_socket_mutator_compare((grpc_socket_mutator *)a, + (grpc_socket_mutator *)b); +} + +static const grpc_arg_pointer_vtable socket_mutator_arg_vtable = { + socket_mutator_arg_copy, socket_mutator_arg_destroy, socket_mutator_cmp}; + +grpc_arg grpc_socket_mutator_to_arg(grpc_socket_mutator *mutator) { + grpc_arg arg; + arg.type = GRPC_ARG_POINTER; + arg.key = GRPC_ARG_SOCKET_MUTATOR; + arg.value.pointer.vtable = &socket_mutator_arg_vtable; + arg.value.pointer.p = mutator; + return arg; +} diff --git a/src/core/lib/iomgr/socket_mutator.h b/src/core/lib/iomgr/socket_mutator.h new file mode 100644 index 0000000000..2f5b6c248e --- /dev/null +++ b/src/core/lib/iomgr/socket_mutator.h @@ -0,0 +1,80 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_SOCKET_MUTATOR_H +#define GRPC_CORE_LIB_IOMGR_SOCKET_MUTATOR_H + +#include <grpc/impl/codegen/grpc_types.h> +#include <grpc/support/sync.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** The virtual table of grpc_socket_mutator */ +typedef struct { + /** Mutates the socket opitons of \a fd */ + bool (*mutate_fd)(int fd, grpc_socket_mutator *mutator); + /** Compare socket mutator \a a and \a b */ + int (*compare)(grpc_socket_mutator *a, grpc_socket_mutator *b); + /** Destroys the socket mutator instance */ + void (*destory)(grpc_socket_mutator *mutator); +} grpc_socket_mutator_vtable; + +/** The Socket Mutator interface allows changes on socket options */ +struct grpc_socket_mutator { + const grpc_socket_mutator_vtable *vtable; + gpr_refcount refcount; +}; + +/** called by concrete implementations to initialize the base struct */ +void grpc_socket_mutator_init(grpc_socket_mutator *mutator, + const grpc_socket_mutator_vtable *vtable); + +/** Wrap \a mutator as a grpc_arg */ +grpc_arg grpc_socket_mutator_to_arg(grpc_socket_mutator *mutator); + +/** Perform the file descriptor mutation operation of \a mutator on \a fd */ +bool grpc_socket_mutator_mutate_fd(grpc_socket_mutator *mutator, int fd); + +/** Compare if \a a and \a b are the same mutator or have same settings */ +int grpc_socket_mutator_compare(grpc_socket_mutator *a, grpc_socket_mutator *b); + +grpc_socket_mutator *grpc_socket_mutator_ref(grpc_socket_mutator *mutator); +void grpc_socket_mutator_unref(grpc_socket_mutator *mutator); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_CORE_LIB_IOMGR_SOCKET_MUTATOR_H */ diff --git a/src/core/lib/iomgr/socket_utils_common_posix.c b/src/core/lib/iomgr/socket_utils_common_posix.c index bc28bbe316..88e9ade253 100644 --- a/src/core/lib/iomgr/socket_utils_common_posix.c +++ b/src/core/lib/iomgr/socket_utils_common_posix.c @@ -209,6 +209,15 @@ grpc_error *grpc_set_socket_low_latency(int fd, int low_latency) { return GRPC_ERROR_NONE; } +/* set a socket using a grpc_socket_mutator */ +grpc_error *grpc_set_socket_with_mutator(int fd, grpc_socket_mutator *mutator) { + GPR_ASSERT(mutator); + if (!grpc_socket_mutator_mutate_fd(mutator, fd)) { + return GRPC_ERROR_CREATE("grpc_socket_mutator failed."); + } + return GRPC_ERROR_NONE; +} + static gpr_once g_probe_ipv6_once = GPR_ONCE_INIT; static int g_ipv6_loopback_available; diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h index 175fb2b717..e84d3781a1 100644 --- a/src/core/lib/iomgr/socket_utils_posix.h +++ b/src/core/lib/iomgr/socket_utils_posix.h @@ -39,7 +39,9 @@ #include <sys/socket.h> #include <unistd.h> +#include <grpc/impl/codegen/grpc_types.h> #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/socket_mutator.h" /* a wrapper for accept or accept4 */ int grpc_accept4(int sockfd, grpc_resolved_address *resolved_addr, int nonblock, @@ -88,6 +90,9 @@ grpc_error *grpc_set_socket_sndbuf(int fd, int buffer_size_bytes); /* Tries to set the socket's receive buffer to given size. */ grpc_error *grpc_set_socket_rcvbuf(int fd, int buffer_size_bytes); +/* Tries to set the socket using a grpc_socket_mutator */ +grpc_error *grpc_set_socket_with_mutator(int fd, grpc_socket_mutator *mutator); + /* An enum to keep track of IPv4/IPv6 socket modes. Currently, this information is only used when a socket is first created, but diff --git a/src/core/lib/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h index 18e6e60ebc..0485661316 100644 --- a/src/core/lib/iomgr/tcp_client.h +++ b/src/core/lib/iomgr/tcp_client.h @@ -34,6 +34,7 @@ #ifndef GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H #define GRPC_CORE_LIB_IOMGR_TCP_CLIENT_H +#include <grpc/impl/codegen/grpc_types.h> #include <grpc/support/time.h> #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/pollset_set.h" diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index bc08c94ee0..13347735df 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -51,6 +51,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_posix.h" #include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/iomgr/socket_mutator.h" #include "src/core/lib/iomgr/socket_utils_posix.h" #include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/iomgr/timer.h" @@ -73,7 +74,8 @@ typedef struct { grpc_channel_args *channel_args; } async_connect; -static grpc_error *prepare_socket(const grpc_resolved_address *addr, int fd) { +static grpc_error *prepare_socket(const grpc_resolved_address *addr, int fd, + const grpc_channel_args *channel_args) { grpc_error *err = GRPC_ERROR_NONE; GPR_ASSERT(fd >= 0); @@ -88,6 +90,16 @@ static grpc_error *prepare_socket(const grpc_resolved_address *addr, int fd) { } err = grpc_set_socket_no_sigpipe_if_possible(fd); if (err != GRPC_ERROR_NONE) goto error; + if (channel_args) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_SOCKET_MUTATOR)) { + GPR_ASSERT(channel_args->args[i].type == GRPC_ARG_POINTER); + grpc_socket_mutator *mutator = channel_args->args[i].value.pointer.p; + err = grpc_set_socket_with_mutator(fd, mutator); + if (err != GRPC_ERROR_NONE) goto error; + } + } + } goto done; error: @@ -287,7 +299,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, GPR_ASSERT(grpc_sockaddr_is_v4mapped(addr, &addr4_copy)); addr = &addr4_copy; } - if ((error = prepare_socket(addr, fd)) != GRPC_ERROR_NONE) { + if ((error = prepare_socket(addr, fd, channel_args)) != GRPC_ERROR_NONE) { grpc_exec_ctx_sched(exec_ctx, closure, error, NULL); return; } diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index 30f7c66f15..4d1e809872 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -37,10 +37,10 @@ #include "src/core/lib/iomgr/sockaddr_windows.h" +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/log_windows.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/useful.h> #include "src/core/lib/channel/channel_args.h" diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 880af93ee1..12a4797e6f 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -46,9 +46,9 @@ #include <sys/types.h> #include <unistd.h> +#include <grpc/slice.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice.h> #include <grpc/support/string_util.h> #include <grpc/support/sync.h> #include <grpc/support/time.h> @@ -56,6 +56,7 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #ifdef GRPC_HAVE_MSG_NOSIGNAL @@ -83,10 +84,10 @@ typedef struct { gpr_atm shutdown_count; /* garbage after the last read */ - gpr_slice_buffer last_read_buffer; + grpc_slice_buffer last_read_buffer; - gpr_slice_buffer *incoming_buffer; - gpr_slice_buffer *outgoing_buffer; + grpc_slice_buffer *incoming_buffer; + grpc_slice_buffer *outgoing_buffer; /** slice within outgoing_buffer to write next */ size_t outgoing_slice_idx; /** byte within outgoing_buffer->slices[outgoing_slice_idx] to write next */ @@ -102,7 +103,7 @@ typedef struct { char *peer_string; - grpc_resource_user resource_user; + grpc_resource_user *resource_user; grpc_resource_user_slice_allocator slice_allocator; } grpc_tcp; @@ -110,28 +111,18 @@ static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, grpc_error *error); static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, grpc_error *error); -static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, - grpc_error *error); - -static void tcp_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx, - grpc_tcp *tcp) { - if (gpr_atm_full_fetch_add(&tcp->shutdown_count, 1) == 0) { - grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user, - grpc_closure_create(tcp_unref_closure, tcp)); - } -} static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - tcp_maybe_shutdown_resource_user(exec_ctx, tcp); grpc_fd_shutdown(exec_ctx, tcp->em_fd); + grpc_resource_user_shutdown(exec_ctx, tcp->resource_user); } static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd, "tcp_unref_orphan"); - gpr_slice_buffer_destroy(&tcp->last_read_buffer); - grpc_resource_user_destroy(exec_ctx, &tcp->resource_user); + grpc_slice_buffer_destroy(&tcp->last_read_buffer); + grpc_resource_user_unref(exec_ctx, tcp->resource_user); gpr_free(tcp->peer_string); gpr_free(tcp); } @@ -168,16 +159,10 @@ static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } #endif -static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - TCP_UNREF(exec_ctx, arg, "resource_user"); -} - static void tcp_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; - tcp_maybe_shutdown_resource_user(exec_ctx, tcp); - gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); + grpc_slice_buffer_reset_and_unref(&tcp->last_read_buffer); TCP_UNREF(exec_ctx, tcp, "destroy"); } @@ -191,8 +176,8 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, gpr_log(GPR_DEBUG, "read: error=%s", str); grpc_error_free_string(str); for (i = 0; i < tcp->incoming_buffer->count; i++) { - char *dump = gpr_dump_slice(tcp->incoming_buffer->slices[i], - GPR_DUMP_HEX | GPR_DUMP_ASCII); + char *dump = grpc_dump_slice(tcp->incoming_buffer->slices[i], + GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "READ %p (peer=%s): %s", tcp, tcp->peer_string, dump); gpr_free(dump); } @@ -216,8 +201,8 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { GPR_TIMER_BEGIN("tcp_continue_read", 0); for (i = 0; i < tcp->incoming_buffer->count; i++) { - iov[i].iov_base = GPR_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); - iov[i].iov_len = GPR_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); + iov[i].iov_base = GRPC_SLICE_START_PTR(tcp->incoming_buffer->slices[i]); + iov[i].iov_len = GRPC_SLICE_LENGTH(tcp->incoming_buffer->slices[i]); } msg.msg_name = NULL; @@ -244,19 +229,19 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { /* We've consumed the edge, request a new one */ grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { - gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer); call_read_cb(exec_ctx, tcp, GRPC_OS_ERROR(errno, "recvmsg")); TCP_UNREF(exec_ctx, tcp, "read"); } } else if (read_bytes == 0) { /* 0 read size ==> end of stream */ - gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); + grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer); call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("Socket closed")); TCP_UNREF(exec_ctx, tcp, "read"); } else { GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); if ((size_t)read_bytes < tcp->incoming_buffer->length) { - gpr_slice_buffer_trim_end( + grpc_slice_buffer_trim_end( tcp->incoming_buffer, tcp->incoming_buffer->length - (size_t)read_bytes, &tcp->last_read_buffer); @@ -275,8 +260,8 @@ static void tcp_read_allocation_done(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { grpc_tcp *tcp = tcpp; if (error != GRPC_ERROR_NONE) { - gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); - gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); + grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer); + grpc_slice_buffer_reset_and_unref(&tcp->last_read_buffer); call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error)); TCP_UNREF(exec_ctx, tcp, "read"); } else { @@ -301,8 +286,8 @@ static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, GPR_ASSERT(!tcp->finished_edge); if (error != GRPC_ERROR_NONE) { - gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer); - gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); + grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer); + grpc_slice_buffer_reset_and_unref(&tcp->last_read_buffer); call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error)); TCP_UNREF(exec_ctx, tcp, "read"); } else { @@ -311,13 +296,13 @@ static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, } static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *incoming_buffer, grpc_closure *cb) { + grpc_slice_buffer *incoming_buffer, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; GPR_ASSERT(tcp->read_cb == NULL); tcp->read_cb = cb; tcp->incoming_buffer = incoming_buffer; - gpr_slice_buffer_reset_and_unref(incoming_buffer); - gpr_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer); + grpc_slice_buffer_reset_and_unref(incoming_buffer); + grpc_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer); TCP_REF(tcp, "read"); if (tcp->finished_edge) { tcp->finished_edge = false; @@ -347,11 +332,11 @@ static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) { iov_size != MAX_WRITE_IOVEC; iov_size++) { iov[iov_size].iov_base = - GPR_SLICE_START_PTR( + GRPC_SLICE_START_PTR( tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) + tcp->outgoing_byte_idx; iov[iov_size].iov_len = - GPR_SLICE_LENGTH( + GRPC_SLICE_LENGTH( tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) - tcp->outgoing_byte_idx; sending_length += iov[iov_size].iov_len; @@ -392,7 +377,7 @@ static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) { size_t slice_length; tcp->outgoing_slice_idx--; - slice_length = GPR_SLICE_LENGTH( + slice_length = GRPC_SLICE_LENGTH( tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]); if (slice_length > trailing) { tcp->outgoing_byte_idx = slice_length - trailing; @@ -442,7 +427,7 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, } static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *buf, grpc_closure *cb) { + grpc_slice_buffer *buf, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; grpc_error *error = GRPC_ERROR_NONE; @@ -451,7 +436,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, for (i = 0; i < buf->count; i++) { char *data = - gpr_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); + grpc_dump_slice(buf->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "WRITE %p (peer=%s): %s", tcp, tcp->peer_string, data); gpr_free(data); } @@ -508,6 +493,11 @@ static char *tcp_get_peer(grpc_endpoint *ep) { return gpr_strdup(tcp->peer_string); } +static int tcp_get_fd(grpc_endpoint *ep) { + grpc_tcp *tcp = (grpc_tcp *)ep; + return tcp->fd; +} + static grpc_workqueue *tcp_get_workqueue(grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; return grpc_fd_get_workqueue(tcp->em_fd); @@ -515,7 +505,7 @@ static grpc_workqueue *tcp_get_workqueue(grpc_endpoint *ep) { static grpc_resource_user *tcp_get_resource_user(grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - return &tcp->resource_user; + return tcp->resource_user; } static const grpc_endpoint_vtable vtable = {tcp_read, @@ -526,7 +516,8 @@ static const grpc_endpoint_vtable vtable = {tcp_read, tcp_shutdown, tcp_destroy, tcp_get_resource_user, - tcp_get_peer}; + tcp_get_peer, + tcp_get_fd}; grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, grpc_resource_quota *resource_quota, @@ -543,20 +534,18 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, tcp->slice_size = slice_size; tcp->iov_size = 1; tcp->finished_edge = true; - /* paired with unref in grpc_tcp_destroy, and with the shutdown for our - * resource_user */ - gpr_ref_init(&tcp->refcount, 2); + /* paired with unref in grpc_tcp_destroy */ + gpr_ref_init(&tcp->refcount, 1); gpr_atm_no_barrier_store(&tcp->shutdown_count, 0); tcp->em_fd = em_fd; tcp->read_closure.cb = tcp_handle_read; tcp->read_closure.cb_arg = tcp; tcp->write_closure.cb = tcp_handle_write; tcp->write_closure.cb_arg = tcp; - gpr_slice_buffer_init(&tcp->last_read_buffer); - grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string); - grpc_resource_user_slice_allocator_init(&tcp->slice_allocator, - &tcp->resource_user, - tcp_read_allocation_done, tcp); + grpc_slice_buffer_init(&tcp->last_read_buffer); + tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string); + grpc_resource_user_slice_allocator_init( + &tcp->slice_allocator, tcp->resource_user, tcp_read_allocation_done, tcp); /* Tell network status tracker about new endpoint */ grpc_network_status_register_endpoint(&tcp->base); @@ -576,8 +565,7 @@ void grpc_tcp_destroy_and_release_fd(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, GPR_ASSERT(ep->vtable == &vtable); tcp->release_fd = fd; tcp->release_fd_cb = done; - tcp_maybe_shutdown_resource_user(exec_ctx, tcp); - gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer); + grpc_slice_buffer_reset_and_unref(&tcp->last_read_buffer); TCP_UNREF(exec_ctx, tcp, "destroy"); } diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index 8e74c9e863..6e2ad1dbe9 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -38,14 +38,17 @@ #include <limits.h> #include <string.h> +#include <grpc/slice_buffer.h> + #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/string_util.h> #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/network_status_tracker.h" +#include "src/core/lib/iomgr/resource_quota.h" #include "src/core/lib/iomgr/tcp_uv.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" int grpc_tcp_trace = 0; @@ -62,15 +65,14 @@ typedef struct { grpc_closure *read_cb; grpc_closure *write_cb; - gpr_slice read_slice; - gpr_slice_buffer *read_slices; - gpr_slice_buffer *write_slices; + grpc_slice read_slice; + grpc_slice_buffer *read_slices; + grpc_slice_buffer *write_slices; uv_buf_t *write_buffers; - grpc_resource_user resource_user; + grpc_resource_user *resource_user; bool shutting_down; - bool resource_user_shutting_down; char *peer_string; grpc_pollset *pollset; @@ -78,23 +80,23 @@ typedef struct { static void uv_close_callback(uv_handle_t *handle) { gpr_free(handle); } -static void tcp_free(grpc_tcp *tcp) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_resource_user_destroy(&exec_ctx, &tcp->resource_user); +static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { + grpc_resource_user_unref(exec_ctx, tcp->resource_user); gpr_free(tcp); - grpc_exec_ctx_finish(&exec_ctx); } /*#define GRPC_TCP_REFCOUNT_DEBUG*/ #ifdef GRPC_TCP_REFCOUNT_DEBUG -#define TCP_UNREF(tcp, reason) tcp_unref((tcp), (reason), __FILE__, __LINE__) -#define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__) -static void tcp_unref(grpc_tcp *tcp, const char *reason, const char *file, - int line) { +#define TCP_UNREF(exec_ctx, tcp, reason) \ + tcp_unref((exec_ctx), (tcp), (reason), __FILE__, __LINE__) +#define TCP_REF(tcp, reason) \ + tcp_ref((exec_ctx), (tcp), (reason), __FILE__, __LINE__) +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, + const char *reason, const char *file, int line) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP unref %p : %s %d -> %d", tcp, reason, tcp->refcount.count, tcp->refcount.count - 1); if (gpr_unref(&tcp->refcount)) { - tcp_free(tcp); + tcp_free(exec_ctx, tcp); } } @@ -105,11 +107,11 @@ static void tcp_ref(grpc_tcp *tcp, const char *reason, const char *file, gpr_ref(&tcp->refcount); } #else -#define TCP_UNREF(tcp, reason) tcp_unref((tcp)) +#define TCP_UNREF(exec_ctx, tcp, reason) tcp_unref((exec_ctx), (tcp)) #define TCP_REF(tcp, reason) tcp_ref((tcp)) -static void tcp_unref(grpc_tcp *tcp) { +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { if (gpr_unref(&tcp->refcount)) { - tcp_free(tcp); + tcp_free(exec_ctx, tcp); } } @@ -122,15 +124,15 @@ static void alloc_uv_buf(uv_handle_t *handle, size_t suggested_size, grpc_tcp *tcp = handle->data; (void)suggested_size; tcp->read_slice = grpc_resource_user_slice_malloc( - &exec_ctx, &tcp->resource_user, GRPC_TCP_DEFAULT_READ_SLICE_SIZE); - buf->base = (char *)GPR_SLICE_START_PTR(tcp->read_slice); - buf->len = GPR_SLICE_LENGTH(tcp->read_slice); + &exec_ctx, tcp->resource_user, GRPC_TCP_DEFAULT_READ_SLICE_SIZE); + buf->base = (char *)GRPC_SLICE_START_PTR(tcp->read_slice); + buf->len = GRPC_SLICE_LENGTH(tcp->read_slice); grpc_exec_ctx_finish(&exec_ctx); } static void read_callback(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { - gpr_slice sub; + grpc_slice sub; grpc_error *error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_tcp *tcp = stream->data; @@ -139,7 +141,7 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, // Nothing happened. Wait for the next callback return; } - TCP_UNREF(tcp, "read"); + TCP_UNREF(&exec_ctx, tcp, "read"); tcp->read_cb = NULL; // TODO(murgatroid99): figure out what the return value here means uv_read_stop(stream); @@ -147,8 +149,8 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, error = GRPC_ERROR_CREATE("EOF"); } else if (nread > 0) { // Successful read - sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread); - gpr_slice_buffer_add(tcp->read_slices, sub); + sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread); + grpc_slice_buffer_add(tcp->read_slices, sub); error = GRPC_ERROR_NONE; if (grpc_tcp_trace) { size_t i; @@ -156,8 +158,8 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, gpr_log(GPR_DEBUG, "read: error=%s", str); grpc_error_free_string(str); for (i = 0; i < tcp->read_slices->count; i++) { - char *dump = gpr_dump_slice(tcp->read_slices->slices[i], - GPR_DUMP_HEX | GPR_DUMP_ASCII); + char *dump = grpc_dump_slice(tcp->read_slices->slices[i], + GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "READ %p (peer=%s): %s", tcp, tcp->peer_string, dump); gpr_free(dump); @@ -172,14 +174,14 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, } static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *read_slices, grpc_closure *cb) { + grpc_slice_buffer *read_slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; int status; grpc_error *error = GRPC_ERROR_NONE; GPR_ASSERT(tcp->read_cb == NULL); tcp->read_cb = cb; tcp->read_slices = read_slices; - gpr_slice_buffer_reset_and_unref(read_slices); + grpc_slice_buffer_reset_and_unref(read_slices); TCP_REF(tcp, "read"); // TODO(murgatroid99): figure out what the return value here means status = @@ -202,7 +204,7 @@ static void write_callback(uv_write_t *req, int status) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_closure *cb = tcp->write_cb; tcp->write_cb = NULL; - TCP_UNREF(tcp, "write"); + TCP_UNREF(&exec_ctx, tcp, "write"); if (status == 0) { error = GRPC_ERROR_NONE; } else { @@ -213,28 +215,28 @@ static void write_callback(uv_write_t *req, int status) { gpr_log(GPR_DEBUG, "write complete on %p: error=%s", tcp, str); } gpr_free(tcp->write_buffers); - grpc_resource_user_free(&exec_ctx, &tcp->resource_user, + grpc_resource_user_free(&exec_ctx, tcp->resource_user, sizeof(uv_buf_t) * tcp->write_slices->count); grpc_exec_ctx_sched(&exec_ctx, cb, error, NULL); grpc_exec_ctx_finish(&exec_ctx); } static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *write_slices, + grpc_slice_buffer *write_slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; uv_buf_t *buffers; unsigned int buffer_count; unsigned int i; - gpr_slice *slice; + grpc_slice *slice; uv_write_t *write_req; if (grpc_tcp_trace) { size_t j; for (j = 0; j < write_slices->count; j++) { - char *data = gpr_dump_slice(write_slices->slices[j], - GPR_DUMP_HEX | GPR_DUMP_ASCII); + char *data = grpc_dump_slice(write_slices->slices[j], + GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "WRITE %p (peer=%s): %s", tcp, tcp->peer_string, data); gpr_free(data); } @@ -259,12 +261,12 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->write_cb = cb; buffer_count = (unsigned int)tcp->write_slices->count; buffers = gpr_malloc(sizeof(uv_buf_t) * buffer_count); - grpc_resource_user_alloc(exec_ctx, &tcp->resource_user, + grpc_resource_user_alloc(exec_ctx, tcp->resource_user, sizeof(uv_buf_t) * buffer_count, NULL); for (i = 0; i < buffer_count; i++) { slice = &tcp->write_slices->slices[i]; - buffers[i].base = (char *)GPR_SLICE_START_PTR(*slice); - buffers[i].len = GPR_SLICE_LENGTH(*slice); + buffers[i].base = (char *)GRPC_SLICE_START_PTR(*slice); + buffers[i].len = GRPC_SLICE_LENGTH(*slice); } tcp->write_buffers = buffers; write_req = &tcp->write_req; @@ -295,22 +297,6 @@ static void uv_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, static void shutdown_callback(uv_shutdown_t *req, int status) {} -static void resource_user_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - TCP_UNREF(arg, "resource_user"); -} - -static void uv_resource_user_maybe_shutdown(grpc_exec_ctx *exec_ctx, - grpc_tcp *tcp) { - if (!tcp->resource_user_shutting_down) { - tcp->resource_user_shutting_down = true; - TCP_REF(tcp, "resource_user"); - grpc_resource_user_shutdown( - exec_ctx, &tcp->resource_user, - grpc_closure_create(resource_user_shutdown_done, tcp)); - } -} - static void uv_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; if (!tcp->shutting_down) { @@ -324,8 +310,7 @@ static void uv_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; uv_close((uv_handle_t *)tcp->handle, uv_close_callback); - uv_resource_user_maybe_shutdown(exec_ctx, tcp); - TCP_UNREF(tcp, "destroy"); + TCP_UNREF(exec_ctx, tcp, "destroy"); } static char *uv_get_peer(grpc_endpoint *ep) { @@ -335,15 +320,18 @@ static char *uv_get_peer(grpc_endpoint *ep) { static grpc_resource_user *uv_get_resource_user(grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - return &tcp->resource_user; + return tcp->resource_user; } static grpc_workqueue *uv_get_workqueue(grpc_endpoint *ep) { return NULL; } +static int uv_get_fd(grpc_endpoint *ep) { return -1; } + static grpc_endpoint_vtable vtable = { uv_endpoint_read, uv_endpoint_write, uv_get_workqueue, uv_add_to_pollset, uv_add_to_pollset_set, uv_endpoint_shutdown, - uv_destroy, uv_get_resource_user, uv_get_peer}; + uv_destroy, uv_get_resource_user, uv_get_peer, + uv_get_fd}; grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, grpc_resource_quota *resource_quota, @@ -364,8 +352,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, gpr_ref_init(&tcp->refcount, 1); tcp->peer_string = gpr_strdup(peer_string); tcp->shutting_down = false; - tcp->resource_user_shutting_down = false; - grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string); + tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string); /* Tell network status tracking code about the new endpoint */ grpc_network_status_register_endpoint(&tcp->base); diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index 46f0491d10..62afbcef51 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -40,10 +40,10 @@ #include "src/core/lib/iomgr/network_status_tracker.h" #include "src/core/lib/iomgr/sockaddr_windows.h" +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/log_windows.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -105,50 +105,39 @@ typedef struct grpc_tcp { grpc_closure *read_cb; grpc_closure *write_cb; - gpr_slice read_slice; - gpr_slice_buffer *write_slices; - gpr_slice_buffer *read_slices; + grpc_slice read_slice; + grpc_slice_buffer *write_slices; + grpc_slice_buffer *read_slices; - grpc_resource_user resource_user; + grpc_resource_user *resource_user; /* The IO Completion Port runs from another thread. We need some mechanism to protect ourselves when requesting a shutdown. */ gpr_mu mu; int shutting_down; - gpr_atm resource_user_shutdown_count; - char *peer_string; } grpc_tcp; -static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, - grpc_error *error); - -static void win_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx, - grpc_tcp *tcp) { - if (gpr_atm_full_fetch_add(&tcp->resource_user_shutdown_count, 1) == 0) { - grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user, - grpc_closure_create(win_unref_closure, tcp)); - } -} - -static void tcp_free(grpc_tcp *tcp) { +static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { grpc_winsocket_destroy(tcp->socket); gpr_mu_destroy(&tcp->mu); gpr_free(tcp->peer_string); + grpc_resource_user_unref(exec_ctx, tcp->resource_user); gpr_free(tcp); } /*#define GRPC_TCP_REFCOUNT_DEBUG*/ #ifdef GRPC_TCP_REFCOUNT_DEBUG -#define TCP_UNREF(tcp, reason) tcp_unref((tcp), (reason), __FILE__, __LINE__) +#define TCP_UNREF(exec_ctx, tcp, reason) \ + tcp_unref((exec_ctx), (tcp), (reason), __FILE__, __LINE__) #define TCP_REF(tcp, reason) tcp_ref((tcp), (reason), __FILE__, __LINE__) -static void tcp_unref(grpc_tcp *tcp, const char *reason, const char *file, - int line) { +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, + const char *reason, const char *file, int line) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "TCP unref %p : %s %d -> %d", tcp, reason, tcp->refcount.count, tcp->refcount.count - 1); if (gpr_unref(&tcp->refcount)) { - tcp_free(tcp); + tcp_free(exec_ctx, tcp); } } @@ -159,28 +148,23 @@ static void tcp_ref(grpc_tcp *tcp, const char *reason, const char *file, gpr_ref(&tcp->refcount); } #else -#define TCP_UNREF(tcp, reason) tcp_unref((tcp)) +#define TCP_UNREF(exec_ctx, tcp, reason) tcp_unref((exec_ctx), (tcp)) #define TCP_REF(tcp, reason) tcp_ref((tcp)) -static void tcp_unref(grpc_tcp *tcp) { +static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { if (gpr_unref(&tcp->refcount)) { - tcp_free(tcp); + tcp_free(exec_ctx, tcp); } } static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); } #endif -static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - TCP_UNREF(arg, "resource_user"); -} - /* Asynchronous callback from the IOCP, or the background thread. */ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { grpc_tcp *tcp = tcpp; grpc_closure *cb = tcp->read_cb; grpc_winsocket *socket = tcp->socket; - gpr_slice sub; + grpc_slice sub; grpc_winsocket_callback_info *info = &socket->read_info; GRPC_ERROR_REF(error); @@ -190,25 +174,25 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { char *utf8_message = gpr_format_message(info->wsa_error); error = GRPC_ERROR_CREATE(utf8_message); gpr_free(utf8_message); - gpr_slice_unref(tcp->read_slice); + grpc_slice_unref(tcp->read_slice); } else { if (info->bytes_transfered != 0 && !tcp->shutting_down) { - sub = gpr_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered); - gpr_slice_buffer_add(tcp->read_slices, sub); + sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, info->bytes_transfered); + grpc_slice_buffer_add(tcp->read_slices, sub); } else { - gpr_slice_unref(tcp->read_slice); + grpc_slice_unref(tcp->read_slice); error = GRPC_ERROR_CREATE("End of TCP stream"); } } } tcp->read_cb = NULL; - TCP_UNREF(tcp, "read"); + TCP_UNREF(exec_ctx, tcp, "read"); grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *read_slices, grpc_closure *cb) { + grpc_slice_buffer *read_slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; grpc_winsocket *handle = tcp->socket; grpc_winsocket_callback_info *info = &handle->read_info; @@ -225,13 +209,13 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->read_cb = cb; tcp->read_slices = read_slices; - gpr_slice_buffer_reset_and_unref(read_slices); + grpc_slice_buffer_reset_and_unref(read_slices); - tcp->read_slice = gpr_slice_malloc(8192); + tcp->read_slice = grpc_slice_malloc(8192); - buffer.len = (ULONG)GPR_SLICE_LENGTH( + buffer.len = (ULONG)GRPC_SLICE_LENGTH( tcp->read_slice); // we know slice size fits in 32bit. - buffer.buf = (char *)GPR_SLICE_START_PTR(tcp->read_slice); + buffer.buf = (char *)GRPC_SLICE_START_PTR(tcp->read_slice); TCP_REF(tcp, "read"); @@ -287,13 +271,13 @@ static void on_write(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) { } } - TCP_UNREF(tcp, "write"); + TCP_UNREF(exec_ctx, tcp, "write"); grpc_exec_ctx_sched(exec_ctx, cb, error, NULL); } /* Initiates a write. */ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, - gpr_slice_buffer *slices, grpc_closure *cb) { + grpc_slice_buffer *slices, grpc_closure *cb) { grpc_tcp *tcp = (grpc_tcp *)ep; grpc_winsocket *socket = tcp->socket; grpc_winsocket_callback_info *info = &socket->write_info; @@ -320,10 +304,10 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, } for (i = 0; i < tcp->write_slices->count; i++) { - len = GPR_SLICE_LENGTH(tcp->write_slices->slices[i]); + len = GRPC_SLICE_LENGTH(tcp->write_slices->slices[i]); GPR_ASSERT(len <= ULONG_MAX); buffers[i].len = (ULONG)len; - buffers[i].buf = (char *)GPR_SLICE_START_PTR(tcp->write_slices->slices[i]); + buffers[i].buf = (char *)GRPC_SLICE_START_PTR(tcp->write_slices->slices[i]); } /* First, let's try a synchronous, non-blocking write. */ @@ -355,7 +339,7 @@ static void win_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, if (status != 0) { int wsa_error = WSAGetLastError(); if (wsa_error != WSA_IO_PENDING) { - TCP_UNREF(tcp, "write"); + TCP_UNREF(exec_ctx, tcp, "write"); grpc_exec_ctx_sched(exec_ctx, cb, GRPC_WSA_ERROR(wsa_error, "WSASend"), NULL); return; @@ -396,15 +380,14 @@ static void win_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { callback. See the comments in on_read and on_write. */ tcp->shutting_down = 1; grpc_winsocket_shutdown(tcp->socket); - win_maybe_shutdown_resource_user(exec_ctx, tcp); gpr_mu_unlock(&tcp->mu); + grpc_resource_user_shutdown(exec_ctx, tcp->resource_user); } static void win_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp *tcp = (grpc_tcp *)ep; - win_maybe_shutdown_resource_user(exec_ctx, tcp); - TCP_UNREF(tcp, "destroy"); + TCP_UNREF(exec_ctx, tcp, "destroy"); } static char *win_get_peer(grpc_endpoint *ep) { @@ -416,9 +399,11 @@ static grpc_workqueue *win_get_workqueue(grpc_endpoint *ep) { return NULL; } static grpc_resource_user *win_get_resource_user(grpc_endpoint *ep) { grpc_tcp *tcp = (grpc_tcp *)ep; - return &tcp->resource_user; + return tcp->resource_user; } +static int win_get_fd(grpc_endpoint *ep) { return -1; } + static grpc_endpoint_vtable vtable = {win_read, win_write, win_get_workqueue, @@ -427,7 +412,8 @@ static grpc_endpoint_vtable vtable = {win_read, win_shutdown, win_destroy, win_get_resource_user, - win_get_peer}; + win_get_peer, + win_get_fd}; grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, grpc_resource_quota *resource_quota, @@ -441,7 +427,7 @@ grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, grpc_closure_init(&tcp->on_read, on_read, tcp); grpc_closure_init(&tcp->on_write, on_write, tcp); tcp->peer_string = gpr_strdup(peer_string); - grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string); + tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string); /* Tell network status tracking code about the new endpoint */ grpc_network_status_register_endpoint(&tcp->base); diff --git a/src/core/lib/iomgr/wakeup_fd_pipe.c b/src/core/lib/iomgr/wakeup_fd_pipe.c index 183f0eb930..e186d63683 100644 --- a/src/core/lib/iomgr/wakeup_fd_pipe.c +++ b/src/core/lib/iomgr/wakeup_fd_pipe.c @@ -95,6 +95,8 @@ static void pipe_destroy(grpc_wakeup_fd* fd_info) { static int pipe_check_availability(void) { grpc_wakeup_fd fd; + fd.read_fd = fd.write_fd = -1; + if (pipe_init(&fd) == GRPC_ERROR_NONE) { pipe_destroy(&fd); return 1; diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 6fb5b5b15a..85b3bc5350 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -141,8 +141,8 @@ grpc_channel_credentials_duplicate_without_call_credentials( /* --- grpc_credentials_md. --- */ typedef struct { - gpr_slice key; - gpr_slice value; + grpc_slice key; + grpc_slice value; } grpc_credentials_md; typedef struct { @@ -157,7 +157,7 @@ grpc_credentials_md_store *grpc_credentials_md_store_create( /* Will ref key and value. */ void grpc_credentials_md_store_add(grpc_credentials_md_store *store, - gpr_slice key, gpr_slice value); + grpc_slice key, grpc_slice value); void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store, const char *key, const char *value); grpc_credentials_md_store *grpc_credentials_md_store_ref( diff --git a/src/core/lib/security/credentials/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c index 6a352aab3a..e6cb567734 100644 --- a/src/core/lib/security/credentials/credentials_metadata.c +++ b/src/core/lib/security/credentials/credentials_metadata.c @@ -59,11 +59,11 @@ grpc_credentials_md_store *grpc_credentials_md_store_create( } void grpc_credentials_md_store_add(grpc_credentials_md_store *store, - gpr_slice key, gpr_slice value) { + grpc_slice key, grpc_slice value) { if (store == NULL) return; store_ensure_capacity(store); - store->entries[store->num_entries].key = gpr_slice_ref(key); - store->entries[store->num_entries].value = gpr_slice_ref(value); + store->entries[store->num_entries].key = grpc_slice_ref(key); + store->entries[store->num_entries].value = grpc_slice_ref(value); store->num_entries++; } @@ -72,9 +72,9 @@ void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store, const char *value) { if (store == NULL) return; store_ensure_capacity(store); - store->entries[store->num_entries].key = gpr_slice_from_copied_string(key); + store->entries[store->num_entries].key = grpc_slice_from_copied_string(key); store->entries[store->num_entries].value = - gpr_slice_from_copied_string(value); + grpc_slice_from_copied_string(value); store->num_entries++; } @@ -91,8 +91,8 @@ void grpc_credentials_md_store_unref(grpc_credentials_md_store *store) { if (store->entries != NULL) { size_t i; for (i = 0; i < store->num_entries; i++) { - gpr_slice_unref(store->entries[i].key); - gpr_slice_unref(store->entries[i].value); + grpc_slice_unref(store->entries[i].key); + grpc_slice_unref(store->entries[i].value); } gpr_free(store->entries); } diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index cb5ba554b0..afe0e3d357 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -45,6 +45,7 @@ #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/security/credentials/jwt/jwt_credentials.h" #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" @@ -174,7 +175,7 @@ static grpc_error *create_default_creds_from_path( grpc_auth_json_key key; grpc_auth_refresh_token token; grpc_call_credentials *result = NULL; - gpr_slice creds_data = gpr_empty_slice(); + grpc_slice creds_data = gpr_empty_slice(); grpc_error *error = GRPC_ERROR_NONE; if (creds_path == NULL) { error = GRPC_ERROR_CREATE("creds_path unset"); @@ -185,9 +186,9 @@ static grpc_error *create_default_creds_from_path( goto end; } json = grpc_json_parse_string_with_len( - (char *)GPR_SLICE_START_PTR(creds_data), GPR_SLICE_LENGTH(creds_data)); + (char *)GRPC_SLICE_START_PTR(creds_data), GRPC_SLICE_LENGTH(creds_data)); if (json == NULL) { - char *dump = gpr_dump_slice(creds_data, GPR_DUMP_HEX | GPR_DUMP_ASCII); + char *dump = grpc_dump_slice(creds_data, GPR_DUMP_HEX | GPR_DUMP_ASCII); error = grpc_error_set_str(GRPC_ERROR_CREATE("Failed to parse JSON"), GRPC_ERROR_STR_RAW_BYTES, dump); gpr_free(dump); @@ -224,7 +225,7 @@ static grpc_error *create_default_creds_from_path( end: GPR_ASSERT((result == NULL) + (error == GRPC_ERROR_NONE) == 1); if (creds_path != NULL) gpr_free(creds_path); - gpr_slice_unref(creds_data); + grpc_slice_unref(creds_data); if (json != NULL) grpc_json_destroy(json); *creds = result; return error; diff --git a/src/core/lib/security/credentials/jwt/json_token.h b/src/core/lib/security/credentials/jwt/json_token.h index 07fc5bf0e0..c13eb55803 100644 --- a/src/core/lib/security/credentials/jwt/json_token.h +++ b/src/core/lib/security/credentials/jwt/json_token.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H #define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include <openssl/rsa.h> #include "src/core/lib/json/json.h" diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c index f87ba0ce8d..3daf0f4ef7 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.c +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c @@ -144,17 +144,44 @@ grpc_service_account_jwt_access_credentials_create_from_auth_json_key( return &c->base; } +static char *redact_private_key(const char *json_key) { + char *json_copy = gpr_strdup(json_key); + grpc_json *json = grpc_json_parse_string(json_copy); + if (!json) { + gpr_free(json_copy); + return gpr_strdup("<Json failed to parse.>"); + } + const char *redacted = "<redacted>"; + grpc_json *current = json->child; + while (current) { + if (current->type == GRPC_JSON_STRING && + strcmp(current->key, "private_key") == 0) { + current->value = (char *)redacted; + break; + } + current = current->next; + } + char *clean_json = grpc_json_dump_to_string(json, 2); + gpr_free(json_copy); + grpc_json_destroy(json); + return clean_json; +} + grpc_call_credentials *grpc_service_account_jwt_access_credentials_create( const char *json_key, gpr_timespec token_lifetime, void *reserved) { - GRPC_API_TRACE( - "grpc_service_account_jwt_access_credentials_create(" - "json_key=%s, " - "token_lifetime=" - "gpr_timespec { tv_sec: %" PRId64 - ", tv_nsec: %d, clock_type: %d }, " - "reserved=%p)", - 5, (json_key, token_lifetime.tv_sec, token_lifetime.tv_nsec, - (int)token_lifetime.clock_type, reserved)); + if (grpc_api_trace) { + char *clean_json = redact_private_key(json_key); + gpr_log(GPR_INFO, + "grpc_service_account_jwt_access_credentials_create(" + "json_key=%s, " + "token_lifetime=" + "gpr_timespec { tv_sec: %" PRId64 + ", tv_nsec: %d, clock_type: %d }, " + "reserved=%p)", + clean_json, token_lifetime.tv_sec, token_lifetime.tv_nsec, + (int)token_lifetime.clock_type, reserved); + gpr_free(clean_json); + } GPR_ASSERT(reserved == NULL); return grpc_service_account_jwt_access_credentials_create_from_auth_json_key( grpc_auth_json_key_create_from_string(json_key), token_lifetime); diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index 43eb642515..42bd89dd0a 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -85,18 +85,18 @@ static const EVP_MD *evp_md_from_alg(const char *alg) { } static grpc_json *parse_json_part_from_jwt(const char *str, size_t len, - gpr_slice *buffer) { + grpc_slice *buffer) { grpc_json *json; *buffer = grpc_base64_decode_with_len(str, len, 1); - if (GPR_SLICE_IS_EMPTY(*buffer)) { + if (GRPC_SLICE_IS_EMPTY(*buffer)) { gpr_log(GPR_ERROR, "Invalid base64."); return NULL; } - json = grpc_json_parse_string_with_len((char *)GPR_SLICE_START_PTR(*buffer), - GPR_SLICE_LENGTH(*buffer)); + json = grpc_json_parse_string_with_len((char *)GRPC_SLICE_START_PTR(*buffer), + GRPC_SLICE_LENGTH(*buffer)); if (json == NULL) { - gpr_slice_unref(*buffer); + grpc_slice_unref(*buffer); gpr_log(GPR_ERROR, "JSON parsing error."); } return json; @@ -129,16 +129,16 @@ typedef struct { const char *kid; const char *typ; /* TODO(jboeuf): Add others as needed (jku, jwk, x5u, x5c and so on...). */ - gpr_slice buffer; + grpc_slice buffer; } jose_header; static void jose_header_destroy(jose_header *h) { - gpr_slice_unref(h->buffer); + grpc_slice_unref(h->buffer); gpr_free(h); } /* Takes ownership of json and buffer. */ -static jose_header *jose_header_from_json(grpc_json *json, gpr_slice buffer) { +static jose_header *jose_header_from_json(grpc_json *json, grpc_slice buffer) { grpc_json *cur; jose_header *h = gpr_malloc(sizeof(jose_header)); memset(h, 0, sizeof(jose_header)); @@ -190,12 +190,12 @@ struct grpc_jwt_claims { gpr_timespec nbf; grpc_json *json; - gpr_slice buffer; + grpc_slice buffer; }; void grpc_jwt_claims_destroy(grpc_jwt_claims *claims) { grpc_json_destroy(claims->json); - gpr_slice_unref(claims->buffer); + grpc_slice_unref(claims->buffer); gpr_free(claims); } @@ -240,7 +240,7 @@ gpr_timespec grpc_jwt_claims_not_before(const grpc_jwt_claims *claims) { } /* Takes ownership of json and buffer even in case of failure. */ -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer) { +grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, grpc_slice buffer) { grpc_json *cur; grpc_jwt_claims *claims = gpr_malloc(sizeof(grpc_jwt_claims)); memset(claims, 0, sizeof(grpc_jwt_claims)); @@ -333,8 +333,8 @@ typedef struct { jose_header *header; grpc_jwt_claims *claims; char *audience; - gpr_slice signature; - gpr_slice signed_data; + grpc_slice signature; + grpc_slice signed_data; void *user_data; grpc_jwt_verification_done_cb user_cb; grpc_http_response responses[HTTP_RESPONSE_COUNT]; @@ -343,7 +343,7 @@ typedef struct { /* Takes ownership of the header, claims and signature. */ static verifier_cb_ctx *verifier_cb_ctx_create( grpc_jwt_verifier *verifier, grpc_pollset *pollset, jose_header *header, - grpc_jwt_claims *claims, const char *audience, gpr_slice signature, + grpc_jwt_claims *claims, const char *audience, grpc_slice signature, const char *signed_jwt, size_t signed_jwt_len, void *user_data, grpc_jwt_verification_done_cb cb) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -355,7 +355,7 @@ static verifier_cb_ctx *verifier_cb_ctx_create( ctx->audience = gpr_strdup(audience); ctx->claims = claims; ctx->signature = signature; - ctx->signed_data = gpr_slice_from_copied_buffer(signed_jwt, signed_jwt_len); + ctx->signed_data = grpc_slice_from_copied_buffer(signed_jwt, signed_jwt_len); ctx->user_data = user_data; ctx->user_cb = cb; grpc_exec_ctx_finish(&exec_ctx); @@ -365,8 +365,8 @@ static verifier_cb_ctx *verifier_cb_ctx_create( void verifier_cb_ctx_destroy(verifier_cb_ctx *ctx) { if (ctx->audience != NULL) gpr_free(ctx->audience); if (ctx->claims != NULL) grpc_jwt_claims_destroy(ctx->claims); - gpr_slice_unref(ctx->signature); - gpr_slice_unref(ctx->signed_data); + grpc_slice_unref(ctx->signature); + grpc_slice_unref(ctx->signed_data); jose_header_destroy(ctx->header); for (size_t i = 0; i < HTTP_RESPONSE_COUNT; i++) { grpc_http_response_destroy(&ctx->responses[i]); @@ -449,17 +449,17 @@ end: static BIGNUM *bignum_from_base64(const char *b64) { BIGNUM *result = NULL; - gpr_slice bin; + grpc_slice bin; if (b64 == NULL) return NULL; bin = grpc_base64_decode(b64, 1); - if (GPR_SLICE_IS_EMPTY(bin)) { + if (GRPC_SLICE_IS_EMPTY(bin)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return NULL; } - result = BN_bin2bn(GPR_SLICE_START_PTR(bin), - TSI_SIZE_AS_SIZE(GPR_SLICE_LENGTH(bin)), NULL); - gpr_slice_unref(bin); + result = BN_bin2bn(GRPC_SLICE_START_PTR(bin), + TSI_SIZE_AS_SIZE(GRPC_SLICE_LENGTH(bin)), NULL); + grpc_slice_unref(bin); return result; } @@ -553,7 +553,7 @@ static EVP_PKEY *find_verification_key(const grpc_json *json, } static int verify_jwt_signature(EVP_PKEY *key, const char *alg, - gpr_slice signature, gpr_slice signed_data) { + grpc_slice signature, grpc_slice signed_data) { EVP_MD_CTX *md_ctx = EVP_MD_CTX_create(); const EVP_MD *md = evp_md_from_alg(alg); int result = 0; @@ -567,13 +567,13 @@ static int verify_jwt_signature(EVP_PKEY *key, const char *alg, gpr_log(GPR_ERROR, "EVP_DigestVerifyInit failed."); goto end; } - if (EVP_DigestVerifyUpdate(md_ctx, GPR_SLICE_START_PTR(signed_data), - GPR_SLICE_LENGTH(signed_data)) != 1) { + if (EVP_DigestVerifyUpdate(md_ctx, GRPC_SLICE_START_PTR(signed_data), + GRPC_SLICE_LENGTH(signed_data)) != 1) { gpr_log(GPR_ERROR, "EVP_DigestVerifyUpdate failed."); goto end; } - if (EVP_DigestVerifyFinal(md_ctx, GPR_SLICE_START_PTR(signature), - GPR_SLICE_LENGTH(signature)) != 1) { + if (EVP_DigestVerifyFinal(md_ctx, GRPC_SLICE_START_PTR(signature), + GRPC_SLICE_LENGTH(signature)) != 1) { gpr_log(GPR_ERROR, "JWT signature verification failed."); goto end; } @@ -799,9 +799,9 @@ void grpc_jwt_verifier_verify(grpc_exec_ctx *exec_ctx, grpc_json *json; jose_header *header = NULL; grpc_jwt_claims *claims = NULL; - gpr_slice header_buffer; - gpr_slice claims_buffer; - gpr_slice signature; + grpc_slice header_buffer; + grpc_slice claims_buffer; + grpc_slice signature; size_t signed_jwt_len; const char *cur = jwt; @@ -824,7 +824,7 @@ void grpc_jwt_verifier_verify(grpc_exec_ctx *exec_ctx, signed_jwt_len = (size_t)(dot - jwt); cur = dot + 1; signature = grpc_base64_decode(cur, 1); - if (GPR_SLICE_IS_EMPTY(signature)) goto error; + if (GRPC_SLICE_IS_EMPTY(signature)) goto error; retrieve_key_and_verify( exec_ctx, verifier_cb_ctx_create(verifier, pollset, header, claims, audience, diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h index b0f6d1c240..f09f9d5d47 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.h +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h @@ -37,7 +37,7 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/json/json.h" -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include <grpc/support/time.h> /* --- Constants. --- */ @@ -129,7 +129,7 @@ void grpc_jwt_verifier_verify(grpc_exec_ctx *exec_ctx, /* --- TESTING ONLY exposed functions. --- */ -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); +grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, grpc_slice buffer); grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, const char *audience); diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index d980577c46..b3625b22c0 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -392,15 +392,32 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token( return &c->base.base; } +static char *create_loggable_refresh_token(grpc_auth_refresh_token *token) { + if (strcmp(token->type, GRPC_AUTH_JSON_TYPE_INVALID) == 0) { + return gpr_strdup("<Invalid json token>"); + } + char *loggable_token = NULL; + gpr_asprintf(&loggable_token, + "{\n type: %s\n client_id: %s\n client_secret: " + "<redacted>\n refresh_token: <redacted>\n}", + token->type, token->client_id); + return loggable_token; +} + grpc_call_credentials *grpc_google_refresh_token_credentials_create( const char *json_refresh_token, void *reserved) { - GRPC_API_TRACE( - "grpc_refresh_token_credentials_create(json_refresh_token=%s, " - "reserved=%p)", - 2, (json_refresh_token, reserved)); + grpc_auth_refresh_token token = + grpc_auth_refresh_token_create_from_string(json_refresh_token); + if (grpc_api_trace) { + char *loggable_token = create_loggable_refresh_token(&token); + gpr_log(GPR_INFO, + "grpc_refresh_token_credentials_create(json_refresh_token=%s, " + "reserved=%p)", + loggable_token, reserved); + gpr_free(loggable_token); + } GPR_ASSERT(reserved == NULL); - return grpc_refresh_token_credentials_create_from_auth_refresh_token( - grpc_auth_refresh_token_create_from_string(json_refresh_token)); + return grpc_refresh_token_credentials_create_from_auth_refresh_token(token); } // @@ -430,9 +447,9 @@ grpc_call_credentials *grpc_access_token_credentials_create( gpr_malloc(sizeof(grpc_access_token_credentials)); char *token_md_value; GRPC_API_TRACE( - "grpc_access_token_credentials_create(access_token=%s, " + "grpc_access_token_credentials_create(access_token=<redacted>, " "reserved=%p)", - 2, (access_token, reserved)); + 1, (reserved)); GPR_ASSERT(reserved == NULL); memset(c, 0, sizeof(grpc_access_token_credentials)); c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2; diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c index 905de3723e..5d950098a0 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.c +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c @@ -93,17 +93,19 @@ static void plugin_md_request_metadata_ready(void *request, } else if (num_md > 0) { md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md)); for (i = 0; i < num_md; i++) { - md_array[i].key = gpr_slice_from_copied_string(md[i].key); + md_array[i].key = grpc_slice_from_copied_string(md[i].key); md_array[i].value = - gpr_slice_from_copied_buffer(md[i].value, md[i].value_length); + grpc_slice_from_copied_buffer(md[i].value, md[i].value_length); } r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK, NULL); for (i = 0; i < num_md; i++) { - gpr_slice_unref(md_array[i].key); - gpr_slice_unref(md_array[i].value); + grpc_slice_unref(md_array[i].key); + grpc_slice_unref(md_array[i].value); } gpr_free(md_array); + } else if (num_md == 0) { + r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL); } } gpr_free(r); diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index b366d1410f..053bf5972c 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -92,7 +92,7 @@ static void bubble_up_error(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_status_code status, const char *error_msg) { call_data *calld = elem->call_data; gpr_log(GPR_ERROR, "Client side authentication failure: %s", error_msg); - gpr_slice error_slice = gpr_slice_from_copied_string(error_msg); + grpc_slice error_slice = grpc_slice_from_copied_string(error_msg); grpc_transport_stream_op_add_close(&calld->op, status, &error_slice); grpc_call_next_op(exec_ctx, elem, &calld->op); } @@ -121,8 +121,8 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, for (i = 0; i < num_md; i++) { grpc_metadata_batch_add_tail( mdb, &calld->md_links[i], - grpc_mdelem_from_slices(gpr_slice_ref(md_elems[i].key), - gpr_slice_ref(md_elems[i].value))); + grpc_mdelem_from_slices(grpc_slice_ref(md_elems[i].key), + grpc_slice_ref(md_elems[i].value))); } grpc_call_next_op(exec_ctx, elem, op); } @@ -341,14 +341,8 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, GRPC_AUTH_CONTEXT_UNREF(chand->auth_context, "client_auth_filter"); } -const grpc_channel_filter grpc_client_auth_filter = {auth_start_transport_op, - grpc_channel_next_op, - sizeof(call_data), - init_call_elem, - set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - grpc_call_next_get_peer, - "client-auth"}; +const grpc_channel_filter grpc_client_auth_filter = { + auth_start_transport_op, grpc_channel_next_op, sizeof(call_data), + init_call_elem, set_pollset_or_pollset_set, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, grpc_channel_next_get_info, "client-auth"}; diff --git a/src/core/lib/security/transport/handshake.c b/src/core/lib/security/transport/handshake.c index fbeec312b6..9623797610 100644 --- a/src/core/lib/security/transport/handshake.c +++ b/src/core/lib/security/transport/handshake.c @@ -36,9 +36,9 @@ #include <stdbool.h> #include <string.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/security/context/security_context.h" #include "src/core/lib/security/transport/secure_endpoint.h" @@ -54,9 +54,9 @@ typedef struct { size_t handshake_buffer_size; grpc_endpoint *wrapped_endpoint; grpc_endpoint *secure_endpoint; - gpr_slice_buffer left_overs; - gpr_slice_buffer incoming; - gpr_slice_buffer outgoing; + grpc_slice_buffer left_overs; + grpc_slice_buffer incoming; + grpc_slice_buffer outgoing; grpc_security_handshake_done_cb cb; void *user_data; grpc_closure on_handshake_data_sent_to_peer; @@ -104,9 +104,9 @@ static void unref_handshake(grpc_security_handshake *h) { if (gpr_unref(&h->refs)) { if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker); if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer); - gpr_slice_buffer_destroy(&h->left_overs); - gpr_slice_buffer_destroy(&h->outgoing); - gpr_slice_buffer_destroy(&h->incoming); + grpc_slice_buffer_destroy(&h->left_overs); + grpc_slice_buffer_destroy(&h->outgoing); + grpc_slice_buffer_destroy(&h->incoming); GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake"); GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake"); gpr_free(h); @@ -125,7 +125,7 @@ static void security_handshake_done(grpc_exec_ctx *exec_ctx, h->auth_context); } else { const char *msg = grpc_error_string(error); - gpr_log(GPR_ERROR, "Security handshake failed: %s", msg); + gpr_log(GPR_DEBUG, "Security handshake failed: %s", msg); grpc_error_free_string(msg); if (h->secure_endpoint != NULL) { @@ -190,7 +190,7 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) { size_t offset = 0; tsi_result result = TSI_OK; - gpr_slice to_send; + grpc_slice to_send; do { size_t to_send_size = h->handshake_buffer_size - offset; @@ -212,9 +212,9 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx, } to_send = - gpr_slice_from_copied_buffer((const char *)h->handshake_buffer, offset); - gpr_slice_buffer_reset_and_unref(&h->outgoing); - gpr_slice_buffer_add(&h->outgoing, to_send); + grpc_slice_from_copied_buffer((const char *)h->handshake_buffer, offset); + grpc_slice_buffer_reset_and_unref(&h->outgoing); + grpc_slice_buffer_add(&h->outgoing, to_send); /* TODO(klempner,jboeuf): This should probably use the client setup deadline */ grpc_endpoint_write(exec_ctx, h->wrapped_endpoint, &h->outgoing, @@ -239,9 +239,9 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, } for (i = 0; i < h->incoming.count; i++) { - consumed_slice_size = GPR_SLICE_LENGTH(h->incoming.slices[i]); + consumed_slice_size = GRPC_SLICE_LENGTH(h->incoming.slices[i]); result = tsi_handshaker_process_bytes_from_peer( - h->handshaker, GPR_SLICE_START_PTR(h->incoming.slices[i]), + h->handshaker, GRPC_SLICE_START_PTR(h->incoming.slices[i]), &consumed_slice_size); if (!tsi_handshaker_is_in_progress(h->handshaker)) break; } @@ -267,7 +267,7 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, /* Handshake is done and successful this point. */ has_left_overs_in_current_slice = - (consumed_slice_size < GPR_SLICE_LENGTH(h->incoming.slices[i])); + (consumed_slice_size < GRPC_SLICE_LENGTH(h->incoming.slices[i])); num_left_overs = (has_left_overs_in_current_slice ? 1 : 0) + h->incoming.count - i - 1; if (num_left_overs == 0) { @@ -277,13 +277,13 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, /* Put the leftovers in our buffer (ownership transfered). */ if (has_left_overs_in_current_slice) { - gpr_slice_buffer_add( + grpc_slice_buffer_add( &h->left_overs, - gpr_slice_split_tail(&h->incoming.slices[i], consumed_slice_size)); - gpr_slice_unref( + grpc_slice_split_tail(&h->incoming.slices[i], consumed_slice_size)); + grpc_slice_unref( h->incoming.slices[i]); /* split_tail above increments refcount. */ } - gpr_slice_buffer_addn( + grpc_slice_buffer_addn( &h->left_overs, &h->incoming.slices[i + 1], num_left_overs - (size_t)has_left_overs_in_current_slice); check_peer(exec_ctx, h); @@ -325,7 +325,7 @@ static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { void grpc_do_security_handshake( grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, bool is_client_side, - grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_security_connector_handshake_list *handshake_node; @@ -344,11 +344,11 @@ void grpc_do_security_handshake( on_handshake_data_sent_to_peer, h); grpc_closure_init(&h->on_handshake_data_received_from_peer, on_handshake_data_received_from_peer, h); - gpr_slice_buffer_init(&h->left_overs); - gpr_slice_buffer_init(&h->outgoing); - gpr_slice_buffer_init(&h->incoming); + grpc_slice_buffer_init(&h->left_overs); + grpc_slice_buffer_init(&h->outgoing); + grpc_slice_buffer_init(&h->incoming); if (read_buffer != NULL) { - gpr_slice_buffer_move_into(read_buffer, &h->incoming); + grpc_slice_buffer_move_into(read_buffer, &h->incoming); gpr_free(read_buffer); } if (!is_client_side) { diff --git a/src/core/lib/security/transport/handshake.h b/src/core/lib/security/transport/handshake.h index 53092f5421..f894540515 100644 --- a/src/core/lib/security/transport/handshake.h +++ b/src/core/lib/security/transport/handshake.h @@ -42,7 +42,7 @@ void grpc_do_security_handshake( grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker, grpc_security_connector *connector, bool is_client_side, - grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake); diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 3924997d31..1b278410e8 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -31,15 +31,22 @@ * */ -#include "src/core/lib/security/transport/secure_endpoint.h" +/* With the addition of a libuv endpoint, sockaddr.h now includes uv.h when + using that endpoint. Because of various transitive includes in uv.h, + including windows.h on Windows, uv.h must be included before other system + headers. Therefore, sockaddr.h must always be included first */ +#include "src/core/lib/iomgr/sockaddr.h" + +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/sync.h> #include "src/core/lib/debug/trace.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/tsi_error.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/tsi/transport_security_interface.h" @@ -54,15 +61,15 @@ typedef struct { grpc_closure *read_cb; grpc_closure *write_cb; grpc_closure on_read; - gpr_slice_buffer *read_buffer; - gpr_slice_buffer source_buffer; + grpc_slice_buffer *read_buffer; + grpc_slice_buffer source_buffer; /* saved handshaker leftover data to unprotect. */ - gpr_slice_buffer leftover_bytes; + grpc_slice_buffer leftover_bytes; /* buffers for read and write */ - gpr_slice read_staging_buffer; + grpc_slice read_staging_buffer; - gpr_slice write_staging_buffer; - gpr_slice_buffer output_buffer; + grpc_slice write_staging_buffer; + grpc_slice_buffer output_buffer; gpr_refcount ref; } secure_endpoint; @@ -73,11 +80,11 @@ static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; grpc_endpoint_destroy(exec_ctx, ep->wrapped_ep); tsi_frame_protector_destroy(ep->protector); - gpr_slice_buffer_destroy(&ep->leftover_bytes); - gpr_slice_unref(ep->read_staging_buffer); - gpr_slice_unref(ep->write_staging_buffer); - gpr_slice_buffer_destroy(&ep->output_buffer); - gpr_slice_buffer_destroy(&ep->source_buffer); + grpc_slice_buffer_destroy(&ep->leftover_bytes); + grpc_slice_unref(ep->read_staging_buffer); + grpc_slice_unref(ep->write_staging_buffer); + grpc_slice_buffer_destroy(&ep->output_buffer); + grpc_slice_buffer_destroy(&ep->source_buffer); gpr_mu_destroy(&ep->protector_mu); gpr_free(ep); } @@ -121,10 +128,10 @@ static void secure_endpoint_ref(secure_endpoint *ep) { gpr_ref(&ep->ref); } static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { - gpr_slice_buffer_add(ep->read_buffer, ep->read_staging_buffer); - ep->read_staging_buffer = gpr_slice_malloc(STAGING_BUFFER_SIZE); - *cur = GPR_SLICE_START_PTR(ep->read_staging_buffer); - *end = GPR_SLICE_END_PTR(ep->read_staging_buffer); + grpc_slice_buffer_add(ep->read_buffer, ep->read_staging_buffer); + ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + *cur = GRPC_SLICE_START_PTR(ep->read_staging_buffer); + *end = GRPC_SLICE_END_PTR(ep->read_staging_buffer); } static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, @@ -132,8 +139,8 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, if (grpc_trace_secure_endpoint) { size_t i; for (i = 0; i < ep->read_buffer->count; i++) { - char *data = gpr_dump_slice(ep->read_buffer->slices[i], - GPR_DUMP_HEX | GPR_DUMP_ASCII); + char *data = grpc_dump_slice(ep->read_buffer->slices[i], + GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "READ %p: %s", ep, data); gpr_free(data); } @@ -149,11 +156,11 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, uint8_t keep_looping = 0; tsi_result result = TSI_OK; secure_endpoint *ep = (secure_endpoint *)user_data; - uint8_t *cur = GPR_SLICE_START_PTR(ep->read_staging_buffer); - uint8_t *end = GPR_SLICE_END_PTR(ep->read_staging_buffer); + uint8_t *cur = GRPC_SLICE_START_PTR(ep->read_staging_buffer); + uint8_t *end = GRPC_SLICE_END_PTR(ep->read_staging_buffer); if (error != GRPC_ERROR_NONE) { - gpr_slice_buffer_reset_and_unref(ep->read_buffer); + grpc_slice_buffer_reset_and_unref(ep->read_buffer); call_read_cb(exec_ctx, ep, GRPC_ERROR_CREATE_REFERENCING( "Secure read failed", &error, 1)); return; @@ -161,9 +168,9 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, /* TODO(yangg) check error, maybe bail out early */ for (i = 0; i < ep->source_buffer.count; i++) { - gpr_slice encrypted = ep->source_buffer.slices[i]; - uint8_t *message_bytes = GPR_SLICE_START_PTR(encrypted); - size_t message_size = GPR_SLICE_LENGTH(encrypted); + grpc_slice encrypted = ep->source_buffer.slices[i]; + uint8_t *message_bytes = GRPC_SLICE_START_PTR(encrypted); + size_t message_size = GRPC_SLICE_LENGTH(encrypted); while (message_size > 0 || keep_looping) { size_t unprotected_buffer_size_written = (size_t)(end - cur); @@ -198,20 +205,20 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, if (result != TSI_OK) break; } - if (cur != GPR_SLICE_START_PTR(ep->read_staging_buffer)) { - gpr_slice_buffer_add( + if (cur != GRPC_SLICE_START_PTR(ep->read_staging_buffer)) { + grpc_slice_buffer_add( ep->read_buffer, - gpr_slice_split_head( + grpc_slice_split_head( &ep->read_staging_buffer, - (size_t)(cur - GPR_SLICE_START_PTR(ep->read_staging_buffer)))); + (size_t)(cur - GRPC_SLICE_START_PTR(ep->read_staging_buffer)))); } /* TODO(yangg) experiment with moving this block after read_cb to see if it helps latency */ - gpr_slice_buffer_reset_and_unref(&ep->source_buffer); + grpc_slice_buffer_reset_and_unref(&ep->source_buffer); if (result != TSI_OK) { - gpr_slice_buffer_reset_and_unref(ep->read_buffer); + grpc_slice_buffer_reset_and_unref(ep->read_buffer); call_read_cb(exec_ctx, ep, grpc_set_tsi_error_result( GRPC_ERROR_CREATE("Unwrap failed"), result)); return; @@ -221,15 +228,15 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *user_data, } static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, - gpr_slice_buffer *slices, grpc_closure *cb) { + grpc_slice_buffer *slices, grpc_closure *cb) { secure_endpoint *ep = (secure_endpoint *)secure_ep; ep->read_cb = cb; ep->read_buffer = slices; - gpr_slice_buffer_reset_and_unref(ep->read_buffer); + grpc_slice_buffer_reset_and_unref(ep->read_buffer); SECURE_ENDPOINT_REF(ep, "read"); if (ep->leftover_bytes.count) { - gpr_slice_buffer_swap(&ep->leftover_bytes, &ep->source_buffer); + grpc_slice_buffer_swap(&ep->leftover_bytes, &ep->source_buffer); GPR_ASSERT(ep->leftover_bytes.count == 0); on_read(exec_ctx, ep, GRPC_ERROR_NONE); return; @@ -241,37 +248,37 @@ static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, static void flush_write_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { - gpr_slice_buffer_add(&ep->output_buffer, ep->write_staging_buffer); - ep->write_staging_buffer = gpr_slice_malloc(STAGING_BUFFER_SIZE); - *cur = GPR_SLICE_START_PTR(ep->write_staging_buffer); - *end = GPR_SLICE_END_PTR(ep->write_staging_buffer); + grpc_slice_buffer_add(&ep->output_buffer, ep->write_staging_buffer); + ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + *cur = GRPC_SLICE_START_PTR(ep->write_staging_buffer); + *end = GRPC_SLICE_END_PTR(ep->write_staging_buffer); } static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, - gpr_slice_buffer *slices, grpc_closure *cb) { + grpc_slice_buffer *slices, grpc_closure *cb) { GPR_TIMER_BEGIN("secure_endpoint.endpoint_write", 0); unsigned i; tsi_result result = TSI_OK; secure_endpoint *ep = (secure_endpoint *)secure_ep; - uint8_t *cur = GPR_SLICE_START_PTR(ep->write_staging_buffer); - uint8_t *end = GPR_SLICE_END_PTR(ep->write_staging_buffer); + uint8_t *cur = GRPC_SLICE_START_PTR(ep->write_staging_buffer); + uint8_t *end = GRPC_SLICE_END_PTR(ep->write_staging_buffer); - gpr_slice_buffer_reset_and_unref(&ep->output_buffer); + grpc_slice_buffer_reset_and_unref(&ep->output_buffer); if (grpc_trace_secure_endpoint) { for (i = 0; i < slices->count; i++) { char *data = - gpr_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); + grpc_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); gpr_log(GPR_DEBUG, "WRITE %p: %s", ep, data); gpr_free(data); } } for (i = 0; i < slices->count; i++) { - gpr_slice plain = slices->slices[i]; - uint8_t *message_bytes = GPR_SLICE_START_PTR(plain); - size_t message_size = GPR_SLICE_LENGTH(plain); + grpc_slice plain = slices->slices[i]; + uint8_t *message_bytes = GRPC_SLICE_START_PTR(plain); + size_t message_size = GRPC_SLICE_LENGTH(plain); while (message_size > 0) { size_t protected_buffer_size_to_send = (size_t)(end - cur); size_t processed_message_size = message_size; @@ -310,18 +317,18 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, flush_write_staging_buffer(ep, &cur, &end); } } while (still_pending_size > 0); - if (cur != GPR_SLICE_START_PTR(ep->write_staging_buffer)) { - gpr_slice_buffer_add( + if (cur != GRPC_SLICE_START_PTR(ep->write_staging_buffer)) { + grpc_slice_buffer_add( &ep->output_buffer, - gpr_slice_split_head( + grpc_slice_split_head( &ep->write_staging_buffer, - (size_t)(cur - GPR_SLICE_START_PTR(ep->write_staging_buffer)))); + (size_t)(cur - GRPC_SLICE_START_PTR(ep->write_staging_buffer)))); } } if (result != TSI_OK) { /* TODO(yangg) do different things according to the error type? */ - gpr_slice_buffer_reset_and_unref(&ep->output_buffer); + grpc_slice_buffer_reset_and_unref(&ep->output_buffer); grpc_exec_ctx_sched( exec_ctx, cb, grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Wrap failed"), result), @@ -365,6 +372,8 @@ static char *endpoint_get_peer(grpc_endpoint *secure_ep) { return grpc_endpoint_get_peer(ep->wrapped_ep); } +static int endpoint_get_fd(grpc_endpoint *secure_ep) { return -1; } + static grpc_workqueue *endpoint_get_workqueue(grpc_endpoint *secure_ep) { secure_endpoint *ep = (secure_endpoint *)secure_ep; return grpc_endpoint_get_workqueue(ep->wrapped_ep); @@ -384,25 +393,26 @@ static const grpc_endpoint_vtable vtable = {endpoint_read, endpoint_shutdown, endpoint_destroy, endpoint_get_resource_user, - endpoint_get_peer}; + endpoint_get_peer, + endpoint_get_fd}; grpc_endpoint *grpc_secure_endpoint_create( struct tsi_frame_protector *protector, grpc_endpoint *transport, - gpr_slice *leftover_slices, size_t leftover_nslices) { + grpc_slice *leftover_slices, size_t leftover_nslices) { size_t i; secure_endpoint *ep = (secure_endpoint *)gpr_malloc(sizeof(secure_endpoint)); ep->base.vtable = &vtable; ep->wrapped_ep = transport; ep->protector = protector; - gpr_slice_buffer_init(&ep->leftover_bytes); + grpc_slice_buffer_init(&ep->leftover_bytes); for (i = 0; i < leftover_nslices; i++) { - gpr_slice_buffer_add(&ep->leftover_bytes, - gpr_slice_ref(leftover_slices[i])); + grpc_slice_buffer_add(&ep->leftover_bytes, + grpc_slice_ref(leftover_slices[i])); } - ep->write_staging_buffer = gpr_slice_malloc(STAGING_BUFFER_SIZE); - ep->read_staging_buffer = gpr_slice_malloc(STAGING_BUFFER_SIZE); - gpr_slice_buffer_init(&ep->output_buffer); - gpr_slice_buffer_init(&ep->source_buffer); + ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + grpc_slice_buffer_init(&ep->output_buffer); + grpc_slice_buffer_init(&ep->source_buffer); ep->read_buffer = NULL; grpc_closure_init(&ep->on_read, on_read, ep); gpr_mu_init(&ep->protector_mu); diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h index d00075b769..a61f40a4fa 100644 --- a/src/core/lib/security/transport/secure_endpoint.h +++ b/src/core/lib/security/transport/secure_endpoint.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H #define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/lib/iomgr/endpoint.h" struct tsi_frame_protector; @@ -44,6 +44,6 @@ extern int grpc_trace_secure_endpoint; /* Takes ownership of protector and to_wrap, and refs leftover_slices. */ grpc_endpoint *grpc_secure_endpoint_create( struct tsi_frame_protector *protector, grpc_endpoint *to_wrap, - gpr_slice *leftover_slices, size_t leftover_nslices); + grpc_slice *leftover_slices, size_t leftover_nslices); #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */ diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index ebf72a3abb..0fbd63a7e1 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -36,10 +36,10 @@ #include <stdbool.h> #include <string.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/host_port.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/string_util.h> #include "src/core/ext/transport/chttp2/alpn/alpn.h" @@ -127,7 +127,7 @@ void grpc_server_security_connector_shutdown( void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, - grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { @@ -142,7 +142,7 @@ void grpc_channel_security_connector_do_handshake( void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { if (sc == NULL || nonsecure_endpoint == NULL) { gpr_free(read_buffer); @@ -316,7 +316,7 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { @@ -328,7 +328,7 @@ static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx, static void fake_server_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base, false, nonsecure_endpoint, read_buffer, deadline, @@ -422,7 +422,7 @@ static grpc_security_status ssl_create_handshaker( static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { @@ -447,7 +447,7 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx, static void ssl_server_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data) { grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; @@ -642,8 +642,8 @@ static grpc_security_connector_vtable ssl_channel_vtable = { static grpc_security_connector_vtable ssl_server_vtable = { ssl_server_destroy, ssl_server_check_peer}; -static gpr_slice compute_default_pem_root_certs_once(void) { - gpr_slice result = gpr_empty_slice(); +static grpc_slice compute_default_pem_root_certs_once(void) { + grpc_slice result = gpr_empty_slice(); /* First try to load the roots from the environment. */ char *default_root_certs_path = @@ -656,17 +656,17 @@ static gpr_slice compute_default_pem_root_certs_once(void) { /* Try overridden roots if needed. */ grpc_ssl_roots_override_result ovrd_res = GRPC_SSL_ROOTS_OVERRIDE_FAIL; - if (GPR_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != NULL) { + if (GRPC_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != NULL) { char *pem_root_certs = NULL; ovrd_res = ssl_roots_override_cb(&pem_root_certs); if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) { GPR_ASSERT(pem_root_certs != NULL); - result = gpr_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free); + result = grpc_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free); } } /* Fall back to installed certs if needed. */ - if (GPR_SLICE_IS_EMPTY(result) && + if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", grpc_load_file(installed_roots_path, 0, &result)); @@ -674,13 +674,13 @@ static gpr_slice compute_default_pem_root_certs_once(void) { return result; } -static gpr_slice default_pem_root_certs; +static grpc_slice default_pem_root_certs; static void init_default_pem_root_certs(void) { default_pem_root_certs = compute_default_pem_root_certs_once(); } -gpr_slice grpc_get_default_ssl_roots_for_testing(void) { +grpc_slice grpc_get_default_ssl_roots_for_testing(void) { return compute_default_pem_root_certs_once(); } @@ -714,8 +714,8 @@ size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) { loading all the roots once for the lifetime of the process. */ static gpr_once once = GPR_ONCE_INIT; gpr_once_init(&once, init_default_pem_root_certs); - *pem_root_certs = GPR_SLICE_START_PTR(default_pem_root_certs); - return GPR_SLICE_LENGTH(default_pem_root_certs); + *pem_root_certs = GRPC_SLICE_START_PTR(default_pem_root_certs); + return GRPC_SLICE_LENGTH(default_pem_root_certs); } grpc_security_status grpc_ssl_channel_security_connector_create( diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index 0b5b44bf1a..dc02692b01 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -144,7 +144,7 @@ struct grpc_channel_security_connector { void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; @@ -157,7 +157,7 @@ void grpc_channel_security_connector_check_call_host( /* Handshake. */ void grpc_channel_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector, - grpc_endpoint *nonsecure_endpoint, gpr_slice_buffer *read_buffer, + grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); /* --- server_security_connector object. --- @@ -176,14 +176,14 @@ struct grpc_server_security_connector { grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); }; void grpc_server_security_connector_do_handshake( grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc, grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint, - gpr_slice_buffer *read_buffer, gpr_timespec deadline, + grpc_slice_buffer *read_buffer, gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data); void grpc_server_security_connector_shutdown( @@ -233,7 +233,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create( size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs); /* Exposed for TESTING ONLY!. */ -gpr_slice grpc_get_default_ssl_roots_for_testing(void); +grpc_slice grpc_get_default_ssl_roots_for_testing(void); /* Config for ssl servers. */ typedef struct { diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index b2c6815af8..eaa1d0720b 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -78,7 +78,7 @@ static grpc_metadata_array metadata_batch_to_md_array( usr_md = &result.metadata[result.count++]; usr_md->key = grpc_mdstr_as_c_string(key); usr_md->value = grpc_mdstr_as_c_string(value); - usr_md->value_length = GPR_SLICE_LENGTH(value->slice); + usr_md->value_length = GRPC_SLICE_LENGTH(value->slice); } return result; } @@ -92,14 +92,14 @@ static grpc_mdelem *remove_consumed_md(void *user_data, grpc_mdelem *md) { /* Maybe we could do a pointer comparison but we do not have any guarantee that the metadata processor used the same pointers for consumed_md in the callback. */ - if (GPR_SLICE_LENGTH(md->key->slice) != strlen(consumed_md->key) || - GPR_SLICE_LENGTH(md->value->slice) != consumed_md->value_length) { + if (GRPC_SLICE_LENGTH(md->key->slice) != strlen(consumed_md->key) || + GRPC_SLICE_LENGTH(md->value->slice) != consumed_md->value_length) { continue; } - if (memcmp(GPR_SLICE_START_PTR(md->key->slice), consumed_md->key, - GPR_SLICE_LENGTH(md->key->slice)) == 0 && - memcmp(GPR_SLICE_START_PTR(md->value->slice), consumed_md->value, - GPR_SLICE_LENGTH(md->value->slice)) == 0) { + if (memcmp(GRPC_SLICE_START_PTR(md->key->slice), consumed_md->key, + GRPC_SLICE_LENGTH(md->key->slice)) == 0 && + memcmp(GRPC_SLICE_START_PTR(md->value->slice), consumed_md->value, + GRPC_SLICE_LENGTH(md->value->slice)) == 0) { return NULL; /* Delete. */ } } @@ -134,14 +134,14 @@ static void on_md_processing_done( grpc_metadata_array_destroy(&calld->md); grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL); } else { - gpr_slice message; + grpc_slice message; grpc_transport_stream_op *close_op = gpr_malloc(sizeof(*close_op)); memset(close_op, 0, sizeof(*close_op)); grpc_metadata_array_destroy(&calld->md); error_details = error_details != NULL ? error_details : "Authentication metadata processing failed."; - message = gpr_slice_from_copied_string(error_details); + message = grpc_slice_from_copied_string(error_details); calld->transport_op->send_initial_metadata = NULL; if (calld->transport_op->send_message != NULL) { grpc_byte_stream_destroy(&exec_ctx, calld->transport_op->send_message); @@ -278,4 +278,5 @@ const grpc_channel_filter grpc_server_auth_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "server-auth"}; diff --git a/src/core/lib/security/util/b64.c b/src/core/lib/security/util/b64.c index 9da42e4e73..4892e8e621 100644 --- a/src/core/lib/security/util/b64.c +++ b/src/core/lib/security/util/b64.c @@ -120,7 +120,7 @@ char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe, return result; } -gpr_slice grpc_base64_decode(const char *b64, int url_safe) { +grpc_slice grpc_base64_decode(const char *b64, int url_safe) { return grpc_base64_decode_with_len(b64, strlen(b64), url_safe); } @@ -182,10 +182,10 @@ static int decode_group(const unsigned char *codes, size_t num_codes, return 1; } -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe) { - gpr_slice result = gpr_slice_malloc(b64_len); - unsigned char *current = GPR_SLICE_START_PTR(result); +grpc_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, + int url_safe) { + grpc_slice result = grpc_slice_malloc(b64_len); + unsigned char *current = GRPC_SLICE_START_PTR(result); size_t result_size = 0; unsigned char codes[4]; size_t num_codes = 0; @@ -224,10 +224,10 @@ gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, !decode_group(codes, num_codes, current, &result_size)) { goto fail; } - GPR_SLICE_SET_LENGTH(result, result_size); + GRPC_SLICE_SET_LENGTH(result, result_size); return result; fail: - gpr_slice_unref(result); + grpc_slice_unref(result); return gpr_empty_slice(); } diff --git a/src/core/lib/security/util/b64.h b/src/core/lib/security/util/b64.h index 6908095287..6ea0b5365b 100644 --- a/src/core/lib/security/util/b64.h +++ b/src/core/lib/security/util/b64.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_SECURITY_UTIL_B64_H #define GRPC_CORE_LIB_SECURITY_UTIL_B64_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> /* Encodes data using base64. It is the caller's responsability to free the returned char * using gpr_free. Returns NULL on NULL input. */ @@ -43,10 +43,10 @@ char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, /* Decodes data according to the base64 specification. Returns an empty slice in case of failure. */ -gpr_slice grpc_base64_decode(const char *b64, int url_safe); +grpc_slice grpc_base64_decode(const char *b64, int url_safe); /* Same as above except that the length is provided by the caller. */ -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe); +grpc_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, + int url_safe); #endif /* GRPC_CORE_LIB_SECURITY_UTIL_B64_H */ diff --git a/src/core/lib/support/percent_encoding.c b/src/core/lib/slice/percent_encoding.c index 3c19f264f9..b9e35f1c71 100644 --- a/src/core/lib/support/percent_encoding.c +++ b/src/core/lib/slice/percent_encoding.c @@ -31,15 +31,15 @@ * */ -#include "src/core/lib/support/percent_encoding.h" +#include "src/core/lib/slice/percent_encoding.h" #include <grpc/support/log.h> -const uint8_t gpr_url_percent_encoding_unreserved_bytes[256 / 8] = { +const uint8_t grpc_url_percent_encoding_unreserved_bytes[256 / 8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0xfe, 0xff, 0xff, 0x87, 0xfe, 0xff, 0xff, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t gpr_compatible_percent_encoding_unreserved_bytes[256 / 8] = { +const uint8_t grpc_compatible_percent_encoding_unreserved_bytes[256 / 8] = { 0x00, 0x00, 0x00, 0x00, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; @@ -49,14 +49,14 @@ static bool is_unreserved_character(uint8_t c, return ((unreserved_bytes[c / 8] >> (c % 8)) & 1) != 0; } -gpr_slice gpr_percent_encode_slice(gpr_slice slice, - const uint8_t *unreserved_bytes) { +grpc_slice grpc_percent_encode_slice(grpc_slice slice, + const uint8_t *unreserved_bytes) { static const uint8_t hex[] = "0123456789ABCDEF"; // first pass: count the number of bytes needed to output this string size_t output_length = 0; - const uint8_t *slice_start = GPR_SLICE_START_PTR(slice); - const uint8_t *slice_end = GPR_SLICE_END_PTR(slice); + const uint8_t *slice_start = GRPC_SLICE_START_PTR(slice); + const uint8_t *slice_end = GRPC_SLICE_END_PTR(slice); const uint8_t *p; bool any_reserved_bytes = false; for (p = slice_start; p < slice_end; p++) { @@ -66,11 +66,11 @@ gpr_slice gpr_percent_encode_slice(gpr_slice slice, } // no unreserved bytes: return the string unmodified if (!any_reserved_bytes) { - return gpr_slice_ref(slice); + return grpc_slice_ref(slice); } // second pass: actually encode - gpr_slice out = gpr_slice_malloc(output_length); - uint8_t *q = GPR_SLICE_START_PTR(out); + grpc_slice out = grpc_slice_malloc(output_length); + uint8_t *q = GRPC_SLICE_START_PTR(out); for (p = slice_start; p < slice_end; p++) { if (is_unreserved_character(*p, unreserved_bytes)) { *q++ = *p; @@ -80,7 +80,7 @@ gpr_slice gpr_percent_encode_slice(gpr_slice slice, *q++ = hex[*p & 15]; } } - GPR_ASSERT(q == GPR_SLICE_END_PTR(out)); + GPR_ASSERT(q == GRPC_SLICE_END_PTR(out)); return out; } @@ -97,11 +97,11 @@ static uint8_t dehex(uint8_t c) { GPR_UNREACHABLE_CODE(return 255); } -bool gpr_strict_percent_decode_slice(gpr_slice slice_in, - const uint8_t *unreserved_bytes, - gpr_slice *slice_out) { - const uint8_t *p = GPR_SLICE_START_PTR(slice_in); - const uint8_t *in_end = GPR_SLICE_END_PTR(slice_in); +bool grpc_strict_percent_decode_slice(grpc_slice slice_in, + const uint8_t *unreserved_bytes, + grpc_slice *slice_out) { + const uint8_t *p = GRPC_SLICE_START_PTR(slice_in); + const uint8_t *in_end = GRPC_SLICE_END_PTR(slice_in); size_t out_length = 0; bool any_percent_encoded_stuff = false; while (p != in_end) { @@ -119,12 +119,12 @@ bool gpr_strict_percent_decode_slice(gpr_slice slice_in, } } if (!any_percent_encoded_stuff) { - *slice_out = gpr_slice_ref(slice_in); + *slice_out = grpc_slice_ref(slice_in); return true; } - p = GPR_SLICE_START_PTR(slice_in); - *slice_out = gpr_slice_malloc(out_length); - uint8_t *q = GPR_SLICE_START_PTR(*slice_out); + p = GRPC_SLICE_START_PTR(slice_in); + *slice_out = grpc_slice_malloc(out_length); + uint8_t *q = GRPC_SLICE_START_PTR(*slice_out); while (p != in_end) { if (*p == '%') { *q++ = (uint8_t)(dehex(p[1]) << 4) | (dehex(p[2])); @@ -133,13 +133,13 @@ bool gpr_strict_percent_decode_slice(gpr_slice slice_in, *q++ = *p++; } } - GPR_ASSERT(q == GPR_SLICE_END_PTR(*slice_out)); + GPR_ASSERT(q == GRPC_SLICE_END_PTR(*slice_out)); return true; } -gpr_slice gpr_permissive_percent_decode_slice(gpr_slice slice_in) { - const uint8_t *p = GPR_SLICE_START_PTR(slice_in); - const uint8_t *in_end = GPR_SLICE_END_PTR(slice_in); +grpc_slice grpc_permissive_percent_decode_slice(grpc_slice slice_in) { + const uint8_t *p = GRPC_SLICE_START_PTR(slice_in); + const uint8_t *in_end = GRPC_SLICE_END_PTR(slice_in); size_t out_length = 0; bool any_percent_encoded_stuff = false; while (p != in_end) { @@ -158,11 +158,11 @@ gpr_slice gpr_permissive_percent_decode_slice(gpr_slice slice_in) { } } if (!any_percent_encoded_stuff) { - return gpr_slice_ref(slice_in); + return grpc_slice_ref(slice_in); } - p = GPR_SLICE_START_PTR(slice_in); - gpr_slice out = gpr_slice_malloc(out_length); - uint8_t *q = GPR_SLICE_START_PTR(out); + p = GRPC_SLICE_START_PTR(slice_in); + grpc_slice out = grpc_slice_malloc(out_length); + uint8_t *q = GRPC_SLICE_START_PTR(out); while (p != in_end) { if (*p == '%') { if (!valid_hex(p + 1, in_end) || !valid_hex(p + 2, in_end)) { @@ -175,6 +175,6 @@ gpr_slice gpr_permissive_percent_decode_slice(gpr_slice slice_in) { *q++ = *p++; } } - GPR_ASSERT(q == GPR_SLICE_END_PTR(out)); + GPR_ASSERT(q == GRPC_SLICE_END_PTR(out)); return out; } diff --git a/src/core/lib/support/percent_encoding.h b/src/core/lib/slice/percent_encoding.h index 000bf14ede..189bdbe312 100644 --- a/src/core/lib/support/percent_encoding.h +++ b/src/core/lib/slice/percent_encoding.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_SUPPORT_PERCENT_ENCODING_H -#define GRPC_CORE_LIB_SUPPORT_PERCENT_ENCODING_H +#ifndef GRPC_CORE_LIB_SLICE_PERCENT_ENCODING_H +#define GRPC_CORE_LIB_SLICE_PERCENT_ENCODING_H /* Percent encoding and decoding of slices. Transforms arbitrary strings into safe-for-transmission strings by using @@ -43,36 +43,36 @@ #include <stdbool.h> -#include <grpc/support/slice.h> +#include <grpc/slice.h> /* URL percent encoding spec bitfield (usabel as 'unreserved_bytes' in - gpr_percent_encode_slice, gpr_strict_percent_decode_slice). + grpc_percent_encode_slice, grpc_strict_percent_decode_slice). Flags [A-Za-z0-9-_.~] as unreserved bytes for the percent encoding routines */ -extern const uint8_t gpr_url_percent_encoding_unreserved_bytes[256 / 8]; +extern const uint8_t grpc_url_percent_encoding_unreserved_bytes[256 / 8]; /* URL percent encoding spec bitfield (usabel as 'unreserved_bytes' in - gpr_percent_encode_slice, gpr_strict_percent_decode_slice). + grpc_percent_encode_slice, grpc_strict_percent_decode_slice). Flags ascii7 non-control characters excluding '%' as unreserved bytes for the percent encoding routines */ -extern const uint8_t gpr_compatible_percent_encoding_unreserved_bytes[256 / 8]; +extern const uint8_t grpc_compatible_percent_encoding_unreserved_bytes[256 / 8]; /* Percent-encode a slice, returning the new slice (this cannot fail): unreserved_bytes is a bitfield indicating which bytes are considered unreserved and thus do not need percent encoding */ -gpr_slice gpr_percent_encode_slice(gpr_slice slice, - const uint8_t *unreserved_bytes); +grpc_slice grpc_percent_encode_slice(grpc_slice slice, + const uint8_t *unreserved_bytes); /* Percent-decode a slice, strictly. If the input is legal (contains no unreserved bytes, and legal % encodings), returns true and sets *slice_out to the decoded slice. If the input is not legal, returns false and leaves *slice_out untouched. unreserved_bytes is a bitfield indicating which bytes are considered unreserved and thus do not need percent encoding */ -bool gpr_strict_percent_decode_slice(gpr_slice slice_in, - const uint8_t *unreserved_bytes, - gpr_slice *slice_out); +bool grpc_strict_percent_decode_slice(grpc_slice slice_in, + const uint8_t *unreserved_bytes, + grpc_slice *slice_out); /* Percent-decode a slice, permissively. If a % triplet can not be decoded, pass it through verbatim. This cannot fail. */ -gpr_slice gpr_permissive_percent_decode_slice(gpr_slice slice_in); +grpc_slice grpc_permissive_percent_decode_slice(grpc_slice slice_in); -#endif /* GRPC_CORE_LIB_SUPPORT_PERCENT_ENCODING_H */ +#endif /* GRPC_CORE_LIB_SLICE_PERCENT_ENCODING_H */ diff --git a/src/core/lib/support/slice.c b/src/core/lib/slice/slice.c index 2959a6716f..52977e6d9a 100644 --- a/src/core/lib/support/slice.c +++ b/src/core/lib/slice/slice.c @@ -31,51 +31,51 @@ * */ +#include <grpc/slice.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice.h> #include <string.h> -gpr_slice gpr_empty_slice(void) { - gpr_slice out; +grpc_slice gpr_empty_slice(void) { + grpc_slice out; out.refcount = 0; out.data.inlined.length = 0; return out; } -gpr_slice gpr_slice_ref(gpr_slice slice) { +grpc_slice grpc_slice_ref(grpc_slice slice) { if (slice.refcount) { slice.refcount->ref(slice.refcount); } return slice; } -void gpr_slice_unref(gpr_slice slice) { +void grpc_slice_unref(grpc_slice slice) { if (slice.refcount) { slice.refcount->unref(slice.refcount); } } -/* gpr_slice_from_static_string support structure - a refcount that does +/* grpc_slice_from_static_string support structure - a refcount that does nothing */ static void noop_ref_or_unref(void *unused) {} -static gpr_slice_refcount noop_refcount = {noop_ref_or_unref, - noop_ref_or_unref}; +static grpc_slice_refcount noop_refcount = {noop_ref_or_unref, + noop_ref_or_unref}; -gpr_slice gpr_slice_from_static_string(const char *s) { - gpr_slice slice; +grpc_slice grpc_slice_from_static_string(const char *s) { + grpc_slice slice; slice.refcount = &noop_refcount; slice.data.refcounted.bytes = (uint8_t *)s; slice.data.refcounted.length = strlen(s); return slice; } -/* gpr_slice_new support structures - we create a refcount object extended +/* grpc_slice_new support structures - we create a refcount object extended with the user provided data pointer & destroy function */ typedef struct new_slice_refcount { - gpr_slice_refcount rc; + grpc_slice_refcount rc; gpr_refcount refs; void (*user_destroy)(void *); void *user_data; @@ -94,10 +94,10 @@ static void new_slice_unref(void *p) { } } -gpr_slice gpr_slice_new_with_user_data(void *p, size_t len, - void (*destroy)(void *), - void *user_data) { - gpr_slice slice; +grpc_slice grpc_slice_new_with_user_data(void *p, size_t len, + void (*destroy)(void *), + void *user_data) { + grpc_slice slice; new_slice_refcount *rc = gpr_malloc(sizeof(new_slice_refcount)); gpr_ref_init(&rc->refs, 1); rc->rc.ref = new_slice_ref; @@ -111,15 +111,15 @@ gpr_slice gpr_slice_new_with_user_data(void *p, size_t len, return slice; } -gpr_slice gpr_slice_new(void *p, size_t len, void (*destroy)(void *)) { +grpc_slice grpc_slice_new(void *p, size_t len, void (*destroy)(void *)) { /* Pass "p" to *destroy when the slice is no longer needed. */ - return gpr_slice_new_with_user_data(p, len, destroy, p); + return grpc_slice_new_with_user_data(p, len, destroy, p); } -/* gpr_slice_new_with_len support structures - we create a refcount object +/* grpc_slice_new_with_len support structures - we create a refcount object extended with the user provided data pointer & destroy function */ typedef struct new_with_len_slice_refcount { - gpr_slice_refcount rc; + grpc_slice_refcount rc; gpr_refcount refs; void *user_data; size_t user_length; @@ -139,9 +139,9 @@ static void new_with_len_unref(void *p) { } } -gpr_slice gpr_slice_new_with_len(void *p, size_t len, - void (*destroy)(void *, size_t)) { - gpr_slice slice; +grpc_slice grpc_slice_new_with_len(void *p, size_t len, + void (*destroy)(void *, size_t)) { + grpc_slice slice; new_with_len_slice_refcount *rc = gpr_malloc(sizeof(new_with_len_slice_refcount)); gpr_ref_init(&rc->refs, 1); @@ -157,18 +157,18 @@ gpr_slice gpr_slice_new_with_len(void *p, size_t len, return slice; } -gpr_slice gpr_slice_from_copied_buffer(const char *source, size_t length) { - gpr_slice slice = gpr_slice_malloc(length); - memcpy(GPR_SLICE_START_PTR(slice), source, length); +grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t length) { + grpc_slice slice = grpc_slice_malloc(length); + memcpy(GRPC_SLICE_START_PTR(slice), source, length); return slice; } -gpr_slice gpr_slice_from_copied_string(const char *source) { - return gpr_slice_from_copied_buffer(source, strlen(source)); +grpc_slice grpc_slice_from_copied_string(const char *source) { + return grpc_slice_from_copied_buffer(source, strlen(source)); } typedef struct { - gpr_slice_refcount base; + grpc_slice_refcount base; gpr_refcount refs; } malloc_refcount; @@ -184,8 +184,8 @@ static void malloc_unref(void *p) { } } -gpr_slice gpr_slice_malloc(size_t length) { - gpr_slice slice; +grpc_slice grpc_slice_malloc(size_t length) { + grpc_slice slice; if (length > sizeof(slice.data.inlined.bytes)) { /* Memory layout used by the slice created here: @@ -221,8 +221,8 @@ gpr_slice gpr_slice_malloc(size_t length) { return slice; } -gpr_slice gpr_slice_sub_no_ref(gpr_slice source, size_t begin, size_t end) { - gpr_slice subset; +grpc_slice grpc_slice_sub_no_ref(grpc_slice source, size_t begin, size_t end) { + grpc_slice subset; GPR_ASSERT(end >= begin); @@ -246,24 +246,24 @@ gpr_slice gpr_slice_sub_no_ref(gpr_slice source, size_t begin, size_t end) { return subset; } -gpr_slice gpr_slice_sub(gpr_slice source, size_t begin, size_t end) { - gpr_slice subset; +grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { + grpc_slice subset; if (end - begin <= sizeof(subset.data.inlined.bytes)) { subset.refcount = NULL; subset.data.inlined.length = (uint8_t)(end - begin); - memcpy(subset.data.inlined.bytes, GPR_SLICE_START_PTR(source) + begin, + memcpy(subset.data.inlined.bytes, GRPC_SLICE_START_PTR(source) + begin, end - begin); } else { - subset = gpr_slice_sub_no_ref(source, begin, end); + subset = grpc_slice_sub_no_ref(source, begin, end); /* Bump the refcount */ subset.refcount->ref(subset.refcount); } return subset; } -gpr_slice gpr_slice_split_tail(gpr_slice *source, size_t split) { - gpr_slice tail; +grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { + grpc_slice tail; if (source->refcount == NULL) { /* inlined data, copy it out */ @@ -297,8 +297,8 @@ gpr_slice gpr_slice_split_tail(gpr_slice *source, size_t split) { return tail; } -gpr_slice gpr_slice_split_head(gpr_slice *source, size_t split) { - gpr_slice head; +grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { + grpc_slice head; if (source->refcount == NULL) { GPR_ASSERT(source->data.inlined.length >= split); @@ -335,23 +335,23 @@ gpr_slice gpr_slice_split_head(gpr_slice *source, size_t split) { return head; } -int gpr_slice_cmp(gpr_slice a, gpr_slice b) { - int d = (int)(GPR_SLICE_LENGTH(a) - GPR_SLICE_LENGTH(b)); +int grpc_slice_cmp(grpc_slice a, grpc_slice b) { + int d = (int)(GRPC_SLICE_LENGTH(a) - GRPC_SLICE_LENGTH(b)); if (d != 0) return d; - return memcmp(GPR_SLICE_START_PTR(a), GPR_SLICE_START_PTR(b), - GPR_SLICE_LENGTH(a)); + return memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b), + GRPC_SLICE_LENGTH(a)); } -int gpr_slice_str_cmp(gpr_slice a, const char *b) { +int grpc_slice_str_cmp(grpc_slice a, const char *b) { size_t b_length = strlen(b); - int d = (int)(GPR_SLICE_LENGTH(a) - b_length); + int d = (int)(GRPC_SLICE_LENGTH(a) - b_length); if (d != 0) return d; - return memcmp(GPR_SLICE_START_PTR(a), b, b_length); + return memcmp(GRPC_SLICE_START_PTR(a), b, b_length); } -int gpr_slice_is_equivalent(gpr_slice a, gpr_slice b) { +int grpc_slice_is_equivalent(grpc_slice a, grpc_slice b) { if (a.refcount == NULL || b.refcount == NULL) { - return gpr_slice_cmp(a, b) == 0; + return grpc_slice_cmp(a, b) == 0; } return a.data.refcounted.length == b.data.refcounted.length && a.data.refcounted.bytes == b.data.refcounted.bytes; diff --git a/src/core/lib/support/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 66f111d767..990ef128bd 100644 --- a/src/core/lib/support/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -31,8 +31,8 @@ * */ +#include <grpc/slice_buffer.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice_buffer.h> #include <string.h> @@ -43,35 +43,35 @@ /* grow a buffer; requires GRPC_SLICE_BUFFER_INLINE_ELEMENTS > 1 */ #define GROW(x) (3 * (x) / 2) -static void maybe_embiggen(gpr_slice_buffer *sb) { +static void maybe_embiggen(grpc_slice_buffer *sb) { if (sb->count == sb->capacity) { sb->capacity = GROW(sb->capacity); GPR_ASSERT(sb->capacity > sb->count); if (sb->slices == sb->inlined) { - sb->slices = gpr_malloc(sb->capacity * sizeof(gpr_slice)); - memcpy(sb->slices, sb->inlined, sb->count * sizeof(gpr_slice)); + sb->slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); + memcpy(sb->slices, sb->inlined, sb->count * sizeof(grpc_slice)); } else { - sb->slices = gpr_realloc(sb->slices, sb->capacity * sizeof(gpr_slice)); + sb->slices = gpr_realloc(sb->slices, sb->capacity * sizeof(grpc_slice)); } } } -void gpr_slice_buffer_init(gpr_slice_buffer *sb) { +void grpc_slice_buffer_init(grpc_slice_buffer *sb) { sb->count = 0; sb->length = 0; sb->capacity = GRPC_SLICE_BUFFER_INLINE_ELEMENTS; sb->slices = sb->inlined; } -void gpr_slice_buffer_destroy(gpr_slice_buffer *sb) { - gpr_slice_buffer_reset_and_unref(sb); +void grpc_slice_buffer_destroy(grpc_slice_buffer *sb) { + grpc_slice_buffer_reset_and_unref(sb); if (sb->slices != sb->inlined) { gpr_free(sb->slices); } } -uint8_t *gpr_slice_buffer_tiny_add(gpr_slice_buffer *sb, size_t n) { - gpr_slice *back; +uint8_t *grpc_slice_buffer_tiny_add(grpc_slice_buffer *sb, size_t n) { + grpc_slice *back; uint8_t *out; sb->length += n; @@ -94,16 +94,16 @@ add_new: return back->data.inlined.bytes; } -size_t gpr_slice_buffer_add_indexed(gpr_slice_buffer *sb, gpr_slice s) { +size_t grpc_slice_buffer_add_indexed(grpc_slice_buffer *sb, grpc_slice s) { size_t out = sb->count; maybe_embiggen(sb); sb->slices[out] = s; - sb->length += GPR_SLICE_LENGTH(s); + sb->length += GRPC_SLICE_LENGTH(s); sb->count = out + 1; return out; } -void gpr_slice_buffer_add(gpr_slice_buffer *sb, gpr_slice s) { +void grpc_slice_buffer_add(grpc_slice_buffer *sb, grpc_slice s) { size_t n = sb->count; /* if both the last slice in the slice buffer and the slice being added are inlined (that is, that they carry their data inside the slice data @@ -111,19 +111,20 @@ void gpr_slice_buffer_add(gpr_slice_buffer *sb, gpr_slice s) { into the back slice, preventing many small slices being passed into writes */ if (!s.refcount && n) { - gpr_slice *back = &sb->slices[n - 1]; - if (!back->refcount && back->data.inlined.length < GPR_SLICE_INLINED_SIZE) { + grpc_slice *back = &sb->slices[n - 1]; + if (!back->refcount && + back->data.inlined.length < GRPC_SLICE_INLINED_SIZE) { if (s.data.inlined.length + back->data.inlined.length <= - GPR_SLICE_INLINED_SIZE) { + GRPC_SLICE_INLINED_SIZE) { memcpy(back->data.inlined.bytes + back->data.inlined.length, s.data.inlined.bytes, s.data.inlined.length); back->data.inlined.length = (uint8_t)(back->data.inlined.length + s.data.inlined.length); } else { - size_t cp1 = GPR_SLICE_INLINED_SIZE - back->data.inlined.length; + size_t cp1 = GRPC_SLICE_INLINED_SIZE - back->data.inlined.length; memcpy(back->data.inlined.bytes + back->data.inlined.length, s.data.inlined.bytes, cp1); - back->data.inlined.length = GPR_SLICE_INLINED_SIZE; + back->data.inlined.length = GRPC_SLICE_INLINED_SIZE; maybe_embiggen(sb); back = &sb->slices[n]; sb->count = n + 1; @@ -136,35 +137,35 @@ void gpr_slice_buffer_add(gpr_slice_buffer *sb, gpr_slice s) { return; /* early out */ } } - gpr_slice_buffer_add_indexed(sb, s); + grpc_slice_buffer_add_indexed(sb, s); } -void gpr_slice_buffer_addn(gpr_slice_buffer *sb, gpr_slice *s, size_t n) { +void grpc_slice_buffer_addn(grpc_slice_buffer *sb, grpc_slice *s, size_t n) { size_t i; for (i = 0; i < n; i++) { - gpr_slice_buffer_add(sb, s[i]); + grpc_slice_buffer_add(sb, s[i]); } } -void gpr_slice_buffer_pop(gpr_slice_buffer *sb) { +void grpc_slice_buffer_pop(grpc_slice_buffer *sb) { if (sb->count != 0) { size_t count = --sb->count; - sb->length -= GPR_SLICE_LENGTH(sb->slices[count]); + sb->length -= GRPC_SLICE_LENGTH(sb->slices[count]); } } -void gpr_slice_buffer_reset_and_unref(gpr_slice_buffer *sb) { +void grpc_slice_buffer_reset_and_unref(grpc_slice_buffer *sb) { size_t i; for (i = 0; i < sb->count; i++) { - gpr_slice_unref(sb->slices[i]); + grpc_slice_unref(sb->slices[i]); } sb->count = 0; sb->length = 0; } -void gpr_slice_buffer_swap(gpr_slice_buffer *a, gpr_slice_buffer *b) { +void grpc_slice_buffer_swap(grpc_slice_buffer *a, grpc_slice_buffer *b) { GPR_SWAP(size_t, a->count, b->count); GPR_SWAP(size_t, a->capacity, b->capacity); GPR_SWAP(size_t, a->length, b->length); @@ -172,111 +173,112 @@ void gpr_slice_buffer_swap(gpr_slice_buffer *a, gpr_slice_buffer *b) { if (a->slices == a->inlined) { if (b->slices == b->inlined) { /* swap contents of inlined buffer */ - gpr_slice temp[GRPC_SLICE_BUFFER_INLINE_ELEMENTS]; - memcpy(temp, a->slices, b->count * sizeof(gpr_slice)); - memcpy(a->slices, b->slices, a->count * sizeof(gpr_slice)); - memcpy(b->slices, temp, b->count * sizeof(gpr_slice)); + grpc_slice temp[GRPC_SLICE_BUFFER_INLINE_ELEMENTS]; + memcpy(temp, a->slices, b->count * sizeof(grpc_slice)); + memcpy(a->slices, b->slices, a->count * sizeof(grpc_slice)); + memcpy(b->slices, temp, b->count * sizeof(grpc_slice)); } else { /* a is inlined, b is not - copy a inlined into b, fix pointers */ a->slices = b->slices; b->slices = b->inlined; - memcpy(b->slices, a->inlined, b->count * sizeof(gpr_slice)); + memcpy(b->slices, a->inlined, b->count * sizeof(grpc_slice)); } } else if (b->slices == b->inlined) { /* b is inlined, a is not - copy b inlined int a, fix pointers */ b->slices = a->slices; a->slices = a->inlined; - memcpy(a->slices, b->inlined, a->count * sizeof(gpr_slice)); + memcpy(a->slices, b->inlined, a->count * sizeof(grpc_slice)); } else { /* no inlining: easy swap */ - GPR_SWAP(gpr_slice *, a->slices, b->slices); + GPR_SWAP(grpc_slice *, a->slices, b->slices); } } -void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst) { +void grpc_slice_buffer_move_into(grpc_slice_buffer *src, + grpc_slice_buffer *dst) { /* anything to move? */ if (src->count == 0) { return; } /* anything in dst? */ if (dst->count == 0) { - gpr_slice_buffer_swap(src, dst); + grpc_slice_buffer_swap(src, dst); return; } /* both buffers have data - copy, and reset src */ - gpr_slice_buffer_addn(dst, src->slices, src->count); + grpc_slice_buffer_addn(dst, src->slices, src->count); src->count = 0; src->length = 0; } -void gpr_slice_buffer_move_first(gpr_slice_buffer *src, size_t n, - gpr_slice_buffer *dst) { +void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst) { size_t src_idx; size_t output_len = dst->length + n; size_t new_input_len = src->length - n; GPR_ASSERT(src->length >= n); if (src->length == n) { - gpr_slice_buffer_move_into(src, dst); + grpc_slice_buffer_move_into(src, dst); return; } src_idx = 0; while (src_idx < src->capacity) { - gpr_slice slice = src->slices[src_idx]; - size_t slice_len = GPR_SLICE_LENGTH(slice); + grpc_slice slice = src->slices[src_idx]; + size_t slice_len = GRPC_SLICE_LENGTH(slice); if (n > slice_len) { - gpr_slice_buffer_add(dst, slice); + grpc_slice_buffer_add(dst, slice); n -= slice_len; src_idx++; } else if (n == slice_len) { - gpr_slice_buffer_add(dst, slice); + grpc_slice_buffer_add(dst, slice); src_idx++; break; } else { /* n < slice_len */ - src->slices[src_idx] = gpr_slice_split_tail(&slice, n); - GPR_ASSERT(GPR_SLICE_LENGTH(slice) == n); - GPR_ASSERT(GPR_SLICE_LENGTH(src->slices[src_idx]) == slice_len - n); - gpr_slice_buffer_add(dst, slice); + src->slices[src_idx] = grpc_slice_split_tail(&slice, n); + GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); + GPR_ASSERT(GRPC_SLICE_LENGTH(src->slices[src_idx]) == slice_len - n); + grpc_slice_buffer_add(dst, slice); break; } } GPR_ASSERT(dst->length == output_len); memmove(src->slices, src->slices + src_idx, - sizeof(gpr_slice) * (src->count - src_idx)); + sizeof(grpc_slice) * (src->count - src_idx)); src->count -= src_idx; src->length = new_input_len; GPR_ASSERT(src->count > 0); } -void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n, - gpr_slice_buffer *garbage) { +void grpc_slice_buffer_trim_end(grpc_slice_buffer *sb, size_t n, + grpc_slice_buffer *garbage) { GPR_ASSERT(n <= sb->length); sb->length -= n; for (;;) { size_t idx = sb->count - 1; - gpr_slice slice = sb->slices[idx]; - size_t slice_len = GPR_SLICE_LENGTH(slice); + grpc_slice slice = sb->slices[idx]; + size_t slice_len = GRPC_SLICE_LENGTH(slice); if (slice_len > n) { - sb->slices[idx] = gpr_slice_split_head(&slice, slice_len - n); - gpr_slice_buffer_add_indexed(garbage, slice); + sb->slices[idx] = grpc_slice_split_head(&slice, slice_len - n); + grpc_slice_buffer_add_indexed(garbage, slice); return; } else if (slice_len == n) { - gpr_slice_buffer_add_indexed(garbage, slice); + grpc_slice_buffer_add_indexed(garbage, slice); sb->count = idx; return; } else { - gpr_slice_buffer_add_indexed(garbage, slice); + grpc_slice_buffer_add_indexed(garbage, slice); n -= slice_len; sb->count = idx; } } } -gpr_slice gpr_slice_buffer_take_first(gpr_slice_buffer *sb) { - gpr_slice slice; +grpc_slice grpc_slice_buffer_take_first(grpc_slice_buffer *sb) { + grpc_slice slice; GPR_ASSERT(sb->count > 0); slice = sb->slices[0]; - memmove(&sb->slices[0], &sb->slices[1], (sb->count - 1) * sizeof(gpr_slice)); + memmove(&sb->slices[0], &sb->slices[1], (sb->count - 1) * sizeof(grpc_slice)); sb->count--; - sb->length -= GPR_SLICE_LENGTH(slice); + sb->length -= GRPC_SLICE_LENGTH(slice); return slice; } diff --git a/src/core/lib/slice/slice_string_helpers.c b/src/core/lib/slice/slice_string_helpers.c new file mode 100644 index 0000000000..4731762ece --- /dev/null +++ b/src/core/lib/slice/slice_string_helpers.c @@ -0,0 +1,89 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/slice/slice_string_helpers.h" + +#include <string.h> + +#include <grpc/support/log.h> + +#include "src/core/lib/support/string.h" + +char *grpc_dump_slice(grpc_slice s, uint32_t flags) { + return gpr_dump((const char *)GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s), + flags); +} + +/** Finds the initial (\a begin) and final (\a end) offsets of the next + * substring from \a str + \a read_offset until the next \a sep or the end of \a + * str. + * + * Returns 1 and updates \a begin and \a end. Returns 0 otherwise. */ +static int slice_find_separator_offset(const grpc_slice str, const char *sep, + const size_t read_offset, size_t *begin, + size_t *end) { + size_t i; + const uint8_t *str_ptr = GRPC_SLICE_START_PTR(str) + read_offset; + const size_t str_len = GRPC_SLICE_LENGTH(str) - read_offset; + const size_t sep_len = strlen(sep); + if (str_len < sep_len) { + return 0; + } + + for (i = 0; i <= str_len - sep_len; i++) { + if (memcmp(str_ptr + i, sep, sep_len) == 0) { + *begin = read_offset; + *end = read_offset + i; + return 1; + } + } + return 0; +} + +void grpc_slice_split(grpc_slice str, const char *sep, grpc_slice_buffer *dst) { + const size_t sep_len = strlen(sep); + size_t begin, end; + + GPR_ASSERT(sep_len > 0); + + if (slice_find_separator_offset(str, sep, 0, &begin, &end) != 0) { + do { + grpc_slice_buffer_add_indexed(dst, grpc_slice_sub(str, begin, end)); + } while (slice_find_separator_offset(str, sep, end + sep_len, &begin, + &end) != 0); + grpc_slice_buffer_add_indexed( + dst, grpc_slice_sub(str, end + sep_len, GRPC_SLICE_LENGTH(str))); + } else { /* no sep found, add whole input */ + grpc_slice_buffer_add_indexed(dst, grpc_slice_ref(str)); + } +} diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h b/src/core/lib/slice/slice_string_helpers.h index 06d6dbf29d..151c720777 100644 --- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.h +++ b/src/core/lib/slice/slice_string_helpers.h @@ -31,11 +31,28 @@ * */ -#ifndef GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H -#define GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H +#ifndef GRPC_CORE_LIB_SLICE_SLICE_STRING_HELPERS_H +#define GRPC_CORE_LIB_SLICE_SLICE_STRING_HELPERS_H -#include "src/core/lib/iomgr/ev_posix.h" +#include <stddef.h> -const grpc_event_engine_vtable *grpc_init_poll_and_epoll_posix(void); +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> +#include <grpc/support/port_platform.h> -#endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_AND_EPOLL_POSIX_H */ +#ifdef __cplusplus +extern "C" { +#endif + +/* Calls gpr_dump on a slice. */ +char *grpc_dump_slice(grpc_slice slice, uint32_t flags); + +/** Split \a str by the separator \a sep. Results are stored in \a dst, which + * should be a properly initialized instance. */ +void grpc_slice_split(grpc_slice str, const char *sep, grpc_slice_buffer *dst); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_CORE_LIB_SLICE_SLICE_STRING_HELPERS_H */ diff --git a/src/core/lib/support/env.h b/src/core/lib/support/env.h index ec3959bc6e..6ada5d9390 100644 --- a/src/core/lib/support/env.h +++ b/src/core/lib/support/env.h @@ -36,8 +36,6 @@ #include <stdio.h> -#include <grpc/support/slice.h> - #ifdef __cplusplus extern "C" { #endif diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index d17fb9da4b..dc243bf0bf 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -120,11 +120,6 @@ char *gpr_dump(const char *buf, size_t len, uint32_t flags) { return out.data; } -char *gpr_dump_slice(gpr_slice s, uint32_t flags) { - return gpr_dump((const char *)GPR_SLICE_START_PTR(s), GPR_SLICE_LENGTH(s), - flags); -} - int gpr_parse_bytes_to_uint32(const char *buf, size_t len, uint32_t *result) { uint32_t out = 0; uint32_t new; @@ -239,50 +234,6 @@ char *gpr_strjoin_sep(const char **strs, size_t nstrs, const char *sep, return out; } -/** Finds the initial (\a begin) and final (\a end) offsets of the next - * substring from \a str + \a read_offset until the next \a sep or the end of \a - * str. - * - * Returns 1 and updates \a begin and \a end. Returns 0 otherwise. */ -static int slice_find_separator_offset(const gpr_slice str, const char *sep, - const size_t read_offset, size_t *begin, - size_t *end) { - size_t i; - const uint8_t *str_ptr = GPR_SLICE_START_PTR(str) + read_offset; - const size_t str_len = GPR_SLICE_LENGTH(str) - read_offset; - const size_t sep_len = strlen(sep); - if (str_len < sep_len) { - return 0; - } - - for (i = 0; i <= str_len - sep_len; i++) { - if (memcmp(str_ptr + i, sep, sep_len) == 0) { - *begin = read_offset; - *end = read_offset + i; - return 1; - } - } - return 0; -} - -void gpr_slice_split(gpr_slice str, const char *sep, gpr_slice_buffer *dst) { - const size_t sep_len = strlen(sep); - size_t begin, end; - - GPR_ASSERT(sep_len > 0); - - if (slice_find_separator_offset(str, sep, 0, &begin, &end) != 0) { - do { - gpr_slice_buffer_add_indexed(dst, gpr_slice_sub(str, begin, end)); - } while (slice_find_separator_offset(str, sep, end + sep_len, &begin, - &end) != 0); - gpr_slice_buffer_add_indexed( - dst, gpr_slice_sub(str, end + sep_len, GPR_SLICE_LENGTH(str))); - } else { /* no sep found, add whole input */ - gpr_slice_buffer_add_indexed(dst, gpr_slice_ref(str)); - } -} - void gpr_strvec_init(gpr_strvec *sv) { memset(sv, 0, sizeof(*sv)); } void gpr_strvec_destroy(gpr_strvec *sv) { diff --git a/src/core/lib/support/string.h b/src/core/lib/support/string.h index 9a94e9471c..13891d0b9e 100644 --- a/src/core/lib/support/string.h +++ b/src/core/lib/support/string.h @@ -36,9 +36,9 @@ #include <stddef.h> +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> #ifdef __cplusplus extern "C" { @@ -54,9 +54,6 @@ extern "C" { Result should be freed with gpr_free() */ char *gpr_dump(const char *buf, size_t len, uint32_t flags); -/* Calls gpr_dump on a slice. */ -char *gpr_dump_slice(gpr_slice slice, uint32_t flags); - /* Parses an array of bytes into an integer (base 10). Returns 1 on success, 0 on failure. */ int gpr_parse_bytes_to_uint32(const char *data, size_t length, @@ -98,10 +95,6 @@ char *gpr_strjoin(const char **strs, size_t nstrs, size_t *total_length); char *gpr_strjoin_sep(const char **strs, size_t nstrs, const char *sep, size_t *total_length); -/** Split \a str by the separator \a sep. Results are stored in \a dst, which - * should be a properly initialized instance. */ -void gpr_slice_split(gpr_slice str, const char *sep, gpr_slice_buffer *dst); - /* A vector of strings... for building up a final string one piece at a time */ typedef struct { char **strs; diff --git a/src/core/lib/support/tmpfile.h b/src/core/lib/support/tmpfile.h index 059142ab0f..8952e5ec3d 100644 --- a/src/core/lib/support/tmpfile.h +++ b/src/core/lib/support/tmpfile.h @@ -36,7 +36,7 @@ #include <stdio.h> -#include <grpc/support/slice.h> +#include <grpc/slice.h> #ifdef __cplusplus extern "C" { diff --git a/src/core/lib/surface/byte_buffer.c b/src/core/lib/surface/byte_buffer.c index 054a6e6c58..d646591a65 100644 --- a/src/core/lib/surface/byte_buffer.c +++ b/src/core/lib/surface/byte_buffer.c @@ -35,22 +35,23 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> -grpc_byte_buffer *grpc_raw_byte_buffer_create(gpr_slice *slices, +grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, size_t nslices) { return grpc_raw_compressed_byte_buffer_create(slices, nslices, GRPC_COMPRESS_NONE); } grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - gpr_slice *slices, size_t nslices, grpc_compression_algorithm compression) { + grpc_slice *slices, size_t nslices, + grpc_compression_algorithm compression) { size_t i; grpc_byte_buffer *bb = gpr_malloc(sizeof(grpc_byte_buffer)); bb->type = GRPC_BB_RAW; bb->data.raw.compression = compression; - gpr_slice_buffer_init(&bb->data.raw.slice_buffer); + grpc_slice_buffer_init(&bb->data.raw.slice_buffer); for (i = 0; i < nslices; i++) { - gpr_slice_ref(slices[i]); - gpr_slice_buffer_add(&bb->data.raw.slice_buffer, slices[i]); + grpc_slice_ref(slices[i]); + grpc_slice_buffer_add(&bb->data.raw.slice_buffer, slices[i]); } return bb; } @@ -58,13 +59,13 @@ grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( grpc_byte_buffer_reader *reader) { grpc_byte_buffer *bb = gpr_malloc(sizeof(grpc_byte_buffer)); - gpr_slice slice; + grpc_slice slice; bb->type = GRPC_BB_RAW; bb->data.raw.compression = GRPC_COMPRESS_NONE; - gpr_slice_buffer_init(&bb->data.raw.slice_buffer); + grpc_slice_buffer_init(&bb->data.raw.slice_buffer); while (grpc_byte_buffer_reader_next(reader, &slice)) { - gpr_slice_buffer_add(&bb->data.raw.slice_buffer, slice); + grpc_slice_buffer_add(&bb->data.raw.slice_buffer, slice); } return bb; } @@ -83,7 +84,7 @@ void grpc_byte_buffer_destroy(grpc_byte_buffer *bb) { if (!bb) return; switch (bb->type) { case GRPC_BB_RAW: - gpr_slice_buffer_destroy(&bb->data.raw.slice_buffer); + grpc_slice_buffer_destroy(&bb->data.raw.slice_buffer); break; } gpr_free(bb); diff --git a/src/core/lib/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c index 310bacb2c9..0089959fbb 100644 --- a/src/core/lib/surface/byte_buffer_reader.c +++ b/src/core/lib/surface/byte_buffer_reader.c @@ -37,9 +37,9 @@ #include <grpc/byte_buffer.h> #include <grpc/compression.h> #include <grpc/grpc.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include "src/core/lib/compression/message_compress.h" @@ -56,11 +56,11 @@ static int is_compressed(grpc_byte_buffer *buffer) { int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer) { - gpr_slice_buffer decompressed_slices_buffer; + grpc_slice_buffer decompressed_slices_buffer; reader->buffer_in = buffer; switch (reader->buffer_in->type) { case GRPC_BB_RAW: - gpr_slice_buffer_init(&decompressed_slices_buffer); + grpc_slice_buffer_init(&decompressed_slices_buffer); if (is_compressed(reader->buffer_in)) { if (grpc_msg_decompress(reader->buffer_in->data.raw.compression, &reader->buffer_in->data.raw.slice_buffer, @@ -76,7 +76,7 @@ int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, grpc_raw_byte_buffer_create(decompressed_slices_buffer.slices, decompressed_slices_buffer.count); } - gpr_slice_buffer_destroy(&decompressed_slices_buffer); + grpc_slice_buffer_destroy(&decompressed_slices_buffer); } else { /* not compressed, use the input buffer as output */ reader->buffer_out = reader->buffer_in; } @@ -98,13 +98,13 @@ void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader) { } int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - gpr_slice *slice) { + grpc_slice *slice) { switch (reader->buffer_in->type) { case GRPC_BB_RAW: { - gpr_slice_buffer *slice_buffer; + grpc_slice_buffer *slice_buffer; slice_buffer = &reader->buffer_out->data.raw.slice_buffer; if (reader->current.index < slice_buffer->count) { - *slice = gpr_slice_ref(slice_buffer->slices[reader->current.index]); + *slice = grpc_slice_ref(slice_buffer->slices[reader->current.index]); reader->current.index += 1; return 1; } @@ -114,18 +114,18 @@ int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, return 0; } -gpr_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) { - gpr_slice in_slice; +grpc_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) { + grpc_slice in_slice; size_t bytes_read = 0; const size_t input_size = grpc_byte_buffer_length(reader->buffer_out); - gpr_slice out_slice = gpr_slice_malloc(input_size); - uint8_t *const outbuf = GPR_SLICE_START_PTR(out_slice); /* just an alias */ + grpc_slice out_slice = grpc_slice_malloc(input_size); + uint8_t *const outbuf = GRPC_SLICE_START_PTR(out_slice); /* just an alias */ while (grpc_byte_buffer_reader_next(reader, &in_slice) != 0) { - const size_t slice_length = GPR_SLICE_LENGTH(in_slice); - memcpy(&(outbuf[bytes_read]), GPR_SLICE_START_PTR(in_slice), slice_length); + const size_t slice_length = GRPC_SLICE_LENGTH(in_slice); + memcpy(&(outbuf[bytes_read]), GRPC_SLICE_START_PTR(in_slice), slice_length); bytes_read += slice_length; - gpr_slice_unref(in_slice); + grpc_slice_unref(in_slice); GPR_ASSERT(bytes_read <= input_size); } return out_slice; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 3f4afd9a29..1e0f3eeca5 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -39,9 +39,9 @@ #include <grpc/compression.h> #include <grpc/grpc.h> +#include <grpc/slice.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -49,6 +49,7 @@ #include "src/core/lib/compression/algorithm_metadata.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" @@ -122,6 +123,7 @@ struct grpc_call { grpc_channel *channel; grpc_call *parent; grpc_call *first_child; + gpr_timespec start_time; /* TODO(ctiller): share with cq if possible? */ gpr_mu mu; @@ -184,7 +186,7 @@ struct grpc_call { grpc_slice_buffer_stream sending_stream; grpc_byte_stream *receiving_stream; grpc_byte_buffer **receiving_buffer; - gpr_slice receiving_slice; + grpc_slice receiving_slice; grpc_closure receiving_slice_ready; grpc_closure receiving_stream_ready; grpc_closure receiving_initial_metadata_ready; @@ -239,6 +241,7 @@ grpc_error *grpc_call_create(const grpc_call_create_args *args, call->channel = args->channel; call->cq = args->cq; call->parent = args->parent_call; + call->start_time = gpr_now(GPR_CLOCK_MONOTONIC); /* Always support no compression */ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); call->is_client = args->server_transport_data == NULL; @@ -311,10 +314,10 @@ grpc_error *grpc_call_create(const grpc_call_create_args *args, GRPC_CHANNEL_INTERNAL_REF(args->channel, "call"); /* initial refcount dropped by grpc_call_destroy */ - grpc_error *error = - grpc_call_stack_init(&exec_ctx, channel_stack, 1, destroy_call, call, - call->context, args->server_transport_data, path, - send_deadline, CALL_STACK_FROM_CALL(call)); + grpc_error *error = grpc_call_stack_init( + &exec_ctx, channel_stack, 1, destroy_call, call, call->context, + args->server_transport_data, path, call->start_time, send_deadline, + CALL_STACK_FROM_CALL(call)); if (error != GRPC_ERROR_NONE) { grpc_status_code status; const char *error_str; @@ -427,6 +430,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, get_final_status(call, set_status_value_directly, &c->final_info.final_status); + c->final_info.stats.latency = + gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->final_info, c); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call"); @@ -493,8 +498,8 @@ static void destroy_encodings_accepted_by_peer(void *p) { return; } static void set_encodings_accepted_by_peer(grpc_call *call, grpc_mdelem *mdel) { size_t i; grpc_compression_algorithm algorithm; - gpr_slice_buffer accept_encoding_parts; - gpr_slice accept_encoding_slice; + grpc_slice_buffer accept_encoding_parts; + grpc_slice accept_encoding_slice; void *accepted_user_data; accepted_user_data = @@ -506,23 +511,23 @@ static void set_encodings_accepted_by_peer(grpc_call *call, grpc_mdelem *mdel) { } accept_encoding_slice = mdel->value->slice; - gpr_slice_buffer_init(&accept_encoding_parts); - gpr_slice_split(accept_encoding_slice, ",", &accept_encoding_parts); + grpc_slice_buffer_init(&accept_encoding_parts); + grpc_slice_split(accept_encoding_slice, ",", &accept_encoding_parts); /* No need to zero call->encodings_accepted_by_peer: grpc_call_create already * zeroes the whole grpc_call */ /* Always support no compression */ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); for (i = 0; i < accept_encoding_parts.count; i++) { - const gpr_slice *accept_encoding_entry_slice = + const grpc_slice *accept_encoding_entry_slice = &accept_encoding_parts.slices[i]; if (grpc_compression_algorithm_parse( - (const char *)GPR_SLICE_START_PTR(*accept_encoding_entry_slice), - GPR_SLICE_LENGTH(*accept_encoding_entry_slice), &algorithm)) { + (const char *)GRPC_SLICE_START_PTR(*accept_encoding_entry_slice), + GRPC_SLICE_LENGTH(*accept_encoding_entry_slice), &algorithm)) { GPR_BITSET(&call->encodings_accepted_by_peer, algorithm); } else { char *accept_encoding_entry_str = - gpr_dump_slice(*accept_encoding_entry_slice, GPR_DUMP_ASCII); + grpc_dump_slice(*accept_encoding_entry_slice, GPR_DUMP_ASCII); gpr_log(GPR_ERROR, "Invalid entry in accept encoding metadata: '%s'. Ignoring.", accept_encoding_entry_str); @@ -530,7 +535,7 @@ static void set_encodings_accepted_by_peer(grpc_call *call, grpc_mdelem *mdel) { } } - gpr_slice_buffer_destroy(&accept_encoding_parts); + grpc_slice_buffer_destroy(&accept_encoding_parts); grpc_mdelem_set_user_data( mdel, destroy_encodings_accepted_by_peer, @@ -551,14 +556,14 @@ static void get_final_details(grpc_call *call, char **out_details, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (call->status[i].is_set) { if (call->status[i].details) { - gpr_slice details = call->status[i].details->slice; - size_t len = GPR_SLICE_LENGTH(details); + grpc_slice details = call->status[i].details->slice; + size_t len = GRPC_SLICE_LENGTH(details); if (len + 1 > *out_details_capacity) { *out_details_capacity = GPR_MAX(len + 1, *out_details_capacity * 3 / 2); *out_details = gpr_realloc(*out_details, *out_details_capacity); } - memcpy(*out_details, GPR_SLICE_START_PTR(details), len); + memcpy(*out_details, GRPC_SLICE_START_PTR(details), len); (*out_details)[len] = 0; } else { goto no_details; @@ -899,7 +904,7 @@ static uint32_t decode_status(grpc_mdelem *md) { status = ((uint32_t)(intptr_t)user_data) - STATUS_OFFSET; } else { if (!gpr_parse_bytes_to_uint32(grpc_mdstr_as_c_string(md->value), - GPR_SLICE_LENGTH(md->value->slice), + GRPC_SLICE_LENGTH(md->value->slice), &status)) { status = GRPC_STATUS_UNKNOWN; /* could not parse status code */ } @@ -952,7 +957,7 @@ static grpc_mdelem *publish_app_metadata(grpc_call *call, grpc_mdelem *elem, mdusr = &dest->metadata[dest->count++]; mdusr->key = grpc_mdstr_as_c_string(elem->key); mdusr->value = grpc_mdstr_as_c_string(elem->value); - mdusr->value_length = GPR_SLICE_LENGTH(elem->value->slice); + mdusr->value_length = GRPC_SLICE_LENGTH(elem->value->slice); GPR_TIMER_END("publish_app_metadata", 0); return elem; } @@ -1084,8 +1089,8 @@ static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, &call->receiving_slice, remaining, &call->receiving_slice_ready)) { - gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); } else { return; } @@ -1098,8 +1103,8 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_call *call = bctl->call; if (error == GRPC_ERROR_NONE) { - gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); continue_receiving_slices(exec_ctx, bctl); } else { if (grpc_trace_operation_failures) { @@ -1460,6 +1465,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_stream_init( &call->sending_stream, &op->data.send_message->data.raw.slice_buffer, op->flags); + /* If the outgoing buffer is already compressed, mark it as so in the + flags. These will be picked up by the compression filter and further + (wasteful) attempts at compression skipped. */ + if (op->data.send_message->data.raw.compression > GRPC_COMPRESS_NONE) { + call->sending_stream.base.flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } stream_op->send_message = &call->sending_stream.base; break; case GRPC_OP_SEND_CLOSE_FROM_CLIENT: diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 92d783b78d..1389df6886 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -175,6 +175,15 @@ char *grpc_channel_get_target(grpc_channel *channel) { return gpr_strdup(channel->target); } +void grpc_channel_get_info(grpc_channel *channel, + const grpc_channel_info *channel_info) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_channel_element *elem = + grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0); + elem->filter->get_channel_info(&exec_ctx, elem, channel_info); + grpc_exec_ctx_finish(&exec_ctx); +} + static grpc_call *grpc_channel_create_call_internal( grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask, grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative, diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c index d32c884e8e..d0df8e7e17 100644 --- a/src/core/lib/surface/lame_client.c +++ b/src/core/lib/surface/lame_client.c @@ -88,6 +88,10 @@ static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { return NULL; } +static void lame_get_channel_info(grpc_exec_ctx *exec_ctx, + grpc_channel_element *elem, + const grpc_channel_info *channel_info) {} + static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_transport_op *op) { @@ -140,6 +144,7 @@ const grpc_channel_filter grpc_lame_filter = { init_channel_elem, destroy_channel_elem, lame_get_peer, + lame_get_channel_info, "lame-client", }; diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 3a90308058..89dd825460 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -264,13 +264,13 @@ static void channel_broadcaster_init(grpc_server *s, channel_broadcaster *cb) { struct shutdown_cleanup_args { grpc_closure closure; - gpr_slice slice; + grpc_slice slice; }; static void shutdown_cleanup(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { struct shutdown_cleanup_args *a = arg; - gpr_slice_unref(a->slice); + grpc_slice_unref(a->slice); gpr_free(a); } @@ -283,7 +283,7 @@ static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel, op->send_goaway = send_goaway; op->set_accept_stream = true; - sc->slice = gpr_slice_from_copied_string("Server shutdown"); + sc->slice = grpc_slice_from_copied_string("Server shutdown"); op->goaway_message = &sc->slice; op->goaway_status = GRPC_STATUS_OK; op->disconnect_with_error = send_disconnect; @@ -459,8 +459,8 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand, } static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) { - gpr_slice slice = value->slice; - size_t len = GPR_SLICE_LENGTH(slice); + grpc_slice slice = value->slice; + size_t len = GRPC_SLICE_LENGTH(slice); if (len + 1 > *capacity) { *capacity = GPR_MAX(len + 1, *capacity * 2); @@ -965,6 +965,7 @@ const grpc_channel_filter grpc_server_top_filter = { init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + grpc_channel_next_get_info, "server", }; diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c index 41242684da..0db8b41aa9 100644 --- a/src/core/lib/surface/version.c +++ b/src/core/lib/surface/version.c @@ -36,6 +36,6 @@ #include <grpc/grpc.h> -const char *grpc_version_string(void) { return "1.1.0-dev"; } +const char *grpc_version_string(void) { return "2.0.0-dev"; } const char *grpc_g_stands_for(void) { return "good"; } diff --git a/src/core/lib/transport/byte_stream.c b/src/core/lib/transport/byte_stream.c index 2f6c75cb6a..2f1d7b7c60 100644 --- a/src/core/lib/transport/byte_stream.c +++ b/src/core/lib/transport/byte_stream.c @@ -38,7 +38,7 @@ #include <grpc/support/log.h> int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, gpr_slice *slice, + grpc_byte_stream *byte_stream, grpc_slice *slice, size_t max_size_hint, grpc_closure *on_complete) { return byte_stream->next(exec_ctx, byte_stream, slice, max_size_hint, on_complete); @@ -53,11 +53,11 @@ void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, static int slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, - gpr_slice *slice, size_t max_size_hint, + grpc_slice *slice, size_t max_size_hint, grpc_closure *on_complete) { grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; GPR_ASSERT(stream->cursor < stream->backing_buffer->count); - *slice = gpr_slice_ref(stream->backing_buffer->slices[stream->cursor]); + *slice = grpc_slice_ref(stream->backing_buffer->slices[stream->cursor]); stream->cursor++; return 1; } @@ -66,7 +66,7 @@ static void slice_buffer_stream_destroy(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream) {} void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, - gpr_slice_buffer *slice_buffer, + grpc_slice_buffer *slice_buffer, uint32_t flags) { GPR_ASSERT(slice_buffer->length <= UINT32_MAX); stream->base.length = (uint32_t)slice_buffer->length; diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index e64dce6283..1fdd5b4d77 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H #define GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H -#include <grpc/support/slice_buffer.h> +#include <grpc/slice_buffer.h> #include "src/core/lib/iomgr/exec_ctx.h" /** Internal bit flag for grpc_begin_message's \a flags signaling the use of @@ -50,7 +50,7 @@ struct grpc_byte_stream { uint32_t length; uint32_t flags; int (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, - gpr_slice *slice, size_t max_size_hint, + grpc_slice *slice, size_t max_size_hint, grpc_closure *on_complete); void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream); }; @@ -65,7 +65,7 @@ struct grpc_byte_stream { * once a slice is returned into *slice, it is owned by the caller. */ int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, gpr_slice *slice, + grpc_byte_stream *byte_stream, grpc_slice *slice, size_t max_size_hint, grpc_closure *on_complete); void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -74,12 +74,12 @@ void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, /* grpc_byte_stream that wraps a slice buffer */ typedef struct grpc_slice_buffer_stream { grpc_byte_stream base; - gpr_slice_buffer *backing_buffer; + grpc_slice_buffer *backing_buffer; size_t cursor; } grpc_slice_buffer_stream; void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, - gpr_slice_buffer *slice_buffer, + grpc_slice_buffer *slice_buffer, uint32_t flags); #endif /* GRPC_CORE_LIB_TRANSPORT_BYTE_STREAM_H */ diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index fdb5307814..89072879d9 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -43,6 +43,8 @@ int grpc_connectivity_state_trace = 0; const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { + case GRPC_CHANNEL_INIT: + return "INIT"; case GRPC_CHANNEL_IDLE: return "IDLE"; case GRPC_CHANNEL_CONNECTING: @@ -159,6 +161,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, grpc_error_free_string(error_string); } switch (state) { + case GRPC_CHANNEL_INIT: case GRPC_CHANNEL_CONNECTING: case GRPC_CHANNEL_IDLE: case GRPC_CHANNEL_READY: diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 4b40c275ad..fac19b91d9 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -51,7 +51,7 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" -gpr_slice (*grpc_chttp2_base64_encode_and_huffman_compress)(gpr_slice input); +grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)(grpc_slice input); /* There are two kinds of mdelem and mdstr instances. * Static instances are declared in static_metadata.{h,c} and @@ -85,16 +85,16 @@ typedef void (*destroy_user_data_func)(void *user_data); /* Shadow structure for grpc_mdstr for non-static values */ typedef struct internal_string { /* must be byte compatible with grpc_mdstr */ - gpr_slice slice; + grpc_slice slice; uint32_t hash; /* private only data */ gpr_atm refcnt; uint8_t has_base64_and_huffman_encoded; - gpr_slice_refcount refcount; + grpc_slice_refcount refcount; - gpr_slice base64_and_huffman; + grpc_slice base64_and_huffman; gpr_atm size_in_decoder_table; @@ -174,7 +174,7 @@ void grpc_mdctx_global_init(void) { grpc_mdstr *elem = &grpc_static_mdstr_table[i]; const char *str = grpc_static_metadata_strings[i]; uint32_t hash = gpr_murmur_hash3(str, strlen(str), g_hash_seed); - *(gpr_slice *)&elem->slice = gpr_slice_from_static_string(str); + *(grpc_slice *)&elem->slice = grpc_slice_from_static_string(str); *(uint32_t *)&elem->hash = hash; for (j = 0;; j++) { size_t idx = (hash + j) % GPR_ARRAY_SIZE(g_static_strtab); @@ -321,7 +321,7 @@ static void internal_destroy_string(strtab_shard *shard, internal_string *is) { internal_string *cur; GPR_TIMER_BEGIN("internal_destroy_string", 0); if (is->has_base64_and_huffman_encoded) { - gpr_slice_unref(is->base64_and_huffman); + grpc_slice_unref(is->base64_and_huffman); } for (prev_next = &shard->strs[TABLE_IDX(is->hash, LOG2_STRTAB_SHARD_COUNT, shard->capacity)], @@ -350,10 +350,10 @@ grpc_mdstr *grpc_mdstr_from_string(const char *str) { return grpc_mdstr_from_buffer((const uint8_t *)str, strlen(str)); } -grpc_mdstr *grpc_mdstr_from_slice(gpr_slice slice) { - grpc_mdstr *result = grpc_mdstr_from_buffer(GPR_SLICE_START_PTR(slice), - GPR_SLICE_LENGTH(slice)); - gpr_slice_unref(slice); +grpc_mdstr *grpc_mdstr_from_slice(grpc_slice slice) { + grpc_mdstr *result = grpc_mdstr_from_buffer(GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_LENGTH(slice)); + grpc_slice_unref(slice); return result; } @@ -373,9 +373,9 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { idx = (hash + i) % GPR_ARRAY_SIZE(g_static_strtab); ss = g_static_strtab[idx]; if (ss == NULL) break; - if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length && + if (ss->hash == hash && GRPC_SLICE_LENGTH(ss->slice) == length && (length == 0 || - 0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) { + 0 == memcmp(buf, GRPC_SLICE_START_PTR(ss->slice), length))) { GPR_TIMER_END("grpc_mdstr_from_buffer", 0); return ss; } @@ -386,8 +386,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { /* search for an existing string */ idx = TABLE_IDX(hash, LOG2_STRTAB_SHARD_COUNT, shard->capacity); for (s = shard->strs[idx]; s; s = s->bucket_next) { - if (s->hash == hash && GPR_SLICE_LENGTH(s->slice) == length && - 0 == memcmp(buf, GPR_SLICE_START_PTR(s->slice), length)) { + if (s->hash == hash && GRPC_SLICE_LENGTH(s->slice) == length && + 0 == memcmp(buf, GRPC_SLICE_START_PTR(s->slice), length)) { if (gpr_atm_full_fetch_add(&s->refcnt, 1) == 0) { /* If we get here, we've added a ref to something that was about to * die - drop it immediately. @@ -404,7 +404,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { } /* not found: create a new string */ - if (length + 1 < GPR_SLICE_INLINED_SIZE) { + if (length + 1 < GRPC_SLICE_INLINED_SIZE) { /* string data goes directly into the slice */ s = gpr_malloc(sizeof(internal_string)); gpr_atm_rel_store(&s->refcnt, 1); @@ -589,7 +589,7 @@ grpc_mdelem *grpc_mdelem_from_strings(const char *key, const char *value) { grpc_mdstr_from_string(value)); } -grpc_mdelem *grpc_mdelem_from_slices(gpr_slice key, gpr_slice value) { +grpc_mdelem *grpc_mdelem_from_slices(grpc_slice key, grpc_slice value) { return grpc_mdelem_from_metadata_strings(grpc_mdstr_from_slice(key), grpc_mdstr_from_slice(value)); } @@ -607,12 +607,12 @@ static size_t get_base64_encoded_size(size_t raw_length) { } size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem *elem) { - size_t overhead_and_key = 32 + GPR_SLICE_LENGTH(elem->key->slice); - size_t value_len = GPR_SLICE_LENGTH(elem->value->slice); + size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(elem->key->slice); + size_t value_len = GRPC_SLICE_LENGTH(elem->value->slice); if (is_mdstr_static(elem->value)) { if (grpc_is_binary_header( - (const char *)GPR_SLICE_START_PTR(elem->key->slice), - GPR_SLICE_LENGTH(elem->key->slice))) { + (const char *)GRPC_SLICE_START_PTR(elem->key->slice), + GRPC_SLICE_LENGTH(elem->key->slice))) { return overhead_and_key + get_base64_encoded_size(value_len); } else { return overhead_and_key + value_len; @@ -622,8 +622,8 @@ size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem *elem) { gpr_atm current_size = gpr_atm_acq_load(&is->size_in_decoder_table); if (current_size == SIZE_IN_DECODER_TABLE_NOT_SET) { if (grpc_is_binary_header( - (const char *)GPR_SLICE_START_PTR(elem->key->slice), - GPR_SLICE_LENGTH(elem->key->slice))) { + (const char *)GRPC_SLICE_START_PTR(elem->key->slice), + GRPC_SLICE_LENGTH(elem->key->slice))) { current_size = (gpr_atm)get_base64_encoded_size(value_len); } else { current_size = (gpr_atm)value_len; @@ -679,7 +679,7 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { } const char *grpc_mdstr_as_c_string(const grpc_mdstr *s) { - return (const char *)GPR_SLICE_START_PTR(s->slice); + return (const char *)GRPC_SLICE_START_PTR(s->slice); } size_t grpc_mdstr_length(const grpc_mdstr *s) { return GRPC_MDSTR_LENGTH(s); } @@ -687,6 +687,11 @@ size_t grpc_mdstr_length(const grpc_mdstr *s) { return GRPC_MDSTR_LENGTH(s); } grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) { internal_string *s = (internal_string *)gs; if (is_mdstr_static(gs)) return gs; +#ifdef GRPC_METADATA_REFCOUNT_DEBUG + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "STR REF:%p:%zu->%zu: '%s'", + (void *)s, gpr_atm_no_barrier_load(&s->refcnt), + gpr_atm_no_barrier_load(&s->refcnt) + 1, grpc_mdstr_as_c_string(gs)); +#endif GPR_ASSERT(gpr_atm_full_fetch_add(&s->refcnt, 1) > 0); return gs; } @@ -694,6 +699,11 @@ grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) { void grpc_mdstr_unref(grpc_mdstr *gs DEBUG_ARGS) { internal_string *s = (internal_string *)gs; if (is_mdstr_static(gs)) return; +#ifdef GRPC_METADATA_REFCOUNT_DEBUG + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "STR UNREF:%p:%zu->%zu: '%s'", + (void *)s, gpr_atm_no_barrier_load(&s->refcnt), + gpr_atm_no_barrier_load(&s->refcnt) - 1, grpc_mdstr_as_c_string(gs)); +#endif if (1 == gpr_atm_full_fetch_add(&s->refcnt, -1)) { strtab_shard *shard = &g_strtab_shard[SHARD_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT)]; @@ -718,8 +728,8 @@ void *grpc_mdelem_get_user_data(grpc_mdelem *md, void (*destroy_func)(void *)) { return result; } -void grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), - void *user_data) { +void *grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), + void *user_data) { internal_metadata *im = (internal_metadata *)md; GPR_ASSERT(!is_mdelem_static(md)); GPR_ASSERT((user_data == NULL) == (destroy_func == NULL)); @@ -730,16 +740,17 @@ void grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), if (destroy_func != NULL) { destroy_func(user_data); } - return; + return (void *)gpr_atm_no_barrier_load(&im->user_data); } gpr_atm_no_barrier_store(&im->user_data, (gpr_atm)user_data); gpr_atm_rel_store(&im->destroy_user_data, (gpr_atm)destroy_func); gpr_mu_unlock(&im->mu_user_data); + return user_data; } -gpr_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *gs) { +grpc_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *gs) { internal_string *s = (internal_string *)gs; - gpr_slice slice; + grpc_slice slice; strtab_shard *shard = &g_strtab_shard[SHARD_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT)]; gpr_mu_lock(&shard->mu); diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 71eff0acf2..a4955a1ea7 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_TRANSPORT_METADATA_H #define GRPC_CORE_LIB_TRANSPORT_METADATA_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include <grpc/support/useful.h> #ifdef __cplusplus @@ -77,7 +77,7 @@ typedef struct grpc_mdelem grpc_mdelem; /* if changing this, make identical changes in internal_string in metadata.c */ struct grpc_mdstr { - const gpr_slice slice; + const grpc_slice slice; const uint32_t hash; /* there is a private part to this in metadata.c */ }; @@ -96,12 +96,12 @@ void grpc_test_only_set_metadata_hash_seed(uint32_t seed); clients may have handy */ grpc_mdstr *grpc_mdstr_from_string(const char *str); /* Unrefs the slice. */ -grpc_mdstr *grpc_mdstr_from_slice(gpr_slice slice); +grpc_mdstr *grpc_mdstr_from_slice(grpc_slice slice); grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *str, size_t length); /* Returns a borrowed slice from the mdstr with its contents base64 encoded and huffman compressed */ -gpr_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *str); +grpc_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *str); /* Constructors for grpc_mdelem instances; take a variety of data types that clients may have handy */ @@ -109,7 +109,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *key, grpc_mdstr *value); grpc_mdelem *grpc_mdelem_from_strings(const char *key, const char *value); /* Unrefs the slices. */ -grpc_mdelem *grpc_mdelem_from_slices(gpr_slice key, gpr_slice value); +grpc_mdelem *grpc_mdelem_from_slices(grpc_slice key, grpc_slice value); grpc_mdelem *grpc_mdelem_from_string_and_buffer(const char *key, const uint8_t *value, size_t value_length); @@ -120,8 +120,8 @@ size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem *elem); is used as a type tag and is checked during user_data fetch. */ void *grpc_mdelem_get_user_data(grpc_mdelem *md, void (*if_destroy_func)(void *)); -void grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), - void *user_data); +void *grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), + void *user_data); /* Reference counting */ //#define GRPC_METADATA_REFCOUNT_DEBUG @@ -149,7 +149,7 @@ void grpc_mdelem_unref(grpc_mdelem *md); Does not promise that the returned string has no embedded nulls however. */ const char *grpc_mdstr_as_c_string(const grpc_mdstr *s); -#define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) +#define GRPC_MDSTR_LENGTH(s) (GRPC_SLICE_LENGTH(s->slice)) /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */ #define GRPC_MDELEM_LENGTH(e) \ @@ -165,8 +165,8 @@ void grpc_mdctx_global_init(void); void grpc_mdctx_global_shutdown(void); /* Implementation provided by chttp2_transport */ -extern gpr_slice (*grpc_chttp2_base64_encode_and_huffman_compress)( - gpr_slice input); +extern grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)( + grpc_slice input); #ifdef __cplusplus } diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h index 0424b4db98..7a9ccb4bc8 100644 --- a/src/core/lib/transport/metadata_batch.h +++ b/src/core/lib/transport/metadata_batch.h @@ -37,8 +37,8 @@ #include <stdbool.h> #include <grpc/grpc.h> +#include <grpc/slice.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> #include <grpc/support/time.h> #include "src/core/lib/transport/metadata.h" diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 75aec7a5b4..b448126da8 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -40,6 +40,7 @@ #include <grpc/support/log.h> #include <grpc/support/sync.h> +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/transport_impl.h" @@ -159,6 +160,11 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, return transport->vtable->get_peer(exec_ctx, transport); } +grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, + grpc_transport *transport) { + return transport->vtable->get_endpoint(exec_ctx, transport); +} + void grpc_transport_stream_op_finish_with_failure(grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op, grpc_error *error) { @@ -207,21 +213,21 @@ void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op, void grpc_transport_stream_op_add_cancellation_with_message( grpc_transport_stream_op *op, grpc_status_code status, - gpr_slice *optional_message) { + grpc_slice *optional_message) { GPR_ASSERT(status != GRPC_STATUS_OK); if (op->cancel_error != GRPC_ERROR_NONE) { if (optional_message) { - gpr_slice_unref(*optional_message); + grpc_slice_unref(*optional_message); } return; } grpc_error *error; if (optional_message != NULL) { - char *msg = gpr_dump_slice(*optional_message, GPR_DUMP_ASCII); + char *msg = grpc_dump_slice(*optional_message, GPR_DUMP_ASCII); error = grpc_error_set_str(GRPC_ERROR_CREATE(msg), GRPC_ERROR_STR_GRPC_MESSAGE, msg); gpr_free(msg); - gpr_slice_unref(*optional_message); + grpc_slice_unref(*optional_message); } else { error = GRPC_ERROR_CREATE("Call cancelled"); } @@ -231,22 +237,22 @@ void grpc_transport_stream_op_add_cancellation_with_message( void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op, grpc_status_code status, - gpr_slice *optional_message) { + grpc_slice *optional_message) { GPR_ASSERT(status != GRPC_STATUS_OK); if (op->cancel_error != GRPC_ERROR_NONE || op->close_error != GRPC_ERROR_NONE) { if (optional_message) { - gpr_slice_unref(*optional_message); + grpc_slice_unref(*optional_message); } return; } grpc_error *error; if (optional_message != NULL) { - char *msg = gpr_dump_slice(*optional_message, GPR_DUMP_ASCII); + char *msg = grpc_dump_slice(*optional_message, GPR_DUMP_ASCII); error = grpc_error_set_str(GRPC_ERROR_CREATE(msg), GRPC_ERROR_STR_GRPC_MESSAGE, msg); gpr_free(msg); - gpr_slice_unref(*optional_message); + grpc_slice_unref(*optional_message); } else { error = GRPC_ERROR_CREATE("Call force closed"); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 50253ebad1..96c26749c8 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -37,6 +37,7 @@ #include <stddef.h> #include "src/core/lib/channel/context.h" +#include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" @@ -181,7 +182,7 @@ typedef struct grpc_transport_op { bool send_goaway; /** what should the goaway contain? */ grpc_status_code goaway_status; - gpr_slice *goaway_message; + grpc_slice *goaway_message; /** set the callback for accepting new streams; this is a permanent callback, unlike the other one-shot closures. If true, the callback is set to set_accept_stream_fn, with its @@ -249,11 +250,11 @@ void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op, void grpc_transport_stream_op_add_cancellation_with_message( grpc_transport_stream_op *op, grpc_status_code status, - gpr_slice *optional_message); + grpc_slice *optional_message); void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op, grpc_status_code status, - gpr_slice *optional_message); + grpc_slice *optional_message); char *grpc_transport_stream_op_string(grpc_transport_stream_op *op); char *grpc_transport_op_string(grpc_transport_op *op); @@ -283,7 +284,7 @@ void grpc_transport_ping(grpc_transport *transport, grpc_closure *cb); /* Advise peer of pending connection termination. */ void grpc_transport_goaway(grpc_transport *transport, grpc_status_code status, - gpr_slice debug_data); + grpc_slice debug_data); /* Close a transport. Aborts all open streams. */ void grpc_transport_close(grpc_transport *transport); @@ -295,6 +296,10 @@ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport); char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *transport); +/* Get the endpoint used by \a transport */ +grpc_endpoint *grpc_transport_get_endpoint(grpc_exec_ctx *exec_ctx, + grpc_transport *transport); + /* Allocate a grpc_transport_op, and preconfigure the on_consumed closure to \a on_consumed and then delete the returned transport op */ grpc_transport_op *grpc_make_transport_op(grpc_closure *on_consumed); diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index fc7140671b..8553148c35 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -74,6 +74,9 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_get_peer */ char *(*get_peer)(grpc_exec_ctx *exec_ctx, grpc_transport *self); + + /* implementation of grpc_transport_get_endpoint */ + grpc_endpoint *(*get_endpoint)(grpc_exec_ctx *exec_ctx, grpc_transport *self); } grpc_transport_vtable; /* an instance of a grpc transport */ diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 533ec52077..58d6ad508e 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -40,6 +40,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/connectivity_state.h" @@ -48,12 +49,12 @@ static void put_metadata(gpr_strvec *b, grpc_mdelem *md) { gpr_strvec_add(b, gpr_strdup("key=")); - gpr_strvec_add(b, - gpr_dump_slice(md->key->slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); + gpr_strvec_add( + b, grpc_dump_slice(md->key->slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); gpr_strvec_add(b, gpr_strdup(" value=")); gpr_strvec_add( - b, gpr_dump_slice(md->value->slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); + b, grpc_dump_slice(md->value->slice, GPR_DUMP_HEX | GPR_DUMP_ASCII)); } static void put_metadata_list(gpr_strvec *b, grpc_metadata_batch md) { @@ -175,8 +176,8 @@ char *grpc_transport_op_string(grpc_transport_op *op) { first = false; char *msg = op->goaway_message == NULL ? "null" - : gpr_dump_slice(*op->goaway_message, - GPR_DUMP_ASCII | GPR_DUMP_HEX); + : grpc_dump_slice(*op->goaway_message, + GPR_DUMP_ASCII | GPR_DUMP_HEX); gpr_asprintf(&tmp, "SEND_GOAWAY:status=%d:msg=%s", op->goaway_status, msg); if (op->goaway_message != NULL) gpr_free(msg); gpr_strvec_add(&b, tmp); |