diff options
Diffstat (limited to 'src/core')
92 files changed, 5743 insertions, 5099 deletions
diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c index 872543057e..daa1a7ac61 100644 --- a/src/core/census/grpc_filter.c +++ b/src/core/census/grpc_filter.c @@ -53,28 +53,24 @@ typedef struct call_data { int error; /* recv callback */ - grpc_stream_op_buffer *recv_ops; + grpc_metadata_batch *recv_initial_metadata; grpc_closure *on_done_recv; + grpc_closure finish_recv; } call_data; typedef struct channel_data { grpc_mdstr *path_str; /* pointer to meta data str with key == ":path" */ } channel_data; -static void extract_and_annotate_method_tag(grpc_stream_op_buffer *sopb, +static void extract_and_annotate_method_tag(grpc_metadata_batch *md, call_data *calld, channel_data *chand) { grpc_linked_mdelem *m; - size_t i; - for (i = 0; i < sopb->nops; i++) { - grpc_stream_op *op = &sopb->ops[i]; - if (op->type != GRPC_OP_METADATA) continue; - for (m = op->data.metadata.list.head; m != NULL; m = m->next) { - if (m->md->key == chand->path_str) { - gpr_log(GPR_DEBUG, "%s", - (const char *)GPR_SLICE_START_PTR(m->md->value->slice)); - /* Add method tag here */ - } + for (m = md->list.head; m != NULL; m = m->next) { + if (m->md->key == chand->path_str) { + gpr_log(GPR_DEBUG, "%s", + (const char *)GPR_SLICE_START_PTR(m->md->value->slice)); + /* Add method tag here */ } } } @@ -83,8 +79,8 @@ static void client_mutate_op(grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - if (op->send_ops) { - extract_and_annotate_method_tag(op->send_ops, calld, chand); + if (op->send_initial_metadata) { + extract_and_annotate_method_tag(op->send_initial_metadata, calld, chand); } } @@ -101,7 +97,7 @@ static void server_on_done_recv(grpc_exec_ctx *exec_ctx, void *ptr, call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; if (success) { - extract_and_annotate_method_tag(calld->recv_ops, calld, chand); + extract_and_annotate_method_tag(calld->recv_initial_metadata, calld, chand); } calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success); } @@ -109,11 +105,11 @@ static void server_on_done_recv(grpc_exec_ctx *exec_ctx, void *ptr, static void server_mutate_op(grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; - if (op->recv_ops) { + if (op->recv_initial_metadata) { /* substitute our callback for the op callback */ - calld->recv_ops = op->recv_ops; - calld->on_done_recv = op->on_done_recv; - op->on_done_recv = calld->on_done_recv; + calld->recv_initial_metadata = op->recv_initial_metadata; + calld->on_done_recv = op->on_complete; + op->on_complete = &calld->finish_recv; } } @@ -128,13 +124,11 @@ static void server_start_transport_op(grpc_exec_ctx *exec_ctx, static void client_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *d = elem->call_data; GPR_ASSERT(d != NULL); memset(d, 0, sizeof(*d)); d->start_ts = gpr_now(GPR_CLOCK_REALTIME); - if (initial_op) client_mutate_op(elem, initial_op); } static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx, @@ -146,15 +140,13 @@ static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx, static void server_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *d = elem->call_data; GPR_ASSERT(d != NULL); memset(d, 0, sizeof(*d)); d->start_ts = gpr_now(GPR_CLOCK_REALTIME); /* TODO(hongyu): call census_tracing_start_op here. */ - grpc_closure_init(d->on_done_recv, server_on_done_recv, elem); - if (initial_op) server_mutate_op(elem, initial_op); + grpc_closure_init(&d->finish_recv, server_on_done_recv, elem); } static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx, @@ -165,12 +157,11 @@ static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx, } static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; GPR_ASSERT(chand != NULL); - chand->path_str = grpc_mdstr_from_string(mdctx, ":path"); + chand->path_str = grpc_mdstr_from_string(args->metadata_context, ":path"); } static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, @@ -183,15 +174,13 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, } const grpc_channel_filter grpc_client_census_filter = { - client_start_transport_op, grpc_channel_next_op, - sizeof(call_data), client_init_call_elem, - client_destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, "census-client"}; + client_start_transport_op, grpc_channel_next_op, sizeof(call_data), + client_init_call_elem, grpc_call_stack_ignore_set_pollset, + client_destroy_call_elem, sizeof(channel_data), init_channel_elem, + destroy_channel_elem, grpc_call_next_get_peer, "census-client"}; const grpc_channel_filter grpc_server_census_filter = { - server_start_transport_op, grpc_channel_next_op, - sizeof(call_data), server_init_call_elem, - server_destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, - grpc_call_next_get_peer, "census-server"}; + server_start_transport_op, grpc_channel_next_op, sizeof(call_data), + server_init_call_elem, grpc_call_stack_ignore_set_pollset, + server_destroy_call_elem, sizeof(channel_data), init_channel_elem, + destroy_channel_elem, grpc_call_next_get_peer, "census-server"}; diff --git a/src/core/channel/channel_stack.c b/src/core/channel/channel_stack.c index abd7f719e7..02e33a09ab 100644 --- a/src/core/channel/channel_stack.c +++ b/src/core/channel/channel_stack.c @@ -104,13 +104,14 @@ grpc_call_element *grpc_call_stack_element(grpc_call_stack *call_stack, void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, const grpc_channel_filter **filters, size_t filter_count, grpc_channel *master, - const grpc_channel_args *args, + const grpc_channel_args *channel_args, grpc_mdctx *metadata_context, grpc_channel_stack *stack) { size_t call_size = ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) + ROUND_UP_TO_ALIGNMENT_SIZE(filter_count * sizeof(grpc_call_element)); grpc_channel_element *elems; + grpc_channel_element_args args; char *user_data; size_t i; @@ -122,11 +123,14 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, /* init per-filter data */ for (i = 0; i < filter_count; i++) { + args.master = master; + args.channel_args = channel_args; + args.metadata_context = metadata_context; + args.is_first = i == 0; + args.is_last = i == (filter_count - 1); elems[i].filter = filters[i]; elems[i].channel_data = user_data; - elems[i].filter->init_channel_elem(exec_ctx, &elems[i], master, args, - metadata_context, i == 0, - i == (filter_count - 1)); + elems[i].filter->init_channel_elem(exec_ctx, &elems[i], &args); user_data += ROUND_UP_TO_ALIGNMENT_SIZE(filters[i]->sizeof_channel_data); call_size += ROUND_UP_TO_ALIGNMENT_SIZE(filters[i]->sizeof_call_data); } @@ -151,33 +155,63 @@ void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx, } void grpc_call_stack_init(grpc_exec_ctx *exec_ctx, - grpc_channel_stack *channel_stack, + grpc_channel_stack *channel_stack, int initial_refs, + grpc_iomgr_cb_func destroy, void *destroy_arg, + grpc_call_context_element *context, const void *transport_server_data, - grpc_transport_stream_op *initial_op, grpc_call_stack *call_stack) { grpc_channel_element *channel_elems = CHANNEL_ELEMS_FROM_STACK(channel_stack); + grpc_call_element_args args; size_t count = channel_stack->count; grpc_call_element *call_elems; char *user_data; size_t i; call_stack->count = count; + gpr_ref_init(&call_stack->refcount.refs, initial_refs); + grpc_closure_init(&call_stack->refcount.destroy, destroy, destroy_arg); call_elems = CALL_ELEMS_FROM_STACK(call_stack); user_data = ((char *)call_elems) + ROUND_UP_TO_ALIGNMENT_SIZE(count * sizeof(grpc_call_element)); /* init per-filter data */ for (i = 0; i < count; i++) { + args.refcount = &call_stack->refcount; + args.server_transport_data = transport_server_data; + args.context = context; call_elems[i].filter = channel_elems[i].filter; call_elems[i].channel_data = channel_elems[i].channel_data; call_elems[i].call_data = user_data; - call_elems[i].filter->init_call_elem(exec_ctx, &call_elems[i], - transport_server_data, initial_op); + call_elems[i].filter->init_call_elem(exec_ctx, &call_elems[i], &args); user_data += ROUND_UP_TO_ALIGNMENT_SIZE(call_elems[i].filter->sizeof_call_data); } } +void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_call_stack *call_stack, + grpc_pollset *pollset) { + size_t count = call_stack->count; + grpc_call_element *call_elems; + char *user_data; + size_t i; + + call_elems = CALL_ELEMS_FROM_STACK(call_stack); + user_data = ((char *)call_elems) + + ROUND_UP_TO_ALIGNMENT_SIZE(count * sizeof(grpc_call_element)); + + /* init per-filter data */ + for (i = 0; i < count; i++) { + call_elems[i].filter->set_pollset(exec_ctx, &call_elems[i], pollset); + user_data += + ROUND_UP_TO_ALIGNMENT_SIZE(call_elems[i].filter->sizeof_call_data); + } +} + +void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_pollset *pollset) {} + void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack) { grpc_call_element *elems = CALL_ELEMS_FROM_STACK(stack); size_t count = stack->count; diff --git a/src/core/channel/channel_stack.h b/src/core/channel/channel_stack.h index 6732cc3018..5d33ab5b42 100644 --- a/src/core/channel/channel_stack.h +++ b/src/core/channel/channel_stack.h @@ -51,6 +51,20 @@ typedef struct grpc_channel_element grpc_channel_element; typedef struct grpc_call_element grpc_call_element; +typedef struct { + grpc_channel *master; + const grpc_channel_args *channel_args; + grpc_mdctx *metadata_context; + int is_first; + int is_last; +} grpc_channel_element_args; + +typedef struct { + grpc_stream_refcount *refcount; + const void *server_transport_data; + grpc_call_context_element *context; +} grpc_call_element_args; + /* Channel filters specify: 1. the amount of memory needed in the channel & call (via the sizeof_XXX members) @@ -84,8 +98,9 @@ typedef struct { transport and is on the server. Most filters want to ignore this argument. */ void (*init_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op); + grpc_call_element_args *args); + void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset); /* Destroy per call data. The filter does not need to do any chaining */ void (*destroy_call_elem)(grpc_exec_ctx *exec_ctx, grpc_call_element *elem); @@ -99,9 +114,7 @@ typedef struct { useful for asserting correct configuration by upper layer code. The filter does not need to do any chaining */ void (*init_channel_elem)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, - grpc_channel *master, const grpc_channel_args *args, - grpc_mdctx *metadata_context, int is_first, - int is_last); + grpc_channel_element_args *args); /* Destroy per channel data. The filter does not need to do any chaining */ void (*destroy_channel_elem)(grpc_exec_ctx *exec_ctx, @@ -141,7 +154,14 @@ typedef struct { /* A call stack tracks a set of related filters for one call, and guarantees they live within a single malloc() allocation */ -typedef struct { size_t count; } grpc_call_stack; +typedef struct { + /* shared refcount for this channel stack. + MUST be the first element: the underlying code calls destroy + with the address of the refcount, but higher layers prefer to think + about the address of the call stack itself. */ + grpc_stream_refcount refcount; + size_t count; +} grpc_call_stack; /* Get a channel element given a channel stack and its index */ grpc_channel_element *grpc_channel_stack_element(grpc_channel_stack *stack, @@ -170,13 +190,35 @@ void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx, expected to be NULL on a client, or an opaque transport owned pointer on the server. */ void grpc_call_stack_init(grpc_exec_ctx *exec_ctx, - grpc_channel_stack *channel_stack, + grpc_channel_stack *channel_stack, int initial_refs, + grpc_iomgr_cb_func destroy, void *destroy_arg, + grpc_call_context_element *context, const void *transport_server_data, - grpc_transport_stream_op *initial_op, grpc_call_stack *call_stack); +/* Set a pollset for a call stack: must occur before the first op is started */ +void grpc_call_stack_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_call_stack *call_stack, + grpc_pollset *pollset); + +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +#define grpc_call_stack_ref(call_stack, reason) \ + grpc_stream_ref(&(call_stack)->refcount, reason) +#define grpc_call_stack_unref(exec_ctx, call_stack, reason) \ + grpc_stream_unref(exec_ctx, &(call_stack)->refcount, reason) +#else +#define grpc_call_stack_ref(call_stack) grpc_stream_ref(&(call_stack)->refcount) +#define grpc_call_stack_unref(exec_ctx, call_stack) \ + grpc_stream_unref(exec_ctx, &(call_stack)->refcount) +#endif + /* Destroy a call stack */ void grpc_call_stack_destroy(grpc_exec_ctx *exec_ctx, grpc_call_stack *stack); +/* Ignore set pollset - used by filters to implement the set_pollset method + if they don't care about pollsets at all. Does nothing. */ +void grpc_call_stack_ignore_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, + grpc_pollset *pollset); /* Call the next operation in a call stack */ void grpc_call_next_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op); diff --git a/src/core/channel/client_channel.c b/src/core/channel/client_channel.c index 9f85557ea1..16d91d4277 100644 --- a/src/core/channel/client_channel.c +++ b/src/core/channel/client_channel.c @@ -43,6 +43,7 @@ #include "src/core/channel/channel_args.h" #include "src/core/channel/connected_channel.h" +#include "src/core/channel/subchannel_call_holder.h" #include "src/core/iomgr/iomgr.h" #include "src/core/profiling/timers.h" #include "src/core/support/string.h" @@ -51,7 +52,7 @@ /* Client channel implementation */ -typedef struct call_data call_data; +typedef grpc_subchannel_call_holder call_data; typedef struct client_channel_channel_data { /** metadata context for this channel */ @@ -98,360 +99,22 @@ typedef struct { grpc_lb_policy *lb_policy; } lb_policy_connectivity_watcher; -typedef enum { - CALL_CREATED, - CALL_WAITING_FOR_SEND, - CALL_WAITING_FOR_CONFIG, - CALL_WAITING_FOR_PICK, - CALL_WAITING_FOR_CALL, - CALL_ACTIVE, - CALL_CANCELLED -} call_state; - -struct call_data { - /* owning element */ - grpc_call_element *elem; - - gpr_mu mu_state; - - call_state state; - gpr_timespec deadline; - grpc_subchannel *picked_channel; - grpc_closure async_setup_task; - grpc_transport_stream_op waiting_op; - /* our child call stack */ - grpc_subchannel_call *subchannel_call; - grpc_linked_mdelem status; - grpc_linked_mdelem details; -}; - -static grpc_closure *merge_into_waiting_op(grpc_call_element *elem, - grpc_transport_stream_op *new_op) - GRPC_MUST_USE_RESULT; - -static void handle_op_after_cancellation(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; - if (op->send_ops) { - grpc_stream_ops_unref_owned_objects(op->send_ops->ops, op->send_ops->nops); - op->on_done_send->cb(exec_ctx, op->on_done_send->cb_arg, 0); - } - if (op->recv_ops) { - char status[GPR_LTOA_MIN_BUFSIZE]; - grpc_metadata_batch mdb; - gpr_ltoa(GRPC_STATUS_CANCELLED, status); - calld->status.md = - grpc_mdelem_from_strings(chand->mdctx, "grpc-status", status); - calld->details.md = - grpc_mdelem_from_strings(chand->mdctx, "grpc-message", "Cancelled"); - calld->status.prev = calld->details.next = NULL; - calld->status.next = &calld->details; - calld->details.prev = &calld->status; - mdb.list.head = &calld->status; - mdb.list.tail = &calld->details; - mdb.garbage.head = mdb.garbage.tail = NULL; - mdb.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); - grpc_sopb_add_metadata(op->recv_ops, mdb); - *op->recv_state = GRPC_STREAM_CLOSED; - op->on_done_recv->cb(exec_ctx, op->on_done_recv->cb_arg, 1); - } - if (op->on_consumed) { - op->on_consumed->cb(exec_ctx, op->on_consumed->cb_arg, 0); - } -} - typedef struct { grpc_closure closure; grpc_call_element *elem; } waiting_call; -static void perform_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op, - int continuation); - -static void continue_with_pick(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - waiting_call *wc = arg; - call_data *calld = wc->elem->call_data; - perform_transport_stream_op(exec_ctx, wc->elem, &calld->waiting_op, 1); - gpr_free(wc); -} - -static void add_to_lb_policy_wait_queue_locked_state_config( - grpc_call_element *elem) { - channel_data *chand = elem->channel_data; - waiting_call *wc = gpr_malloc(sizeof(*wc)); - grpc_closure_init(&wc->closure, continue_with_pick, wc); - wc->elem = elem; - grpc_closure_list_add(&chand->waiting_for_config_closures, &wc->closure, 1); -} - -static int is_empty(void *p, int len) { - char *ptr = p; - int i; - for (i = 0; i < len; i++) { - if (ptr[i] != 0) return 0; - } - return 1; -} - -static void started_call_locked(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - call_data *calld = arg; - grpc_transport_stream_op op; - int have_waiting; - - if (calld->state == CALL_CANCELLED && calld->subchannel_call != NULL) { - memset(&op, 0, sizeof(op)); - op.cancel_with_status = GRPC_STATUS_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, &op); - } else if (calld->state == CALL_WAITING_FOR_CALL) { - have_waiting = !is_empty(&calld->waiting_op, sizeof(calld->waiting_op)); - if (calld->subchannel_call != NULL) { - calld->state = CALL_ACTIVE; - gpr_mu_unlock(&calld->mu_state); - if (have_waiting) { - grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, - &calld->waiting_op); - } - } else { - calld->state = CALL_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - if (have_waiting) { - handle_op_after_cancellation(exec_ctx, calld->elem, &calld->waiting_op); - } - } - } else { - GPR_ASSERT(calld->state == CALL_CANCELLED); - gpr_mu_unlock(&calld->mu_state); - } -} - -static void started_call(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - call_data *calld = arg; - gpr_mu_lock(&calld->mu_state); - started_call_locked(exec_ctx, arg, iomgr_success); -} - -static void picked_target(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - call_data *calld = arg; - grpc_pollset *pollset; - grpc_subchannel_call_create_status call_creation_status; - - GPR_TIMER_BEGIN("picked_target", 0); - - if (calld->picked_channel == NULL) { - /* treat this like a cancellation */ - calld->waiting_op.cancel_with_status = GRPC_STATUS_UNAVAILABLE; - perform_transport_stream_op(exec_ctx, calld->elem, &calld->waiting_op, 1); - } else { - gpr_mu_lock(&calld->mu_state); - if (calld->state == CALL_CANCELLED) { - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, calld->elem, &calld->waiting_op); - } else { - GPR_ASSERT(calld->state == CALL_WAITING_FOR_PICK); - calld->state = CALL_WAITING_FOR_CALL; - pollset = calld->waiting_op.bind_pollset; - grpc_closure_init(&calld->async_setup_task, started_call, calld); - call_creation_status = grpc_subchannel_create_call( - exec_ctx, calld->picked_channel, pollset, &calld->subchannel_call, - &calld->async_setup_task); - if (call_creation_status == GRPC_SUBCHANNEL_CALL_CREATE_READY) { - started_call_locked(exec_ctx, calld, iomgr_success); - } else { - gpr_mu_unlock(&calld->mu_state); - } - } - } - - GPR_TIMER_END("picked_target", 0); -} - -static grpc_closure *merge_into_waiting_op(grpc_call_element *elem, - grpc_transport_stream_op *new_op) { - call_data *calld = elem->call_data; - grpc_closure *consumed_op = NULL; - grpc_transport_stream_op *waiting_op = &calld->waiting_op; - GPR_ASSERT((waiting_op->send_ops != NULL) + (new_op->send_ops != NULL) <= 1); - GPR_ASSERT((waiting_op->recv_ops != NULL) + (new_op->recv_ops != NULL) <= 1); - if (new_op->send_ops != NULL) { - waiting_op->send_ops = new_op->send_ops; - waiting_op->is_last_send = new_op->is_last_send; - waiting_op->on_done_send = new_op->on_done_send; - } - if (new_op->recv_ops != NULL) { - waiting_op->recv_ops = new_op->recv_ops; - waiting_op->recv_state = new_op->recv_state; - waiting_op->on_done_recv = new_op->on_done_recv; - } - if (new_op->on_consumed != NULL) { - if (waiting_op->on_consumed != NULL) { - consumed_op = waiting_op->on_consumed; - } - waiting_op->on_consumed = new_op->on_consumed; - } - if (new_op->cancel_with_status != GRPC_STATUS_OK) { - waiting_op->cancel_with_status = new_op->cancel_with_status; - } - return consumed_op; -} - static char *cc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - grpc_subchannel_call *subchannel_call; - char *result; - - gpr_mu_lock(&calld->mu_state); - if (calld->state == CALL_ACTIVE) { - subchannel_call = calld->subchannel_call; - GRPC_SUBCHANNEL_CALL_REF(subchannel_call, "get_peer"); - gpr_mu_unlock(&calld->mu_state); - result = grpc_subchannel_call_get_peer(exec_ctx, subchannel_call); - GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, subchannel_call, "get_peer"); - return result; - } else { - gpr_mu_unlock(&calld->mu_state); - return grpc_channel_get_target(chand->master); - } -} - -static void perform_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op, - int continuation) { - call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; - grpc_subchannel_call *subchannel_call; - grpc_lb_policy *lb_policy; - grpc_transport_stream_op op2; - GPR_TIMER_BEGIN("perform_transport_stream_op", 0); - GPR_ASSERT(elem->filter == &grpc_client_channel_filter); - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - - gpr_mu_lock(&calld->mu_state); - switch (calld->state) { - case CALL_ACTIVE: - GPR_ASSERT(!continuation); - subchannel_call = calld->subchannel_call; - gpr_mu_unlock(&calld->mu_state); - grpc_subchannel_call_process_op(exec_ctx, subchannel_call, op); - break; - case CALL_CANCELLED: - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - break; - case CALL_WAITING_FOR_SEND: - GPR_ASSERT(!continuation); - grpc_exec_ctx_enqueue(exec_ctx, merge_into_waiting_op(elem, op), 1); - if (!calld->waiting_op.send_ops && - calld->waiting_op.cancel_with_status == GRPC_STATUS_OK) { - gpr_mu_unlock(&calld->mu_state); - break; - } - *op = calld->waiting_op; - memset(&calld->waiting_op, 0, sizeof(calld->waiting_op)); - continuation = 1; - /* fall through */ - case CALL_WAITING_FOR_CONFIG: - case CALL_WAITING_FOR_PICK: - case CALL_WAITING_FOR_CALL: - if (!continuation) { - if (op->cancel_with_status != GRPC_STATUS_OK) { - calld->state = CALL_CANCELLED; - op2 = calld->waiting_op; - memset(&calld->waiting_op, 0, sizeof(calld->waiting_op)); - if (op->on_consumed) { - calld->waiting_op.on_consumed = op->on_consumed; - op->on_consumed = NULL; - } else if (op2.on_consumed) { - calld->waiting_op.on_consumed = op2.on_consumed; - op2.on_consumed = NULL; - } - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - handle_op_after_cancellation(exec_ctx, elem, &op2); - } else { - grpc_exec_ctx_enqueue(exec_ctx, merge_into_waiting_op(elem, op), 1); - gpr_mu_unlock(&calld->mu_state); - } - break; - } - /* fall through */ - case CALL_CREATED: - if (op->cancel_with_status != GRPC_STATUS_OK) { - calld->state = CALL_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - } else { - calld->waiting_op = *op; - - if (op->send_ops == NULL) { - /* need to have some send ops before we can select the - lb target */ - calld->state = CALL_WAITING_FOR_SEND; - gpr_mu_unlock(&calld->mu_state); - } else { - gpr_mu_lock(&chand->mu_config); - lb_policy = chand->lb_policy; - if (lb_policy) { - grpc_transport_stream_op *waiting_op = &calld->waiting_op; - grpc_pollset *bind_pollset = waiting_op->bind_pollset; - grpc_metadata_batch *initial_metadata = - &waiting_op->send_ops->ops[0].data.metadata; - GRPC_LB_POLICY_REF(lb_policy, "pick"); - gpr_mu_unlock(&chand->mu_config); - calld->state = CALL_WAITING_FOR_PICK; - - GPR_ASSERT(waiting_op->bind_pollset); - GPR_ASSERT(waiting_op->send_ops); - GPR_ASSERT(waiting_op->send_ops->nops >= 1); - GPR_ASSERT(waiting_op->send_ops->ops[0].type == GRPC_OP_METADATA); - gpr_mu_unlock(&calld->mu_state); - - grpc_closure_init(&calld->async_setup_task, picked_target, calld); - grpc_lb_policy_pick(exec_ctx, lb_policy, bind_pollset, - initial_metadata, &calld->picked_channel, - &calld->async_setup_task); - - GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick"); - } else if (chand->resolver != NULL) { - calld->state = CALL_WAITING_FOR_CONFIG; - add_to_lb_policy_wait_queue_locked_state_config(elem); - if (!chand->started_resolving && chand->resolver != NULL) { - GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver"); - chand->started_resolving = 1; - grpc_resolver_next(exec_ctx, chand->resolver, - &chand->incoming_configuration, - &chand->on_config_changed); - } - gpr_mu_unlock(&chand->mu_config); - gpr_mu_unlock(&calld->mu_state); - } else { - calld->state = CALL_CANCELLED; - gpr_mu_unlock(&chand->mu_config); - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - } - } - } - break; - } - - GPR_TIMER_END("perform_transport_stream_op", 0); + return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data, + chand->master); } static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { - perform_transport_stream_op(exec_ctx, elem, op, 0); + GRPC_CALL_LOG_OP(GPR_INFO, elem, op); + grpc_subchannel_call_holder_perform_op(exec_ctx, elem->call_data, op); } static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand, @@ -593,11 +256,9 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, op->connectivity_state = NULL; } - if (!is_empty(op, sizeof(*op))) { - lb_policy = chand->lb_policy; - if (lb_policy) { - GRPC_LB_POLICY_REF(lb_policy, "broadcast"); - } + lb_policy = chand->lb_policy; + if (lb_policy) { + GRPC_LB_POLICY_REF(lb_policy, "broadcast"); } if (op->disconnect && chand->resolver != NULL) { @@ -624,67 +285,110 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx, } } -/* Constructor for call_data */ -static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { +typedef struct { + grpc_metadata_batch *initial_metadata; + grpc_subchannel **subchannel; + grpc_closure *on_ready; + grpc_call_element *elem; + grpc_closure closure; +} continue_picking_args; + +static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg, + grpc_metadata_batch *initial_metadata, + grpc_subchannel **subchannel, + grpc_closure *on_ready); + +static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg, int success) { + continue_picking_args *cpa = arg; + if (!success) { + grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 0); + } else if (cpa->subchannel == NULL) { + /* cancelled, do nothing */ + } else if (cc_pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata, + cpa->subchannel, cpa->on_ready)) { + grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 1); + } + gpr_free(cpa); +} + +static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp, + grpc_metadata_batch *initial_metadata, + grpc_subchannel **subchannel, + grpc_closure *on_ready) { + grpc_call_element *elem = elemp; + channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; + continue_picking_args *cpa; + grpc_closure *closure; - /* TODO(ctiller): is there something useful we can do here? */ - GPR_ASSERT(initial_op == NULL); + GPR_ASSERT(subchannel); - GPR_ASSERT(elem->filter == &grpc_client_channel_filter); - GPR_ASSERT(server_transport_data == NULL); - gpr_mu_init(&calld->mu_state); - calld->elem = elem; - calld->state = CALL_CREATED; - calld->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + gpr_mu_lock(&chand->mu_config); + if (initial_metadata == NULL) { + if (chand->lb_policy != NULL) { + grpc_lb_policy_cancel_pick(exec_ctx, chand->lb_policy, subchannel); + } + for (closure = chand->waiting_for_config_closures.head; closure != NULL; + closure = grpc_closure_next(closure)) { + cpa = closure->cb_arg; + if (cpa->subchannel == subchannel) { + cpa->subchannel = NULL; + grpc_exec_ctx_enqueue(exec_ctx, cpa->on_ready, 0); + } + } + gpr_mu_unlock(&chand->mu_config); + return 1; + } + if (chand->lb_policy != NULL) { + int r = grpc_lb_policy_pick(exec_ctx, chand->lb_policy, calld->pollset, + initial_metadata, subchannel, on_ready); + gpr_mu_unlock(&chand->mu_config); + return r; + } + if (chand->resolver != NULL && !chand->started_resolving) { + chand->started_resolving = 1; + GRPC_CHANNEL_INTERNAL_REF(chand->master, "resolver"); + grpc_resolver_next(exec_ctx, chand->resolver, + &chand->incoming_configuration, + &chand->on_config_changed); + } + cpa = gpr_malloc(sizeof(*cpa)); + cpa->initial_metadata = initial_metadata; + cpa->subchannel = subchannel; + cpa->on_ready = on_ready; + cpa->elem = elem; + grpc_closure_init(&cpa->closure, continue_picking, cpa); + grpc_closure_list_add(&chand->waiting_for_config_closures, &cpa->closure, 1); + gpr_mu_unlock(&chand->mu_config); + return 0; +} + +/* Constructor for call_data */ +static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_call_element_args *args) { + grpc_subchannel_call_holder_init(elem->call_data, cc_pick_subchannel, elem); } /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - call_data *calld = elem->call_data; - grpc_subchannel_call *subchannel_call; - - /* if the call got activated, we need to destroy the child stack also, and - remove it from the in-flight requests tracked by the child_entry we - picked */ - gpr_mu_lock(&calld->mu_state); - switch (calld->state) { - case CALL_ACTIVE: - subchannel_call = calld->subchannel_call; - gpr_mu_unlock(&calld->mu_state); - GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, subchannel_call, "client_channel"); - break; - case CALL_CREATED: - case CALL_CANCELLED: - gpr_mu_unlock(&calld->mu_state); - break; - case CALL_WAITING_FOR_PICK: - case CALL_WAITING_FOR_CONFIG: - case CALL_WAITING_FOR_CALL: - case CALL_WAITING_FOR_SEND: - GPR_UNREACHABLE_CODE(return ); - } + grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data); } /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, - grpc_mdctx *metadata_context, int is_first, - int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; memset(chand, 0, sizeof(*chand)); - GPR_ASSERT(is_last); + GPR_ASSERT(args->is_last); GPR_ASSERT(elem->filter == &grpc_client_channel_filter); gpr_mu_init(&chand->mu_config); - chand->mdctx = metadata_context; - chand->master = master; + chand->mdctx = args->metadata_context; + chand->master = args->master; grpc_pollset_set_init(&chand->pollset_set); grpc_closure_init(&chand->on_config_changed, cc_on_config_changed, chand); @@ -709,10 +413,16 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, gpr_mu_destroy(&chand->mu_config); } +static void cc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset) { + call_data *calld = elem->call_data; + calld->pollset = pollset; +} + const grpc_channel_filter grpc_client_channel_filter = { cc_start_transport_stream_op, cc_start_transport_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, cc_get_peer, "client-channel", + init_call_elem, cc_set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, cc_get_peer, "client-channel", }; void grpc_client_channel_set_resolver(grpc_exec_ctx *exec_ctx, diff --git a/src/core/channel/client_uchannel.c b/src/core/channel/client_uchannel.c index 510677a844..ec6b02381a 100644 --- a/src/core/channel/client_uchannel.c +++ b/src/core/channel/client_uchannel.c @@ -39,6 +39,7 @@ #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/compress_filter.h" +#include "src/core/channel/subchannel_call_holder.h" #include "src/core/iomgr/iomgr.h" #include "src/core/support/string.h" #include "src/core/surface/channel.h" @@ -52,8 +53,6 @@ /** Microchannel (uchannel) implementation: a lightweight channel without any * load-balancing mechanisms meant for communication from within the core. */ -typedef struct call_data call_data; - typedef struct client_uchannel_channel_data { /** metadata context for this channel */ grpc_mdctx *mdctx; @@ -80,85 +79,7 @@ typedef struct client_uchannel_channel_data { gpr_mu mu_state; } channel_data; -typedef enum { - CALL_CREATED, - CALL_WAITING_FOR_SEND, - CALL_WAITING_FOR_CALL, - CALL_ACTIVE, - CALL_CANCELLED -} call_state; - -struct call_data { - /* owning element */ - grpc_call_element *elem; - - gpr_mu mu_state; - - call_state state; - gpr_timespec deadline; - grpc_closure async_setup_task; - grpc_transport_stream_op waiting_op; - /* our child call stack */ - grpc_subchannel_call *subchannel_call; - grpc_linked_mdelem status; - grpc_linked_mdelem details; -}; - -static grpc_closure *merge_into_waiting_op(grpc_call_element *elem, - grpc_transport_stream_op *new_op) - GRPC_MUST_USE_RESULT; - -static void handle_op_after_cancellation(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op) { - call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; - if (op->send_ops) { - grpc_stream_ops_unref_owned_objects(op->send_ops->ops, op->send_ops->nops); - op->on_done_send->cb(exec_ctx, op->on_done_send->cb_arg, 0); - } - if (op->recv_ops) { - char status[GPR_LTOA_MIN_BUFSIZE]; - grpc_metadata_batch mdb; - gpr_ltoa(GRPC_STATUS_CANCELLED, status); - calld->status.md = - grpc_mdelem_from_strings(chand->mdctx, "grpc-status", status); - calld->details.md = - grpc_mdelem_from_strings(chand->mdctx, "grpc-message", "Cancelled"); - calld->status.prev = calld->details.next = NULL; - calld->status.next = &calld->details; - calld->details.prev = &calld->status; - mdb.list.head = &calld->status; - mdb.list.tail = &calld->details; - mdb.garbage.head = mdb.garbage.tail = NULL; - mdb.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); - grpc_sopb_add_metadata(op->recv_ops, mdb); - *op->recv_state = GRPC_STREAM_CLOSED; - op->on_done_recv->cb(exec_ctx, op->on_done_recv->cb_arg, 1); - } - if (op->on_consumed) { - op->on_consumed->cb(exec_ctx, op->on_consumed->cb_arg, 0); - } -} - -typedef struct { - grpc_closure closure; - grpc_call_element *elem; -} waiting_call; - -static void perform_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op, - int continuation); - -static int is_empty(void *p, int len) { - char *ptr = p; - int i; - for (i = 0; i < len; i++) { - if (ptr[i] != 0) return 0; - } - return 1; -} +typedef grpc_subchannel_call_holder call_data; static void monitor_subchannel(grpc_exec_ctx *exec_ctx, void *arg, int iomgr_success) { @@ -171,201 +92,17 @@ static void monitor_subchannel(grpc_exec_ctx *exec_ctx, void *arg, &chand->connectivity_cb); } -static void started_call_locked(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - call_data *calld = arg; - grpc_transport_stream_op op; - int have_waiting; - - if (calld->state == CALL_CANCELLED && iomgr_success == 0) { - have_waiting = !is_empty(&calld->waiting_op, sizeof(calld->waiting_op)); - gpr_mu_unlock(&calld->mu_state); - if (have_waiting) { - handle_op_after_cancellation(exec_ctx, calld->elem, &calld->waiting_op); - } - } else if (calld->state == CALL_CANCELLED && calld->subchannel_call != NULL) { - memset(&op, 0, sizeof(op)); - op.cancel_with_status = GRPC_STATUS_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, &op); - } else if (calld->state == CALL_WAITING_FOR_CALL) { - have_waiting = !is_empty(&calld->waiting_op, sizeof(calld->waiting_op)); - if (calld->subchannel_call != NULL) { - calld->state = CALL_ACTIVE; - gpr_mu_unlock(&calld->mu_state); - if (have_waiting) { - grpc_subchannel_call_process_op(exec_ctx, calld->subchannel_call, - &calld->waiting_op); - } - } else { - calld->state = CALL_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - if (have_waiting) { - handle_op_after_cancellation(exec_ctx, calld->elem, &calld->waiting_op); - } - } - } else { - GPR_ASSERT(calld->state == CALL_CANCELLED); - gpr_mu_unlock(&calld->mu_state); - have_waiting = !is_empty(&calld->waiting_op, sizeof(calld->waiting_op)); - if (have_waiting) { - handle_op_after_cancellation(exec_ctx, calld->elem, &calld->waiting_op); - } - } -} - -static void started_call(grpc_exec_ctx *exec_ctx, void *arg, - int iomgr_success) { - call_data *calld = arg; - gpr_mu_lock(&calld->mu_state); - started_call_locked(exec_ctx, arg, iomgr_success); -} - -static grpc_closure *merge_into_waiting_op(grpc_call_element *elem, - grpc_transport_stream_op *new_op) { - call_data *calld = elem->call_data; - grpc_closure *consumed_op = NULL; - grpc_transport_stream_op *waiting_op = &calld->waiting_op; - GPR_ASSERT((waiting_op->send_ops != NULL) + (new_op->send_ops != NULL) <= 1); - GPR_ASSERT((waiting_op->recv_ops != NULL) + (new_op->recv_ops != NULL) <= 1); - if (new_op->send_ops != NULL) { - waiting_op->send_ops = new_op->send_ops; - waiting_op->is_last_send = new_op->is_last_send; - waiting_op->on_done_send = new_op->on_done_send; - } - if (new_op->recv_ops != NULL) { - waiting_op->recv_ops = new_op->recv_ops; - waiting_op->recv_state = new_op->recv_state; - waiting_op->on_done_recv = new_op->on_done_recv; - } - if (new_op->on_consumed != NULL) { - if (waiting_op->on_consumed != NULL) { - consumed_op = waiting_op->on_consumed; - } - waiting_op->on_consumed = new_op->on_consumed; - } - if (new_op->cancel_with_status != GRPC_STATUS_OK) { - waiting_op->cancel_with_status = new_op->cancel_with_status; - } - return consumed_op; -} - static char *cuc_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; - grpc_subchannel_call *subchannel_call; - char *result; - - gpr_mu_lock(&calld->mu_state); - if (calld->state == CALL_ACTIVE) { - subchannel_call = calld->subchannel_call; - GRPC_SUBCHANNEL_CALL_REF(subchannel_call, "get_peer"); - gpr_mu_unlock(&calld->mu_state); - result = grpc_subchannel_call_get_peer(exec_ctx, subchannel_call); - GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, subchannel_call, "get_peer"); - return result; - } else { - gpr_mu_unlock(&calld->mu_state); - return grpc_channel_get_target(chand->master); - } -} - -static void perform_transport_stream_op(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_transport_stream_op *op, - int continuation) { - call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; - grpc_subchannel_call *subchannel_call; - grpc_transport_stream_op op2; - GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter); - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - - gpr_mu_lock(&calld->mu_state); - /* make sure the wrapped subchannel has been set (see - * grpc_client_uchannel_set_subchannel) */ - GPR_ASSERT(chand->subchannel != NULL); - - switch (calld->state) { - case CALL_ACTIVE: - GPR_ASSERT(!continuation); - subchannel_call = calld->subchannel_call; - gpr_mu_unlock(&calld->mu_state); - grpc_subchannel_call_process_op(exec_ctx, subchannel_call, op); - break; - case CALL_CANCELLED: - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - break; - case CALL_WAITING_FOR_SEND: - GPR_ASSERT(!continuation); - grpc_exec_ctx_enqueue(exec_ctx, merge_into_waiting_op(elem, op), 1); - if (!calld->waiting_op.send_ops && - calld->waiting_op.cancel_with_status == GRPC_STATUS_OK) { - gpr_mu_unlock(&calld->mu_state); - break; - } - *op = calld->waiting_op; - memset(&calld->waiting_op, 0, sizeof(calld->waiting_op)); - continuation = 1; - /* fall through */ - case CALL_WAITING_FOR_CALL: - if (!continuation) { - if (op->cancel_with_status != GRPC_STATUS_OK) { - calld->state = CALL_CANCELLED; - op2 = calld->waiting_op; - memset(&calld->waiting_op, 0, sizeof(calld->waiting_op)); - if (op->on_consumed) { - calld->waiting_op.on_consumed = op->on_consumed; - op->on_consumed = NULL; - } else if (op2.on_consumed) { - calld->waiting_op.on_consumed = op2.on_consumed; - op2.on_consumed = NULL; - } - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - handle_op_after_cancellation(exec_ctx, elem, &op2); - grpc_subchannel_cancel_waiting_call(exec_ctx, chand->subchannel, 1); - } else { - grpc_exec_ctx_enqueue(exec_ctx, merge_into_waiting_op(elem, op), 1); - gpr_mu_unlock(&calld->mu_state); - } - break; - } - /* fall through */ - case CALL_CREATED: - if (op->cancel_with_status != GRPC_STATUS_OK) { - calld->state = CALL_CANCELLED; - gpr_mu_unlock(&calld->mu_state); - handle_op_after_cancellation(exec_ctx, elem, op); - } else { - calld->waiting_op = *op; - if (op->send_ops == NULL) { - calld->state = CALL_WAITING_FOR_SEND; - gpr_mu_unlock(&calld->mu_state); - } else { - grpc_subchannel_call_create_status call_creation_status; - grpc_pollset *pollset = calld->waiting_op.bind_pollset; - calld->state = CALL_WAITING_FOR_CALL; - grpc_closure_init(&calld->async_setup_task, started_call, calld); - call_creation_status = grpc_subchannel_create_call( - exec_ctx, chand->subchannel, pollset, &calld->subchannel_call, - &calld->async_setup_task); - if (call_creation_status == GRPC_SUBCHANNEL_CALL_CREATE_READY) { - started_call_locked(exec_ctx, calld, 1); - } else { - gpr_mu_unlock(&calld->mu_state); - } - } - } - break; - } + return grpc_subchannel_call_holder_get_peer(exec_ctx, elem->call_data, + chand->master); } static void cuc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { - perform_transport_stream_op(exec_ctx, elem, op, 0); + GRPC_CALL_LOG_OP(GPR_INFO, elem, op); + grpc_subchannel_call_holder_perform_op(exec_ctx, elem->call_data, op); } static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx, @@ -392,64 +129,40 @@ static void cuc_start_transport_op(grpc_exec_ctx *exec_ctx, } } +static int cuc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *arg, + grpc_metadata_batch *initial_metadata, + grpc_subchannel **subchannel, + grpc_closure *on_ready) { + channel_data *chand = arg; + GPR_ASSERT(initial_metadata != NULL); + *subchannel = chand->subchannel; + return 1; +} + /* Constructor for call_data */ static void cuc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { - call_data *calld = elem->call_data; - memset(calld, 0, sizeof(call_data)); - - /* TODO(ctiller): is there something useful we can do here? */ - GPR_ASSERT(initial_op == NULL); - - GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter); - GPR_ASSERT(server_transport_data == NULL); - gpr_mu_init(&calld->mu_state); - calld->elem = elem; - calld->state = CALL_CREATED; - calld->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + grpc_call_element_args *args) { + grpc_subchannel_call_holder_init(elem->call_data, cuc_pick_subchannel, + elem->channel_data); } /* Destructor for call_data */ static void cuc_destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { - call_data *calld = elem->call_data; - grpc_subchannel_call *subchannel_call; - - /* if the call got activated, we need to destroy the child stack also, and - remove it from the in-flight requests tracked by the child_entry we - picked */ - gpr_mu_lock(&calld->mu_state); - switch (calld->state) { - case CALL_ACTIVE: - subchannel_call = calld->subchannel_call; - gpr_mu_unlock(&calld->mu_state); - GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, subchannel_call, "client_uchannel"); - break; - case CALL_CREATED: - case CALL_CANCELLED: - gpr_mu_unlock(&calld->mu_state); - break; - case CALL_WAITING_FOR_CALL: - case CALL_WAITING_FOR_SEND: - GPR_UNREACHABLE_CODE(return ); - } + grpc_subchannel_call_holder_destroy(exec_ctx, elem->call_data); } /* Constructor for channel_data */ static void cuc_init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, - grpc_channel *master, - const grpc_channel_args *args, - grpc_mdctx *metadata_context, int is_first, - int is_last) { + grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; memset(chand, 0, sizeof(*chand)); grpc_closure_init(&chand->connectivity_cb, monitor_subchannel, chand); - GPR_ASSERT(is_last); + GPR_ASSERT(args->is_last); GPR_ASSERT(elem->filter == &grpc_client_uchannel_filter); - chand->mdctx = metadata_context; - chand->master = master; + chand->mdctx = args->metadata_context; + chand->master = args->master; grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE, "client_uchannel"); gpr_mu_init(&chand->mu_state); @@ -465,17 +178,17 @@ static void cuc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, gpr_mu_destroy(&chand->mu_state); } +static void cuc_set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset) { + call_data *calld = elem->call_data; + calld->pollset = pollset; +} + const grpc_channel_filter grpc_client_uchannel_filter = { - cuc_start_transport_stream_op, - cuc_start_transport_op, - sizeof(call_data), - cuc_init_call_elem, - cuc_destroy_call_elem, - sizeof(channel_data), - cuc_init_channel_elem, - cuc_destroy_channel_elem, - cuc_get_peer, - "client-uchannel", + cuc_start_transport_stream_op, cuc_start_transport_op, sizeof(call_data), + cuc_init_call_elem, cuc_set_pollset, cuc_destroy_call_elem, + sizeof(channel_data), cuc_init_channel_elem, cuc_destroy_channel_elem, + cuc_get_peer, "client-uchannel", }; grpc_connectivity_state grpc_client_uchannel_check_connectivity_state( diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c index 20b5084044..f23d8052f3 100644 --- a/src/core/channel/compress_filter.c +++ b/src/core/channel/compress_filter.c @@ -50,13 +50,20 @@ typedef struct call_data { grpc_linked_mdelem compression_algorithm_storage; grpc_linked_mdelem accept_encoding_storage; gpr_uint32 remaining_slice_bytes; - /**< Input data to be read, as per BEGIN_MESSAGE */ - int written_initial_metadata; /**< Already processed initial md? */ /** Compression algorithm we'll try to use. It may be given by incoming * metadata, or by the channel's default compression settings. */ grpc_compression_algorithm compression_algorithm; /** If true, contents of \a compression_algorithm are authoritative */ int has_compression_algorithm; + + grpc_transport_stream_op send_op; + gpr_uint32 send_length; + gpr_uint32 send_flags; + gpr_slice incoming_slice; + grpc_slice_buffer_stream replacement_stream; + grpc_closure *post_send; + grpc_closure send_done; + grpc_closure got_slice; } call_data; typedef struct channel_data { @@ -76,24 +83,6 @@ typedef struct channel_data { grpc_compression_options compression_options; } channel_data; -/** Compress \a slices in place using \a algorithm. Returns 1 if compression did - * actually happen, 0 otherwise (for example if the compressed output size was - * larger than the raw input). - * - * Returns 1 if the data was actually compress and 0 otherwise. */ -static int compress_send_sb(grpc_compression_algorithm algorithm, - gpr_slice_buffer *slices) { - int did_compress; - gpr_slice_buffer tmp; - gpr_slice_buffer_init(&tmp); - did_compress = grpc_msg_compress(algorithm, slices, &tmp); - if (did_compress) { - gpr_slice_buffer_swap(slices, &tmp); - } - gpr_slice_buffer_destroy(&tmp); - return did_compress; -} - /** For each \a md element from the incoming metadata, filter out the entry for * "grpc-encoding", using its value to populate the call data's * compression_algorithm field. */ @@ -127,7 +116,9 @@ static grpc_mdelem *compression_md_filter(void *user_data, grpc_mdelem *md) { return md; } -static int skip_compression(channel_data *channeld, call_data *calld) { +static int skip_compression(grpc_call_element *elem) { + call_data *calld = elem->call_data; + channel_data *channeld = elem->channel_data; if (calld->has_compression_algorithm) { if (calld->compression_algorithm == GRPC_COMPRESS_NONE) { return 1; @@ -138,169 +129,127 @@ static int skip_compression(channel_data *channeld, call_data *calld) { return channeld->default_compression_algorithm == GRPC_COMPRESS_NONE; } -/** Assembles a new grpc_stream_op_buffer with the compressed slices, modifying - * the associated GRPC_OP_BEGIN_MESSAGE accordingly (new compressed length, - * flags indicating compression is in effect) and replaces \a send_ops with it. - * */ -static void finish_compressed_sopb(grpc_stream_op_buffer *send_ops, - grpc_call_element *elem) { - size_t i; +/** Filter initial metadata */ +static void process_send_initial_metadata( + grpc_call_element *elem, grpc_metadata_batch *initial_metadata) { call_data *calld = elem->call_data; - int new_slices_added = 0; /* GPR_FALSE */ - grpc_metadata_batch metadata; - grpc_stream_op_buffer new_send_ops; - grpc_sopb_init(&new_send_ops); - - for (i = 0; i < send_ops->nops; i++) { - grpc_stream_op *sop = &send_ops->ops[i]; - switch (sop->type) { - case GRPC_OP_BEGIN_MESSAGE: - GPR_ASSERT(calld->slices.length <= GPR_UINT32_MAX); - grpc_sopb_add_begin_message( - &new_send_ops, (gpr_uint32)calld->slices.length, - sop->data.begin_message.flags | GRPC_WRITE_INTERNAL_COMPRESS); - break; - case GRPC_OP_SLICE: - /* Once we reach the slices section of the original buffer, simply add - * all the new (compressed) slices. We obviously want to do this only - * once, hence the "new_slices_added" guard. */ - if (!new_slices_added) { - size_t j; - for (j = 0; j < calld->slices.count; ++j) { - grpc_sopb_add_slice(&new_send_ops, - gpr_slice_ref(calld->slices.slices[j])); - } - new_slices_added = 1; /* GPR_TRUE */ - } - break; - case GRPC_OP_METADATA: - /* move the metadata to the new buffer. */ - grpc_metadata_batch_move(&metadata, &sop->data.metadata); - grpc_sopb_add_metadata(&new_send_ops, metadata); - break; - case GRPC_NO_OP: - break; - } + channel_data *channeld = elem->channel_data; + /* Parse incoming request for compression. If any, it'll be available + * at calld->compression_algorithm */ + grpc_metadata_batch_filter(initial_metadata, compression_md_filter, elem); + if (!calld->has_compression_algorithm) { + /* If no algorithm was found in the metadata and we aren't + * exceptionally skipping compression, fall back to the channel + * default */ + calld->compression_algorithm = channeld->default_compression_algorithm; + calld->has_compression_algorithm = 1; /* GPR_TRUE */ } - grpc_sopb_swap(send_ops, &new_send_ops); - grpc_sopb_destroy(&new_send_ops); + /* hint compression algorithm */ + grpc_metadata_batch_add_tail( + initial_metadata, &calld->compression_algorithm_storage, + GRPC_MDELEM_REF( + channeld + ->mdelem_compression_algorithms[calld->compression_algorithm])); + + /* convey supported compression algorithms */ + grpc_metadata_batch_add_tail( + initial_metadata, &calld->accept_encoding_storage, + GRPC_MDELEM_REF(channeld->mdelem_accept_encoding)); } -/** Filter's "main" function, called for any incoming grpc_transport_stream_op - * instance that holds a non-zero number of send operations, accesible to this - * function in \a send_ops. */ -static void process_send_ops(grpc_call_element *elem, - grpc_stream_op_buffer *send_ops) { - call_data *calld = elem->call_data; - channel_data *channeld = elem->channel_data; - size_t i; - int did_compress = 0; +static void continue_send_message(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem); - /* In streaming calls, we need to reset the previously accumulated slices */ +static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, int success) { + grpc_call_element *elem = elemp; + call_data *calld = elem->call_data; gpr_slice_buffer_reset_and_unref(&calld->slices); - for (i = 0; i < send_ops->nops; ++i) { - grpc_stream_op *sop = &send_ops->ops[i]; - switch (sop->type) { - case GRPC_OP_BEGIN_MESSAGE: - /* buffer up slices until we've processed all the expected ones (as - * given by GRPC_OP_BEGIN_MESSAGE) */ - calld->remaining_slice_bytes = sop->data.begin_message.length; - if (sop->data.begin_message.flags & GRPC_WRITE_NO_COMPRESS) { - calld->has_compression_algorithm = 1; /* GPR_TRUE */ - calld->compression_algorithm = GRPC_COMPRESS_NONE; - } - break; - case GRPC_OP_METADATA: - if (!calld->written_initial_metadata) { - /* Parse incoming request for compression. If any, it'll be available - * at calld->compression_algorithm */ - grpc_metadata_batch_filter(&(sop->data.metadata), - compression_md_filter, elem); - if (!calld->has_compression_algorithm) { - /* If no algorithm was found in the metadata and we aren't - * exceptionally skipping compression, fall back to the channel - * default */ - calld->compression_algorithm = - channeld->default_compression_algorithm; - calld->has_compression_algorithm = 1; /* GPR_TRUE */ - } - /* hint compression algorithm */ - grpc_metadata_batch_add_tail( - &(sop->data.metadata), &calld->compression_algorithm_storage, - GRPC_MDELEM_REF(channeld->mdelem_compression_algorithms - [calld->compression_algorithm])); - - /* convey supported compression algorithms */ - grpc_metadata_batch_add_tail( - &(sop->data.metadata), &calld->accept_encoding_storage, - GRPC_MDELEM_REF(channeld->mdelem_accept_encoding)); - - calld->written_initial_metadata = 1; /* GPR_TRUE */ - } - break; - case GRPC_OP_SLICE: - if (skip_compression(channeld, calld)) continue; - GPR_ASSERT(calld->remaining_slice_bytes > 0); - /* Increase input ref count, gpr_slice_buffer_add takes ownership. */ - gpr_slice_buffer_add(&calld->slices, gpr_slice_ref(sop->data.slice)); - GPR_ASSERT(GPR_SLICE_LENGTH(sop->data.slice) <= - calld->remaining_slice_bytes); - calld->remaining_slice_bytes -= - (gpr_uint32)GPR_SLICE_LENGTH(sop->data.slice); - if (calld->remaining_slice_bytes == 0) { - did_compress = - compress_send_sb(calld->compression_algorithm, &calld->slices); - } - break; - case GRPC_NO_OP: - break; - } - } + calld->post_send->cb(exec_ctx, calld->post_send->cb_arg, success); +} - /* Modify the send_ops stream_op_buffer depending on whether compression was - * carried out */ +static void finish_send_message(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + call_data *calld = elem->call_data; + int did_compress; + gpr_slice_buffer tmp; + gpr_slice_buffer_init(&tmp); + did_compress = + grpc_msg_compress(calld->compression_algorithm, &calld->slices, &tmp); if (did_compress) { - finish_compressed_sopb(send_ops, elem); + gpr_slice_buffer_swap(&calld->slices, &tmp); + calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } + gpr_slice_buffer_destroy(&tmp); + + grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices, + calld->send_flags); + calld->send_op.send_message = &calld->replacement_stream.base; + calld->post_send = calld->send_op.on_complete; + calld->send_op.on_complete = &calld->send_done; + + grpc_call_next_op(exec_ctx, elem, &calld->send_op); +} + +static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, int success) { + grpc_call_element *elem = elemp; + call_data *calld = elem->call_data; + gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + if (calld->send_length == calld->slices.length) { + finish_send_message(exec_ctx, elem); + } else { + continue_send_message(exec_ctx, elem); + } +} + +static void continue_send_message(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + call_data *calld = elem->call_data; + while (grpc_byte_stream_next(exec_ctx, calld->send_op.send_message, + &calld->incoming_slice, ~(size_t)0, + &calld->got_slice)) { + gpr_slice_buffer_add(&calld->slices, calld->incoming_slice); + if (calld->send_length == calld->slices.length) { + finish_send_message(exec_ctx, elem); + break; + } } } -/* Called either: - - in response to an API call (or similar) from above, to send something - - a network event (or similar) from below, to receive something - op contains type and call direction information, in addition to the data - that is being sent or received. */ static void compress_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { + call_data *calld = elem->call_data; + GPR_TIMER_BEGIN("compress_start_transport_stream_op", 0); - if (op->send_ops && op->send_ops->nops > 0) { - process_send_ops(elem, op->send_ops); + if (op->send_initial_metadata) { + process_send_initial_metadata(elem, op->send_initial_metadata); + } + if (op->send_message != NULL && !skip_compression(elem) && + 0 == (op->send_message->flags & GRPC_WRITE_NO_COMPRESS)) { + calld->send_op = *op; + calld->send_length = op->send_message->length; + calld->send_flags = op->send_message->flags; + continue_send_message(exec_ctx, elem); + } else { + /* pass control down the stack */ + grpc_call_next_op(exec_ctx, elem, op); } GPR_TIMER_END("compress_start_transport_stream_op", 0); - - /* pass control down the stack */ - grpc_call_next_op(exec_ctx, elem, op); } /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; /* initialize members */ gpr_slice_buffer_init(&calld->slices); calld->has_compression_algorithm = 0; - calld->written_initial_metadata = 0; /* GPR_FALSE */ - - if (initial_op) { - if (initial_op->send_ops && initial_op->send_ops->nops > 0) { - process_send_ops(elem, initial_op->send_ops); - } - } + grpc_closure_init(&calld->got_slice, got_slice, elem); + grpc_closure_init(&calld->send_done, send_done, elem); } /* Destructor for call_data */ @@ -313,9 +262,8 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *channeld = elem->channel_data; grpc_compression_algorithm algo_idx; const char *supported_algorithms_names[GRPC_COMPRESS_ALGORITHMS_COUNT - 1]; @@ -325,24 +273,25 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_compression_options_init(&channeld->compression_options); channeld->compression_options.enabled_algorithms_bitset = - (gpr_uint32)grpc_channel_args_compression_algorithm_get_states(args); + (gpr_uint32)grpc_channel_args_compression_algorithm_get_states( + args->channel_args); channeld->default_compression_algorithm = - grpc_channel_args_get_compression_algorithm(args); + grpc_channel_args_get_compression_algorithm(args->channel_args); /* Make sure the default isn't disabled. */ GPR_ASSERT(grpc_compression_options_is_algorithm_enabled( &channeld->compression_options, channeld->default_compression_algorithm)); channeld->compression_options.default_compression_algorithm = channeld->default_compression_algorithm; - channeld->mdstr_request_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY); + channeld->mdstr_request_compression_algorithm_key = grpc_mdstr_from_string( + args->metadata_context, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY); channeld->mdstr_outgoing_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, "grpc-encoding"); + grpc_mdstr_from_string(args->metadata_context, "grpc-encoding"); channeld->mdstr_compression_capabilities_key = - grpc_mdstr_from_string(mdctx, "grpc-accept-encoding"); + grpc_mdstr_from_string(args->metadata_context, "grpc-accept-encoding"); for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { char *algorithm_name; @@ -354,9 +303,9 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(grpc_compression_algorithm_name(algo_idx, &algorithm_name) != 0); channeld->mdelem_compression_algorithms[algo_idx] = grpc_mdelem_from_metadata_strings( - mdctx, + args->metadata_context, GRPC_MDSTR_REF(channeld->mdstr_outgoing_compression_algorithm_key), - grpc_mdstr_from_string(mdctx, algorithm_name)); + grpc_mdstr_from_string(args->metadata_context, algorithm_name)); if (algo_idx > 0) { supported_algorithms_names[supported_algorithms_idx++] = algorithm_name; } @@ -369,11 +318,12 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, &accept_encoding_str_len); channeld->mdelem_accept_encoding = grpc_mdelem_from_metadata_strings( - mdctx, GRPC_MDSTR_REF(channeld->mdstr_compression_capabilities_key), - grpc_mdstr_from_string(mdctx, accept_encoding_str)); + args->metadata_context, + GRPC_MDSTR_REF(channeld->mdstr_compression_capabilities_key), + grpc_mdstr_from_string(args->metadata_context, accept_encoding_str)); gpr_free(accept_encoding_str); - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); } /* Destructor for channel data */ @@ -393,5 +343,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_compress_filter = { compress_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, grpc_call_next_get_peer, "compress"}; + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, "compress"}; diff --git a/src/core/channel/connected_channel.c b/src/core/channel/connected_channel.c index 6d4d7be632..0e1efd965a 100644 --- a/src/core/channel/connected_channel.c +++ b/src/core/channel/connected_channel.c @@ -83,8 +83,7 @@ static void con_start_transport_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; int r; @@ -92,10 +91,18 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, GPR_ASSERT(elem->filter == &grpc_connected_channel_filter); r = grpc_transport_init_stream(exec_ctx, chand->transport, TRANSPORT_STREAM_FROM_CALL_DATA(calld), - server_transport_data, initial_op); + args->refcount, args->server_transport_data); GPR_ASSERT(r == 0); } +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset) { + call_data *calld = elem->call_data; + channel_data *chand = elem->channel_data; + grpc_transport_set_pollset(exec_ctx, chand->transport, + TRANSPORT_STREAM_FROM_CALL_DATA(calld), pollset); +} + /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { @@ -108,11 +115,10 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *cd = (channel_data *)elem->channel_data; - GPR_ASSERT(is_last); + GPR_ASSERT(args->is_last); GPR_ASSERT(elem->filter == &grpc_connected_channel_filter); cd->transport = NULL; } @@ -132,8 +138,8 @@ static char *con_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { const grpc_channel_filter grpc_connected_channel_filter = { con_start_transport_stream_op, con_start_transport_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, con_get_peer, "connected", + init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, con_get_peer, "connected", }; void grpc_connected_channel_bind_transport(grpc_channel_stack *channel_stack, @@ -154,3 +160,8 @@ void grpc_connected_channel_bind_transport(grpc_channel_stack *channel_stack, channel. */ channel_stack->call_stack_size += grpc_transport_stream_size(transport); } + +grpc_stream *grpc_connected_channel_get_stream(grpc_call_element *elem) { + call_data *calld = elem->call_data; + return TRANSPORT_STREAM_FROM_CALL_DATA(calld); +} diff --git a/src/core/channel/connected_channel.h b/src/core/channel/connected_channel.h index eac6eb7ebe..95c1834bfa 100644 --- a/src/core/channel/connected_channel.h +++ b/src/core/channel/connected_channel.h @@ -46,4 +46,6 @@ extern const grpc_channel_filter grpc_connected_channel_filter; void grpc_connected_channel_bind_transport(grpc_channel_stack* channel_stack, grpc_transport* transport); +grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem); + #endif /* GRPC_INTERNAL_CORE_CHANNEL_CONNECTED_CHANNEL_H */ diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c index f78a5cc315..3a0f68f30f 100644 --- a/src/core/channel/http_client_filter.c +++ b/src/core/channel/http_client_filter.c @@ -45,10 +45,8 @@ typedef struct call_data { grpc_linked_mdelem te_trailers; grpc_linked_mdelem content_type; grpc_linked_mdelem user_agent; - int sent_initial_metadata; - int got_initial_metadata; - grpc_stream_op_buffer *recv_ops; + grpc_metadata_batch *recv_initial_metadata; /** Closure to call when finished with the hc_on_recv hook */ grpc_closure *on_done_recv; @@ -91,18 +89,11 @@ static grpc_mdelem *client_recv_filter(void *user_data, grpc_mdelem *md) { static void hc_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, int success) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - size_t i; - size_t nops = calld->recv_ops->nops; - grpc_stream_op *ops = calld->recv_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *op = &ops[i]; - client_recv_filter_args a; - if (op->type != GRPC_OP_METADATA) continue; - calld->got_initial_metadata = 1; - a.elem = elem; - a.exec_ctx = exec_ctx; - grpc_metadata_batch_filter(&op->data.metadata, client_recv_filter, &a); - } + client_recv_filter_args a; + a.elem = elem; + a.exec_ctx = exec_ctx; + grpc_metadata_batch_filter(calld->recv_initial_metadata, client_recv_filter, + &a); calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success); } @@ -123,40 +114,29 @@ static void hc_mutate_op(grpc_call_element *elem, /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; - size_t i; - if (op->send_ops && !calld->sent_initial_metadata) { - size_t nops = op->send_ops->nops; - grpc_stream_op *ops = op->send_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *stream_op = &ops[i]; - if (stream_op->type != GRPC_OP_METADATA) continue; - calld->sent_initial_metadata = 1; - grpc_metadata_batch_filter(&stream_op->data.metadata, client_strip_filter, - elem); - /* Send : prefixed headers, which have to be before any application - layer headers. */ - grpc_metadata_batch_add_head(&stream_op->data.metadata, &calld->method, - GRPC_MDELEM_REF(channeld->method)); - grpc_metadata_batch_add_head(&stream_op->data.metadata, &calld->scheme, - GRPC_MDELEM_REF(channeld->scheme)); - grpc_metadata_batch_add_tail(&stream_op->data.metadata, - &calld->te_trailers, - GRPC_MDELEM_REF(channeld->te_trailers)); - grpc_metadata_batch_add_tail(&stream_op->data.metadata, - &calld->content_type, - GRPC_MDELEM_REF(channeld->content_type)); - grpc_metadata_batch_add_tail(&stream_op->data.metadata, - &calld->user_agent, - GRPC_MDELEM_REF(channeld->user_agent)); - break; - } + if (op->send_initial_metadata != NULL) { + grpc_metadata_batch_filter(op->send_initial_metadata, client_strip_filter, + elem); + /* Send : prefixed headers, which have to be before any application + layer headers. */ + grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->method, + GRPC_MDELEM_REF(channeld->method)); + grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->scheme, + GRPC_MDELEM_REF(channeld->scheme)); + grpc_metadata_batch_add_tail(op->send_initial_metadata, &calld->te_trailers, + GRPC_MDELEM_REF(channeld->te_trailers)); + grpc_metadata_batch_add_tail(op->send_initial_metadata, + &calld->content_type, + GRPC_MDELEM_REF(channeld->content_type)); + grpc_metadata_batch_add_tail(op->send_initial_metadata, &calld->user_agent, + GRPC_MDELEM_REF(channeld->user_agent)); } - if (op->recv_ops && !calld->got_initial_metadata) { + if (op->recv_initial_metadata != NULL) { /* substitute our callback for the higher callback */ - calld->recv_ops = op->recv_ops; - calld->on_done_recv = op->on_done_recv; - op->on_done_recv = &calld->hc_on_recv; + calld->recv_initial_metadata = op->recv_initial_metadata; + calld->on_done_recv = op->on_complete; + op->on_complete = &calld->hc_on_recv; } } @@ -172,14 +152,10 @@ static void hc_start_transport_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *calld = elem->call_data; - calld->sent_initial_metadata = 0; - calld->got_initial_metadata = 0; calld->on_done_recv = NULL; grpc_closure_init(&calld->hc_on_recv, hc_on_recv, elem); - if (initial_op) hc_mutate_op(elem, initial_op); } /* Destructor for call_data */ @@ -250,28 +226,31 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *channel_args, - grpc_mdctx *mdctx, int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { /* grab pointers to our data from the channel element */ channel_data *channeld = elem->channel_data; /* The first and the last filters tend to be implemented differently to handle the case that there's no 'next' filter to call on the up or down path */ - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); /* initialize members */ - channeld->te_trailers = grpc_mdelem_from_strings(mdctx, "te", "trailers"); - channeld->method = grpc_mdelem_from_strings(mdctx, ":method", "POST"); - channeld->scheme = grpc_mdelem_from_strings(mdctx, ":scheme", - scheme_from_args(channel_args)); - channeld->content_type = - grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); - channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200"); + channeld->te_trailers = + grpc_mdelem_from_strings(args->metadata_context, "te", "trailers"); + channeld->method = + grpc_mdelem_from_strings(args->metadata_context, ":method", "POST"); + channeld->scheme = grpc_mdelem_from_strings( + args->metadata_context, ":scheme", scheme_from_args(args->channel_args)); + channeld->content_type = grpc_mdelem_from_strings( + args->metadata_context, "content-type", "application/grpc"); + channeld->status = + grpc_mdelem_from_strings(args->metadata_context, ":status", "200"); channeld->user_agent = grpc_mdelem_from_metadata_strings( - mdctx, grpc_mdstr_from_string(mdctx, "user-agent"), - user_agent_from_args(mdctx, channel_args)); + args->metadata_context, + grpc_mdstr_from_string(args->metadata_context, "user-agent"), + user_agent_from_args(args->metadata_context, args->channel_args)); } /* Destructor for channel data */ @@ -290,6 +269,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_http_client_filter = { hc_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, - "http-client"}; + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, "http-client"}; diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c index 99e5066a4e..2adfe2bb61 100644 --- a/src/core/channel/http_server_filter.c +++ b/src/core/channel/http_server_filter.c @@ -39,7 +39,6 @@ #include "src/core/profiling/timers.h" typedef struct call_data { - gpr_uint8 got_initial_metadata; gpr_uint8 seen_path; gpr_uint8 seen_post; gpr_uint8 sent_status; @@ -49,7 +48,7 @@ typedef struct call_data { grpc_linked_mdelem status; grpc_linked_mdelem content_type; - grpc_stream_op_buffer *recv_ops; + grpc_metadata_batch *recv_initial_metadata; /** Closure to call when finished with the hs_on_recv hook */ grpc_closure *on_done_recv; /** Receive closures are chained: we inject this closure as the on_done_recv @@ -154,43 +153,35 @@ static void hs_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, int success) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; if (success) { - size_t i; - size_t nops = calld->recv_ops->nops; - grpc_stream_op *ops = calld->recv_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *op = &ops[i]; - server_filter_args a; - if (op->type != GRPC_OP_METADATA) continue; - calld->got_initial_metadata = 1; - a.elem = elem; - a.exec_ctx = exec_ctx; - grpc_metadata_batch_filter(&op->data.metadata, server_filter, &a); - /* Have we seen the required http2 transport headers? - (:method, :scheme, content-type, with :path and :authority covered - at the channel level right now) */ - if (calld->seen_post && calld->seen_scheme && calld->seen_te_trailers && - calld->seen_path && calld->seen_authority) { - /* do nothing */ - } else { - if (!calld->seen_path) { - gpr_log(GPR_ERROR, "Missing :path header"); - } - if (!calld->seen_authority) { - gpr_log(GPR_ERROR, "Missing :authority header"); - } - if (!calld->seen_post) { - gpr_log(GPR_ERROR, "Missing :method header"); - } - if (!calld->seen_scheme) { - gpr_log(GPR_ERROR, "Missing :scheme header"); - } - if (!calld->seen_te_trailers) { - gpr_log(GPR_ERROR, "Missing te trailers header"); - } - /* Error this call out */ - success = 0; - grpc_call_element_send_cancel(exec_ctx, elem); + server_filter_args a; + a.elem = elem; + a.exec_ctx = exec_ctx; + grpc_metadata_batch_filter(calld->recv_initial_metadata, server_filter, &a); + /* Have we seen the required http2 transport headers? + (:method, :scheme, content-type, with :path and :authority covered + at the channel level right now) */ + if (calld->seen_post && calld->seen_scheme && calld->seen_te_trailers && + calld->seen_path && calld->seen_authority) { + /* do nothing */ + } else { + if (!calld->seen_path) { + gpr_log(GPR_ERROR, "Missing :path header"); + } + if (!calld->seen_authority) { + gpr_log(GPR_ERROR, "Missing :authority header"); + } + if (!calld->seen_post) { + gpr_log(GPR_ERROR, "Missing :method header"); } + if (!calld->seen_scheme) { + gpr_log(GPR_ERROR, "Missing :scheme header"); + } + if (!calld->seen_te_trailers) { + gpr_log(GPR_ERROR, "Missing te trailers header"); + } + /* Error this call out */ + success = 0; + grpc_call_element_send_cancel(exec_ctx, elem); } } calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success); @@ -201,29 +192,21 @@ static void hs_mutate_op(grpc_call_element *elem, /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; - size_t i; - if (op->send_ops && !calld->sent_status) { - size_t nops = op->send_ops->nops; - grpc_stream_op *ops = op->send_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *stream_op = &ops[i]; - if (stream_op->type != GRPC_OP_METADATA) continue; - calld->sent_status = 1; - grpc_metadata_batch_add_head(&stream_op->data.metadata, &calld->status, - GRPC_MDELEM_REF(channeld->status_ok)); - grpc_metadata_batch_add_tail(&stream_op->data.metadata, - &calld->content_type, - GRPC_MDELEM_REF(channeld->content_type)); - break; - } + if (op->send_initial_metadata != NULL && !calld->sent_status) { + calld->sent_status = 1; + grpc_metadata_batch_add_head(op->send_initial_metadata, &calld->status, + GRPC_MDELEM_REF(channeld->status_ok)); + grpc_metadata_batch_add_tail(op->send_initial_metadata, + &calld->content_type, + GRPC_MDELEM_REF(channeld->content_type)); } - if (op->recv_ops && !calld->got_initial_metadata) { + if (op->recv_initial_metadata) { /* substitute our callback for the higher callback */ - calld->recv_ops = op->recv_ops; - calld->on_done_recv = op->on_done_recv; - op->on_done_recv = &calld->hs_on_recv; + calld->recv_initial_metadata = op->recv_initial_metadata; + calld->on_done_recv = op->on_complete; + op->on_complete = &calld->hs_on_recv; } } @@ -239,14 +222,12 @@ static void hs_start_transport_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; /* initialize members */ memset(calld, 0, sizeof(*calld)); grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem); - if (initial_op) hs_mutate_op(elem, initial_op); } /* Destructor for call_data */ @@ -255,34 +236,39 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { /* grab pointers to our data from the channel element */ channel_data *channeld = elem->channel_data; /* The first and the last filters tend to be implemented differently to handle the case that there's no 'next' filter to call on the up or down path */ - GPR_ASSERT(!is_first); - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); /* initialize members */ - channeld->te_trailers = grpc_mdelem_from_strings(mdctx, "te", "trailers"); - channeld->status_ok = grpc_mdelem_from_strings(mdctx, ":status", "200"); + channeld->te_trailers = + grpc_mdelem_from_strings(args->metadata_context, "te", "trailers"); + channeld->status_ok = + grpc_mdelem_from_strings(args->metadata_context, ":status", "200"); channeld->status_not_found = - grpc_mdelem_from_strings(mdctx, ":status", "404"); - channeld->method_post = grpc_mdelem_from_strings(mdctx, ":method", "POST"); - channeld->http_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "http"); - channeld->https_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "https"); - channeld->grpc_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "grpc"); - channeld->path_key = grpc_mdstr_from_string(mdctx, ":path"); - channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority"); - channeld->host_key = grpc_mdstr_from_string(mdctx, "host"); - channeld->content_type = - grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); + grpc_mdelem_from_strings(args->metadata_context, ":status", "404"); + channeld->method_post = + grpc_mdelem_from_strings(args->metadata_context, ":method", "POST"); + channeld->http_scheme = + grpc_mdelem_from_strings(args->metadata_context, ":scheme", "http"); + channeld->https_scheme = + grpc_mdelem_from_strings(args->metadata_context, ":scheme", "https"); + channeld->grpc_scheme = + grpc_mdelem_from_strings(args->metadata_context, ":scheme", "grpc"); + channeld->path_key = grpc_mdstr_from_string(args->metadata_context, ":path"); + channeld->authority_key = + grpc_mdstr_from_string(args->metadata_context, ":authority"); + channeld->host_key = grpc_mdstr_from_string(args->metadata_context, "host"); + channeld->content_type = grpc_mdelem_from_strings( + args->metadata_context, "content-type", "application/grpc"); - channeld->mdctx = mdctx; + channeld->mdctx = args->metadata_context; } /* Destructor for channel data */ @@ -306,6 +292,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_http_server_filter = { hs_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, - "http-server"}; + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, "http-server"}; diff --git a/src/core/channel/noop_filter.c b/src/core/channel/noop_filter.c index 48f6b1c650..2fbf1c06bb 100644 --- a/src/core/channel/noop_filter.c +++ b/src/core/channel/noop_filter.c @@ -73,16 +73,13 @@ static void noop_start_transport_stream_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; /* initialize members */ calld->unused = channeld->unused; - - if (initial_op) noop_mutate_op(elem, initial_op); } /* Destructor for call_data */ @@ -91,17 +88,15 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { /* grab pointers to our data from the channel element */ channel_data *channeld = elem->channel_data; - /* The first and the last filters tend to be implemented differently to - handle the case that there's no 'next' filter to call on the up or down + /* The last filter tends to be implemented differently to + handle the case that there's no 'next' filter to call on the down path */ - GPR_ASSERT(!is_first); - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); /* initialize members */ channeld->unused = 0; @@ -118,5 +113,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_no_op_filter = { noop_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, grpc_call_next_get_peer, "no-op"}; + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, "no-op"}; diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c new file mode 100644 index 0000000000..7251714519 --- /dev/null +++ b/src/core/channel/subchannel_call_holder.c @@ -0,0 +1,283 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/channel/subchannel_call_holder.h" + +#include <grpc/support/alloc.h> + +#include "src/core/profiling/timers.h" + +#define GET_CALL(holder) \ + ((grpc_subchannel_call *)(gpr_atm_acq_load(&(holder)->subchannel_call))) + +#define CANCELLED_CALL ((grpc_subchannel_call *)1) + +static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *holder, + int success); +static void call_ready(grpc_exec_ctx *exec_ctx, void *holder, int success); +static void retry_ops(grpc_exec_ctx *exec_ctx, void *retry_ops_args, + int success); + +static void add_waiting_locked(grpc_subchannel_call_holder *holder, + grpc_transport_stream_op *op); +static void fail_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder); +static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder); + +void grpc_subchannel_call_holder_init( + grpc_subchannel_call_holder *holder, + grpc_subchannel_call_holder_pick_subchannel pick_subchannel, + void *pick_subchannel_arg) { + gpr_atm_rel_store(&holder->subchannel_call, 0); + holder->pick_subchannel = pick_subchannel; + holder->pick_subchannel_arg = pick_subchannel_arg; + gpr_mu_init(&holder->mu); + holder->subchannel = NULL; + holder->waiting_ops = NULL; + holder->waiting_ops_count = 0; + holder->waiting_ops_capacity = 0; + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; +} + +void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder) { + grpc_subchannel_call *call = GET_CALL(holder); + if (call != NULL && call != CANCELLED_CALL) { + GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "holder"); + } + GPR_ASSERT(holder->creation_phase == + GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING); + gpr_mu_destroy(&holder->mu); + GPR_ASSERT(holder->waiting_ops_count == 0); + gpr_free(holder->waiting_ops); +} + +void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder, + grpc_transport_stream_op *op) { + /* try to (atomically) get the call */ + grpc_subchannel_call *call = GET_CALL(holder); + GPR_TIMER_BEGIN("grpc_subchannel_call_holder_perform_op", 0); + if (call == CANCELLED_CALL) { + grpc_transport_stream_op_finish_with_failure(exec_ctx, op); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); + return; + } + if (call != NULL) { + grpc_subchannel_call_process_op(exec_ctx, call, op); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); + return; + } + /* we failed; lock and figure out what to do */ + gpr_mu_lock(&holder->mu); +retry: + /* need to recheck that another thread hasn't set the call */ + call = GET_CALL(holder); + if (call == CANCELLED_CALL) { + gpr_mu_unlock(&holder->mu); + grpc_transport_stream_op_finish_with_failure(exec_ctx, op); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); + return; + } + if (call != NULL) { + gpr_mu_unlock(&holder->mu); + grpc_subchannel_call_process_op(exec_ctx, call, op); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); + return; + } + /* if this is a cancellation, then we can raise our cancelled flag */ + if (op->cancel_with_status != GRPC_STATUS_OK) { + if (!gpr_atm_rel_cas(&holder->subchannel_call, 0, 1)) { + goto retry; + } else { + switch (holder->creation_phase) { + case GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING: + fail_locked(exec_ctx, holder); + break; + case GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL: + grpc_subchannel_cancel_create_call(exec_ctx, holder->subchannel, + &holder->subchannel_call); + break; + case GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL: + holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, NULL, + &holder->subchannel, NULL); + break; + } + gpr_mu_unlock(&holder->mu); + grpc_transport_stream_op_finish_with_failure(exec_ctx, op); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); + return; + } + } + /* if we don't have a subchannel, try to get one */ + if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && + holder->subchannel == NULL && op->send_initial_metadata != NULL) { + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL; + grpc_closure_init(&holder->next_step, subchannel_ready, holder); + if (holder->pick_subchannel(exec_ctx, holder->pick_subchannel_arg, + op->send_initial_metadata, &holder->subchannel, + &holder->next_step)) { + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + } + } + /* if we've got a subchannel, then let's ask it to create a call */ + if (holder->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING && + holder->subchannel != NULL) { + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL; + grpc_closure_init(&holder->next_step, call_ready, holder); + if (grpc_subchannel_create_call(exec_ctx, holder->subchannel, + holder->pollset, &holder->subchannel_call, + &holder->next_step)) { + /* got one immediately - continue the op (and any waiting ops) */ + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + retry_waiting_locked(exec_ctx, holder); + goto retry; + } + } + /* nothing to be done but wait */ + add_waiting_locked(holder, op); + gpr_mu_unlock(&holder->mu); + GPR_TIMER_END("grpc_subchannel_call_holder_perform_op", 0); +} + +static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, int success) { + grpc_subchannel_call_holder *holder = arg; + grpc_subchannel_call *call; + gpr_mu_lock(&holder->mu); + GPR_ASSERT(holder->creation_phase == + GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL); + call = GET_CALL(holder); + GPR_ASSERT(call == NULL || call == CANCELLED_CALL); + if (holder->subchannel == NULL) { + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + fail_locked(exec_ctx, holder); + } else { + grpc_closure_init(&holder->next_step, call_ready, holder); + if (grpc_subchannel_create_call(exec_ctx, holder->subchannel, + holder->pollset, &holder->subchannel_call, + &holder->next_step)) { + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + /* got one immediately - continue the op (and any waiting ops) */ + retry_waiting_locked(exec_ctx, holder); + } + } + gpr_mu_unlock(&holder->mu); +} + +static void call_ready(grpc_exec_ctx *exec_ctx, void *arg, int success) { + grpc_subchannel_call_holder *holder = arg; + GPR_TIMER_BEGIN("call_ready", 0); + gpr_mu_lock(&holder->mu); + GPR_ASSERT(holder->creation_phase == + GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL); + holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; + if (GET_CALL(holder) != NULL) { + retry_waiting_locked(exec_ctx, holder); + } else { + fail_locked(exec_ctx, holder); + } + gpr_mu_unlock(&holder->mu); + GPR_TIMER_END("call_ready", 0); +} + +typedef struct { + grpc_transport_stream_op *ops; + size_t nops; + grpc_subchannel_call *call; +} retry_ops_args; + +static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder) { + retry_ops_args *a = gpr_malloc(sizeof(*a)); + a->ops = holder->waiting_ops; + a->nops = holder->waiting_ops_count; + a->call = GET_CALL(holder); + if (a->call == CANCELLED_CALL) { + gpr_free(a); + fail_locked(exec_ctx, holder); + return; + } + holder->waiting_ops = NULL; + holder->waiting_ops_count = 0; + holder->waiting_ops_capacity = 0; + GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops"); + grpc_exec_ctx_enqueue(exec_ctx, grpc_closure_create(retry_ops, a), 1); +} + +static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, int success) { + retry_ops_args *a = args; + size_t i; + for (i = 0; i < a->nops; i++) { + grpc_subchannel_call_process_op(exec_ctx, a->call, &a->ops[i]); + } + GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops"); + gpr_free(a->ops); + gpr_free(a); +} + +static void add_waiting_locked(grpc_subchannel_call_holder *holder, + grpc_transport_stream_op *op) { + GPR_TIMER_BEGIN("add_waiting_locked", 0); + if (holder->waiting_ops_count == holder->waiting_ops_capacity) { + holder->waiting_ops_capacity = GPR_MAX(3, 2 * holder->waiting_ops_capacity); + holder->waiting_ops = + gpr_realloc(holder->waiting_ops, holder->waiting_ops_capacity * + sizeof(*holder->waiting_ops)); + } + holder->waiting_ops[holder->waiting_ops_count++] = *op; + GPR_TIMER_END("add_waiting_locked", 0); +} + +static void fail_locked(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder) { + size_t i; + for (i = 0; i < holder->waiting_ops_count; i++) { + grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].on_complete, 0); + grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].recv_message_ready, + 0); + } + holder->waiting_ops_count = 0; +} + +char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder, + grpc_channel *master) { + grpc_subchannel_call *subchannel_call = GET_CALL(holder); + + if (subchannel_call) { + return grpc_subchannel_call_get_peer(exec_ctx, subchannel_call); + } else { + return grpc_channel_get_target(master); + } +} diff --git a/src/core/channel/subchannel_call_holder.h b/src/core/channel/subchannel_call_holder.h new file mode 100644 index 0000000000..bda051c566 --- /dev/null +++ b/src/core/channel/subchannel_call_holder.h @@ -0,0 +1,98 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CHANNEL_SUBCHANNEL_CALL_HOLDER_H +#define GRPC_INTERNAL_CORE_CHANNEL_SUBCHANNEL_CALL_HOLDER_H + +#include "src/core/client_config/subchannel.h" + +/** Pick a subchannel for grpc_subchannel_call_holder; + Return 1 if subchannel is available immediately (in which case on_ready + should not be called), or 0 otherwise (in which case on_ready should be + called when the subchannel is available) */ +typedef int (*grpc_subchannel_call_holder_pick_subchannel)( + grpc_exec_ctx *exec_ctx, void *arg, grpc_metadata_batch *initial_metadata, + grpc_subchannel **subchannel, grpc_closure *on_ready); + +typedef enum { + GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING, + GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL, + GRPC_SUBCHANNEL_CALL_HOLDER_CREATING_CALL +} grpc_subchannel_call_holder_creation_phase; + +/** Wrapper for holding a pointer to grpc_subchannel_call, and the + associated machinery to create such a pointer. + Handles queueing of stream ops until a call object is ready, waiting + for initial metadata before trying to create a call object, + and handling cancellation gracefully. + + Both the channel and uchannel filter use this as their call_data. */ +typedef struct grpc_subchannel_call_holder { + /** either 0 for no call, 1 for cancelled, or a pointer to a + grpc_subchannel_call */ + gpr_atm subchannel_call; + /** Helper function to choose the subchannel on which to create + the call object. Channel filter delegates to the load + balancing policy (once it's ready); uchannel returns + immediately */ + grpc_subchannel_call_holder_pick_subchannel pick_subchannel; + void *pick_subchannel_arg; + + gpr_mu mu; + + grpc_subchannel_call_holder_creation_phase creation_phase; + grpc_subchannel *subchannel; + grpc_pollset *pollset; + + grpc_transport_stream_op *waiting_ops; + size_t waiting_ops_count; + size_t waiting_ops_capacity; + + grpc_closure next_step; +} grpc_subchannel_call_holder; + +void grpc_subchannel_call_holder_init( + grpc_subchannel_call_holder *holder, + grpc_subchannel_call_holder_pick_subchannel pick_subchannel, + void *pick_subchannel_arg); +void grpc_subchannel_call_holder_destroy(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder); + +void grpc_subchannel_call_holder_perform_op(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder, + grpc_transport_stream_op *op); +char *grpc_subchannel_call_holder_get_peer(grpc_exec_ctx *exec_ctx, + grpc_subchannel_call_holder *holder, + grpc_channel *master); + +#endif diff --git a/src/core/client_config/connector.h b/src/core/client_config/connector.h index e9b8be4b53..a649f143ae 100644 --- a/src/core/client_config/connector.h +++ b/src/core/client_config/connector.h @@ -51,6 +51,8 @@ typedef struct { /** address to connect to */ const struct sockaddr *addr; size_t addr_len; + /** initial connect string to send */ + gpr_slice initial_connect_string; /** deadline for connection */ gpr_timespec deadline; /** channel arguments (to be passed to transport) */ diff --git a/src/core/client_config/default_initial_connect_string.c b/src/core/client_config/default_initial_connect_string.c new file mode 100644 index 0000000000..6a4e23e6f2 --- /dev/null +++ b/src/core/client_config/default_initial_connect_string.c @@ -0,0 +1,39 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/slice.h> +#include "src/core/iomgr/sockaddr.h" + +void grpc_set_default_initial_connect_string(struct sockaddr **addr, + size_t *addr_len, + gpr_slice *initial_str) {} diff --git a/src/core/surface/byte_buffer_queue.h b/src/core/client_config/initial_connect_string.c index 2c3b22d24e..19afa1675a 100644 --- a/src/core/surface/byte_buffer_queue.h +++ b/src/core/client_config/initial_connect_string.c @@ -31,32 +31,23 @@ * */ -#ifndef GRPC_INTERNAL_CORE_SURFACE_BYTE_BUFFER_QUEUE_H -#define GRPC_INTERNAL_CORE_SURFACE_BYTE_BUFFER_QUEUE_H +#include "src/core/client_config/initial_connect_string.h" -#include <grpc/byte_buffer.h> +#include <stddef.h> -/* TODO(ctiller): inline an element or two into this struct to avoid per-call - allocations */ -typedef struct { - grpc_byte_buffer **data; - size_t count; - size_t capacity; -} grpc_bbq_array; +extern void grpc_set_default_initial_connect_string(struct sockaddr **addr, + size_t *addr_len, + gpr_slice *initial_str); -/* should be initialized by zeroing memory */ -typedef struct { - size_t drain_pos; - grpc_bbq_array filling; - grpc_bbq_array draining; - size_t bytes; -} grpc_byte_buffer_queue; +static grpc_set_initial_connect_string_func g_set_initial_connect_string_func = + grpc_set_default_initial_connect_string; -void grpc_bbq_destroy(grpc_byte_buffer_queue *q); -grpc_byte_buffer *grpc_bbq_pop(grpc_byte_buffer_queue *q); -void grpc_bbq_flush(grpc_byte_buffer_queue *q); -int grpc_bbq_empty(grpc_byte_buffer_queue *q); -void grpc_bbq_push(grpc_byte_buffer_queue *q, grpc_byte_buffer *bb); -size_t grpc_bbq_bytes(grpc_byte_buffer_queue *q); +void grpc_test_set_initial_connect_string_function( + grpc_set_initial_connect_string_func func) { + g_set_initial_connect_string_func = func; +} -#endif /* GRPC_INTERNAL_CORE_SURFACE_BYTE_BUFFER_QUEUE_H */ +void grpc_set_initial_connect_string(struct sockaddr **addr, size_t *addr_len, + gpr_slice *initial_str) { + g_set_initial_connect_string_func(addr, addr_len, initial_str); +} diff --git a/src/core/client_config/initial_connect_string.h b/src/core/client_config/initial_connect_string.h new file mode 100644 index 0000000000..b6dca7134a --- /dev/null +++ b/src/core/client_config/initial_connect_string.h @@ -0,0 +1,50 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H +#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H + +#include <grpc/support/slice.h> +#include "src/core/iomgr/sockaddr.h" + +typedef void (*grpc_set_initial_connect_string_func)(struct sockaddr **addr, + size_t *addr_len, + gpr_slice *initial_str); +void grpc_test_set_initial_connect_string_function( + grpc_set_initial_connect_string_func func); + +/** Set a string to be sent once connected. Optionally reset addr. */ +void grpc_set_initial_connect_string(struct sockaddr **addr, size_t *addr_len, + gpr_slice *connect_string); + +#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_INITIAL_CONNECT_STRING_H */ diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index e5bf0680ff..93312abb00 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -130,6 +130,30 @@ void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { } } +static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_subchannel **target) { + pick_first_lb_policy *p = (pick_first_lb_policy *)pol; + pending_pick *pp; + gpr_mu_lock(&p->mu); + pp = p->pending_picks; + p->pending_picks = NULL; + while (pp != NULL) { + pending_pick *next = pp->next; + if (pp->target == target) { + grpc_subchannel_del_interested_party( + exec_ctx, p->subchannels[p->checking_subchannel], pp->pollset); + *target = NULL; + grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 0); + gpr_free(pp); + } else { + pp->next = p->pending_picks; + p->pending_picks = pp; + } + pp = next; + } + gpr_mu_unlock(&p->mu); +} + static void start_picking(grpc_exec_ctx *exec_ctx, pick_first_lb_policy *p) { p->started_picking = 1; p->checking_subchannel = 0; @@ -149,16 +173,16 @@ void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_mu_unlock(&p->mu); } -void pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_pollset *pollset, grpc_metadata_batch *initial_metadata, - grpc_subchannel **target, grpc_closure *on_complete) { +int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, + grpc_metadata_batch *initial_metadata, grpc_subchannel **target, + grpc_closure *on_complete) { pick_first_lb_policy *p = (pick_first_lb_policy *)pol; pending_pick *pp; gpr_mu_lock(&p->mu); if (p->selected) { gpr_mu_unlock(&p->mu); *target = p->selected; - grpc_exec_ctx_enqueue(exec_ctx, on_complete, 1); + return 1; } else { if (!p->started_picking) { start_picking(exec_ctx, p); @@ -172,6 +196,7 @@ void pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pp->on_complete = on_complete; p->pending_picks = pp; gpr_mu_unlock(&p->mu); + return 0; } } @@ -365,8 +390,8 @@ void pf_notify_on_state_change(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } static const grpc_lb_policy_vtable pick_first_lb_policy_vtable = { - pf_destroy, pf_shutdown, pf_pick, pf_exit_idle, pf_broadcast, - pf_check_connectivity, pf_notify_on_state_change}; + pf_destroy, pf_shutdown, pf_pick, pf_cancel_pick, pf_exit_idle, + pf_broadcast, pf_check_connectivity, pf_notify_on_state_change}; static void pick_first_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policies/round_robin.c b/src/core/client_config/lb_policies/round_robin.c index d0b60a0df2..1ffe32fff2 100644 --- a/src/core/client_config/lb_policies/round_robin.c +++ b/src/core/client_config/lb_policies/round_robin.c @@ -264,6 +264,33 @@ void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_mu_unlock(&p->mu); } +static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_subchannel **target) { + round_robin_lb_policy *p = (round_robin_lb_policy *)pol; + pending_pick *pp; + size_t i; + gpr_mu_lock(&p->mu); + pp = p->pending_picks; + p->pending_picks = NULL; + while (pp != NULL) { + pending_pick *next = pp->next; + if (pp->target == target) { + for (i = 0; i < p->num_subchannels; i++) { + grpc_subchannel_add_interested_party(exec_ctx, p->subchannels[i], + pp->pollset); + } + *target = NULL; + grpc_exec_ctx_enqueue(exec_ctx, pp->on_complete, 0); + gpr_free(pp); + } else { + pp->next = p->pending_picks; + p->pending_picks = pp; + } + pp = next; + } + gpr_mu_unlock(&p->mu); +} + static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { size_t i; p->started_picking = 1; @@ -286,9 +313,9 @@ void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_mu_unlock(&p->mu); } -void rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_pollset *pollset, grpc_metadata_batch *initial_metadata, - grpc_subchannel **target, grpc_closure *on_complete) { +int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_pollset *pollset, + grpc_metadata_batch *initial_metadata, grpc_subchannel **target, + grpc_closure *on_complete) { size_t i; round_robin_lb_policy *p = (round_robin_lb_policy *)pol; pending_pick *pp; @@ -303,7 +330,7 @@ void rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, } /* only advance the last picked pointer if the selection was used */ advance_last_picked_locked(p); - on_complete->cb(exec_ctx, on_complete->cb_arg, 1); + return 1; } else { if (!p->started_picking) { start_picking(exec_ctx, p); @@ -319,6 +346,7 @@ void rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pp->on_complete = on_complete; p->pending_picks = pp; gpr_mu_unlock(&p->mu); + return 0; } } @@ -487,8 +515,8 @@ static void rr_notify_on_state_change(grpc_exec_ctx *exec_ctx, } static const grpc_lb_policy_vtable round_robin_lb_policy_vtable = { - rr_destroy, rr_shutdown, rr_pick, rr_exit_idle, rr_broadcast, - rr_check_connectivity, rr_notify_on_state_change}; + rr_destroy, rr_shutdown, rr_pick, rr_cancel_pick, rr_exit_idle, + rr_broadcast, rr_check_connectivity, rr_notify_on_state_change}; static void round_robin_factory_ref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/client_config/lb_policy.c b/src/core/client_config/lb_policy.c index c955186f7f..36a2454309 100644 --- a/src/core/client_config/lb_policy.c +++ b/src/core/client_config/lb_policy.c @@ -68,12 +68,17 @@ void grpc_lb_policy_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy) { policy->vtable->shutdown(exec_ctx, policy); } -void grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_pollset *pollset, - grpc_metadata_batch *initial_metadata, - grpc_subchannel **target, grpc_closure *on_complete) { - policy->vtable->pick(exec_ctx, policy, pollset, initial_metadata, target, - on_complete); +int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_pollset *pollset, + grpc_metadata_batch *initial_metadata, + grpc_subchannel **target, grpc_closure *on_complete) { + return policy->vtable->pick(exec_ctx, policy, pollset, initial_metadata, + target, on_complete); +} + +void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_subchannel **target) { + policy->vtable->cancel_pick(exec_ctx, policy, target); } void grpc_lb_policy_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, diff --git a/src/core/client_config/lb_policy.h b/src/core/client_config/lb_policy.h index 0eefe64991..a696c3ce64 100644 --- a/src/core/client_config/lb_policy.h +++ b/src/core/client_config/lb_policy.h @@ -56,9 +56,11 @@ struct grpc_lb_policy_vtable { void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); /** implement grpc_lb_policy_pick */ - void (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_pollset *pollset, grpc_metadata_batch *initial_metadata, - grpc_subchannel **target, grpc_closure *on_complete); + int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_pollset *pollset, grpc_metadata_batch *initial_metadata, + grpc_subchannel **target, grpc_closure *on_complete); + void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_subchannel **target); /** try to enter a READY connectivity state */ void (*exit_idle)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); @@ -106,10 +108,13 @@ void grpc_lb_policy_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); target for this rpc, and 'return' it by calling \a on_complete after setting \a target. Picking can be asynchronous. Any IO should be done under \a pollset. */ -void grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_pollset *pollset, - grpc_metadata_batch *initial_metadata, - grpc_subchannel **target, grpc_closure *on_complete); +int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_pollset *pollset, + grpc_metadata_batch *initial_metadata, + grpc_subchannel **target, grpc_closure *on_complete); + +void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + grpc_subchannel **target); void grpc_lb_policy_broadcast(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_transport_op *op); diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 0401dd3868..ff211bc2e6 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -40,9 +40,12 @@ #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/connected_channel.h" +#include "src/core/client_config/initial_connect_string.h" #include "src/core/iomgr/timer.h" -#include "src/core/transport/connectivity_state.h" +#include "src/core/profiling/timers.h" #include "src/core/surface/channel.h" +#include "src/core/transport/connectivity_state.h" +#include "src/core/transport/connectivity_state.h" #define GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS 20 #define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1 @@ -69,7 +72,7 @@ typedef struct waiting_for_connect { struct waiting_for_connect *next; grpc_closure *notify; grpc_pollset *pollset; - grpc_subchannel_call **target; + gpr_atm *target; grpc_subchannel *subchannel; grpc_closure continuation; } waiting_for_connect; @@ -85,6 +88,8 @@ struct grpc_subchannel { /** address to connect to */ struct sockaddr *addr; size_t addr_len; + /** initial string to send to peer */ + gpr_slice initial_connect_string; /** metadata context */ grpc_mdctx *mdctx; /** master channel - the grpc_channel instance that ultimately owns @@ -137,14 +142,16 @@ struct grpc_subchannel { struct grpc_subchannel_call { connection *connection; - gpr_refcount refs; }; #define SUBCHANNEL_CALL_TO_CALL_STACK(call) ((grpc_call_stack *)((call) + 1)) #define CHANNEL_STACK_FROM_CONNECTION(con) ((grpc_channel_stack *)((con) + 1)) +#define CALLSTACK_TO_SUBCHANNEL_CALL(callstack) \ + (((grpc_subchannel_call *)(callstack)) - 1) static grpc_subchannel_call *create_call(grpc_exec_ctx *exec_ctx, - connection *con); + connection *con, + grpc_pollset *pollset); static void connectivity_state_changed_locked(grpc_exec_ctx *exec_ctx, grpc_subchannel *c, const char *reason); @@ -163,7 +170,7 @@ static grpc_subchannel *connection_unref_locked( connection *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) GRPC_MUST_USE_RESULT; static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c); -#ifdef GRPC_SUBCHANNEL_REFCOUNT_DEBUG +#ifdef GRPC_STREAM_REFCOUNT_DEBUG #define SUBCHANNEL_REF_LOCKED(p, r) \ subchannel_ref_locked((p), __FILE__, __LINE__, (r)) #define SUBCHANNEL_UNREF_LOCKED(p, r) \ @@ -173,6 +180,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c); #define CONNECTION_UNREF_LOCKED(cl, p, r) \ connection_unref_locked((cl), (p), __FILE__, __LINE__, (r)) #define REF_PASS_ARGS , file, line, reason +#define REF_PASS_REASON , reason #define REF_LOG(name, p) \ gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "%s: %p ref %d -> %d %s", \ (name), (p), (p)->refs, (p)->refs + 1, reason) @@ -185,6 +193,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c); #define CONNECTION_REF_LOCKED(p, r) connection_ref_locked((p)) #define CONNECTION_UNREF_LOCKED(cl, p, r) connection_unref_locked((cl), (p)) #define REF_PASS_ARGS +#define REF_PASS_REASON #define REF_LOG(name, p) \ do { \ } while (0) @@ -261,6 +270,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { gpr_free((void *)c->filters); grpc_channel_args_destroy(c->args); gpr_free(c->addr); + gpr_slice_unref(c->initial_connect_string); grpc_mdctx_unref(c->mdctx); grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); @@ -299,6 +309,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, c->addr = gpr_malloc(args->addr_len); memcpy(c->addr, args->addr, args->addr_len); c->addr_len = args->addr_len; + grpc_set_initial_connect_string(&c->addr, &c->addr_len, + &c->initial_connect_string); c->args = grpc_channel_args_copy(args->args); c->mdctx = args->mdctx; c->master = args->master; @@ -312,9 +324,9 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, return c; } -void grpc_subchannel_cancel_waiting_call(grpc_exec_ctx *exec_ctx, - grpc_subchannel *subchannel, - int iomgr_success) { +static void cancel_waiting_calls(grpc_exec_ctx *exec_ctx, + grpc_subchannel *subchannel, + int iomgr_success) { waiting_for_connect *w4c; gpr_mu_lock(&subchannel->mu); w4c = subchannel->waiting; @@ -335,6 +347,37 @@ void grpc_subchannel_cancel_waiting_call(grpc_exec_ctx *exec_ctx, } } +void grpc_subchannel_cancel_create_call(grpc_exec_ctx *exec_ctx, + grpc_subchannel *subchannel, + gpr_atm *target) { + waiting_for_connect *w4c; + int unref_count = 0; + gpr_mu_lock(&subchannel->mu); + w4c = subchannel->waiting; + subchannel->waiting = NULL; + while (w4c != NULL) { + waiting_for_connect *next = w4c->next; + if (w4c->target == target) { + grpc_subchannel_del_interested_party(exec_ctx, w4c->subchannel, + w4c->pollset); + grpc_exec_ctx_enqueue(exec_ctx, w4c->notify, 0); + + unref_count++; + gpr_free(w4c); + } else { + w4c->next = subchannel->waiting; + subchannel->waiting = w4c; + } + + w4c = next; + } + gpr_mu_unlock(&subchannel->mu); + + while (unref_count-- > 0) { + GRPC_SUBCHANNEL_UNREF(exec_ctx, subchannel, "waiting_for_connect"); + } +} + static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connect_in_args args; @@ -343,6 +386,7 @@ static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { args.addr_len = c->addr_len; args.deadline = compute_connect_deadline(c); args.channel_args = c->args; + args.initial_connect_string = c->initial_connect_string; grpc_connector_connect(exec_ctx, c->connector, &args, &c->connecting_result, &c->connected); @@ -358,29 +402,35 @@ static void start_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { static void continue_creating_call(grpc_exec_ctx *exec_ctx, void *arg, int iomgr_success) { - grpc_subchannel_call_create_status call_creation_status; + int call_creation_finished_ok; waiting_for_connect *w4c = arg; grpc_subchannel_del_interested_party(exec_ctx, w4c->subchannel, w4c->pollset); - call_creation_status = grpc_subchannel_create_call( + call_creation_finished_ok = grpc_subchannel_create_call( exec_ctx, w4c->subchannel, w4c->pollset, w4c->target, w4c->notify); - GPR_ASSERT(call_creation_status == GRPC_SUBCHANNEL_CALL_CREATE_READY); + GPR_ASSERT(call_creation_finished_ok == 1); w4c->notify->cb(exec_ctx, w4c->notify->cb_arg, iomgr_success); GRPC_SUBCHANNEL_UNREF(exec_ctx, w4c->subchannel, "waiting_for_connect"); gpr_free(w4c); } -grpc_subchannel_call_create_status grpc_subchannel_create_call( - grpc_exec_ctx *exec_ctx, grpc_subchannel *c, grpc_pollset *pollset, - grpc_subchannel_call **target, grpc_closure *notify) { +int grpc_subchannel_create_call(grpc_exec_ctx *exec_ctx, grpc_subchannel *c, + grpc_pollset *pollset, gpr_atm *target, + grpc_closure *notify) { connection *con; + grpc_subchannel_call *call; + GPR_TIMER_BEGIN("grpc_subchannel_create_call", 0); gpr_mu_lock(&c->mu); if (c->active != NULL) { con = c->active; CONNECTION_REF_LOCKED(con, "call"); gpr_mu_unlock(&c->mu); - *target = create_call(exec_ctx, con); - return GRPC_SUBCHANNEL_CALL_CREATE_READY; + call = create_call(exec_ctx, con, pollset); + if (!gpr_atm_rel_cas(target, 0, (gpr_atm)(gpr_uintptr)call)) { + GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, call, "failed to set"); + } + GPR_TIMER_END("grpc_subchannel_create_call", 0); + return 1; } else { waiting_for_connect *w4c = gpr_malloc(sizeof(*w4c)); w4c->next = c->waiting; @@ -405,7 +455,8 @@ grpc_subchannel_call_create_status grpc_subchannel_create_call( } else { gpr_mu_unlock(&c->mu); } - return GRPC_SUBCHANNEL_CALL_CREATE_PENDING; + GPR_TIMER_END("grpc_subchannel_create_call", 0); + return 0; } } @@ -653,10 +704,25 @@ static double generate_uniform_random_number(grpc_subchannel *c) { /* Update backoff_delta and next_attempt in subchannel */ static void update_reconnect_parameters(grpc_subchannel *c) { + size_t i; gpr_int32 backoff_delta_millis, jitter; gpr_int32 max_backoff_millis = GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000; double jitter_range; + + if (c->args) { + for (i = 0; i < c->args->num_args; i++) { + if (0 == strcmp(c->args->args[i].key, + "grpc.testing.fixed_reconnect_backoff")) { + GPR_ASSERT(c->args->args[i].type == GRPC_ARG_INTEGER); + c->next_attempt = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(c->args->args[i].value.integer, GPR_TIMESPAN)); + return; + } + } + } + backoff_delta_millis = (gpr_int32)(gpr_time_to_millis(c->backoff_delta) * GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER); @@ -687,7 +753,7 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, int iomgr_success) { update_reconnect_parameters(c); continue_connect(exec_ctx, c); } else { - grpc_subchannel_cancel_waiting_call(exec_ctx, c, iomgr_success); + cancel_waiting_calls(exec_ctx, c, iomgr_success); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->master, "connecting"); GRPC_SUBCHANNEL_UNREF(exec_ctx, c, "connecting"); } @@ -747,26 +813,40 @@ static void connectivity_state_changed_locked(grpc_exec_ctx *exec_ctx, * grpc_subchannel_call implementation */ +static void subchannel_call_destroy(grpc_exec_ctx *exec_ctx, void *call, + int success) { + grpc_subchannel_call *c = call; + gpr_mu *mu = &c->connection->subchannel->mu; + grpc_subchannel *destroy; + GPR_TIMER_BEGIN("grpc_subchannel_call_unref.destroy", 0); + grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c)); + gpr_mu_lock(mu); + destroy = CONNECTION_UNREF_LOCKED(exec_ctx, c->connection, "call"); + gpr_mu_unlock(mu); + gpr_free(c); + if (destroy != NULL) { + subchannel_destroy(exec_ctx, destroy); + } + GPR_TIMER_END("grpc_subchannel_call_unref.destroy", 0); +} + void grpc_subchannel_call_ref(grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { - gpr_ref(&c->refs); +#ifdef GRPC_STREAM_REFCOUNT_DEBUG + grpc_call_stack_ref(SUBCHANNEL_CALL_TO_CALL_STACK(c), reason); +#else + grpc_call_stack_ref(SUBCHANNEL_CALL_TO_CALL_STACK(c)); +#endif } void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx, grpc_subchannel_call *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { - if (gpr_unref(&c->refs)) { - gpr_mu *mu = &c->connection->subchannel->mu; - grpc_subchannel *destroy; - grpc_call_stack_destroy(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c)); - gpr_mu_lock(mu); - destroy = CONNECTION_UNREF_LOCKED(exec_ctx, c->connection, "call"); - gpr_mu_unlock(mu); - gpr_free(c); - if (destroy != NULL) { - subchannel_destroy(exec_ctx, destroy); - } - } +#ifdef GRPC_STREAM_REFCOUNT_DEBUG + grpc_call_stack_unref(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c), reason); +#else + grpc_call_stack_unref(exec_ctx, SUBCHANNEL_CALL_TO_CALL_STACK(c)); +#endif } char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx, @@ -785,14 +865,16 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx, } static grpc_subchannel_call *create_call(grpc_exec_ctx *exec_ctx, - connection *con) { + connection *con, + grpc_pollset *pollset) { grpc_channel_stack *chanstk = CHANNEL_STACK_FROM_CONNECTION(con); grpc_subchannel_call *call = gpr_malloc(sizeof(grpc_subchannel_call) + chanstk->call_stack_size); grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(call); call->connection = con; - gpr_ref_init(&call->refs, 1); - grpc_call_stack_init(exec_ctx, chanstk, NULL, NULL, callstk); + grpc_call_stack_init(exec_ctx, chanstk, 1, subchannel_call_destroy, call, + NULL, NULL, callstk); + grpc_call_stack_set_pollset(exec_ctx, callstk, pollset); return call; } @@ -803,3 +885,8 @@ grpc_mdctx *grpc_subchannel_get_mdctx(grpc_subchannel *subchannel) { grpc_channel *grpc_subchannel_get_master(grpc_subchannel *subchannel) { return subchannel->master; } + +grpc_call_stack *grpc_subchannel_call_get_call_stack( + grpc_subchannel_call *subchannel_call) { + return SUBCHANNEL_CALL_TO_CALL_STACK(subchannel_call); +} diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h index ec1cc7cc69..381b7689b8 100644 --- a/src/core/client_config/subchannel.h +++ b/src/core/client_config/subchannel.h @@ -44,7 +44,7 @@ typedef struct grpc_subchannel grpc_subchannel; typedef struct grpc_subchannel_call grpc_subchannel_call; typedef struct grpc_subchannel_args grpc_subchannel_args; -#ifdef GRPC_SUBCHANNEL_REFCOUNT_DEBUG +#ifdef GRPC_STREAM_REFCOUNT_DEBUG #define GRPC_SUBCHANNEL_REF(p, r) \ grpc_subchannel_ref((p), __FILE__, __LINE__, (r)) #define GRPC_SUBCHANNEL_UNREF(cl, p, r) \ @@ -75,27 +75,22 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx, grpc_subchannel_call *call GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -typedef enum { - GRPC_SUBCHANNEL_CALL_CREATE_READY, - GRPC_SUBCHANNEL_CALL_CREATE_PENDING -} grpc_subchannel_call_create_status; - /** construct a subchannel call (possibly asynchronously). * - * If the returned status is \a GRPC_SUBCHANNEL_CALL_CREATE_READY, the call will - * return immediately and \a target will point to a connected \a subchannel_call - * instance. Note that \a notify will \em not be invoked in this case. - * Otherwise, if the returned status is GRPC_SUBCHANNEL_CALL_CREATE_PENDING, the - * subchannel call will be created asynchronously, invoking the \a notify - * callback upon completion. */ -grpc_subchannel_call_create_status grpc_subchannel_create_call( - grpc_exec_ctx *exec_ctx, grpc_subchannel *subchannel, grpc_pollset *pollset, - grpc_subchannel_call **target, grpc_closure *notify); + * If the returned status is 1, the call will return immediately and \a target + * will point to a connected \a subchannel_call instance. Note that \a notify + * will \em not be invoked in this case. + * Otherwise, if the returned status is 0, the subchannel call will be created + * asynchronously, invoking the \a notify callback upon completion. */ +int grpc_subchannel_create_call(grpc_exec_ctx *exec_ctx, + grpc_subchannel *subchannel, + grpc_pollset *pollset, gpr_atm *target, + grpc_closure *notify); /** cancel \a call in the waiting state. */ -void grpc_subchannel_cancel_waiting_call(grpc_exec_ctx *exec_ctx, - grpc_subchannel *subchannel, - int iomgr_success); +void grpc_subchannel_cancel_create_call(grpc_exec_ctx *exec_ctx, + grpc_subchannel *subchannel, + gpr_atm *target); /** process a transport level op */ void grpc_subchannel_process_transport_op(grpc_exec_ctx *exec_ctx, @@ -138,6 +133,9 @@ void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx, char *grpc_subchannel_call_get_peer(grpc_exec_ctx *exec_ctx, grpc_subchannel_call *subchannel_call); +grpc_call_stack *grpc_subchannel_call_get_call_stack( + grpc_subchannel_call *subchannel_call); + struct grpc_subchannel_args { /** Channel filters for this channel - wrapped factories will likely want to mutate this */ diff --git a/src/core/iomgr/closure.c b/src/core/iomgr/closure.c index b4f1817de4..4aae52a454 100644 --- a/src/core/iomgr/closure.c +++ b/src/core/iomgr/closure.c @@ -39,18 +39,17 @@ void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb, void *cb_arg) { closure->cb = cb; closure->cb_arg = cb_arg; - closure->next = NULL; + closure->final_data = 0; } void grpc_closure_list_add(grpc_closure_list *closure_list, grpc_closure *closure, int success) { if (closure == NULL) return; - closure->next = NULL; - closure->success = success; + closure->final_data = (success != 0); if (closure_list->head == NULL) { closure_list->head = closure; } else { - closure_list->tail->next = closure; + closure_list->tail->final_data |= (gpr_uintptr)closure; } closure_list->tail = closure; } @@ -66,22 +65,12 @@ void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst) { if (dst->head == NULL) { *dst = *src; } else { - dst->tail->next = src->head; + dst->tail->final_data |= (gpr_uintptr)src->head; dst->tail = src->tail; } src->head = src->tail = NULL; } -grpc_closure *grpc_closure_list_pop(grpc_closure_list *list) { - grpc_closure *head; - if (list->head == NULL) { - return NULL; - } - head = list->head; - list->head = list->head->next; - return head; -} - typedef struct { grpc_iomgr_cb_func cb; void *cb_arg; @@ -103,3 +92,7 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg) { grpc_closure_init(&wc->wrapper, closure_wrapper, wc); return &wc->wrapper; } + +grpc_closure *grpc_closure_next(grpc_closure *closure) { + return (grpc_closure *)(closure->final_data & ~(gpr_uintptr)1); +} diff --git a/src/core/iomgr/closure.h b/src/core/iomgr/closure.h index 7a9f7ccad0..a1d738bf5a 100644 --- a/src/core/iomgr/closure.h +++ b/src/core/iomgr/closure.h @@ -34,7 +34,7 @@ #ifndef GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H #define GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H -#include <stddef.h> +#include <grpc/support/port_platform.h> struct grpc_closure; typedef struct grpc_closure grpc_closure; @@ -64,13 +64,10 @@ struct grpc_closure { /** Arguments to be passed to "cb". */ void *cb_arg; - /** Internal. A boolean indication to "cb" on the state of the iomgr. - * For instance, closures created during a shutdown would have this field set - * to false. */ - int success; - - /**< Internal. Do not touch */ - struct grpc_closure *next; + /** Once enqueued, contains in the lower bit the success of the closure, + and in the upper bits the pointer to the next closure in the list. + Before enqueing for execution, this is usable for scratch data. */ + gpr_uintptr final_data; }; /** Initializes \a closure with \a cb and \a cb_arg. */ @@ -91,10 +88,10 @@ void grpc_closure_list_add(grpc_closure_list *list, grpc_closure *closure, /** append all closures from \a src to \a dst and empty \a src. */ void grpc_closure_list_move(grpc_closure_list *src, grpc_closure_list *dst); -/** pop (return and remove) the head closure from \a list. */ -grpc_closure *grpc_closure_list_pop(grpc_closure_list *list); - /** return whether \a list is empty. */ int grpc_closure_list_empty(grpc_closure_list list); +/** return the next pointer for a queued closure list */ +grpc_closure *grpc_closure_next(grpc_closure *closure); + #endif /* GRPC_INTERNAL_CORE_IOMGR_CLOSURE_H */ diff --git a/src/core/iomgr/exec_ctx.c b/src/core/iomgr/exec_ctx.c index 410b34c521..e95eaf267a 100644 --- a/src/core/iomgr/exec_ctx.c +++ b/src/core/iomgr/exec_ctx.c @@ -44,10 +44,11 @@ int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { grpc_closure *c = exec_ctx->closure_list.head; exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; while (c != NULL) { - grpc_closure *next = c->next; + int success = (int)(c->final_data & 1); + grpc_closure *next = (grpc_closure *)(c->final_data & ~(gpr_uintptr)1); did_something++; GPR_TIMER_BEGIN("grpc_exec_ctx_flush.cb", 0); - c->cb(exec_ctx, c->cb_arg, c->success); + c->cb(exec_ctx, c->cb_arg, success); GPR_TIMER_END("grpc_exec_ctx_flush.cb", 0); c = next; } diff --git a/src/core/iomgr/executor.c b/src/core/iomgr/executor.c index 457e5cdbac..00c68f7828 100644 --- a/src/core/iomgr/executor.c +++ b/src/core/iomgr/executor.c @@ -63,8 +63,6 @@ void grpc_executor_init() { /* thread body */ static void closure_exec_thread_func(void *ignored) { - grpc_closure *closure; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (1) { gpr_mu_lock(&g_executor.mu); @@ -72,16 +70,16 @@ static void closure_exec_thread_func(void *ignored) { gpr_mu_unlock(&g_executor.mu); break; } - closure = grpc_closure_list_pop(&g_executor.closures); - if (closure == NULL) { + if (grpc_closure_list_empty(g_executor.closures)) { /* no more work, time to die */ GPR_ASSERT(g_executor.busy == 1); g_executor.busy = 0; gpr_mu_unlock(&g_executor.mu); break; + } else { + grpc_exec_ctx_enqueue_list(&exec_ctx, &g_executor.closures); } gpr_mu_unlock(&g_executor.mu); - closure->cb(&exec_ctx, closure->cb_arg, closure->success); grpc_exec_ctx_flush(&exec_ctx); } grpc_exec_ctx_finish(&exec_ctx); @@ -125,7 +123,6 @@ void grpc_executor_enqueue(grpc_closure *closure, int success) { void grpc_executor_shutdown() { int pending_join; - grpc_closure *closure; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_mu_lock(&g_executor.mu); @@ -136,9 +133,7 @@ void grpc_executor_shutdown() { * list below because we aren't accepting new work */ /* Execute pending callbacks, some may be performing cleanups */ - while ((closure = grpc_closure_list_pop(&g_executor.closures)) != NULL) { - closure->cb(&exec_ctx, closure->cb_arg, closure->success); - } + grpc_exec_ctx_enqueue_list(&exec_ctx, &g_executor.closures); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(grpc_closure_list_empty(g_executor.closures)); if (pending_join) { diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index d15553a12a..c6b0214dea 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -55,8 +55,13 @@ #endif void grpc_pollset_init(grpc_pollset *pollset); +/* Begin shutting down the pollset, and call closure when done. + * GRPC_POLLSET_MU(pollset) must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); +/** Reset the pollset to its initial state (perhaps with some cached objects); + * must have been previously shutdown */ +void grpc_pollset_reset(grpc_pollset *pollset); void grpc_pollset_destroy(grpc_pollset *pollset); /* Do some work on a pollset. diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index 2aafd21dfb..1f1bf47e98 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -47,21 +47,13 @@ #include "src/core/support/block_annotate.h" #include "src/core/profiling/timers.h" -typedef struct wakeup_fd_hdl { - grpc_wakeup_fd wakeup_fd; - struct wakeup_fd_hdl *next; -} wakeup_fd_hdl; - typedef struct { grpc_pollset *pollset; grpc_fd *fd; grpc_closure closure; } delayed_add; -typedef struct { - int epoll_fd; - wakeup_fd_hdl *free_wakeup_fds; -} pollset_hdr; +typedef struct { int epoll_fd; } pollset_hdr; static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { @@ -174,7 +166,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( timeout_ms = grpc_poll_deadline_to_millis_timeout(deadline, now); - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfds[0].events = POLLIN; pfds[0].revents = 0; pfds[1].fd = h->epoll_fd; @@ -197,7 +189,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( /* do nothing */ } else { if (pfds[0].revents) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } if (pfds[1].revents) { do { diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index faa6c14491..09f04b64b9 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -124,7 +124,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfds[0].events = POLLIN; pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfds[1].events = POLLIN; pfds[1].revents = 0; for (i = 0; i < h->fd_count; i++) { @@ -174,7 +174,7 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); } if (pfds[1].revents & POLLIN_CHECK) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } for (i = 2; i < pfd_count; i++) { if (watchers[i].fd == NULL) { diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 6f478ccacb..c179248214 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -111,7 +111,7 @@ void grpc_pollset_kick_ext(grpc_pollset *p, for (specific_worker = p->root_worker.next; specific_worker != &p->root_worker; specific_worker = specific_worker->next) { - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } p->kicked_without_pollers = 1; GPR_TIMER_END("grpc_pollset_kick_ext.broadcast", 0); @@ -122,14 +122,14 @@ void grpc_pollset_kick_ext(grpc_pollset *p, specific_worker->reevaluate_polling_on_wakeup = 1; } specific_worker->kicked_specifically = 1; - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } else if ((flags & GRPC_POLLSET_CAN_KICK_SELF) != 0) { GPR_TIMER_MARK("kick_yoself", 0); if ((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) != 0) { specific_worker->reevaluate_polling_on_wakeup = 1; } specific_worker->kicked_specifically = 1; - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } } else if (gpr_tls_get(&g_current_thread_poller) != (gpr_intptr)p) { GPR_ASSERT((flags & GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) == 0); @@ -151,7 +151,7 @@ void grpc_pollset_kick_ext(grpc_pollset *p, if (specific_worker != NULL) { GPR_TIMER_MARK("finally_kick", 0); push_back_worker(p, specific_worker); - grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd); + grpc_wakeup_fd_wakeup(&specific_worker->wakeup_fd->fd); } } else { GPR_TIMER_MARK("kicked_no_pollers", 0); @@ -177,9 +177,9 @@ void grpc_pollset_global_init(void) { void grpc_pollset_global_shutdown(void) { grpc_wakeup_fd_destroy(&grpc_global_wakeup_fd); - grpc_wakeup_fd_global_destroy(); gpr_tls_destroy(&g_current_thread_poller); gpr_tls_destroy(&g_current_thread_worker); + grpc_wakeup_fd_global_destroy(); } void grpc_kick_poller(void) { grpc_wakeup_fd_wakeup(&grpc_global_wakeup_fd); } @@ -194,7 +194,36 @@ void grpc_pollset_init(grpc_pollset *pollset) { pollset->in_flight_cbs = 0; pollset->shutting_down = 0; pollset->called_shutdown = 0; + pollset->kicked_without_pollers = 0; pollset->idle_jobs.head = pollset->idle_jobs.tail = NULL; + pollset->local_wakeup_cache = NULL; + pollset->kicked_without_pollers = 0; + become_basic_pollset(pollset, NULL); +} + +void grpc_pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!grpc_pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + gpr_mu_destroy(&pollset->mu); + while (pollset->local_wakeup_cache) { + grpc_cached_wakeup_fd *next = pollset->local_wakeup_cache->next; + grpc_wakeup_fd_destroy(&pollset->local_wakeup_cache->fd); + gpr_free(pollset->local_wakeup_cache); + pollset->local_wakeup_cache = next; + } +} + +void grpc_pollset_reset(grpc_pollset *pollset) { + GPR_ASSERT(pollset->shutting_down); + GPR_ASSERT(pollset->in_flight_cbs == 0); + GPR_ASSERT(!grpc_pollset_has_workers(pollset)); + GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); + pollset->vtable->destroy(pollset); + pollset->shutting_down = 0; + pollset->called_shutdown = 0; + pollset->kicked_without_pollers = 0; become_basic_pollset(pollset, NULL); } @@ -244,13 +273,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* this must happen before we (potentially) drop pollset->mu */ worker->next = worker->prev = NULL; worker->reevaluate_polling_on_wakeup = 0; + if (pollset->local_wakeup_cache != NULL) { + worker->wakeup_fd = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker->wakeup_fd->next; + } else { + worker->wakeup_fd = gpr_malloc(sizeof(*worker->wakeup_fd)); + grpc_wakeup_fd_init(&worker->wakeup_fd->fd); + } worker->kicked_specifically = 0; - /* TODO(ctiller): pool these */ - grpc_wakeup_fd_init(&worker->wakeup_fd); /* If there's work waiting for the pollset to be idle, and the pollset is idle, then do that work */ if (!grpc_pollset_has_workers(pollset) && !grpc_closure_list_empty(pollset->idle_jobs)) { + GPR_TIMER_MARK("grpc_pollset_work.idle_jobs", 0); grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); goto done; } @@ -259,16 +294,19 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, May update deadline to ensure timely wakeups. TODO(ctiller): can this work be localized? */ if (grpc_timer_check(exec_ctx, now, &deadline)) { + GPR_TIMER_MARK("grpc_pollset_work.alarm_triggered", 0); gpr_mu_unlock(&pollset->mu); locked = 0; goto done; } /* If we're shutting down then we don't execute any extended work */ if (pollset->shutting_down) { + GPR_TIMER_MARK("grpc_pollset_work.shutting_down", 0); goto done; } /* Give do_promote priority so we don't starve it out */ if (pollset->in_flight_cbs) { + GPR_TIMER_MARK("grpc_pollset_work.in_flight_cbs", 0); gpr_mu_unlock(&pollset->mu); locked = 0; goto done; @@ -293,6 +331,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, locked = 0; gpr_tls_set(&g_current_thread_poller, 0); } else { + GPR_TIMER_MARK("grpc_pollset_work.kicked_without_pollers", 0); pollset->kicked_without_pollers = 0; } /* Finished execution - start cleaning up. @@ -323,7 +362,10 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, remove_worker(pollset, worker); gpr_tls_set(&g_current_thread_worker, 0); } - grpc_wakeup_fd_destroy(&worker->wakeup_fd); + /* release wakeup fd to the local pool */ + worker->wakeup_fd->next = pollset->local_wakeup_cache; + pollset->local_wakeup_cache = worker->wakeup_fd; + /* check shutdown conditions */ if (pollset->shutting_down) { if (grpc_pollset_has_workers(pollset)) { grpc_pollset_kick(pollset, NULL); @@ -338,8 +380,8 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */ gpr_mu_lock(&pollset->mu); } else if (!grpc_closure_list_empty(pollset->idle_jobs)) { - gpr_mu_unlock(&pollset->mu); grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); + gpr_mu_unlock(&pollset->mu); grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } @@ -349,35 +391,20 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { - int call_shutdown = 0; - gpr_mu_lock(&pollset->mu); GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; - if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && - !grpc_pollset_has_workers(pollset)) { - pollset->called_shutdown = 1; - call_shutdown = 1; - } + pollset->shutdown_done = closure; + grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!grpc_pollset_has_workers(pollset)) { grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs); } - pollset->shutdown_done = closure; - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - gpr_mu_unlock(&pollset->mu); - - if (call_shutdown) { + if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 && + !grpc_pollset_has_workers(pollset)) { + pollset->called_shutdown = 1; finish_shutdown(exec_ctx, pollset); } } -void grpc_pollset_destroy(grpc_pollset *pollset) { - GPR_ASSERT(pollset->shutting_down); - GPR_ASSERT(pollset->in_flight_cbs == 0); - GPR_ASSERT(!grpc_pollset_has_workers(pollset)); - pollset->vtable->destroy(pollset); - gpr_mu_destroy(&pollset->mu); -} - int grpc_poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { gpr_timespec timeout; @@ -557,7 +584,7 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, pfd[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfd[0].events = POLLIN; pfd[0].revents = 0; - pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd); + pfd[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker->wakeup_fd->fd); pfd[1].events = POLLIN; pfd[1].revents = 0; nfds = 2; @@ -599,7 +626,7 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd); } if (pfd[1].revents & POLLIN_CHECK) { - grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd); + grpc_wakeup_fd_consume_wakeup(&worker->wakeup_fd->fd); } if (nfds > 2) { grpc_fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK, diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h index 95ebeab1c2..e4593728bd 100644 --- a/src/core/iomgr/pollset_posix.h +++ b/src/core/iomgr/pollset_posix.h @@ -48,8 +48,13 @@ typedef struct grpc_pollset_vtable grpc_pollset_vtable; use the struct tag */ struct grpc_fd; +typedef struct grpc_cached_wakeup_fd { + grpc_wakeup_fd fd; + struct grpc_cached_wakeup_fd *next; +} grpc_cached_wakeup_fd; + typedef struct grpc_pollset_worker { - grpc_wakeup_fd wakeup_fd; + grpc_cached_wakeup_fd *wakeup_fd; int reevaluate_polling_on_wakeup; int kicked_specifically; struct grpc_pollset_worker *next; @@ -74,6 +79,8 @@ typedef struct grpc_pollset { int fd; void *ptr; } data; + /* Local cache of eventfds for workers */ + grpc_cached_wakeup_fd *local_wakeup_cache; } grpc_pollset; struct grpc_pollset_vtable { diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 9f74580273..deb661548d 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -35,6 +35,7 @@ #ifdef GPR_WINSOCK_SOCKET +#include <grpc/support/log.h> #include <grpc/support/thd.h> #include "src/core/iomgr/timer_internal.h" @@ -112,7 +113,6 @@ void grpc_pollset_init(grpc_pollset *pollset) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { - gpr_mu_lock(&grpc_polling_mu); pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { @@ -120,11 +120,20 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } else { pollset->on_shutdown = closure; } - gpr_mu_unlock(&grpc_polling_mu); } void grpc_pollset_destroy(grpc_pollset *pollset) {} +void grpc_pollset_reset(grpc_pollset *pollset) { + GPR_ASSERT(pollset->shutting_down); + GPR_ASSERT( + !has_workers(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET)); + pollset->shutting_down = 0; + pollset->is_iocp_worker = 0; + pollset->kicked_without_pollers = 0; + pollset->on_shutdown = NULL; +} + void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec now, gpr_timespec deadline) { diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h index 882635f638..3294e13797 100644 --- a/src/core/iomgr/tcp_server.h +++ b/src/core/iomgr/tcp_server.h @@ -39,6 +39,9 @@ /* Forward decl of grpc_tcp_server */ typedef struct grpc_tcp_server grpc_tcp_server; +/* Forward decl of grpc_tcp_listener */ +typedef struct grpc_tcp_listener grpc_tcp_listener; + /* Called for newly connected TCP connections. */ typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *ep); @@ -51,19 +54,17 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, grpc_pollset **pollsets, size_t pollset_count, grpc_tcp_server_cb on_accept_cb, void *cb_arg); -/* Add a port to the server, returning port number on success, or negative - on failure. +/* Add a port to the server, returning the newly created listener on success, + or a null pointer on failure. The :: and 0.0.0.0 wildcard addresses are treated identically, accepting both IPv4 and IPv6 connections, but :: is the preferred style. This usually creates one socket, but possibly two on systems which support IPv6, - but not dualstack sockets. - - For raw access to the underlying sockets, see grpc_tcp_server_get_fd(). */ + but not dualstack sockets. */ /* TODO(ctiller): deprecate this, and make grpc_tcp_server_add_ports to handle all of the multiple socket port matching logic in one place */ -int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - size_t addr_len); +grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, + const void *addr, size_t addr_len); /* Returns the file descriptor of the Nth listening socket on this server, or -1 if the index is out of bounds. @@ -75,4 +76,8 @@ int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index); void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server, grpc_closure *closure); +int grpc_tcp_listener_get_port(grpc_tcp_listener *listener); +void grpc_tcp_listener_ref(grpc_tcp_listener *listener); +void grpc_tcp_listener_unref(grpc_tcp_listener *listener); + #endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */ diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index 99c76dcbe9..0ece77c4e8 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -67,14 +67,13 @@ #include <grpc/support/sync.h> #include <grpc/support/time.h> -#define INIT_PORT_CAP 2 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 static gpr_once s_init_max_accept_queue_size; static int s_max_accept_queue_size; /* one listening port */ -typedef struct { +struct grpc_tcp_listener { int fd; grpc_fd *emfd; grpc_tcp_server *server; @@ -84,9 +83,18 @@ typedef struct { struct sockaddr_un un; } addr; size_t addr_len; + int port; grpc_closure read_closure; grpc_closure destroyed_closure; -} server_port; + gpr_refcount refs; + struct grpc_tcp_listener *next; + /* When we add a listener, more than one can be created, mainly because of + IPv6. A sibling will still be in the normal list, but will be flagged + as such. Any action, such as ref or unref, will affect all of the + siblings in the list. */ + struct grpc_tcp_listener *sibling; + int is_sibling; +}; static void unlink_if_unix_domain_socket(const struct sockaddr_un *un) { struct stat st; @@ -112,10 +120,9 @@ struct grpc_tcp_server { /* is this server shutting down? (boolean) */ int shutdown; - /* all listening ports */ - server_port *ports; - size_t nports; - size_t port_capacity; + /* linked list of server ports */ + grpc_tcp_listener *head; + unsigned nports; /* shutdown callback */ grpc_closure *shutdown_complete; @@ -134,9 +141,8 @@ grpc_tcp_server *grpc_tcp_server_create(void) { s->shutdown = 0; s->on_accept_cb = NULL; s->on_accept_cb_arg = NULL; - s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); + s->head = NULL; s->nports = 0; - s->port_capacity = INIT_PORT_CAP; return s; } @@ -145,7 +151,12 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { gpr_mu_destroy(&s->mu); - gpr_free(s->ports); + while (s->head) { + grpc_tcp_listener *sp = s->head; + s->head = sp->next; + grpc_tcp_listener_unref(sp); + } + gpr_free(s); } @@ -166,8 +177,6 @@ static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) { events will be received on them - at this point it's safe to destroy things */ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { - size_t i; - /* delete ALL the things */ gpr_mu_lock(&s->mu); @@ -176,9 +185,9 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { return; } - if (s->nports) { - for (i = 0; i < s->nports; i++) { - server_port *sp = &s->ports[i]; + if (s->head) { + grpc_tcp_listener *sp; + for (sp = s->head; sp; sp = sp->next) { if (sp->addr.sockaddr.sa_family == AF_UNIX) { unlink_if_unix_domain_socket(&sp->addr.un); } @@ -196,7 +205,6 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, grpc_closure *closure) { - size_t i; gpr_mu_lock(&s->mu); GPR_ASSERT(!s->shutdown); @@ -206,8 +214,9 @@ void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, /* shutdown all fd's */ if (s->active_ports) { - for (i = 0; i < s->nports; i++) { - grpc_fd_shutdown(exec_ctx, s->ports[i].emfd); + grpc_tcp_listener *sp; + for (sp = s->head; sp; sp = sp->next) { + grpc_fd_shutdown(exec_ctx, sp->emfd); } gpr_mu_unlock(&s->mu); } else { @@ -298,7 +307,7 @@ error: /* event manager callback when reads are ready */ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { - server_port *sp = arg; + grpc_tcp_listener *sp = arg; grpc_fd *fdobj; size_t i; @@ -364,9 +373,10 @@ error: } } -static int add_socket_to_server(grpc_tcp_server *s, int fd, - const struct sockaddr *addr, size_t addr_len) { - server_port *sp; +static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, int fd, + const struct sockaddr *addr, + size_t addr_len) { + grpc_tcp_listener *sp = NULL; int port; char *addr_str; char *name; @@ -376,32 +386,34 @@ static int add_socket_to_server(grpc_tcp_server *s, int fd, grpc_sockaddr_to_string(&addr_str, (struct sockaddr *)&addr, 1); gpr_asprintf(&name, "tcp-server-listener:%s", addr_str); gpr_mu_lock(&s->mu); + s->nports++; GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); - /* append it to the list under a lock */ - if (s->nports == s->port_capacity) { - s->port_capacity *= 2; - s->ports = gpr_realloc(s->ports, sizeof(server_port) * s->port_capacity); - } - sp = &s->ports[s->nports++]; + sp = gpr_malloc(sizeof(grpc_tcp_listener)); + sp->next = s->head; + s->head = sp; sp->server = s; sp->fd = fd; sp->emfd = grpc_fd_create(fd, name); memcpy(sp->addr.untyped, addr, addr_len); sp->addr_len = addr_len; + sp->port = port; + sp->is_sibling = 0; + sp->sibling = NULL; + gpr_ref_init(&sp->refs, 1); GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); gpr_free(addr_str); gpr_free(name); } - return port; + return sp; } -int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - size_t addr_len) { - int allocated_port1 = -1; - int allocated_port2 = -1; - unsigned i; +grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, + const void *addr, size_t addr_len) { + int allocated_port = -1; + grpc_tcp_listener *sp; + grpc_tcp_listener *sp2 = NULL; int fd; grpc_dualstack_mode dsmode; struct sockaddr_in6 addr6_v4mapped; @@ -420,9 +432,9 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, /* Check if this is a wildcard port, and if so, try to keep the port the same as some previously created listener. */ if (grpc_sockaddr_get_port(addr) == 0) { - for (i = 0; i < s->nports; i++) { + for (sp = s->head; sp; sp = sp->next) { sockname_len = sizeof(sockname_temp); - if (0 == getsockname(s->ports[i].fd, (struct sockaddr *)&sockname_temp, + if (0 == getsockname(sp->fd, (struct sockaddr *)&sockname_temp, &sockname_len)) { port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); if (port > 0) { @@ -436,6 +448,8 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, } } + sp = NULL; + if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = (const struct sockaddr *)&addr6_v4mapped; addr_len = sizeof(addr6_v4mapped); @@ -449,14 +463,16 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, addr = (struct sockaddr *)&wild6; addr_len = sizeof(wild6); fd = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode); - allocated_port1 = add_socket_to_server(s, fd, addr, addr_len); + sp = add_socket_to_server(s, fd, addr, addr_len); + allocated_port = sp->port; if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { goto done; } /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */ - if (port == 0 && allocated_port1 > 0) { - grpc_sockaddr_set_port((struct sockaddr *)&wild4, allocated_port1); + if (port == 0 && allocated_port > 0) { + grpc_sockaddr_set_port((struct sockaddr *)&wild4, allocated_port); + sp2 = sp; } addr = (struct sockaddr *)&wild4; addr_len = sizeof(wild4); @@ -471,22 +487,32 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, addr = (struct sockaddr *)&addr4_copy; addr_len = sizeof(addr4_copy); } - allocated_port2 = add_socket_to_server(s, fd, addr, addr_len); + sp = add_socket_to_server(s, fd, addr, addr_len); + sp->sibling = sp2; + if (sp2) sp2->is_sibling = 1; done: gpr_free(allocated_addr); - return allocated_port1 >= 0 ? allocated_port1 : allocated_port2; + return sp; } int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { - return (port_index < s->nports) ? s->ports[port_index].fd : -1; + grpc_tcp_listener *sp; + for (sp = s->head; sp && port_index != 0; sp = sp->next, port_index--) + ; + if (port_index == 0 && sp) { + return sp->fd; + } else { + return -1; + } } void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, grpc_pollset **pollsets, size_t pollset_count, grpc_tcp_server_cb on_accept_cb, void *on_accept_cb_arg) { - size_t i, j; + size_t i; + grpc_tcp_listener *sp; GPR_ASSERT(on_accept_cb); gpr_mu_lock(&s->mu); GPR_ASSERT(!s->on_accept_cb); @@ -495,17 +521,40 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, s->on_accept_cb_arg = on_accept_cb_arg; s->pollsets = pollsets; s->pollset_count = pollset_count; - for (i = 0; i < s->nports; i++) { - for (j = 0; j < pollset_count; j++) { - grpc_pollset_add_fd(exec_ctx, pollsets[j], s->ports[i].emfd); + for (sp = s->head; sp; sp = sp->next) { + for (i = 0; i < pollset_count; i++) { + grpc_pollset_add_fd(exec_ctx, pollsets[i], sp->emfd); } - s->ports[i].read_closure.cb = on_read; - s->ports[i].read_closure.cb_arg = &s->ports[i]; - grpc_fd_notify_on_read(exec_ctx, s->ports[i].emfd, - &s->ports[i].read_closure); + sp->read_closure.cb = on_read; + sp->read_closure.cb_arg = sp; + grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); s->active_ports++; } gpr_mu_unlock(&s->mu); } +int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + return sp->port; +} + +void grpc_tcp_listener_ref(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + gpr_ref(&sp->refs); +} + +void grpc_tcp_listener_unref(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + if (sp->is_sibling) return; + if (gpr_unref(&sp->refs)) { + grpc_tcp_listener *sibling = sp->sibling; + while (sibling) { + sp = sibling; + sibling = sp->sibling; + gpr_free(sp); + } + gpr_free(listener); + } +} + #endif diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index 3fea8b5b35..a2425cd4d2 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -35,7 +35,8 @@ #ifdef GPR_WINSOCK_SOCKET -#define _GNU_SOURCE +#include <io.h> + #include "src/core/iomgr/sockaddr_utils.h" #include <grpc/support/alloc.h> @@ -51,25 +52,29 @@ #include "src/core/iomgr/tcp_server.h" #include "src/core/iomgr/tcp_windows.h" -#define INIT_PORT_CAP 2 #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 /* one listening port */ -typedef struct server_port { +struct grpc_tcp_listener { /* This seemingly magic number comes from AcceptEx's documentation. each address buffer needs to have at least 16 more bytes at their end. */ gpr_uint8 addresses[(sizeof(struct sockaddr_in6) + 16) * 2]; /* This will hold the socket for the next accept. */ SOCKET new_socket; - /* The listener winsocked. */ + /* The listener winsocket. */ grpc_winsocket *socket; + /* The actual TCP port number. */ + int port; grpc_tcp_server *server; /* The cached AcceptEx for that port. */ LPFN_ACCEPTEX AcceptEx; int shutting_down; /* closure for socket notification of accept being ready */ grpc_closure on_accept; -} server_port; + gpr_refcount refs; + /* linked list */ + struct grpc_tcp_listener *next; +}; /* the overall server */ struct grpc_tcp_server { @@ -82,10 +87,8 @@ struct grpc_tcp_server { /* active port count: how many ports are actually still listening */ int active_ports; - /* all listening ports */ - server_port *ports; - size_t nports; - size_t port_capacity; + /* linked list of server ports */ + grpc_tcp_listener *head; /* shutdown callback */ grpc_closure *shutdown_complete; @@ -99,9 +102,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) { s->active_ports = 0; s->on_accept_cb = NULL; s->on_accept_cb_arg = NULL; - s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); - s->nports = 0; - s->port_capacity = INIT_PORT_CAP; + s->head = NULL; s->shutdown_complete = NULL; return s; } @@ -109,26 +110,26 @@ grpc_tcp_server *grpc_tcp_server_create(void) { static void dont_care_about_shutdown_completion(void *arg) {} static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { - size_t i; - grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); /* Now that the accepts have been aborted, we can destroy the sockets. The IOCP won't get notified on these, so we can flag them as already closed by the system. */ - for (i = 0; i < s->nports; i++) { - server_port *sp = &s->ports[i]; + while (s->head) { + grpc_tcp_listener *sp = s->head; + s->head = sp->next; + sp->next = NULL; grpc_winsocket_destroy(sp->socket); + grpc_tcp_listener_unref(sp); } - gpr_free(s->ports); gpr_free(s); } /* Public function. Stops and destroys a grpc_tcp_server. */ void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, grpc_closure *shutdown_complete) { - size_t i; int immediately_done = 0; + grpc_tcp_listener *sp; gpr_mu_lock(&s->mu); s->shutdown_complete = shutdown_complete; @@ -138,8 +139,7 @@ void grpc_tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, if (s->active_ports == 0) { immediately_done = 1; } - for (i = 0; i < s->nports; i++) { - server_port *sp = &s->ports[i]; + for (sp = s->head; sp; sp = sp->next) { sp->shutting_down = 1; grpc_winsocket_shutdown(sp->socket); } @@ -199,7 +199,7 @@ error: } static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx, - server_port *sp) { + grpc_tcp_listener *sp) { int notify = 0; sp->shutting_down = 0; gpr_mu_lock(&sp->server->mu); @@ -216,7 +216,7 @@ static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx, /* In order to do an async accept, we need to create a socket first which will be the one assigned to the new incoming connection. */ -static void start_accept(grpc_exec_ctx *exec_ctx, server_port *port) { +static void start_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *port) { SOCKET sock = INVALID_SOCKET; char *message; char *utf8_message; @@ -276,7 +276,7 @@ failure: /* Event manager callback when reads are ready. */ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { - server_port *sp = arg; + grpc_tcp_listener *sp = arg; SOCKET sock = sp->new_socket; grpc_winsocket_callback_info *info = &sp->socket->read_info; grpc_endpoint *ep = NULL; @@ -351,16 +351,17 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, int from_iocp) { start_accept(exec_ctx, sp); } -static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, - const struct sockaddr *addr, size_t addr_len) { - server_port *sp; +static grpc_tcp_listener *add_socket_to_server(grpc_tcp_server *s, SOCKET sock, + const struct sockaddr *addr, + size_t addr_len) { + grpc_tcp_listener *sp = NULL; int port; int status; GUID guid = WSAID_ACCEPTEX; DWORD ioctl_num_bytes; LPFN_ACCEPTEX AcceptEx; - if (sock == INVALID_SOCKET) return -1; + if (sock == INVALID_SOCKET) return NULL; /* We need to grab the AcceptEx pointer for that port, as it may be interface-dependent. We'll cache it to avoid doing that again. */ @@ -373,37 +374,34 @@ static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock, gpr_log(GPR_ERROR, "on_connect error: %s", utf8_message); gpr_free(utf8_message); closesocket(sock); - return -1; + return NULL; } port = prepare_socket(sock, addr, addr_len); if (port >= 0) { gpr_mu_lock(&s->mu); GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server"); - /* append it to the list under a lock */ - if (s->nports == s->port_capacity) { - /* too many ports, and we need to store their address in a closure */ - /* TODO(ctiller): make server_port a linked list */ - abort(); - } - sp = &s->ports[s->nports++]; + sp = gpr_malloc(sizeof(grpc_tcp_listener)); + sp->next = s->head; + s->head = sp; sp->server = s; sp->socket = grpc_winsocket_create(sock, "listener"); sp->shutting_down = 0; sp->AcceptEx = AcceptEx; sp->new_socket = INVALID_SOCKET; + sp->port = port; + gpr_ref_init(&sp->refs, 1); grpc_closure_init(&sp->on_accept, on_accept, sp); GPR_ASSERT(sp->socket); gpr_mu_unlock(&s->mu); } - return port; + return sp; } -int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, - size_t addr_len) { - int allocated_port = -1; - unsigned i; +grpc_tcp_listener *grpc_tcp_server_add_port(grpc_tcp_server *s, + const void *addr, size_t addr_len) { + grpc_tcp_listener *sp; SOCKET sock; struct sockaddr_in6 addr6_v4mapped; struct sockaddr_in6 wildcard; @@ -415,9 +413,9 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, /* Check if this is a wildcard port, and if so, try to keep the port the same as some previously created listener. */ if (grpc_sockaddr_get_port(addr) == 0) { - for (i = 0; i < s->nports; i++) { + for (sp = s->head; sp; sp = sp->next) { sockname_len = sizeof(sockname_temp); - if (0 == getsockname(s->ports[i].socket->socket, + if (0 == getsockname(sp->socket->socket, (struct sockaddr *)&sockname_temp, &sockname_len)) { port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp); if (port > 0) { @@ -452,33 +450,56 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr, gpr_free(utf8_message); } - allocated_port = add_socket_to_server(s, sock, addr, addr_len); + sp = add_socket_to_server(s, sock, addr, addr_len); gpr_free(allocated_addr); - return allocated_port; + return sp; } -SOCKET -grpc_tcp_server_get_socket(grpc_tcp_server *s, unsigned index) { - return (index < s->nports) ? s->ports[index].socket->socket : INVALID_SOCKET; +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { + grpc_tcp_listener *sp; + for (sp = s->head; sp && port_index != 0; sp = sp->next, port_index--) + ; + if (port_index == 0 && sp) { + return _open_osfhandle(sp->socket->socket, 0); + } else { + return -1; + } } void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, grpc_pollset **pollset, size_t pollset_count, grpc_tcp_server_cb on_accept_cb, void *on_accept_cb_arg) { - size_t i; + grpc_tcp_listener *sp; GPR_ASSERT(on_accept_cb); gpr_mu_lock(&s->mu); GPR_ASSERT(!s->on_accept_cb); GPR_ASSERT(s->active_ports == 0); s->on_accept_cb = on_accept_cb; s->on_accept_cb_arg = on_accept_cb_arg; - for (i = 0; i < s->nports; i++) { - start_accept(exec_ctx, s->ports + i); + for (sp = s->head; sp; sp = sp->next) { + start_accept(exec_ctx, sp); s->active_ports++; } gpr_mu_unlock(&s->mu); } +int grpc_tcp_listener_get_port(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + return sp->port; +} + +void grpc_tcp_listener_ref(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + gpr_ref(&sp->refs); +} + +void grpc_tcp_listener_unref(grpc_tcp_listener *listener) { + grpc_tcp_listener *sp = listener; + if (gpr_unref(&sp->refs)) { + gpr_free(listener); + } +} + #endif /* GPR_WINSOCK_SOCKET */ diff --git a/src/core/iomgr/workqueue_posix.c b/src/core/iomgr/workqueue_posix.c index 0a0f3c364e..c087b887b8 100644 --- a/src/core/iomgr/workqueue_posix.c +++ b/src/core/iomgr/workqueue_posix.c @@ -129,8 +129,6 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, int success) { void grpc_workqueue_push(grpc_workqueue *workqueue, grpc_closure *closure, int success) { - closure->success = success; - closure->next = NULL; gpr_mu_lock(&workqueue->mu); if (grpc_closure_list_empty(workqueue->closure_list)) { grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd); diff --git a/src/core/profiling/basic_timers.c b/src/core/profiling/basic_timers.c index b49cdd07b3..f0fce7858d 100644 --- a/src/core/profiling/basic_timers.c +++ b/src/core/profiling/basic_timers.c @@ -50,60 +50,197 @@ typedef struct gpr_timer_entry { gpr_timespec tm; const char *tagstr; const char *file; - int line; + short line; char type; gpr_uint8 important; + int thd; } gpr_timer_entry; -#define MAX_COUNT (1024 * 1024 / sizeof(gpr_timer_entry)) +#define MAX_COUNT 1000000 -static __thread gpr_timer_entry g_log[MAX_COUNT]; -static __thread int g_count; +typedef struct gpr_timer_log { + size_t num_entries; + struct gpr_timer_log *next; + struct gpr_timer_log *prev; + gpr_timer_entry log[MAX_COUNT]; +} gpr_timer_log; + +typedef struct gpr_timer_log_list { + gpr_timer_log *head; + /* valid iff head!=NULL */ + gpr_timer_log *tail; +} gpr_timer_log_list; + +static __thread gpr_timer_log *g_thread_log; static gpr_once g_once_init = GPR_ONCE_INIT; static FILE *output_file; +static const char *output_filename = "latency_trace.txt"; +static pthread_mutex_t g_mu; +static pthread_cond_t g_cv; +static gpr_timer_log_list g_in_progress_logs; +static gpr_timer_log_list g_done_logs; +static int g_shutdown; +static gpr_thd_id g_writing_thread; +static __thread int g_thread_id; +static int g_next_thread_id; -static void close_output() { fclose(output_file); } +static int timer_log_push_back(gpr_timer_log_list *list, gpr_timer_log *log) { + if (list->head == NULL) { + list->head = list->tail = log; + log->next = log->prev = NULL; + return 1; + } else { + log->prev = list->tail; + log->next = NULL; + list->tail->next = log; + list->tail = log; + return 0; + } +} -static void init_output() { - output_file = fopen("latency_trace.txt", "w"); - GPR_ASSERT(output_file); - atexit(close_output); +static gpr_timer_log *timer_log_pop_front(gpr_timer_log_list *list) { + gpr_timer_log *out = list->head; + if (out != NULL) { + list->head = out->next; + if (list->head != NULL) { + list->head->prev = NULL; + } else { + list->tail = NULL; + } + } + return out; } -static void log_report() { - int i; - gpr_once_init(&g_once_init, init_output); - for (i = 0; i < g_count; i++) { - gpr_timer_entry *entry = &(g_log[i]); +static void timer_log_remove(gpr_timer_log_list *list, gpr_timer_log *log) { + if (log->prev == NULL) { + list->head = log->next; + if (list->head != NULL) { + list->head->prev = NULL; + } + } else { + log->prev->next = log->next; + } + if (log->next == NULL) { + list->tail = log->prev; + if (list->tail != NULL) { + list->tail->next = NULL; + } + } else { + log->next->prev = log->prev; + } +} + +static void write_log(gpr_timer_log *log) { + size_t i; + if (output_file == NULL) { + output_file = fopen(output_filename, "w"); + } + for (i = 0; i < log->num_entries; i++) { + gpr_timer_entry *entry = &(log->log[i]); + if (gpr_time_cmp(entry->tm, gpr_time_0(entry->tm.clock_type)) < 0) { + entry->tm = gpr_time_0(entry->tm.clock_type); + } fprintf(output_file, - "{\"t\": %ld.%09d, \"thd\": \"%p\", \"type\": \"%c\", \"tag\": " + "{\"t\": %ld.%09d, \"thd\": \"%d\", \"type\": \"%c\", \"tag\": " "\"%s\", \"file\": \"%s\", \"line\": %d, \"imp\": %d}\n", - entry->tm.tv_sec, entry->tm.tv_nsec, - (void *)(gpr_intptr)gpr_thd_currentid(), entry->type, entry->tagstr, - entry->file, entry->line, entry->important); + entry->tm.tv_sec, entry->tm.tv_nsec, entry->thd, entry->type, + entry->tagstr, entry->file, entry->line, entry->important); + } +} + +static void writing_thread(void *unused) { + gpr_timer_log *log; + pthread_mutex_lock(&g_mu); + for (;;) { + while ((log = timer_log_pop_front(&g_done_logs)) == NULL && !g_shutdown) { + pthread_cond_wait(&g_cv, &g_mu); + } + if (log != NULL) { + pthread_mutex_unlock(&g_mu); + write_log(log); + free(log); + pthread_mutex_lock(&g_mu); + } + if (g_shutdown) { + pthread_mutex_unlock(&g_mu); + return; + } } +} - /* Now clear out the log */ - g_count = 0; +static void flush_logs(gpr_timer_log_list *list) { + gpr_timer_log *log; + while ((log = timer_log_pop_front(list)) != NULL) { + write_log(log); + free(log); + } +} + +static void finish_writing() { + pthread_mutex_lock(&g_mu); + g_shutdown = 1; + pthread_cond_signal(&g_cv); + pthread_mutex_unlock(&g_mu); + gpr_thd_join(g_writing_thread); + + gpr_log(GPR_INFO, "flushing logs"); + + pthread_mutex_lock(&g_mu); + flush_logs(&g_done_logs); + flush_logs(&g_in_progress_logs); + pthread_mutex_unlock(&g_mu); + + if (output_file) { + fclose(output_file); + } +} + +void gpr_timers_set_log_filename(const char *filename) { + output_filename = filename; +} + +static void init_output() { + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + gpr_thd_new(&g_writing_thread, writing_thread, NULL, &options); + atexit(finish_writing); +} + +static void rotate_log() { + gpr_timer_log *new = malloc(sizeof(*new)); + gpr_once_init(&g_once_init, init_output); + new->num_entries = 0; + pthread_mutex_lock(&g_mu); + if (g_thread_log != NULL) { + timer_log_remove(&g_in_progress_logs, g_thread_log); + if (timer_log_push_back(&g_done_logs, g_thread_log)) { + pthread_cond_signal(&g_cv); + } + } else { + g_thread_id = g_next_thread_id++; + } + timer_log_push_back(&g_in_progress_logs, new); + pthread_mutex_unlock(&g_mu); + g_thread_log = new; } static void gpr_timers_log_add(const char *tagstr, marker_type type, int important, const char *file, int line) { gpr_timer_entry *entry; - /* TODO (vpai) : Improve concurrency */ - if (g_count == MAX_COUNT) { - log_report(); + if (g_thread_log == NULL || g_thread_log->num_entries == MAX_COUNT) { + rotate_log(); } - entry = &g_log[g_count++]; + entry = &g_thread_log->log[g_thread_log->num_entries++]; entry->tm = gpr_now(GPR_CLOCK_PRECISE); entry->tagstr = tagstr; entry->type = type; entry->file = file; - entry->line = line; + entry->line = (short)line; entry->important = important != 0; + entry->thd = g_thread_id; } /* Latency profiler API implementation. */ @@ -131,4 +268,6 @@ void gpr_timers_global_destroy(void) {} void gpr_timers_global_init(void) {} void gpr_timers_global_destroy(void) {} + +void gpr_timers_set_log_filename(const char *filename) {} #endif /* GRPC_BASIC_PROFILER */ diff --git a/src/core/profiling/timers.h b/src/core/profiling/timers.h index 0d112e7248..6a188dc566 100644 --- a/src/core/profiling/timers.h +++ b/src/core/profiling/timers.h @@ -48,6 +48,8 @@ void gpr_timer_begin(const char *tagstr, int important, const char *file, void gpr_timer_end(const char *tagstr, int important, const char *file, int line); +void gpr_timers_set_log_filename(const char *filename); + #if !(defined(GRPC_STAP_PROFILER) + defined(GRPC_BASIC_PROFILER)) /* No profiling. No-op all the things. */ #define GPR_TIMER_MARK(tag, important) \ diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c index 635982b252..be2522a7a0 100644 --- a/src/core/security/client_auth_filter.c +++ b/src/core/security/client_auth_filter.c @@ -50,7 +50,7 @@ /* We can have a per-call credentials. */ typedef struct { - grpc_credentials *creds; + grpc_call_credentials *creds; grpc_mdstr *host; grpc_mdstr *method; /* pollset bound to this call; if we need to make external @@ -59,8 +59,6 @@ typedef struct { progress */ grpc_pollset *pollset; grpc_transport_stream_op op; - size_t op_md_idx; - int sent_initial_metadata; gpr_uint8 security_context_set; grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT]; char *service_url; @@ -108,9 +106,8 @@ static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data, return; } GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT); - GPR_ASSERT(op->send_ops && op->send_ops->nops > calld->op_md_idx && - op->send_ops->ops[calld->op_md_idx].type == GRPC_OP_METADATA); - mdb = &op->send_ops->ops[calld->op_md_idx].data.metadata; + GPR_ASSERT(op->send_initial_metadata != NULL); + mdb = op->send_initial_metadata; for (i = 0; i < num_md; i++) { grpc_metadata_batch_add_tail( mdb, &calld->md_links[i], @@ -146,39 +143,35 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx, channel_data *chand = elem->channel_data; grpc_client_security_context *ctx = (grpc_client_security_context *)op->context[GRPC_CONTEXT_SECURITY].value; - grpc_credentials *channel_creds = + grpc_call_credentials *channel_call_creds = chand->security_connector->request_metadata_creds; - int channel_creds_has_md = - (channel_creds != NULL) && - grpc_credentials_has_request_metadata(channel_creds); - int call_creds_has_md = (ctx != NULL) && (ctx->creds != NULL) && - grpc_credentials_has_request_metadata(ctx->creds); + int call_creds_has_md = (ctx != NULL) && (ctx->creds != NULL); - if (!channel_creds_has_md && !call_creds_has_md) { + if (channel_call_creds == NULL && !call_creds_has_md) { /* Skip sending metadata altogether. */ grpc_call_next_op(exec_ctx, elem, op); return; } - if (channel_creds_has_md && call_creds_has_md) { - calld->creds = - grpc_composite_credentials_create(channel_creds, ctx->creds, NULL); + if (channel_call_creds != NULL && call_creds_has_md) { + calld->creds = grpc_composite_call_credentials_create(channel_call_creds, + ctx->creds, NULL); if (calld->creds == NULL) { bubble_up_error(exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT, "Incompatible credentials set on channel and call."); return; } } else { - calld->creds = - grpc_credentials_ref(call_creds_has_md ? ctx->creds : channel_creds); + calld->creds = grpc_call_credentials_ref( + call_creds_has_md ? ctx->creds : channel_call_creds); } build_service_url(chand->security_connector->base.url_scheme, calld); calld->op = *op; /* Copy op (originates from the caller's stack). */ GPR_ASSERT(calld->pollset); - grpc_credentials_get_request_metadata(exec_ctx, calld->creds, calld->pollset, - calld->service_url, - on_credentials_metadata, elem); + grpc_call_credentials_get_request_metadata(exec_ctx, calld->creds, + calld->pollset, calld->service_url, + on_credentials_metadata, elem); } static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data, @@ -209,7 +202,6 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; grpc_linked_mdelem *l; - size_t i; grpc_client_security_context *sec_ctx = NULL; if (calld->security_context_set == 0 && @@ -228,53 +220,41 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, chand->security_connector->base.auth_context, "client_auth_filter"); } - if (op->bind_pollset != NULL) { - calld->pollset = op->bind_pollset; - } - - if (op->send_ops != NULL && !calld->sent_initial_metadata) { - size_t nops = op->send_ops->nops; - grpc_stream_op *ops = op->send_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *sop = &ops[i]; - if (sop->type != GRPC_OP_METADATA) continue; - calld->op_md_idx = i; - calld->sent_initial_metadata = 1; - for (l = sop->data.metadata.list.head; l != NULL; l = l->next) { - grpc_mdelem *md = l->md; - /* Pointer comparison is OK for md_elems created from the same context. - */ - if (md->key == chand->authority_string) { - if (calld->host != NULL) GRPC_MDSTR_UNREF(calld->host); - calld->host = GRPC_MDSTR_REF(md->value); - } else if (md->key == chand->path_string) { - if (calld->method != NULL) GRPC_MDSTR_UNREF(calld->method); - calld->method = GRPC_MDSTR_REF(md->value); - } + if (op->send_initial_metadata != NULL) { + for (l = op->send_initial_metadata->list.head; l != NULL; l = l->next) { + grpc_mdelem *md = l->md; + /* Pointer comparison is OK for md_elems created from the same context. + */ + if (md->key == chand->authority_string) { + if (calld->host != NULL) GRPC_MDSTR_UNREF(calld->host); + calld->host = GRPC_MDSTR_REF(md->value); + } else if (md->key == chand->path_string) { + if (calld->method != NULL) GRPC_MDSTR_UNREF(calld->method); + calld->method = GRPC_MDSTR_REF(md->value); } - if (calld->host != NULL) { - grpc_security_status status; - const char *call_host = grpc_mdstr_as_c_string(calld->host); - calld->op = *op; /* Copy op (originates from the caller's stack). */ - status = grpc_channel_security_connector_check_call_host( - exec_ctx, chand->security_connector, call_host, on_host_checked, - elem); - if (status != GRPC_SECURITY_OK) { - if (status == GRPC_SECURITY_ERROR) { - char *error_msg; - gpr_asprintf(&error_msg, - "Invalid host %s set in :authority metadata.", - call_host); - bubble_up_error(exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT, - error_msg); - gpr_free(error_msg); - } - return; /* early exit */ + } + if (calld->host != NULL) { + grpc_security_status status; + const char *call_host = grpc_mdstr_as_c_string(calld->host); + calld->op = *op; /* Copy op (originates from the caller's stack). */ + status = grpc_channel_security_connector_check_call_host( + exec_ctx, chand->security_connector, call_host, on_host_checked, + elem); + if (status != GRPC_SECURITY_OK) { + if (status == GRPC_SECURITY_ERROR) { + char *error_msg; + gpr_asprintf(&error_msg, + "Invalid host %s set in :authority metadata.", + call_host); + bubble_up_error(exec_ctx, elem, GRPC_STATUS_INVALID_ARGUMENT, + error_msg); + gpr_free(error_msg); } + return; /* early exit */ } - send_security_metadata(exec_ctx, elem, op); - return; /* early exit */ } + send_security_metadata(exec_ctx, elem, op); + return; /* early exit */ } /* pass control down the stack */ @@ -283,18 +263,22 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *calld = elem->call_data; memset(calld, 0, sizeof(*calld)); - GPR_ASSERT(!initial_op || !initial_op->send_ops); +} + +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset) { + call_data *calld = elem->call_data; + calld->pollset = pollset; } /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { call_data *calld = elem->call_data; - grpc_credentials_unref(calld->creds); + grpc_call_credentials_unref(calld->creds); if (calld->host != NULL) { GRPC_MDSTR_UNREF(calld->host); } @@ -306,18 +290,17 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, - grpc_mdctx *metadata_context, int is_first, - int is_last) { - grpc_security_connector *sc = grpc_find_security_connector_in_args(args); + grpc_channel_element *elem, + grpc_channel_element_args *args) { + grpc_security_connector *sc = + grpc_find_security_connector_in_args(args->channel_args); /* grab pointers to our data from the channel element */ channel_data *chand = elem->channel_data; /* The first and the last filters tend to be implemented differently to handle the case that there's no 'next' filter to call on the up or down path */ - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); GPR_ASSERT(sc != NULL); /* initialize members */ @@ -325,7 +308,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, chand->security_connector = (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF( sc, "client_auth_filter"); - chand->md_ctx = metadata_context; + chand->md_ctx = args->metadata_context; chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority"); chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path"); chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message"); @@ -356,6 +339,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_client_auth_filter = { auth_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, "client-auth"}; diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c index 5e155d83b9..f7a2e73785 100644 --- a/src/core/security/credentials.c +++ b/src/core/security/credentials.c @@ -54,18 +54,18 @@ /* -- Common. -- */ struct grpc_credentials_metadata_request { - grpc_credentials *creds; + grpc_call_credentials *creds; grpc_credentials_metadata_cb cb; void *user_data; }; static grpc_credentials_metadata_request * -grpc_credentials_metadata_request_create(grpc_credentials *creds, +grpc_credentials_metadata_request_create(grpc_call_credentials *creds, grpc_credentials_metadata_cb cb, void *user_data) { grpc_credentials_metadata_request *r = gpr_malloc(sizeof(grpc_credentials_metadata_request)); - r->creds = grpc_credentials_ref(creds); + r->creds = grpc_call_credentials_ref(creds); r->cb = cb; r->user_data = user_data; return r; @@ -73,44 +73,56 @@ grpc_credentials_metadata_request_create(grpc_credentials *creds, static void grpc_credentials_metadata_request_destroy( grpc_credentials_metadata_request *r) { - grpc_credentials_unref(r->creds); + grpc_call_credentials_unref(r->creds); gpr_free(r); } -grpc_credentials *grpc_credentials_ref(grpc_credentials *creds) { +grpc_channel_credentials *grpc_channel_credentials_ref( + grpc_channel_credentials *creds) { if (creds == NULL) return NULL; gpr_ref(&creds->refcount); return creds; } -void grpc_credentials_unref(grpc_credentials *creds) { +void grpc_channel_credentials_unref(grpc_channel_credentials *creds) { if (creds == NULL) return; if (gpr_unref(&creds->refcount)) { - creds->vtable->destruct(creds); + if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds); gpr_free(creds); } } -void grpc_credentials_release(grpc_credentials *creds) { - GRPC_API_TRACE("grpc_credentials_release(creds=%p)", 1, (creds)); - grpc_credentials_unref(creds); +void grpc_channel_credentials_release(grpc_channel_credentials *creds) { + GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds)); + grpc_channel_credentials_unref(creds); } -int grpc_credentials_has_request_metadata(grpc_credentials *creds) { - if (creds == NULL) return 0; - return creds->vtable->has_request_metadata(creds); +grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) { + if (creds == NULL) return NULL; + gpr_ref(&creds->refcount); + return creds; +} + +void grpc_call_credentials_unref(grpc_call_credentials *creds) { + if (creds == NULL) return; + if (gpr_unref(&creds->refcount)) { + if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds); + gpr_free(creds); + } } -int grpc_credentials_has_request_metadata_only(grpc_credentials *creds) { - if (creds == NULL) return 0; - return creds->vtable->has_request_metadata_only(creds); +void grpc_call_credentials_release(grpc_call_credentials *creds) { + GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds)); + grpc_call_credentials_unref(creds); } -void grpc_credentials_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { - if (creds == NULL || !grpc_credentials_has_request_metadata(creds) || - creds->vtable->get_request_metadata == NULL) { +void grpc_call_credentials_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { + if (creds == NULL || creds->vtable->get_request_metadata == NULL) { if (cb != NULL) { cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK); } @@ -120,19 +132,17 @@ void grpc_credentials_get_request_metadata( user_data); } -grpc_security_status grpc_credentials_create_security_connector( - grpc_credentials *creds, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, - grpc_channel_security_connector **sc, grpc_channel_args **new_args) { +grpc_security_status grpc_channel_credentials_create_security_connector( + grpc_channel_credentials *channel_creds, const char *target, + const grpc_channel_args *args, grpc_channel_security_connector **sc, + grpc_channel_args **new_args) { *new_args = NULL; - if (creds == NULL || creds->vtable->create_security_connector == NULL || - grpc_credentials_has_request_metadata_only(creds)) { - gpr_log(GPR_ERROR, - "Invalid credentials for creating a security connector."); + if (channel_creds == NULL) { return GRPC_SECURITY_ERROR; } - return creds->vtable->create_security_connector( - creds, target, args, request_metadata_creds, sc, new_args); + GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL); + return channel_creds->vtable->create_security_connector( + channel_creds, NULL, target, args, sc, new_args); } grpc_server_credentials *grpc_server_credentials_ref( @@ -145,7 +155,7 @@ grpc_server_credentials *grpc_server_credentials_ref( void grpc_server_credentials_unref(grpc_server_credentials *creds) { if (creds == NULL) return; if (gpr_unref(&creds->refcount)) { - creds->vtable->destruct(creds); + if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds); if (creds->processor.destroy != NULL && creds->processor.state != NULL) { creds->processor.destroy(creds->processor.state); } @@ -200,8 +210,7 @@ grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) { return arg; } -grpc_server_credentials *grpc_server_credentials_from_arg( - const grpc_arg *arg) { +grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) { if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL; if (arg->type != GRPC_ARG_POINTER) { gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, @@ -225,7 +234,7 @@ grpc_server_credentials *grpc_find_server_credentials_in_args( /* -- Ssl credentials. -- */ -static void ssl_destruct(grpc_credentials *creds) { +static void ssl_destruct(grpc_channel_credentials *creds) { grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds; if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key); @@ -254,15 +263,9 @@ static void ssl_server_destruct(grpc_server_credentials *creds) { if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); } -static int ssl_has_request_metadata(const grpc_credentials *creds) { return 0; } - -static int ssl_has_request_metadata_only(const grpc_credentials *creds) { - return 0; -} - static grpc_security_status ssl_create_security_connector( - grpc_credentials *creds, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, + grpc_channel_credentials *creds, grpc_call_credentials *call_creds, + const char *target, const grpc_channel_args *args, grpc_channel_security_connector **sc, grpc_channel_args **new_args) { grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds; grpc_security_status status = GRPC_SECURITY_OK; @@ -279,7 +282,7 @@ static grpc_security_status ssl_create_security_connector( } } status = grpc_ssl_channel_security_connector_create( - request_metadata_creds, &c->config, target, overridden_target_name, sc); + call_creds, &c->config, target, overridden_target_name, sc); if (status != GRPC_SECURITY_OK) { return status; } @@ -296,9 +299,8 @@ static grpc_security_status ssl_server_create_security_connector( return grpc_ssl_server_security_connector_create(&c->config, sc); } -static grpc_credentials_vtable ssl_vtable = { - ssl_destruct, ssl_has_request_metadata, ssl_has_request_metadata_only, NULL, - ssl_create_security_connector}; +static grpc_channel_credentials_vtable ssl_vtable = { + ssl_destruct, ssl_create_security_connector}; static grpc_server_credentials_vtable ssl_server_vtable = { ssl_server_destruct, ssl_server_create_security_connector}; @@ -363,7 +365,7 @@ static void ssl_build_server_config( } } -grpc_credentials *grpc_ssl_credentials_create( +grpc_channel_credentials *grpc_ssl_credentials_create( const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, void *reserved) { grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials)); @@ -374,7 +376,7 @@ grpc_credentials *grpc_ssl_credentials_create( 3, (pem_root_certs, pem_key_cert_pair, reserved)); GPR_ASSERT(reserved == NULL); memset(c, 0, sizeof(grpc_ssl_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_SSL; + c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL; c->base.vtable = &ssl_vtable; gpr_ref_init(&c->base.refcount, 1); ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config); @@ -394,7 +396,7 @@ grpc_server_credentials *grpc_ssl_server_credentials_create( force_client_auth, reserved)); GPR_ASSERT(reserved == NULL); memset(c, 0, sizeof(grpc_ssl_server_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_SSL; + c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL; gpr_ref_init(&c->base.refcount, 1); c->base.vtable = &ssl_server_vtable; ssl_build_server_config(pem_root_certs, pem_key_cert_pairs, @@ -416,7 +418,7 @@ static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) { c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME); } -static void jwt_destruct(grpc_credentials *creds) { +static void jwt_destruct(grpc_call_credentials *creds) { grpc_service_account_jwt_access_credentials *c = (grpc_service_account_jwt_access_credentials *)creds; grpc_auth_json_key_destruct(&c->key); @@ -424,15 +426,12 @@ static void jwt_destruct(grpc_credentials *creds) { gpr_mu_destroy(&c->cache_mu); } -static int jwt_has_request_metadata(const grpc_credentials *creds) { return 1; } - -static int jwt_has_request_metadata_only(const grpc_credentials *creds) { - return 1; -} - -static void jwt_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { +static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { grpc_service_account_jwt_access_credentials *c = (grpc_service_account_jwt_access_credentials *)creds; gpr_timespec refresh_threshold = gpr_time_from_seconds( @@ -484,11 +483,10 @@ static void jwt_get_request_metadata( } } -static grpc_credentials_vtable jwt_vtable = { - jwt_destruct, jwt_has_request_metadata, jwt_has_request_metadata_only, - jwt_get_request_metadata, NULL}; +static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct, + jwt_get_request_metadata}; -grpc_credentials * +grpc_call_credentials * grpc_service_account_jwt_access_credentials_create_from_auth_json_key( grpc_auth_json_key key, gpr_timespec token_lifetime) { grpc_service_account_jwt_access_credentials *c; @@ -498,7 +496,7 @@ grpc_service_account_jwt_access_credentials_create_from_auth_json_key( } c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials)); memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_JWT; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT; gpr_ref_init(&c->base.refcount, 1); c->base.vtable = &jwt_vtable; c->key = key; @@ -508,7 +506,7 @@ grpc_service_account_jwt_access_credentials_create_from_auth_json_key( return &c->base; } -grpc_credentials *grpc_service_account_jwt_access_credentials_create( +grpc_call_credentials *grpc_service_account_jwt_access_credentials_create( const char *json_key, gpr_timespec token_lifetime, void *reserved) { GRPC_API_TRACE( "grpc_service_account_jwt_access_credentials_create(" @@ -525,7 +523,7 @@ grpc_credentials *grpc_service_account_jwt_access_credentials_create( /* -- Oauth2TokenFetcher credentials -- */ -static void oauth2_token_fetcher_destruct(grpc_credentials *creds) { +static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) { grpc_oauth2_token_fetcher_credentials *c = (grpc_oauth2_token_fetcher_credentials *)creds; grpc_credentials_md_store_unref(c->access_token_md); @@ -533,16 +531,6 @@ static void oauth2_token_fetcher_destruct(grpc_credentials *creds) { grpc_httpcli_context_destroy(&c->httpcli_context); } -static int oauth2_token_fetcher_has_request_metadata( - const grpc_credentials *creds) { - return 1; -} - -static int oauth2_token_fetcher_has_request_metadata_only( - const grpc_credentials *creds) { - return 1; -} - grpc_credentials_status grpc_oauth2_token_fetcher_credentials_parse_server_response( const grpc_httpcli_response *response, grpc_credentials_md_store **token_md, @@ -660,8 +648,9 @@ static void on_oauth2_token_fetcher_http_response( } static void oauth2_token_fetcher_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { + grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds, + grpc_pollset *pollset, const char *service_url, + grpc_credentials_metadata_cb cb, void *user_data) { grpc_oauth2_token_fetcher_credentials *c = (grpc_oauth2_token_fetcher_credentials *)creds; gpr_timespec refresh_threshold = gpr_time_from_seconds( @@ -694,7 +683,7 @@ static void oauth2_token_fetcher_get_request_metadata( static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c, grpc_fetch_oauth2_func fetch_func) { memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_OAUTH2; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2; gpr_ref_init(&c->base.refcount, 1); gpr_mu_init(&c->mu); c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME); @@ -704,10 +693,8 @@ static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c, /* -- GoogleComputeEngine credentials. -- */ -static grpc_credentials_vtable compute_engine_vtable = { - oauth2_token_fetcher_destruct, oauth2_token_fetcher_has_request_metadata, - oauth2_token_fetcher_has_request_metadata_only, - oauth2_token_fetcher_get_request_metadata, NULL}; +static grpc_call_credentials_vtable compute_engine_vtable = { + oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata}; static void compute_engine_fetch_oauth2( grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req, @@ -724,7 +711,7 @@ static void compute_engine_fetch_oauth2( response_cb, metadata_req); } -grpc_credentials *grpc_google_compute_engine_credentials_create( +grpc_call_credentials *grpc_google_compute_engine_credentials_create( void *reserved) { grpc_oauth2_token_fetcher_credentials *c = gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials)); @@ -738,17 +725,15 @@ grpc_credentials *grpc_google_compute_engine_credentials_create( /* -- GoogleRefreshToken credentials. -- */ -static void refresh_token_destruct(grpc_credentials *creds) { +static void refresh_token_destruct(grpc_call_credentials *creds) { grpc_google_refresh_token_credentials *c = (grpc_google_refresh_token_credentials *)creds; grpc_auth_refresh_token_destruct(&c->refresh_token); oauth2_token_fetcher_destruct(&c->base.base); } -static grpc_credentials_vtable refresh_token_vtable = { - refresh_token_destruct, oauth2_token_fetcher_has_request_metadata, - oauth2_token_fetcher_has_request_metadata_only, - oauth2_token_fetcher_get_request_metadata, NULL}; +static grpc_call_credentials_vtable refresh_token_vtable = { + refresh_token_destruct, oauth2_token_fetcher_get_request_metadata}; static void refresh_token_fetch_oauth2( grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req, @@ -774,7 +759,8 @@ static void refresh_token_fetch_oauth2( gpr_free(body); } -grpc_credentials *grpc_refresh_token_credentials_create_from_auth_refresh_token( +grpc_call_credentials * +grpc_refresh_token_credentials_create_from_auth_refresh_token( grpc_auth_refresh_token refresh_token) { grpc_google_refresh_token_credentials *c; if (!grpc_auth_refresh_token_is_valid(&refresh_token)) { @@ -789,7 +775,7 @@ grpc_credentials *grpc_refresh_token_credentials_create_from_auth_refresh_token( return &c->base.base; } -grpc_credentials *grpc_google_refresh_token_credentials_create( +grpc_call_credentials *grpc_google_refresh_token_credentials_create( const char *json_refresh_token, void *reserved) { GRPC_API_TRACE( "grpc_refresh_token_credentials_create(json_refresh_token=%s, " @@ -802,20 +788,11 @@ grpc_credentials *grpc_google_refresh_token_credentials_create( /* -- Metadata-only credentials. -- */ -static void md_only_test_destruct(grpc_credentials *creds) { +static void md_only_test_destruct(grpc_call_credentials *creds) { grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds; grpc_credentials_md_store_unref(c->md_store); } -static int md_only_test_has_request_metadata(const grpc_credentials *creds) { - return 1; -} - -static int md_only_test_has_request_metadata_only( - const grpc_credentials *creds) { - return 1; -} - static void on_simulated_token_fetch_done(void *user_data) { grpc_credentials_metadata_request *r = (grpc_credentials_metadata_request *)user_data; @@ -827,9 +804,12 @@ static void on_simulated_token_fetch_done(void *user_data) { grpc_exec_ctx_finish(&exec_ctx); } -static void md_only_test_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { +static void md_only_test_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds; if (c->is_async) { @@ -842,18 +822,15 @@ static void md_only_test_get_request_metadata( } } -static grpc_credentials_vtable md_only_test_vtable = { - md_only_test_destruct, md_only_test_has_request_metadata, - md_only_test_has_request_metadata_only, md_only_test_get_request_metadata, - NULL}; +static grpc_call_credentials_vtable md_only_test_vtable = { + md_only_test_destruct, md_only_test_get_request_metadata}; -grpc_credentials *grpc_md_only_test_credentials_create(const char *md_key, - const char *md_value, - int is_async) { +grpc_call_credentials *grpc_md_only_test_credentials_create( + const char *md_key, const char *md_value, int is_async) { grpc_md_only_test_credentials *c = gpr_malloc(sizeof(grpc_md_only_test_credentials)); memset(c, 0, sizeof(grpc_md_only_test_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_OAUTH2; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2; c->base.vtable = &md_only_test_vtable; gpr_ref_init(&c->base.refcount, 1); c->md_store = grpc_credentials_md_store_create(1); @@ -864,34 +841,26 @@ grpc_credentials *grpc_md_only_test_credentials_create(const char *md_key, /* -- Oauth2 Access Token credentials. -- */ -static void access_token_destruct(grpc_credentials *creds) { +static void access_token_destruct(grpc_call_credentials *creds) { grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds; grpc_credentials_md_store_unref(c->access_token_md); } -static int access_token_has_request_metadata(const grpc_credentials *creds) { - return 1; -} - -static int access_token_has_request_metadata_only( - const grpc_credentials *creds) { - return 1; -} - -static void access_token_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { +static void access_token_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds; cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK); } -static grpc_credentials_vtable access_token_vtable = { - access_token_destruct, access_token_has_request_metadata, - access_token_has_request_metadata_only, access_token_get_request_metadata, - NULL}; +static grpc_call_credentials_vtable access_token_vtable = { + access_token_destruct, access_token_get_request_metadata}; -grpc_credentials *grpc_access_token_credentials_create(const char *access_token, - void *reserved) { +grpc_call_credentials *grpc_access_token_credentials_create( + const char *access_token, void *reserved) { grpc_access_token_credentials *c = gpr_malloc(sizeof(grpc_access_token_credentials)); char *token_md_value; @@ -901,7 +870,7 @@ grpc_credentials *grpc_access_token_credentials_create(const char *access_token, 2, (access_token, reserved)); GPR_ASSERT(reserved == NULL); memset(c, 0, sizeof(grpc_access_token_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_OAUTH2; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2; c->base.vtable = &access_token_vtable; gpr_ref_init(&c->base.refcount, 1); c->access_token_md = grpc_credentials_md_store_create(1); @@ -914,31 +883,11 @@ grpc_credentials *grpc_access_token_credentials_create(const char *access_token, /* -- Fake transport security credentials. -- */ -static void fake_transport_security_credentials_destruct( - grpc_credentials *creds) { - /* Nothing to do here. */ -} - -static void fake_transport_security_server_credentials_destruct( - grpc_server_credentials *creds) { - /* Nothing to do here. */ -} - -static int fake_transport_security_has_request_metadata( - const grpc_credentials *creds) { - return 0; -} - -static int fake_transport_security_has_request_metadata_only( - const grpc_credentials *creds) { - return 0; -} - static grpc_security_status fake_transport_security_create_security_connector( - grpc_credentials *c, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, + grpc_channel_credentials *c, grpc_call_credentials *call_creds, + const char *target, const grpc_channel_args *args, grpc_channel_security_connector **sc, grpc_channel_args **new_args) { - *sc = grpc_fake_channel_security_connector_create(request_metadata_creds, 1); + *sc = grpc_fake_channel_security_connector_create(call_creds, 1); return GRPC_SECURITY_OK; } @@ -949,21 +898,19 @@ fake_transport_security_server_create_security_connector( return GRPC_SECURITY_OK; } -static grpc_credentials_vtable fake_transport_security_credentials_vtable = { - fake_transport_security_credentials_destruct, - fake_transport_security_has_request_metadata, - fake_transport_security_has_request_metadata_only, NULL, - fake_transport_security_create_security_connector}; +static grpc_channel_credentials_vtable + fake_transport_security_credentials_vtable = { + NULL, fake_transport_security_create_security_connector}; static grpc_server_credentials_vtable fake_transport_security_server_credentials_vtable = { - fake_transport_security_server_credentials_destruct, - fake_transport_security_server_create_security_connector}; + NULL, fake_transport_security_server_create_security_connector}; -grpc_credentials *grpc_fake_transport_security_credentials_create(void) { - grpc_credentials *c = gpr_malloc(sizeof(grpc_credentials)); - memset(c, 0, sizeof(grpc_credentials)); - c->type = GRPC_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY; +grpc_channel_credentials *grpc_fake_transport_security_credentials_create( + void) { + grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials)); + memset(c, 0, sizeof(grpc_channel_credentials)); + c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY; c->vtable = &fake_transport_security_credentials_vtable; gpr_ref_init(&c->refcount, 1); return c; @@ -973,69 +920,46 @@ grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( void) { grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials)); memset(c, 0, sizeof(grpc_server_credentials)); - c->type = GRPC_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY; + c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY; gpr_ref_init(&c->refcount, 1); c->vtable = &fake_transport_security_server_credentials_vtable; return c; } -/* -- Composite credentials. -- */ +/* -- Composite call credentials. -- */ typedef struct { - grpc_composite_credentials *composite_creds; + grpc_composite_call_credentials *composite_creds; size_t creds_index; grpc_credentials_md_store *md_elems; char *service_url; void *user_data; grpc_pollset *pollset; grpc_credentials_metadata_cb cb; -} grpc_composite_credentials_metadata_context; +} grpc_composite_call_credentials_metadata_context; -static void composite_destruct(grpc_credentials *creds) { - grpc_composite_credentials *c = (grpc_composite_credentials *)creds; +static void composite_call_destruct(grpc_call_credentials *creds) { + grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds; size_t i; for (i = 0; i < c->inner.num_creds; i++) { - grpc_credentials_unref(c->inner.creds_array[i]); + grpc_call_credentials_unref(c->inner.creds_array[i]); } gpr_free(c->inner.creds_array); } -static int composite_has_request_metadata(const grpc_credentials *creds) { - const grpc_composite_credentials *c = - (const grpc_composite_credentials *)creds; - size_t i; - for (i = 0; i < c->inner.num_creds; i++) { - if (grpc_credentials_has_request_metadata(c->inner.creds_array[i])) { - return 1; - } - } - return 0; -} - -static int composite_has_request_metadata_only(const grpc_credentials *creds) { - const grpc_composite_credentials *c = - (const grpc_composite_credentials *)creds; - size_t i; - for (i = 0; i < c->inner.num_creds; i++) { - if (!grpc_credentials_has_request_metadata_only(c->inner.creds_array[i])) { - return 0; - } - } - return 1; -} - -static void composite_md_context_destroy( - grpc_composite_credentials_metadata_context *ctx) { +static void composite_call_md_context_destroy( + grpc_composite_call_credentials_metadata_context *ctx) { grpc_credentials_md_store_unref(ctx->md_elems); if (ctx->service_url != NULL) gpr_free(ctx->service_url); gpr_free(ctx); } -static void composite_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_credentials_md *md_elems, size_t num_md, - grpc_credentials_status status) { - grpc_composite_credentials_metadata_context *ctx = - (grpc_composite_credentials_metadata_context *)user_data; +static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_credentials_md *md_elems, + size_t num_md, + grpc_credentials_status status) { + grpc_composite_call_credentials_metadata_context *ctx = + (grpc_composite_call_credentials_metadata_context *)user_data; if (status != GRPC_CREDENTIALS_OK) { ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status); return; @@ -1051,158 +975,114 @@ static void composite_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data, } /* See if we need to get some more metadata. */ - while (ctx->creds_index < ctx->composite_creds->inner.num_creds) { - grpc_credentials *inner_creds = + if (ctx->creds_index < ctx->composite_creds->inner.num_creds) { + grpc_call_credentials *inner_creds = ctx->composite_creds->inner.creds_array[ctx->creds_index++]; - if (grpc_credentials_has_request_metadata(inner_creds)) { - grpc_credentials_get_request_metadata(exec_ctx, inner_creds, ctx->pollset, - ctx->service_url, - composite_metadata_cb, ctx); - return; - } + grpc_call_credentials_get_request_metadata(exec_ctx, inner_creds, + ctx->pollset, ctx->service_url, + composite_call_metadata_cb, ctx); + return; } /* We're done!. */ ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries, ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK); - composite_md_context_destroy(ctx); + composite_call_md_context_destroy(ctx); } -static void composite_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { - grpc_composite_credentials *c = (grpc_composite_credentials *)creds; - grpc_composite_credentials_metadata_context *ctx; - if (!grpc_credentials_has_request_metadata(creds)) { - cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK); - return; - } - ctx = gpr_malloc(sizeof(grpc_composite_credentials_metadata_context)); - memset(ctx, 0, sizeof(grpc_composite_credentials_metadata_context)); +static void composite_call_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { + grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds; + grpc_composite_call_credentials_metadata_context *ctx; + + ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context)); + memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context)); ctx->service_url = gpr_strdup(service_url); ctx->user_data = user_data; ctx->cb = cb; ctx->composite_creds = c; ctx->pollset = pollset; ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds); - while (ctx->creds_index < c->inner.num_creds) { - grpc_credentials *inner_creds = c->inner.creds_array[ctx->creds_index++]; - if (grpc_credentials_has_request_metadata(inner_creds)) { - grpc_credentials_get_request_metadata(exec_ctx, inner_creds, pollset, - service_url, composite_metadata_cb, - ctx); - return; - } - } - GPR_ASSERT(0); /* Should have exited before. */ -} - -static grpc_security_status composite_create_security_connector( - grpc_credentials *creds, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, - grpc_channel_security_connector **sc, grpc_channel_args **new_args) { - grpc_composite_credentials *c = (grpc_composite_credentials *)creds; - if (c->connector_creds == NULL) { - gpr_log(GPR_ERROR, - "Cannot create security connector, missing connector credentials."); - return GRPC_SECURITY_ERROR; - } - return grpc_credentials_create_security_connector(c->connector_creds, target, - args, creds, sc, new_args); + grpc_call_credentials_get_request_metadata( + exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset, service_url, + composite_call_metadata_cb, ctx); } -static grpc_credentials_vtable composite_credentials_vtable = { - composite_destruct, composite_has_request_metadata, - composite_has_request_metadata_only, composite_get_request_metadata, - composite_create_security_connector}; +static grpc_call_credentials_vtable composite_call_credentials_vtable = { + composite_call_destruct, composite_call_get_request_metadata}; -static grpc_credentials_array get_creds_array(grpc_credentials **creds_addr) { - grpc_credentials_array result; - grpc_credentials *creds = *creds_addr; +static grpc_call_credentials_array get_creds_array( + grpc_call_credentials **creds_addr) { + grpc_call_credentials_array result; + grpc_call_credentials *creds = *creds_addr; result.creds_array = creds_addr; result.num_creds = 1; - if (strcmp(creds->type, GRPC_CREDENTIALS_TYPE_COMPOSITE) == 0) { - result = *grpc_composite_credentials_get_credentials(creds); + if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) { + result = *grpc_composite_call_credentials_get_credentials(creds); } return result; } -grpc_credentials *grpc_composite_credentials_create(grpc_credentials *creds1, - grpc_credentials *creds2, - void *reserved) { +grpc_call_credentials *grpc_composite_call_credentials_create( + grpc_call_credentials *creds1, grpc_call_credentials *creds2, + void *reserved) { size_t i; size_t creds_array_byte_size; - grpc_credentials_array creds1_array; - grpc_credentials_array creds2_array; - grpc_composite_credentials *c; + grpc_call_credentials_array creds1_array; + grpc_call_credentials_array creds2_array; + grpc_composite_call_credentials *c; GRPC_API_TRACE( - "grpc_composite_credentials_create(creds1=%p, creds2=%p, " + "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, " "reserved=%p)", 3, (creds1, creds2, reserved)); GPR_ASSERT(reserved == NULL); GPR_ASSERT(creds1 != NULL); GPR_ASSERT(creds2 != NULL); - c = gpr_malloc(sizeof(grpc_composite_credentials)); - memset(c, 0, sizeof(grpc_composite_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_COMPOSITE; - c->base.vtable = &composite_credentials_vtable; + c = gpr_malloc(sizeof(grpc_composite_call_credentials)); + memset(c, 0, sizeof(grpc_composite_call_credentials)); + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE; + c->base.vtable = &composite_call_credentials_vtable; gpr_ref_init(&c->base.refcount, 1); creds1_array = get_creds_array(&creds1); creds2_array = get_creds_array(&creds2); c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds; - creds_array_byte_size = c->inner.num_creds * sizeof(grpc_credentials *); + creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *); c->inner.creds_array = gpr_malloc(creds_array_byte_size); memset(c->inner.creds_array, 0, creds_array_byte_size); for (i = 0; i < creds1_array.num_creds; i++) { - grpc_credentials *cur_creds = creds1_array.creds_array[i]; - if (!grpc_credentials_has_request_metadata_only(cur_creds)) { - if (c->connector_creds == NULL) { - c->connector_creds = cur_creds; - } else { - gpr_log(GPR_ERROR, "Cannot compose multiple connector credentials."); - goto fail; - } - } - c->inner.creds_array[i] = grpc_credentials_ref(cur_creds); + grpc_call_credentials *cur_creds = creds1_array.creds_array[i]; + c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds); } for (i = 0; i < creds2_array.num_creds; i++) { - grpc_credentials *cur_creds = creds2_array.creds_array[i]; - if (!grpc_credentials_has_request_metadata_only(cur_creds)) { - if (c->connector_creds == NULL) { - c->connector_creds = cur_creds; - } else { - gpr_log(GPR_ERROR, "Cannot compose multiple connector credentials."); - goto fail; - } - } + grpc_call_credentials *cur_creds = creds2_array.creds_array[i]; c->inner.creds_array[i + creds1_array.num_creds] = - grpc_credentials_ref(cur_creds); + grpc_call_credentials_ref(cur_creds); } return &c->base; - -fail: - grpc_credentials_unref(&c->base); - return NULL; } -const grpc_credentials_array *grpc_composite_credentials_get_credentials( - grpc_credentials *creds) { - const grpc_composite_credentials *c = - (const grpc_composite_credentials *)creds; - GPR_ASSERT(strcmp(creds->type, GRPC_CREDENTIALS_TYPE_COMPOSITE) == 0); +const grpc_call_credentials_array * +grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) { + const grpc_composite_call_credentials *c = + (const grpc_composite_call_credentials *)creds; + GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0); return &c->inner; } -grpc_credentials *grpc_credentials_contains_type( - grpc_credentials *creds, const char *type, - grpc_credentials **composite_creds) { +grpc_call_credentials *grpc_credentials_contains_type( + grpc_call_credentials *creds, const char *type, + grpc_call_credentials **composite_creds) { size_t i; if (strcmp(creds->type, type) == 0) { if (composite_creds != NULL) *composite_creds = NULL; return creds; - } else if (strcmp(creds->type, GRPC_CREDENTIALS_TYPE_COMPOSITE) == 0) { - const grpc_credentials_array *inner_creds_array = - grpc_composite_credentials_get_credentials(creds); + } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) { + const grpc_call_credentials_array *inner_creds_array = + grpc_composite_call_credentials_get_credentials(creds); for (i = 0; i < inner_creds_array->num_creds; i++) { if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) { if (composite_creds != NULL) *composite_creds = creds; @@ -1215,30 +1095,26 @@ grpc_credentials *grpc_credentials_contains_type( /* -- IAM credentials. -- */ -static void iam_destruct(grpc_credentials *creds) { +static void iam_destruct(grpc_call_credentials *creds) { grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds; grpc_credentials_md_store_unref(c->iam_md); } -static int iam_has_request_metadata(const grpc_credentials *creds) { return 1; } - -static int iam_has_request_metadata_only(const grpc_credentials *creds) { - return 1; -} - -static void iam_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { +static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds; cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries, GRPC_CREDENTIALS_OK); } -static grpc_credentials_vtable iam_vtable = { - iam_destruct, iam_has_request_metadata, iam_has_request_metadata_only, - iam_get_request_metadata, NULL}; +static grpc_call_credentials_vtable iam_vtable = {iam_destruct, + iam_get_request_metadata}; -grpc_credentials *grpc_google_iam_credentials_create( +grpc_call_credentials *grpc_google_iam_credentials_create( const char *token, const char *authority_selector, void *reserved) { grpc_google_iam_credentials *c; GRPC_API_TRACE( @@ -1250,7 +1126,7 @@ grpc_credentials *grpc_google_iam_credentials_create( GPR_ASSERT(authority_selector != NULL); c = gpr_malloc(sizeof(grpc_google_iam_credentials)); memset(c, 0, sizeof(grpc_google_iam_credentials)); - c->base.type = GRPC_CREDENTIALS_TYPE_IAM; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM; c->base.vtable = &iam_vtable; gpr_ref_init(&c->base.refcount, 1); c->iam_md = grpc_credentials_md_store_create(2); @@ -1268,21 +1144,13 @@ typedef struct { grpc_credentials_metadata_cb cb; } grpc_metadata_plugin_request; -static void plugin_destruct(grpc_credentials *creds) { +static void plugin_destruct(grpc_call_credentials *creds) { grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds; if (c->plugin.state != NULL && c->plugin.destroy != NULL) { c->plugin.destroy(c->plugin.state); } } -static int plugin_has_request_metadata(const grpc_credentials *creds) { - return 1; -} - -static int plugin_has_request_metadata_only(const grpc_credentials *creds) { - return 1; -} - static void plugin_md_request_metadata_ready(void *request, const grpc_metadata *md, size_t num_md, @@ -1321,9 +1189,12 @@ static void plugin_md_request_metadata_ready(void *request, grpc_exec_ctx_finish(&exec_ctx); } -static void plugin_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data) { +static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data) { grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds; if (c->plugin.get_metadata != NULL) { grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request)); @@ -1337,18 +1208,75 @@ static void plugin_get_request_metadata( } } -static grpc_credentials_vtable plugin_vtable = { - plugin_destruct, plugin_has_request_metadata, - plugin_has_request_metadata_only, plugin_get_request_metadata, NULL}; +static grpc_call_credentials_vtable plugin_vtable = { + plugin_destruct, plugin_get_request_metadata}; -grpc_credentials *grpc_metadata_credentials_create_from_plugin( +grpc_call_credentials *grpc_metadata_credentials_create_from_plugin( grpc_metadata_credentials_plugin plugin, void *reserved) { grpc_plugin_credentials *c = gpr_malloc(sizeof(*c)); + GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1, + (reserved)); GPR_ASSERT(reserved == NULL); memset(c, 0, sizeof(*c)); - c->base.type = GRPC_CREDENTIALS_TYPE_METADATA_PLUGIN; + c->base.type = GRPC_CALL_CREDENTIALS_TYPE_METADATA_PLUGIN; c->base.vtable = &plugin_vtable; gpr_ref_init(&c->base.refcount, 1); c->plugin = plugin; return &c->base; } + +/* -- Composite channel credentials. -- */ + +static void composite_channel_destruct(grpc_channel_credentials *creds) { + grpc_composite_channel_credentials *c = + (grpc_composite_channel_credentials *)creds; + grpc_channel_credentials_unref(c->inner_creds); + grpc_call_credentials_unref(c->call_creds); +} + +static grpc_security_status composite_channel_create_security_connector( + grpc_channel_credentials *creds, grpc_call_credentials *call_creds, + const char *target, const grpc_channel_args *args, + grpc_channel_security_connector **sc, grpc_channel_args **new_args) { + grpc_composite_channel_credentials *c = + (grpc_composite_channel_credentials *)creds; + grpc_security_status status = GRPC_SECURITY_ERROR; + + GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL && + c->inner_creds->vtable != NULL && + c->inner_creds->vtable->create_security_connector != NULL); + /* If we are passed a call_creds, create a call composite to pass it + downstream. */ + if (call_creds != NULL) { + grpc_call_credentials *composite_call_creds = + grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL); + status = c->inner_creds->vtable->create_security_connector( + c->inner_creds, composite_call_creds, target, args, sc, new_args); + grpc_call_credentials_unref(composite_call_creds); + } else { + status = c->inner_creds->vtable->create_security_connector( + c->inner_creds, c->call_creds, target, args, sc, new_args); + } + return status; +} + +static grpc_channel_credentials_vtable composite_channel_credentials_vtable = { + composite_channel_destruct, composite_channel_create_security_connector}; + +grpc_channel_credentials *grpc_composite_channel_credentials_create( + grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds, + void *reserved) { + grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c)); + memset(c, 0, sizeof(*c)); + GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL); + GRPC_API_TRACE( + "grpc_composite_channel_credentials_create(channel_creds=%p, " + "call_creds=%p, reserved=%p)", + 3, (channel_creds, call_creds, reserved)); + c->base.type = channel_creds->type; + c->base.vtable = &composite_channel_credentials_vtable; + gpr_ref_init(&c->base.refcount, 1); + c->inner_creds = grpc_channel_credentials_ref(channel_creds); + c->call_creds = grpc_call_credentials_ref(call_creds); + return &c->base; +} diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h index 01203b08f1..132060910e 100644 --- a/src/core/security/credentials.h +++ b/src/core/security/credentials.h @@ -34,7 +34,7 @@ #ifndef GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H #define GRPC_INTERNAL_CORE_SECURITY_CREDENTIALS_H -#include "src/core/transport/stream_op.h" +#include "src/core/transport/metadata_batch.h" #include <grpc/grpc.h> #include <grpc/grpc_security.h> #include <grpc/support/sync.h> @@ -54,15 +54,17 @@ typedef enum { #define GRPC_FAKE_TRANSPORT_SECURITY_TYPE "fake" -#define GRPC_CREDENTIALS_TYPE_SSL "Ssl" -#define GRPC_CREDENTIALS_TYPE_OAUTH2 "Oauth2" -#define GRPC_CREDENTIALS_TYPE_METADATA_PLUGIN "Plugin" -#define GRPC_CREDENTIALS_TYPE_JWT "Jwt" -#define GRPC_CREDENTIALS_TYPE_IAM "Iam" -#define GRPC_CREDENTIALS_TYPE_COMPOSITE "Composite" -#define GRPC_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY "FakeTransportSecurity" +#define GRPC_CHANNEL_CREDENTIALS_TYPE_SSL "Ssl" +#define GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY \ + "FakeTransportSecurity" -#define GRPC_AUTHORIZATION_METADATA_KEY "Authorization" +#define GRPC_CALL_CREDENTIALS_TYPE_OAUTH2 "Oauth2" +#define GRPC_CALL_CREDENTIALS_TYPE_METADATA_PLUGIN "Plugin" +#define GRPC_CALL_CREDENTIALS_TYPE_JWT "Jwt" +#define GRPC_CALL_CREDENTIALS_TYPE_IAM "Iam" +#define GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE "Composite" + +#define GRPC_AUTHORIZATION_METADATA_KEY "authorization" #define GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY \ "x-goog-iam-authorization-token" #define GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY "x-goog-iam-authority-selector" @@ -87,6 +89,41 @@ typedef enum { #define GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING \ "client_id=%s&client_secret=%s&refresh_token=%s&grant_type=refresh_token" +/* --- Google utils --- */ + +/* It is the caller's responsibility to gpr_free the result if not NULL. */ +char *grpc_get_well_known_google_credentials_file_path(void); + +/* --- grpc_channel_credentials. --- */ + +typedef struct { + void (*destruct)(grpc_channel_credentials *c); + + grpc_security_status (*create_security_connector)( + grpc_channel_credentials *c, grpc_call_credentials *call_creds, + const char *target, const grpc_channel_args *args, + grpc_channel_security_connector **sc, grpc_channel_args **new_args); +} grpc_channel_credentials_vtable; + +struct grpc_channel_credentials { + const grpc_channel_credentials_vtable *vtable; + const char *type; + gpr_refcount refcount; +}; + +grpc_channel_credentials *grpc_channel_credentials_ref( + grpc_channel_credentials *creds); +void grpc_channel_credentials_unref(grpc_channel_credentials *creds); + +/* Creates a security connector for the channel. May also create new channel + args for the channel to be used in place of the passed in const args if + returned non NULL. In that case the caller is responsible for destroying + new_args after channel creation. */ +grpc_security_status grpc_channel_credentials_create_security_connector( + grpc_channel_credentials *creds, const char *target, + const grpc_channel_args *args, grpc_channel_security_connector **sc, + grpc_channel_args **new_args); + /* --- grpc_credentials_md. --- */ typedef struct { @@ -113,16 +150,7 @@ grpc_credentials_md_store *grpc_credentials_md_store_ref( grpc_credentials_md_store *store); void grpc_credentials_md_store_unref(grpc_credentials_md_store *store); -/* --- grpc_credentials. --- */ - -/* Creates a fake transport security credentials object for testing. */ -grpc_credentials *grpc_fake_transport_security_credentials_create(void); -/* Creates a fake server transport security credentials object for testing. */ -grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( - void); - -/* It is the caller's responsibility to gpr_free the result if not NULL. */ -char *grpc_get_well_known_google_credentials_file_path(void); +/* --- grpc_call_credentials. --- */ typedef void (*grpc_credentials_metadata_cb)(grpc_exec_ctx *exec_ctx, void *user_data, @@ -131,57 +159,45 @@ typedef void (*grpc_credentials_metadata_cb)(grpc_exec_ctx *exec_ctx, grpc_credentials_status status); typedef struct { - void (*destruct)(grpc_credentials *c); - int (*has_request_metadata)(const grpc_credentials *c); - int (*has_request_metadata_only)(const grpc_credentials *c); - void (*get_request_metadata)(grpc_exec_ctx *exec_ctx, grpc_credentials *c, - grpc_pollset *pollset, const char *service_url, + void (*destruct)(grpc_call_credentials *c); + void (*get_request_metadata)(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *c, grpc_pollset *pollset, + const char *service_url, grpc_credentials_metadata_cb cb, void *user_data); - grpc_security_status (*create_security_connector)( - grpc_credentials *c, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, - grpc_channel_security_connector **sc, grpc_channel_args **new_args); -} grpc_credentials_vtable; +} grpc_call_credentials_vtable; -struct grpc_credentials { - const grpc_credentials_vtable *vtable; +struct grpc_call_credentials { + const grpc_call_credentials_vtable *vtable; const char *type; gpr_refcount refcount; }; -grpc_credentials *grpc_credentials_ref(grpc_credentials *creds); -void grpc_credentials_unref(grpc_credentials *creds); -int grpc_credentials_has_request_metadata(grpc_credentials *creds); -int grpc_credentials_has_request_metadata_only(grpc_credentials *creds); -void grpc_credentials_get_request_metadata( - grpc_exec_ctx *exec_ctx, grpc_credentials *creds, grpc_pollset *pollset, - const char *service_url, grpc_credentials_metadata_cb cb, void *user_data); - -/* Creates a security connector for the channel. May also create new channel - args for the channel to be used in place of the passed in const args if - returned non NULL. In that case the caller is responsible for destroying - new_args after channel creation. */ -grpc_security_status grpc_credentials_create_security_connector( - grpc_credentials *creds, const char *target, const grpc_channel_args *args, - grpc_credentials *request_metadata_creds, - grpc_channel_security_connector **sc, grpc_channel_args **new_args); +grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds); +void grpc_call_credentials_unref(grpc_call_credentials *creds); +void grpc_call_credentials_get_request_metadata(grpc_exec_ctx *exec_ctx, + grpc_call_credentials *creds, + grpc_pollset *pollset, + const char *service_url, + grpc_credentials_metadata_cb cb, + void *user_data); typedef struct { - grpc_credentials **creds_array; + grpc_call_credentials **creds_array; size_t num_creds; -} grpc_credentials_array; +} grpc_call_credentials_array; -const grpc_credentials_array *grpc_composite_credentials_get_credentials( - grpc_credentials *composite_creds); +const grpc_call_credentials_array * +grpc_composite_call_credentials_get_credentials( + grpc_call_credentials *composite_creds); /* Returns creds if creds is of the specified type or the inner creds of the specified type (if found), if the creds is of type COMPOSITE. If composite_creds is not NULL, *composite_creds will point to creds if of type COMPOSITE in case of success. */ -grpc_credentials *grpc_credentials_contains_type( - grpc_credentials *creds, const char *type, - grpc_credentials **composite_creds); +grpc_call_credentials *grpc_credentials_contains_type( + grpc_call_credentials *creds, const char *type, + grpc_call_credentials **composite_creds); /* Exposed for testing only. */ grpc_credentials_status @@ -192,19 +208,19 @@ void grpc_flush_cached_google_default_credentials(void); /* Metadata-only credentials with the specified key and value where asynchronicity can be simulated for testing. */ -grpc_credentials *grpc_md_only_test_credentials_create(const char *md_key, - const char *md_value, - int is_async); +grpc_call_credentials *grpc_md_only_test_credentials_create( + const char *md_key, const char *md_value, int is_async); /* Private constructor for jwt credentials from an already parsed json key. Takes ownership of the key. */ -grpc_credentials * +grpc_call_credentials * grpc_service_account_jwt_access_credentials_create_from_auth_json_key( grpc_auth_json_key key, gpr_timespec token_lifetime); /* Private constructor for refresh token credentials from an already parsed refresh token. Takes ownership of the refresh token. */ -grpc_credentials *grpc_refresh_token_credentials_create_from_auth_refresh_token( +grpc_call_credentials * +grpc_refresh_token_credentials_create_from_auth_refresh_token( grpc_auth_refresh_token token); /* --- grpc_server_credentials. --- */ @@ -237,10 +253,18 @@ grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg); grpc_server_credentials *grpc_find_server_credentials_in_args( const grpc_channel_args *args); +/* -- Fake transport security credentials. -- */ + +/* Creates a fake transport security credentials object for testing. */ +grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); +/* Creates a fake server transport security credentials object for testing. */ +grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( + void); + /* -- Ssl credentials. -- */ typedef struct { - grpc_credentials base; + grpc_channel_credentials base; grpc_ssl_config config; } grpc_ssl_credentials; @@ -249,10 +273,18 @@ typedef struct { grpc_ssl_server_config config; } grpc_ssl_server_credentials; +/* -- Channel composite credentials. -- */ + +typedef struct { + grpc_channel_credentials base; + grpc_channel_credentials *inner_creds; + grpc_call_credentials *call_creds; +} grpc_composite_channel_credentials; + /* -- Jwt credentials -- */ typedef struct { - grpc_credentials base; + grpc_call_credentials base; /* Have a simple cache for now with just 1 entry. We could have a map based on the service_url for a more sophisticated one. */ @@ -283,7 +315,7 @@ typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); typedef struct { - grpc_credentials base; + grpc_call_credentials base; gpr_mu mu; grpc_credentials_md_store *access_token_md; gpr_timespec token_expiration; @@ -301,14 +333,14 @@ typedef struct { /* -- Oauth2 Access Token credentials. -- */ typedef struct { - grpc_credentials base; + grpc_call_credentials base; grpc_credentials_md_store *access_token_md; } grpc_access_token_credentials; /* -- Metadata-only Test credentials. -- */ typedef struct { - grpc_credentials base; + grpc_call_credentials base; grpc_credentials_md_store *md_store; int is_async; } grpc_md_only_test_credentials; @@ -316,22 +348,21 @@ typedef struct { /* -- GoogleIAM credentials. -- */ typedef struct { - grpc_credentials base; + grpc_call_credentials base; grpc_credentials_md_store *iam_md; } grpc_google_iam_credentials; /* -- Composite credentials. -- */ typedef struct { - grpc_credentials base; - grpc_credentials_array inner; - grpc_credentials *connector_creds; -} grpc_composite_credentials; + grpc_call_credentials base; + grpc_call_credentials_array inner; +} grpc_composite_call_credentials; /* -- Plugin credentials. -- */ typedef struct { - grpc_credentials base; + grpc_call_credentials base; grpc_metadata_credentials_plugin plugin; grpc_credentials_md_store *plugin_md; } grpc_plugin_credentials; diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c index 45135305b2..6a54fe4e47 100644 --- a/src/core/security/google_default_credentials.c +++ b/src/core/security/google_default_credentials.c @@ -50,7 +50,7 @@ /* -- Default credentials. -- */ -static grpc_credentials *default_credentials = NULL; +static grpc_channel_credentials *default_credentials = NULL; static int compute_engine_detection_done = 0; static gpr_mu g_mu; static gpr_once g_once = GPR_ONCE_INIT; @@ -138,11 +138,11 @@ static int is_stack_running_on_compute_engine(void) { } /* Takes ownership of creds_path if not NULL. */ -static grpc_credentials *create_default_creds_from_path(char *creds_path) { +static grpc_call_credentials *create_default_creds_from_path(char *creds_path) { grpc_json *json = NULL; grpc_auth_json_key key; grpc_auth_refresh_token token; - grpc_credentials *result = NULL; + grpc_call_credentials *result = NULL; gpr_slice creds_data = gpr_empty_slice(); int file_ok = 0; if (creds_path == NULL) goto end; @@ -176,9 +176,9 @@ end: return result; } -grpc_credentials *grpc_google_default_credentials_create(void) { - grpc_credentials *result = NULL; - int serving_cached_credentials = 0; +grpc_channel_credentials *grpc_google_default_credentials_create(void) { + grpc_channel_credentials *result = NULL; + grpc_call_credentials *call_creds = NULL; GRPC_API_TRACE("grpc_google_default_credentials_create(void)", 0, ()); @@ -187,20 +187,19 @@ grpc_credentials *grpc_google_default_credentials_create(void) { gpr_mu_lock(&g_mu); if (default_credentials != NULL) { - result = grpc_credentials_ref(default_credentials); - serving_cached_credentials = 1; + result = grpc_channel_credentials_ref(default_credentials); goto end; } /* First, try the environment variable. */ - result = create_default_creds_from_path( + call_creds = create_default_creds_from_path( gpr_getenv(GRPC_GOOGLE_CREDENTIALS_ENV_VAR)); - if (result != NULL) goto end; + if (call_creds != NULL) goto end; /* Then the well-known file. */ - result = create_default_creds_from_path( + call_creds = create_default_creds_from_path( grpc_get_well_known_google_credentials_file_path()); - if (result != NULL) goto end; + if (call_creds != NULL) goto end; /* At last try to see if we're on compute engine (do the detection only once since it requires a network test). */ @@ -208,21 +207,28 @@ grpc_credentials *grpc_google_default_credentials_create(void) { int need_compute_engine_creds = is_stack_running_on_compute_engine(); compute_engine_detection_done = 1; if (need_compute_engine_creds) { - result = grpc_google_compute_engine_credentials_create(NULL); + call_creds = grpc_google_compute_engine_credentials_create(NULL); } } end: - if (!serving_cached_credentials && result != NULL) { - /* Blend with default ssl credentials and add a global reference so that it - can be cached and re-served. */ - grpc_credentials *ssl_creds = grpc_ssl_credentials_create(NULL, NULL, NULL); - default_credentials = grpc_credentials_ref( - grpc_composite_credentials_create(ssl_creds, result, NULL)); - GPR_ASSERT(default_credentials != NULL); - grpc_credentials_unref(ssl_creds); - grpc_credentials_unref(result); - result = default_credentials; + if (result == NULL) { + if (call_creds != NULL) { + /* Blend with default ssl credentials and add a global reference so that + it + can be cached and re-served. */ + grpc_channel_credentials *ssl_creds = + grpc_ssl_credentials_create(NULL, NULL, NULL); + default_credentials = grpc_channel_credentials_ref( + grpc_composite_channel_credentials_create(ssl_creds, call_creds, + NULL)); + GPR_ASSERT(default_credentials != NULL); + grpc_channel_credentials_unref(ssl_creds); + grpc_call_credentials_unref(call_creds); + result = default_credentials; + } else { + gpr_log(GPR_ERROR, "Could not create google default credentials."); + } } gpr_mu_unlock(&g_mu); return result; @@ -232,7 +238,7 @@ void grpc_flush_cached_google_default_credentials(void) { gpr_once_init(&g_once, init_default_credentials); gpr_mu_lock(&g_mu); if (default_credentials != NULL) { - grpc_credentials_unref(default_credentials); + grpc_channel_credentials_unref(default_credentials); default_credentials = NULL; } gpr_mu_unlock(&g_mu); diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index 7c4cf6f04d..3c54a4deae 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -203,16 +203,6 @@ grpc_security_connector *grpc_find_security_connector_in_args( return NULL; } -static int check_request_metadata_creds(grpc_credentials *creds) { - if (creds != NULL && !grpc_credentials_has_request_metadata(creds)) { - gpr_log(GPR_ERROR, - "Incompatible credentials for channel security connector: needs to " - "set request metadata."); - return 0; - } - return 1; -} - /* -- Fake implementation. -- */ typedef struct { @@ -222,7 +212,7 @@ typedef struct { static void fake_channel_destroy(grpc_security_connector *sc) { grpc_channel_security_connector *c = (grpc_channel_security_connector *)sc; - grpc_credentials_unref(c->request_metadata_creds); + grpc_call_credentials_unref(c->request_metadata_creds); GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector"); gpr_free(sc); } @@ -306,7 +296,8 @@ static grpc_security_connector_vtable fake_server_vtable = { fake_server_destroy, fake_server_do_handshake, fake_check_peer}; grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_credentials *request_metadata_creds, int call_host_check_is_async) { + grpc_call_credentials *request_metadata_creds, + int call_host_check_is_async) { grpc_fake_channel_security_connector *c = gpr_malloc(sizeof(grpc_fake_channel_security_connector)); memset(c, 0, sizeof(grpc_fake_channel_security_connector)); @@ -314,8 +305,8 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( c->base.base.is_client_side = 1; c->base.base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; c->base.base.vtable = &fake_channel_vtable; - GPR_ASSERT(check_request_metadata_creds(request_metadata_creds)); - c->base.request_metadata_creds = grpc_credentials_ref(request_metadata_creds); + c->base.request_metadata_creds = + grpc_call_credentials_ref(request_metadata_creds); c->base.check_call_host = fake_channel_check_call_host; c->call_host_check_is_async = call_host_check_is_async; return &c->base; @@ -349,7 +340,7 @@ typedef struct { static void ssl_channel_destroy(grpc_security_connector *sc) { grpc_ssl_channel_security_connector *c = (grpc_ssl_channel_security_connector *)sc; - grpc_credentials_unref(c->base.request_metadata_creds); + grpc_call_credentials_unref(c->base.request_metadata_creds); if (c->handshaker_factory != NULL) { tsi_ssl_handshaker_factory_destroy(c->handshaker_factory); } @@ -580,9 +571,9 @@ size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) { } grpc_security_status grpc_ssl_channel_security_connector_create( - grpc_credentials *request_metadata_creds, const grpc_ssl_config *config, - const char *target_name, const char *overridden_target_name, - grpc_channel_security_connector **sc) { + grpc_call_credentials *request_metadata_creds, + const grpc_ssl_config *config, const char *target_name, + const char *overridden_target_name, grpc_channel_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); const unsigned char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); @@ -606,9 +597,6 @@ grpc_security_status grpc_ssl_channel_security_connector_create( gpr_log(GPR_ERROR, "An ssl channel needs a config and a target name."); goto error; } - if (!check_request_metadata_creds(request_metadata_creds)) { - goto error; - } if (config->pem_root_certs == NULL) { pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); if (pem_root_certs == NULL || pem_root_certs_size == 0) { @@ -627,7 +615,8 @@ grpc_security_status grpc_ssl_channel_security_connector_create( c->base.base.vtable = &ssl_channel_vtable; c->base.base.is_client_side = 1; c->base.base.url_scheme = GRPC_SSL_URL_SCHEME; - c->base.request_metadata_creds = grpc_credentials_ref(request_metadata_creds); + c->base.request_metadata_creds = + grpc_call_credentials_ref(request_metadata_creds); c->base.check_call_host = ssl_channel_check_call_host; gpr_split_host_port(target_name, &c->target_name, &port); gpr_free(port); diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h index 9218a3caab..c5f00c5563 100644 --- a/src/core/security/security_connector.h +++ b/src/core/security/security_connector.h @@ -145,7 +145,7 @@ typedef struct grpc_channel_security_connector grpc_channel_security_connector; struct grpc_channel_security_connector { grpc_security_connector base; /* requires is_client_side to be non 0. */ - grpc_credentials *request_metadata_creds; + grpc_call_credentials *request_metadata_creds; grpc_security_status (*check_call_host)(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, const char *host, @@ -167,7 +167,8 @@ grpc_security_status grpc_channel_security_connector_check_call_host( /* For TESTING ONLY! Creates a fake connector that emulates real channel security. */ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_credentials *request_metadata_creds, int call_host_check_is_async); + grpc_call_credentials *request_metadata_creds, + int call_host_check_is_async); /* For TESTING ONLY! Creates a fake connector that emulates real server security. */ @@ -197,9 +198,9 @@ typedef struct { specific error code otherwise. */ grpc_security_status grpc_ssl_channel_security_connector_create( - grpc_credentials *request_metadata_creds, const grpc_ssl_config *config, - const char *target_name, const char *overridden_target_name, - grpc_channel_security_connector **sc); + grpc_call_credentials *request_metadata_creds, + const grpc_ssl_config *config, const char *target_name, + const char *overridden_target_name, grpc_channel_security_connector **sc); /* Gets the default ssl roots. */ size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs); diff --git a/src/core/security/security_context.c b/src/core/security/security_context.c index f544c1d943..2068c97d78 100644 --- a/src/core/security/security_context.c +++ b/src/core/security/security_context.c @@ -46,7 +46,7 @@ /* --- grpc_call --- */ grpc_call_error grpc_call_set_credentials(grpc_call *call, - grpc_credentials *creds) { + grpc_call_credentials *creds) { grpc_client_security_context *ctx = NULL; GRPC_API_TRACE("grpc_call_set_credentials(call=%p, creds=%p)", 2, (call, creds)); @@ -54,20 +54,16 @@ grpc_call_error grpc_call_set_credentials(grpc_call *call, gpr_log(GPR_ERROR, "Method is client-side only."); return GRPC_CALL_ERROR_NOT_ON_SERVER; } - if (creds != NULL && !grpc_credentials_has_request_metadata_only(creds)) { - gpr_log(GPR_ERROR, "Incompatible credentials to set on a call."); - return GRPC_CALL_ERROR; - } ctx = (grpc_client_security_context *)grpc_call_context_get( call, GRPC_CONTEXT_SECURITY); if (ctx == NULL) { ctx = grpc_client_security_context_create(); - ctx->creds = grpc_credentials_ref(creds); + ctx->creds = grpc_call_credentials_ref(creds); grpc_call_context_set(call, GRPC_CONTEXT_SECURITY, ctx, grpc_client_security_context_destroy); } else { - grpc_credentials_unref(ctx->creds); - ctx->creds = grpc_credentials_ref(creds); + grpc_call_credentials_unref(ctx->creds); + ctx->creds = grpc_call_credentials_ref(creds); } return GRPC_CALL_OK; } @@ -101,7 +97,7 @@ grpc_client_security_context *grpc_client_security_context_create(void) { void grpc_client_security_context_destroy(void *ctx) { grpc_client_security_context *c = (grpc_client_security_context *)ctx; - grpc_credentials_unref(c->creds); + grpc_call_credentials_unref(c->creds); GRPC_AUTH_CONTEXT_UNREF(c->auth_context, "client_security_context"); gpr_free(ctx); } @@ -324,8 +320,7 @@ grpc_arg grpc_auth_context_to_arg(grpc_auth_context *p) { return arg; } -grpc_auth_context *grpc_auth_context_from_arg( - const grpc_arg *arg) { +grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg) { if (strcmp(arg->key, GRPC_AUTH_CONTEXT_ARG) != 0) return NULL; if (arg->type != GRPC_ARG_POINTER) { gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, @@ -340,8 +335,7 @@ grpc_auth_context *grpc_find_auth_context_in_args( size_t i; if (args == NULL) return NULL; for (i = 0; i < args->num_args; i++) { - grpc_auth_context *p = - grpc_auth_context_from_arg(&args->args[i]); + grpc_auth_context *p = grpc_auth_context_from_arg(&args->args[i]); if (p != NULL) return p; } return NULL; diff --git a/src/core/security/security_context.h b/src/core/security/security_context.h index 2bbdc4be97..794258edbc 100644 --- a/src/core/security/security_context.h +++ b/src/core/security/security_context.h @@ -85,7 +85,7 @@ void grpc_auth_property_reset(grpc_auth_property *property); Internal client-side security context. */ typedef struct { - grpc_credentials *creds; + grpc_call_credentials *creds; grpc_auth_context *auth_context; } grpc_client_security_context; diff --git a/src/core/security/server_auth_filter.c b/src/core/security/server_auth_filter.c index 2e18369fe8..fee962b576 100644 --- a/src/core/security/server_auth_filter.c +++ b/src/core/security/server_auth_filter.c @@ -41,8 +41,7 @@ #include <grpc/support/log.h> typedef struct call_data { - gpr_uint8 got_client_metadata; - grpc_stream_op_buffer *recv_ops; + grpc_metadata_batch *recv_initial_metadata; /* Closure to call when finished with the auth_on_recv hook. */ grpc_closure *on_done_recv; /* Receive closures are chained: we inject this closure as the on_done_recv @@ -53,7 +52,6 @@ typedef struct call_data { grpc_metadata_array md; const grpc_metadata *consumed_md; size_t num_consumed_md; - grpc_stream_op *md_op; grpc_auth_context *auth_context; } call_data; @@ -128,20 +126,28 @@ static void on_md_processing_done( if (status == GRPC_STATUS_OK) { calld->consumed_md = consumed_md; calld->num_consumed_md = num_consumed_md; - grpc_metadata_batch_filter(&calld->md_op->data.metadata, remove_consumed_md, + grpc_metadata_batch_filter(calld->recv_initial_metadata, remove_consumed_md, elem); grpc_metadata_array_destroy(&calld->md); calld->on_done_recv->cb(&exec_ctx, calld->on_done_recv->cb_arg, 1); } else { gpr_slice message; + grpc_transport_stream_op close_op; + memset(&close_op, 0, sizeof(close_op)); grpc_metadata_array_destroy(&calld->md); error_details = error_details != NULL ? error_details : "Authentication metadata processing failed."; message = gpr_slice_from_copied_string(error_details); - grpc_sopb_reset(calld->recv_ops); - grpc_transport_stream_op_add_close(&calld->transport_op, status, &message); - grpc_call_next_op(&exec_ctx, elem, &calld->transport_op); + calld->transport_op.send_initial_metadata = NULL; + if (calld->transport_op.send_message != NULL) { + grpc_byte_stream_destroy(calld->transport_op.send_message); + calld->transport_op.send_message = NULL; + } + calld->transport_op.send_trailing_metadata = NULL; + grpc_transport_stream_op_add_close(&close_op, status, &message); + grpc_call_next_op(&exec_ctx, elem, &close_op); + calld->on_done_recv->cb(&exec_ctx, calld->on_done_recv->cb_arg, 0); } grpc_exec_ctx_finish(&exec_ctx); @@ -153,16 +159,8 @@ static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data, call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; if (success) { - size_t i; - size_t nops = calld->recv_ops->nops; - grpc_stream_op *ops = calld->recv_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *op = &ops[i]; - if (op->type != GRPC_OP_METADATA || calld->got_client_metadata) continue; - calld->got_client_metadata = 1; - if (chand->creds->processor.process == NULL) continue; - calld->md_op = op; - calld->md = metadata_batch_to_md_array(&op->data.metadata); + if (chand->creds->processor.process != NULL) { + calld->md = metadata_batch_to_md_array(calld->recv_initial_metadata); chand->creds->processor.process( chand->creds->processor.state, calld->auth_context, calld->md.metadata, calld->md.count, on_md_processing_done, elem); @@ -176,11 +174,11 @@ static void set_recv_ops_md_callbacks(grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; - if (op->recv_ops && !calld->got_client_metadata) { + if (op->recv_initial_metadata != NULL) { /* substitute our callback for the higher callback */ - calld->recv_ops = op->recv_ops; - calld->on_done_recv = op->on_done_recv; - op->on_done_recv = &calld->auth_on_recv; + calld->recv_initial_metadata = op->recv_initial_metadata; + calld->on_done_recv = op->on_complete; + op->on_complete = &calld->auth_on_recv; calld->transport_op = *op; } } @@ -199,8 +197,7 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, /* Constructor for call_data */ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { /* grab pointers to our data from the call element */ call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; @@ -210,47 +207,39 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, memset(calld, 0, sizeof(*calld)); grpc_closure_init(&calld->auth_on_recv, auth_on_recv, elem); - GPR_ASSERT(initial_op && initial_op->context != NULL && - initial_op->context[GRPC_CONTEXT_SECURITY].value == NULL); - - /* Create a security context for the call and reference the auth context from - the channel. */ - if (initial_op->context[GRPC_CONTEXT_SECURITY].value != NULL) { - initial_op->context[GRPC_CONTEXT_SECURITY].destroy( - initial_op->context[GRPC_CONTEXT_SECURITY].value); + if (args->context[GRPC_CONTEXT_SECURITY].value != NULL) { + args->context[GRPC_CONTEXT_SECURITY].destroy( + args->context[GRPC_CONTEXT_SECURITY].value); } + server_ctx = grpc_server_security_context_create(); - server_ctx->auth_context = - grpc_auth_context_create(chand->auth_context); - server_ctx->auth_context->pollset = initial_op->bind_pollset; - initial_op->context[GRPC_CONTEXT_SECURITY].value = server_ctx; - initial_op->context[GRPC_CONTEXT_SECURITY].destroy = - grpc_server_security_context_destroy; + server_ctx->auth_context = grpc_auth_context_create(chand->auth_context); calld->auth_context = server_ctx->auth_context; - /* Set the metadata callbacks. */ - set_recv_ops_md_callbacks(elem, initial_op); + args->context[GRPC_CONTEXT_SECURITY].value = server_ctx; + args->context[GRPC_CONTEXT_SECURITY].destroy = + grpc_server_security_context_destroy; } +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_pollset *pollset) {} + /* Destructor for call_data */ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {} /* Constructor for channel_data */ static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { - grpc_auth_context *auth_context = grpc_find_auth_context_in_args(args); - grpc_server_credentials *creds = grpc_find_server_credentials_in_args(args); + grpc_channel_element *elem, + grpc_channel_element_args *args) { + grpc_auth_context *auth_context = + grpc_find_auth_context_in_args(args->channel_args); + grpc_server_credentials *creds = + grpc_find_server_credentials_in_args(args->channel_args); /* grab pointers to our data from the channel element */ channel_data *chand = elem->channel_data; - /* The first and the last filters tend to be implemented differently to - handle the case that there's no 'next' filter to call on the up or down - path */ - GPR_ASSERT(!is_first); - GPR_ASSERT(!is_last); + GPR_ASSERT(!args->is_last); GPR_ASSERT(auth_context != NULL); GPR_ASSERT(creds != NULL); @@ -258,7 +247,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, chand->auth_context = GRPC_AUTH_CONTEXT_REF(auth_context, "server_auth_filter"); chand->creds = grpc_server_credentials_ref(creds); - chand->mdctx = mdctx; + chand->mdctx = args->metadata_context; } /* Destructor for channel data */ @@ -272,6 +261,6 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, const grpc_channel_filter grpc_server_auth_filter = { auth_start_transport_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), - init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, + init_call_elem, set_pollset, destroy_call_elem, sizeof(channel_data), + init_channel_elem, destroy_channel_elem, grpc_call_next_get_peer, "server-auth"}; diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index 82c639e830..1ea269bf8f 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -94,8 +94,7 @@ static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep, grpc_channel_args *args_copy; grpc_arg args_to_add[2]; args_to_add[0] = grpc_server_credentials_to_arg(state->creds); - args_to_add[1] = - grpc_auth_context_to_arg(state->sc->auth_context); + args_to_add[1] = grpc_auth_context_to_arg(state->sc->auth_context); args_copy = grpc_channel_args_copy_and_add( grpc_server_get_channel_args(state->server), args_to_add, GPR_ARRAY_SIZE(args_to_add)); @@ -250,9 +249,11 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, } for (i = 0; i < resolved->naddrs; i++) { - port_temp = grpc_tcp_server_add_port( + grpc_tcp_listener *listener; + listener = grpc_tcp_server_add_port( tcp, (struct sockaddr *)&resolved->addrs[i].addr, resolved->addrs[i].len); + port_temp = grpc_tcp_listener_get_port(listener); if (port_temp >= 0) { if (port_num == -1) { port_num = port_temp; diff --git a/src/core/support/avl.c b/src/core/support/avl.c new file mode 100644 index 0000000000..9734c9987f --- /dev/null +++ b/src/core/support/avl.c @@ -0,0 +1,288 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/avl.h> + +#include <assert.h> +#include <stdlib.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/string_util.h> +#include <grpc/support/useful.h> + +gpr_avl gpr_avl_create(const gpr_avl_vtable *vtable) { + gpr_avl out; + out.vtable = vtable; + out.root = NULL; + return out; +} + +static gpr_avl_node *ref_node(gpr_avl_node *node) { + if (node) { + gpr_ref(&node->refs); + } + return node; +} + +static void unref_node(const gpr_avl_vtable *vtable, gpr_avl_node *node) { + if (node == NULL) { + return; + } + if (gpr_unref(&node->refs)) { + vtable->destroy_key(node->key); + vtable->destroy_value(node->value); + unref_node(vtable, node->left); + unref_node(vtable, node->right); + gpr_free(node); + } +} + +static long node_height(gpr_avl_node *node) { + return node == NULL ? 0 : node->height; +} + +#ifndef NDEBUG +static long calculate_height(gpr_avl_node *node) { + return node == NULL ? 0 : 1 + GPR_MAX(calculate_height(node->left), + calculate_height(node->right)); +} + +static gpr_avl_node *assert_invariants(gpr_avl_node *n) { + if (n == NULL) return NULL; + assert_invariants(n->left); + assert_invariants(n->right); + assert(calculate_height(n) == n->height); + assert(labs(node_height(n->left) - node_height(n->right)) <= 1); + return n; +} +#else +static gpr_avl_node *assert_invariants(gpr_avl_node *n) { return n; } +#endif + +gpr_avl_node *new_node(void *key, void *value, gpr_avl_node *left, + gpr_avl_node *right) { + gpr_avl_node *node = gpr_malloc(sizeof(*node)); + gpr_ref_init(&node->refs, 1); + node->key = key; + node->value = value; + node->left = assert_invariants(left); + node->right = assert_invariants(right); + node->height = 1 + GPR_MAX(node_height(left), node_height(right)); + return node; +} + +static gpr_avl_node *get(const gpr_avl_vtable *vtable, gpr_avl_node *node, + void *key) { + long cmp; + + if (node == NULL) { + return NULL; + } + + cmp = vtable->compare_keys(node->key, key); + if (cmp == 0) { + return node; + } else if (cmp > 0) { + return get(vtable, node->left, key); + } else { + return get(vtable, node->right, key); + } +} + +void *gpr_avl_get(gpr_avl avl, void *key) { + gpr_avl_node *node = get(avl.vtable, avl.root, key); + return node ? node->value : NULL; +} + +static gpr_avl_node *rotate_left(const gpr_avl_vtable *vtable, void *key, + void *value, gpr_avl_node *left, + gpr_avl_node *right) { + gpr_avl_node *n = + new_node(vtable->copy_key(right->key), vtable->copy_value(right->value), + new_node(key, value, left, ref_node(right->left)), + ref_node(right->right)); + unref_node(vtable, right); + return n; +} + +static gpr_avl_node *rotate_right(const gpr_avl_vtable *vtable, void *key, + void *value, gpr_avl_node *left, + gpr_avl_node *right) { + gpr_avl_node *n = new_node( + vtable->copy_key(left->key), vtable->copy_value(left->value), + ref_node(left->left), new_node(key, value, ref_node(left->right), right)); + unref_node(vtable, left); + return n; +} + +static gpr_avl_node *rotate_left_right(const gpr_avl_vtable *vtable, void *key, + void *value, gpr_avl_node *left, + gpr_avl_node *right) { + /* rotate_right(..., rotate_left(left), right) */ + gpr_avl_node *n = new_node( + vtable->copy_key(left->right->key), + vtable->copy_value(left->right->value), + new_node(vtable->copy_key(left->key), vtable->copy_value(left->value), + ref_node(left->left), ref_node(left->right->left)), + new_node(key, value, ref_node(left->right->right), right)); + unref_node(vtable, left); + return n; +} + +static gpr_avl_node *rotate_right_left(const gpr_avl_vtable *vtable, void *key, + void *value, gpr_avl_node *left, + gpr_avl_node *right) { + /* rotate_left(..., left, rotate_right(right)) */ + gpr_avl_node *n = new_node( + vtable->copy_key(right->left->key), + vtable->copy_value(right->left->value), + new_node(key, value, left, ref_node(right->left->left)), + new_node(vtable->copy_key(right->key), vtable->copy_key(right->value), + ref_node(right->left->right), ref_node(right->right))); + unref_node(vtable, right); + return n; +} + +static gpr_avl_node *rebalance(const gpr_avl_vtable *vtable, void *key, + void *value, gpr_avl_node *left, + gpr_avl_node *right) { + switch (node_height(left) - node_height(right)) { + case 2: + if (node_height(left->left) - node_height(left->right) == -1) { + return assert_invariants( + rotate_left_right(vtable, key, value, left, right)); + } else { + return assert_invariants(rotate_right(vtable, key, value, left, right)); + } + case -2: + if (node_height(right->left) - node_height(right->right) == 1) { + return assert_invariants( + rotate_right_left(vtable, key, value, left, right)); + } else { + return assert_invariants(rotate_left(vtable, key, value, left, right)); + } + default: + return assert_invariants(new_node(key, value, left, right)); + } +} + +static gpr_avl_node *add(const gpr_avl_vtable *vtable, gpr_avl_node *node, + void *key, void *value) { + long cmp; + if (node == NULL) { + return new_node(key, value, NULL, NULL); + } + cmp = vtable->compare_keys(node->key, key); + if (cmp == 0) { + return new_node(key, value, ref_node(node->left), ref_node(node->right)); + } else if (cmp > 0) { + return rebalance( + vtable, vtable->copy_key(node->key), vtable->copy_value(node->value), + add(vtable, node->left, key, value), ref_node(node->right)); + } else { + return rebalance(vtable, vtable->copy_key(node->key), + vtable->copy_value(node->value), ref_node(node->left), + add(vtable, node->right, key, value)); + } +} + +gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value) { + gpr_avl_node *old_root = avl.root; + avl.root = add(avl.vtable, avl.root, key, value); + assert_invariants(avl.root); + unref_node(avl.vtable, old_root); + return avl; +} + +static gpr_avl_node *in_order_head(gpr_avl_node *node) { + while (node->left != NULL) { + node = node->left; + } + return node; +} + +static gpr_avl_node *in_order_tail(gpr_avl_node *node) { + while (node->right != NULL) { + node = node->right; + } + return node; +} + +static gpr_avl_node *remove(const gpr_avl_vtable *vtable, gpr_avl_node *node, + void *key) { + long cmp; + if (node == NULL) { + return NULL; + } + cmp = vtable->compare_keys(node->key, key); + if (cmp == 0) { + if (node->left == NULL) { + return ref_node(node->right); + } else if (node->right == NULL) { + return ref_node(node->left); + } else if (node->left->height < node->right->height) { + gpr_avl_node *h = in_order_head(node->right); + return rebalance(vtable, vtable->copy_key(h->key), + vtable->copy_value(h->value), ref_node(node->left), + remove(vtable, node->right, h->key)); + } else { + gpr_avl_node *h = in_order_tail(node->left); + return rebalance( + vtable, vtable->copy_key(h->key), vtable->copy_value(h->value), + remove(vtable, node->left, h->key), ref_node(node->right)); + } + } else if (cmp > 0) { + return rebalance(vtable, vtable->copy_key(node->key), + vtable->copy_value(node->value), + remove(vtable, node->left, key), ref_node(node->right)); + } else { + return rebalance(vtable, vtable->copy_key(node->key), + vtable->copy_value(node->value), ref_node(node->left), + remove(vtable, node->right, key)); + } +} + +gpr_avl gpr_avl_remove(gpr_avl avl, void *key) { + gpr_avl_node *old_root = avl.root; + avl.root = remove(avl.vtable, avl.root, key); + assert_invariants(avl.root); + unref_node(avl.vtable, old_root); + return avl; +} + +gpr_avl gpr_avl_ref(gpr_avl avl) { + ref_node(avl.root); + return avl; +} + +void gpr_avl_unref(gpr_avl avl) { unref_node(avl.vtable, avl.root); } diff --git a/src/core/support/slice_buffer.c b/src/core/support/slice_buffer.c index 310fbe1350..856d3a2439 100644 --- a/src/core/support/slice_buffer.c +++ b/src/core/support/slice_buffer.c @@ -31,6 +31,7 @@ * */ +#include <grpc/support/port_platform.h> #include <grpc/support/slice_buffer.h> #include <string.h> @@ -208,6 +209,44 @@ void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst) { src->length = 0; } +void gpr_slice_buffer_move_first(gpr_slice_buffer *src, size_t n, + gpr_slice_buffer *dst) { + size_t src_idx; + size_t output_len = dst->length + n; + size_t new_input_len = src->length - n; + GPR_ASSERT(src->length >= n); + if (src->length == n) { + gpr_slice_buffer_move_into(src, dst); + return; + } + src_idx = 0; + while (src_idx < src->capacity) { + gpr_slice slice = src->slices[src_idx]; + size_t slice_len = GPR_SLICE_LENGTH(slice); + if (n > slice_len) { + gpr_slice_buffer_add(dst, slice); + n -= slice_len; + src_idx++; + } else if (n == slice_len) { + gpr_slice_buffer_add(dst, slice); + src_idx++; + break; + } else { /* n < slice_len */ + src->slices[src_idx] = gpr_slice_split_tail(&slice, n); + GPR_ASSERT(GPR_SLICE_LENGTH(slice) == n); + GPR_ASSERT(GPR_SLICE_LENGTH(src->slices[src_idx]) == slice_len - n); + gpr_slice_buffer_add(dst, slice); + break; + } + } + GPR_ASSERT(dst->length == output_len); + memmove(src->slices, src->slices + src_idx, + sizeof(gpr_slice) * (src->count - src_idx)); + src->count -= src_idx; + src->length = new_input_len; + GPR_ASSERT(src->count > 0); +} + void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n, gpr_slice_buffer *garbage) { GPR_ASSERT(n <= sb->length); @@ -231,3 +270,13 @@ void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n, } } } + +gpr_slice gpr_slice_buffer_take_first(gpr_slice_buffer *sb) { + gpr_slice slice; + GPR_ASSERT(sb->count > 0); + slice = sb->slices[0]; + memmove(&sb->slices[0], &sb->slices[1], (sb->count - 1) * sizeof(gpr_slice)); + sb->count--; + sb->length -= GPR_SLICE_LENGTH(slice); + return slice; +} diff --git a/src/core/support/sync_posix.c b/src/core/support/sync_posix.c index 39c96feb13..d6a0f7c325 100644 --- a/src/core/support/sync_posix.c +++ b/src/core/support/sync_posix.c @@ -59,8 +59,11 @@ void gpr_mu_unlock(gpr_mu* mu) { } int gpr_mu_trylock(gpr_mu* mu) { - int err = pthread_mutex_trylock(mu); + int err; + GPR_TIMER_BEGIN("gpr_mu_trylock", 0); + err = pthread_mutex_trylock(mu); GPR_ASSERT(err == 0 || err == EBUSY); + GPR_TIMER_END("gpr_mu_trylock", 0); return err == 0; } diff --git a/src/core/support/thd_posix.c b/src/core/support/thd_posix.c index c36d94d044..653a1c88c1 100644 --- a/src/core/support/thd_posix.c +++ b/src/core/support/thd_posix.c @@ -53,7 +53,7 @@ struct thd_arg { /* Body of every thread started via gpr_thd_new. */ static void *thread_body(void *v) { struct thd_arg a = *(struct thd_arg *)v; - gpr_free(v); + free(v); (*a.body)(a.arg); return NULL; } @@ -63,7 +63,10 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, int thread_started; pthread_attr_t attr; pthread_t p; - struct thd_arg *a = gpr_malloc(sizeof(*a)); + /* don't use gpr_malloc as we may cause an infinite recursion with + * the profiling code */ + struct thd_arg *a = malloc(sizeof(*a)); + GPR_ASSERT(a != NULL); a->body = thd_body; a->arg = arg; @@ -78,7 +81,7 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, thread_started = (pthread_create(&p, &attr, &thread_body, a) == 0); GPR_ASSERT(pthread_attr_destroy(&attr) == 0); if (!thread_started) { - gpr_free(a); + free(a); } *t = (gpr_thd_id)p; return thread_started; diff --git a/src/core/surface/byte_buffer_queue.c b/src/core/surface/byte_buffer_queue.c deleted file mode 100644 index e47dc4f4ce..0000000000 --- a/src/core/surface/byte_buffer_queue.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/surface/byte_buffer_queue.h" -#include <grpc/support/alloc.h> -#include <grpc/support/useful.h> - -static void bba_destroy(grpc_bbq_array *array, size_t start_pos) { - size_t i; - for (i = start_pos; i < array->count; i++) { - grpc_byte_buffer_destroy(array->data[i]); - } - gpr_free(array->data); -} - -/* Append an operation to an array, expanding as needed */ -static void bba_push(grpc_bbq_array *a, grpc_byte_buffer *buffer) { - if (a->count == a->capacity) { - a->capacity = GPR_MAX(a->capacity * 2, 8); - a->data = gpr_realloc(a->data, sizeof(grpc_byte_buffer *) * a->capacity); - } - a->data[a->count++] = buffer; -} - -void grpc_bbq_destroy(grpc_byte_buffer_queue *q) { - bba_destroy(&q->filling, 0); - bba_destroy(&q->draining, q->drain_pos); -} - -int grpc_bbq_empty(grpc_byte_buffer_queue *q) { - return (q->drain_pos == q->draining.count && q->filling.count == 0); -} - -void grpc_bbq_push(grpc_byte_buffer_queue *q, grpc_byte_buffer *buffer) { - q->bytes += grpc_byte_buffer_length(buffer); - bba_push(&q->filling, buffer); -} - -void grpc_bbq_flush(grpc_byte_buffer_queue *q) { - grpc_byte_buffer *bb; - while ((bb = grpc_bbq_pop(q))) { - grpc_byte_buffer_destroy(bb); - } -} - -size_t grpc_bbq_bytes(grpc_byte_buffer_queue *q) { return q->bytes; } - -grpc_byte_buffer *grpc_bbq_pop(grpc_byte_buffer_queue *q) { - grpc_bbq_array temp_array; - grpc_byte_buffer *out; - - if (q->drain_pos == q->draining.count) { - if (q->filling.count == 0) { - return NULL; - } - q->draining.count = 0; - q->drain_pos = 0; - /* swap arrays */ - temp_array = q->filling; - q->filling = q->draining; - q->draining = temp_array; - } - - out = q->draining.data[q->drain_pos++]; - q->bytes -= grpc_byte_buffer_length(out); - return out; -} diff --git a/src/core/surface/byte_buffer_reader.c b/src/core/surface/byte_buffer_reader.c index 9f830df68c..57417f41b0 100644 --- a/src/core/surface/byte_buffer_reader.c +++ b/src/core/surface/byte_buffer_reader.c @@ -121,4 +121,3 @@ gpr_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) { } return out_slice; } - diff --git a/src/core/surface/call.c b/src/core/surface/call.c index 81ff215c0c..86d5fca388 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -31,6 +31,7 @@ * */ #include <assert.h> +#include <limits.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -46,12 +47,11 @@ #include "src/core/profiling/timers.h" #include "src/core/support/string.h" #include "src/core/surface/api_trace.h" -#include "src/core/surface/byte_buffer_queue.h" #include "src/core/surface/call.h" #include "src/core/surface/channel.h" #include "src/core/surface/completion_queue.h" -/** The maximum number of completions possible. +/** The maximum number of concurrent batches possible. Based upon the maximum number of individually queueable ops in the batch api: - initial metadata send @@ -60,7 +60,7 @@ - initial metadata recv - message recv - status/close recv (depending on client/server) */ -#define MAX_CONCURRENT_COMPLETIONS 6 +#define MAX_CONCURRENT_BATCHES 6 typedef struct { grpc_ioreq_completion_func on_complete; @@ -68,24 +68,7 @@ typedef struct { int success; } completed_request; -/* See request_set in grpc_call below for a description */ -#define REQSET_EMPTY 'X' -#define REQSET_DONE 'Y' - -#define MAX_SEND_INITIAL_METADATA_COUNT 3 - -typedef struct { - /* Overall status of the operation: starts OK, may degrade to - non-OK */ - gpr_uint8 success; - /* a bit mask of which request ops are needed (1u << opid) */ - gpr_uint16 need_mask; - /* a bit mask of which request ops are now completed */ - gpr_uint16 complete_mask; - /* Completion function to call at the end of the operation */ - grpc_ioreq_completion_func on_complete; - void *user_data; -} reqinfo_master; +#define MAX_SEND_EXTRA_METADATA_COUNT 3 /* Status data for a request can come from several sources; this enumerates them all, and acts as a priority sorting for which @@ -130,6 +113,23 @@ typedef enum { WRITE_STATE_WRITE_CLOSED } write_state; +typedef struct batch_control { + grpc_call *call; + grpc_cq_completion cq_completion; + grpc_closure finish_batch; + void *notify_tag; + gpr_refcount steps_to_complete; + + gpr_uint8 send_initial_metadata; + gpr_uint8 send_message; + gpr_uint8 send_final_op; + gpr_uint8 recv_initial_metadata; + gpr_uint8 recv_message; + gpr_uint8 recv_final_op; + gpr_uint8 is_notify_tag_closure; + gpr_uint8 success; +} batch_control; + struct grpc_call { grpc_completion_queue *cq; grpc_channel *channel; @@ -138,98 +138,39 @@ struct grpc_call { grpc_mdctx *metadata_context; /* TODO(ctiller): share with cq if possible? */ gpr_mu mu; - gpr_mu completion_mu; - /* how far through the stream have we read? */ - read_state read_state; - /* how far through the stream have we written? */ - write_state write_state; /* client or server call */ gpr_uint8 is_client; /* is the alarm set */ gpr_uint8 have_alarm; - /* are we currently performing a send operation */ - gpr_uint8 sending; - /* are we currently performing a recv operation */ - gpr_uint8 receiving; - /* are we currently completing requests */ - gpr_uint8 completing; /** has grpc_call_destroy been called */ gpr_uint8 destroy_called; - /* pairs with completed_requests */ - gpr_uint8 num_completed_requests; - /* are we currently reading a message? */ - gpr_uint8 reading_message; - /* have we bound a pollset yet? */ - gpr_uint8 bound_pollset; - /* is an error status set */ - gpr_uint8 error_status_set; - /** bitmask of allocated completion events in completions */ - gpr_uint8 allocated_completions; /** flag indicating that cancellation is inherited */ gpr_uint8 cancellation_is_inherited; + /** bitmask of live batches */ + gpr_uint8 used_batches; + /** which ops are in-flight */ + gpr_uint8 sent_initial_metadata; + gpr_uint8 sending_message; + gpr_uint8 sent_final_op; + gpr_uint8 received_initial_metadata; + gpr_uint8 receiving_message; + gpr_uint8 received_final_op; + + batch_control active_batches[MAX_CONCURRENT_BATCHES]; + + /* first idx: is_receiving, second idx: is_trailing */ + grpc_metadata_batch metadata_batch[2][2]; - /* flags with bits corresponding to write states allowing us to determine - what was sent */ - gpr_uint16 last_send_contains; - /* cancel with this status on the next outgoing transport op */ - grpc_status_code cancel_with_status; - - /* Active ioreqs. - request_set and request_data contain one element per active ioreq - operation. - - request_set[op] is an integer specifying a set of operations to which - the request belongs: - - if it is < GRPC_IOREQ_OP_COUNT, then this operation is pending - completion, and the integer represents to which group of operations - the ioreq belongs. Each group is represented by one master, and the - integer in request_set is an index into masters to find the master - data. - - if it is REQSET_EMPTY, the ioreq op is inactive and available to be - started - - finally, if request_set[op] is REQSET_DONE, then the operation is - complete and unavailable to be started again - - request_data[op] is the request data as supplied by the initiator of - a request, and is valid iff request_set[op] <= GRPC_IOREQ_OP_COUNT. - The set fields are as per the request type specified by op. - - Finally, one element of masters is set per active _set_ of ioreq - operations. It describes work left outstanding, result status, and - what work to perform upon operation completion. As one ioreq of each - op type can be active at once, by convention we choose the first element - of the group to be the master -- ie the master of in-progress operation - op is masters[request_set[op]]. This allows constant time allocation - and a strong upper bound of a count of masters to be calculated. */ - gpr_uint8 request_set[GRPC_IOREQ_OP_COUNT]; - grpc_ioreq_data request_data[GRPC_IOREQ_OP_COUNT]; - gpr_uint32 request_flags[GRPC_IOREQ_OP_COUNT]; - reqinfo_master masters[GRPC_IOREQ_OP_COUNT]; - - /* Dynamic array of ioreq's that have completed: the count of - elements is queued in num_completed_requests. - This list is built up under lock(), and flushed entirely during - unlock(). - We know the upper bound of the number of elements as we can only - have one ioreq of each type active at once. */ - completed_request completed_requests[GRPC_IOREQ_OP_COUNT]; - /* Incoming buffer of messages */ - grpc_byte_buffer_queue incoming_queue; /* Buffered read metadata waiting to be returned to the application. Element 0 is initial metadata, element 1 is trailing metadata. */ - grpc_metadata_array buffered_metadata[2]; - /* All metadata received - unreffed at once at the end of the call */ - grpc_mdelem **owned_metadata; - size_t owned_metadata_count; - size_t owned_metadata_capacity; + grpc_metadata_array *buffered_metadata[2]; /* Received call statuses from various sources */ received_status status[STATUS_SOURCE_COUNT]; /* Compression algorithm for the call */ grpc_compression_algorithm compression_algorithm; - /* Supported encodings (compression algorithms), a bitset */ gpr_uint32 encodings_accepted_by_peer; @@ -239,35 +180,36 @@ struct grpc_call { /* Deadline alarm - if have_alarm is non-zero */ grpc_timer alarm; - /* Call refcount - to keep the call alive during asynchronous operations */ - gpr_refcount internal_refcount; - - grpc_linked_mdelem send_initial_metadata[MAX_SEND_INITIAL_METADATA_COUNT]; - grpc_linked_mdelem status_link; - grpc_linked_mdelem details_link; - size_t send_initial_metadata_count; + /* for the client, extra metadata is initial metadata; for the + server, it's trailing metadata */ + grpc_linked_mdelem send_extra_metadata[MAX_SEND_EXTRA_METADATA_COUNT]; + int send_extra_metadata_count; gpr_timespec send_deadline; - grpc_stream_op_buffer send_ops; - grpc_stream_op_buffer recv_ops; - grpc_stream_state recv_state; - - gpr_slice_buffer incoming_message; - gpr_uint32 incoming_message_length; - gpr_uint32 incoming_message_flags; - grpc_closure destroy_closure; - grpc_closure on_done_recv; - grpc_closure on_done_send; - grpc_closure on_done_bind; - - /** completion events - for completion queue use */ - grpc_cq_completion completions[MAX_CONCURRENT_COMPLETIONS]; - /** siblings: children of the same parent form a list, and this list is protected under parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; + + grpc_slice_buffer_stream sending_stream; + grpc_byte_stream *receiving_stream; + grpc_byte_buffer **receiving_buffer; + gpr_slice receiving_slice; + grpc_closure receiving_slice_ready; + grpc_closure receiving_stream_ready; + gpr_uint32 test_only_last_message_flags; + + union { + struct { + grpc_status_code *status; + char **status_details; + size_t *status_details_capacity; + } client; + struct { + int *cancelled; + } server; + } final_op; }; #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) @@ -279,20 +221,15 @@ struct grpc_call { static void set_deadline_alarm(grpc_exec_ctx *exec_ctx, grpc_call *call, gpr_timespec deadline); -static void call_on_done_recv(grpc_exec_ctx *exec_ctx, void *call, int success); -static void call_on_done_send(grpc_exec_ctx *exec_ctx, void *call, int success); -static int fill_send_ops(grpc_call *call, grpc_transport_stream_op *op); static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_transport_stream_op *op); -static void recv_metadata(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_metadata_batch *metadata); -static void finish_read_ops(grpc_call *call); -static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status, +static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + grpc_status_code status, const char *description); -static void finished_loose_op(grpc_exec_ctx *exec_ctx, void *call, int success); - -static void lock(grpc_call *call); -static void unlock(grpc_exec_ctx *exec_ctx, grpc_call *call); +static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack, + int success); +static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, + int success); grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, gpr_uint32 propagation_mask, @@ -301,9 +238,7 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, grpc_mdelem **add_initial_metadata, size_t add_initial_metadata_count, gpr_timespec send_deadline) { - size_t i; - grpc_transport_stream_op initial_op; - grpc_transport_stream_op *initial_op_ptr = NULL; + size_t i, j; grpc_channel_stack *channel_stack = grpc_channel_get_channel_stack(channel); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_call *call; @@ -311,51 +246,37 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, call = gpr_malloc(sizeof(grpc_call) + channel_stack->call_stack_size); memset(call, 0, sizeof(grpc_call)); gpr_mu_init(&call->mu); - gpr_mu_init(&call->completion_mu); call->channel = channel; call->cq = cq; - if (cq != NULL) { - GRPC_CQ_INTERNAL_REF(cq, "bind"); - } call->parent = parent_call; call->is_client = server_transport_data == NULL; - for (i = 0; i < GRPC_IOREQ_OP_COUNT; i++) { - call->request_set[i] = REQSET_EMPTY; - } if (call->is_client) { - call->request_set[GRPC_IOREQ_SEND_TRAILING_METADATA] = REQSET_DONE; - call->request_set[GRPC_IOREQ_SEND_STATUS] = REQSET_DONE; + GPR_ASSERT(add_initial_metadata_count < MAX_SEND_EXTRA_METADATA_COUNT); + for (i = 0; i < add_initial_metadata_count; i++) { + call->send_extra_metadata[i].md = add_initial_metadata[i]; + } + call->send_extra_metadata_count = (int)add_initial_metadata_count; + } else { + GPR_ASSERT(add_initial_metadata_count == 0); + call->send_extra_metadata_count = 0; } - GPR_ASSERT(add_initial_metadata_count < MAX_SEND_INITIAL_METADATA_COUNT); - for (i = 0; i < add_initial_metadata_count; i++) { - call->send_initial_metadata[i].md = add_initial_metadata[i]; + for (i = 0; i < 2; i++) { + for (j = 0; j < 2; j++) { + call->metadata_batch[i][j].deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + } } - call->send_initial_metadata_count = add_initial_metadata_count; call->send_deadline = send_deadline; GRPC_CHANNEL_INTERNAL_REF(channel, "call"); call->metadata_context = grpc_channel_get_metadata_context(channel); - grpc_sopb_init(&call->send_ops); - grpc_sopb_init(&call->recv_ops); - gpr_slice_buffer_init(&call->incoming_message); - grpc_closure_init(&call->on_done_recv, call_on_done_recv, call); - grpc_closure_init(&call->on_done_send, call_on_done_send, call); - grpc_closure_init(&call->on_done_bind, finished_loose_op, call); - /* dropped in destroy and when READ_STATE_STREAM_CLOSED received */ - gpr_ref_init(&call->internal_refcount, 2); - /* server hack: start reads immediately so we can get initial metadata. - TODO(ctiller): figure out a cleaner solution */ - if (!call->is_client) { - memset(&initial_op, 0, sizeof(initial_op)); - initial_op.recv_ops = &call->recv_ops; - initial_op.recv_state = &call->recv_state; - initial_op.on_done_recv = &call->on_done_recv; - initial_op.context = call->context; - call->receiving = 1; - GRPC_CALL_INTERNAL_REF(call, "receiving"); - initial_op_ptr = &initial_op; + /* initial refcount dropped by grpc_call_destroy */ + grpc_call_stack_init(&exec_ctx, channel_stack, 1, destroy_call, call, + call->context, server_transport_data, + CALL_STACK_FROM_CALL(call)); + if (cq != NULL) { + GRPC_CQ_INTERNAL_REF(cq, "bind"); + grpc_call_stack_set_pollset(&exec_ctx, CALL_STACK_FROM_CALL(call), + grpc_cq_pollset(cq)); } - grpc_call_stack_init(&exec_ctx, channel_stack, server_transport_data, - initial_op_ptr, CALL_STACK_FROM_CALL(call)); if (parent_call != NULL) { GRPC_CALL_INTERNAL_REF(parent_call, "child"); GPR_ASSERT(call->is_client); @@ -408,90 +329,62 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call, void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_completion_queue *cq) { - lock(call); + GPR_ASSERT(cq); call->cq = cq; - if (cq) { - GRPC_CQ_INTERNAL_REF(cq, "bind"); - } - unlock(exec_ctx, call); + GRPC_CQ_INTERNAL_REF(cq, "bind"); + grpc_call_stack_set_pollset(exec_ctx, CALL_STACK_FROM_CALL(call), + grpc_cq_pollset(cq)); } grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call) { return call->cq; } -static grpc_cq_completion *allocate_completion(grpc_call *call) { - gpr_uint8 i; - gpr_mu_lock(&call->completion_mu); - for (i = 0; i < GPR_ARRAY_SIZE(call->completions); i++) { - if (call->allocated_completions & (1u << i)) { - continue; - } - /* NB: the following integer arithmetic operation needs to be in its - * expanded form due to the "integral promotion" performed (see section - * 3.2.1.1 of the C89 draft standard). A cast to the smaller container type - * is then required to avoid the compiler warning */ - call->allocated_completions = - (gpr_uint8)(call->allocated_completions | (1u << i)); - gpr_mu_unlock(&call->completion_mu); - return &call->completions[i]; - } - GPR_UNREACHABLE_CODE(return NULL); - return NULL; +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +void grpc_call_internal_ref(grpc_call *c, const char *reason) { + grpc_call_stack_ref(CALL_STACK_FROM_CALL(c), reason); } - -static void done_completion(grpc_exec_ctx *exec_ctx, void *call, - grpc_cq_completion *completion) { - grpc_call *c = call; - gpr_mu_lock(&c->completion_mu); - c->allocated_completions &= - (gpr_uint8) ~(1u << (completion - c->completions)); - gpr_mu_unlock(&c->completion_mu); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, c, "completion"); +void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c, + const char *reason) { + grpc_call_stack_unref(exec_ctx, CALL_STACK_FROM_CALL(c), reason); } - -#ifdef GRPC_CALL_REF_COUNT_DEBUG -void grpc_call_internal_ref(grpc_call *c, const char *reason) { - gpr_log(GPR_DEBUG, "CALL: ref %p %d -> %d [%s]", c, - c->internal_refcount.count, c->internal_refcount.count + 1, reason); #else void grpc_call_internal_ref(grpc_call *c) { -#endif - gpr_ref(&c->internal_refcount); + grpc_call_stack_ref(CALL_STACK_FROM_CALL(c)); } +void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c) { + grpc_call_stack_unref(exec_ctx, CALL_STACK_FROM_CALL(c)); +} +#endif -static void destroy_call(grpc_exec_ctx *exec_ctx, grpc_call *call) { +static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, int success) { size_t i; + int ii; grpc_call *c = call; GPR_TIMER_BEGIN("destroy_call", 0); + for (i = 0; i < 2; i++) { + grpc_metadata_batch_destroy( + &c->metadata_batch[1 /* is_receiving */][i /* is_initial */]); + } + if (c->receiving_stream != NULL) { + grpc_byte_stream_destroy(c->receiving_stream); + } grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c)); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, c->channel, "call"); gpr_mu_destroy(&c->mu); - gpr_mu_destroy(&c->completion_mu); for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (c->status[i].details) { GRPC_MDSTR_UNREF(c->status[i].details); } } - for (i = 0; i < c->owned_metadata_count; i++) { - GRPC_MDELEM_UNREF(c->owned_metadata[i]); - } - gpr_free(c->owned_metadata); - for (i = 0; i < GPR_ARRAY_SIZE(c->buffered_metadata); i++) { - gpr_free(c->buffered_metadata[i].metadata); - } - for (i = 0; i < c->send_initial_metadata_count; i++) { - GRPC_MDELEM_UNREF(c->send_initial_metadata[i].md); + for (ii = 0; ii < c->send_extra_metadata_count; ii++) { + GRPC_MDELEM_UNREF(c->send_extra_metadata[ii].md); } for (i = 0; i < GRPC_CONTEXT_COUNT; i++) { if (c->context[i].destroy) { c->context[i].destroy(c->context[i].value); } } - grpc_sopb_destroy(&c->send_ops); - grpc_sopb_destroy(&c->recv_ops); - grpc_bbq_destroy(&c->incoming_queue); - gpr_slice_buffer_destroy(&c->incoming_message); if (c->cq) { GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); } @@ -499,30 +392,14 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, grpc_call *call) { GPR_TIMER_END("destroy_call", 0); } -#ifdef GRPC_CALL_REF_COUNT_DEBUG -void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c, - const char *reason) { - gpr_log(GPR_DEBUG, "CALL: unref %p %d -> %d [%s]", c, - c->internal_refcount.count, c->internal_refcount.count - 1, reason); -#else -void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c) { -#endif - if (gpr_unref(&c->internal_refcount)) { - destroy_call(exec_ctx, c); - } -} - static void set_status_code(grpc_call *call, status_source source, gpr_uint32 status) { if (call->status[source].is_set) return; call->status[source].is_set = 1; call->status[source].code = (grpc_status_code)status; - call->error_status_set = status != GRPC_STATUS_OK; - if (status != GRPC_STATUS_OK && !grpc_bbq_empty(&call->incoming_queue)) { - grpc_bbq_flush(&call->incoming_queue); - } + /* TODO(ctiller): what to do about the flush that was previously here */ } static void set_compression_algorithm(grpc_call *call, @@ -539,6 +416,14 @@ grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( return algorithm; } +gpr_uint32 grpc_call_test_only_get_message_flags(grpc_call *call) { + gpr_uint32 flags; + gpr_mu_lock(&call->mu); + flags = call->test_only_last_message_flags; + gpr_mu_unlock(&call->mu); + return flags; +} + static void destroy_encodings_accepted_by_peer(void *p) { return; } static void set_encodings_accepted_by_peer(grpc_call *call, grpc_mdelem *mdel) { @@ -596,14 +481,6 @@ gpr_uint32 grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) { return encodings_accepted_by_peer; } -gpr_uint32 grpc_call_test_only_get_message_flags(grpc_call *call) { - gpr_uint32 flags; - gpr_mu_lock(&call->mu); - flags = call->incoming_message_flags; - gpr_mu_unlock(&call->mu); - return flags; -} - static void set_status_details(grpc_call *call, status_source source, grpc_mdstr *status) { if (call->status[source].details != NULL) { @@ -612,149 +489,39 @@ static void set_status_details(grpc_call *call, status_source source, call->status[source].details = status; } -static int is_op_live(grpc_call *call, grpc_ioreq_op op) { - gpr_uint8 set = call->request_set[op]; - reqinfo_master *master; - if (set >= GRPC_IOREQ_OP_COUNT) return 0; - master = &call->masters[set]; - return (master->complete_mask & (1u << op)) == 0; -} - -static void lock(grpc_call *call) { gpr_mu_lock(&call->mu); } - -static int need_more_data(grpc_call *call) { - if (call->read_state == READ_STATE_STREAM_CLOSED) return 0; - /* TODO(ctiller): this needs some serious cleanup */ - return is_op_live(call, GRPC_IOREQ_RECV_INITIAL_METADATA) || - (is_op_live(call, GRPC_IOREQ_RECV_MESSAGE) && - grpc_bbq_empty(&call->incoming_queue)) || - is_op_live(call, GRPC_IOREQ_RECV_TRAILING_METADATA) || - is_op_live(call, GRPC_IOREQ_RECV_STATUS) || - is_op_live(call, GRPC_IOREQ_RECV_STATUS_DETAILS) || - (is_op_live(call, GRPC_IOREQ_RECV_CLOSE) && - grpc_bbq_empty(&call->incoming_queue)) || - (call->write_state == WRITE_STATE_INITIAL && !call->is_client) || - (call->cancel_with_status != GRPC_STATUS_OK) || call->destroy_called; -} - -static void unlock(grpc_exec_ctx *exec_ctx, grpc_call *call) { - grpc_transport_stream_op op; - completed_request completed_requests[GRPC_IOREQ_OP_COUNT]; - int completing_requests = 0; - int start_op = 0; - int i; - const size_t MAX_RECV_PEEK_AHEAD = 65536; - size_t buffered_bytes; - - GPR_TIMER_BEGIN("unlock", 0); - - memset(&op, 0, sizeof(op)); - - op.cancel_with_status = call->cancel_with_status; - start_op = op.cancel_with_status != GRPC_STATUS_OK; - call->cancel_with_status = GRPC_STATUS_OK; /* reset */ - - if (!call->receiving && need_more_data(call)) { - if (grpc_bbq_empty(&call->incoming_queue) && call->reading_message) { - op.max_recv_bytes = call->incoming_message_length - - call->incoming_message.length + MAX_RECV_PEEK_AHEAD; - } else { - buffered_bytes = grpc_bbq_bytes(&call->incoming_queue); - if (buffered_bytes > MAX_RECV_PEEK_AHEAD) { - op.max_recv_bytes = 0; - } else { - op.max_recv_bytes = MAX_RECV_PEEK_AHEAD - buffered_bytes; - } - } - /* TODO(ctiller): 1024 is basically to cover a bug - I don't understand yet */ - if (op.max_recv_bytes > 1024) { - op.recv_ops = &call->recv_ops; - op.recv_state = &call->recv_state; - op.on_done_recv = &call->on_done_recv; - call->receiving = 1; - GRPC_CALL_INTERNAL_REF(call, "receiving"); - start_op = 1; - } - } - - if (!call->sending) { - if (fill_send_ops(call, &op)) { - call->sending = 1; - GRPC_CALL_INTERNAL_REF(call, "sending"); - start_op = 1; - } - } - - if (!call->bound_pollset && call->cq && (!call->is_client || start_op)) { - call->bound_pollset = 1; - op.bind_pollset = grpc_cq_pollset(call->cq); - start_op = 1; - } - - if (!call->completing && call->num_completed_requests != 0) { - completing_requests = call->num_completed_requests; - memcpy(completed_requests, call->completed_requests, - sizeof(completed_requests)); - call->num_completed_requests = 0; - call->completing = 1; - GRPC_CALL_INTERNAL_REF(call, "completing"); - } - - gpr_mu_unlock(&call->mu); - - if (start_op) { - execute_op(exec_ctx, call, &op); - } - - if (completing_requests > 0) { - for (i = 0; i < completing_requests; i++) { - completed_requests[i].on_complete(exec_ctx, call, - completed_requests[i].success, - completed_requests[i].user_data); - } - lock(call); - call->completing = 0; - unlock(exec_ctx, call); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completing"); - } - - GPR_TIMER_END("unlock", 0); -} - -static void get_final_status(grpc_call *call, grpc_ioreq_data out) { +static void get_final_status(grpc_call *call, + void (*set_value)(grpc_status_code code, + void *user_data), + void *set_value_user_data) { int i; for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (call->status[i].is_set) { - out.recv_status.set_value(call->status[i].code, - out.recv_status.user_data); + set_value(call->status[i].code, set_value_user_data); return; } } if (call->is_client) { - out.recv_status.set_value(GRPC_STATUS_UNKNOWN, out.recv_status.user_data); + set_value(GRPC_STATUS_UNKNOWN, set_value_user_data); } else { - out.recv_status.set_value(GRPC_STATUS_OK, out.recv_status.user_data); + set_value(GRPC_STATUS_OK, set_value_user_data); } } -static void get_final_details(grpc_call *call, grpc_ioreq_data out) { +static void get_final_details(grpc_call *call, char **out_details, + size_t *out_details_capacity) { int i; for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (call->status[i].is_set) { if (call->status[i].details) { gpr_slice details = call->status[i].details->slice; size_t len = GPR_SLICE_LENGTH(details); - if (len + 1 > *out.recv_status_details.details_capacity) { - *out.recv_status_details.details_capacity = GPR_MAX( - len + 1, *out.recv_status_details.details_capacity * 3 / 2); - *out.recv_status_details.details = - gpr_realloc(*out.recv_status_details.details, - *out.recv_status_details.details_capacity); + if (len + 1 > *out_details_capacity) { + *out_details_capacity = + GPR_MAX(len + 1, *out_details_capacity * 3 / 2); + *out_details = gpr_realloc(*out_details, *out_details_capacity); } - memcpy(*out.recv_status_details.details, GPR_SLICE_START_PTR(details), - len); - (*out.recv_status_details.details)[len] = 0; + memcpy(*out_details, GPR_SLICE_START_PTR(details), len); + (*out_details)[len] = 0; } else { goto no_details; } @@ -763,328 +530,41 @@ static void get_final_details(grpc_call *call, grpc_ioreq_data out) { } no_details: - if (0 == *out.recv_status_details.details_capacity) { - *out.recv_status_details.details_capacity = 8; - *out.recv_status_details.details = - gpr_malloc(*out.recv_status_details.details_capacity); - } - **out.recv_status_details.details = 0; -} - -static void finish_live_ioreq_op(grpc_call *call, grpc_ioreq_op op, - int success) { - completed_request *cr; - gpr_uint8 master_set = call->request_set[op]; - reqinfo_master *master; - size_t i; - /* ioreq is live: we need to do something */ - master = &call->masters[master_set]; - /* NB: the following integer arithmetic operation needs to be in its - * expanded form due to the "integral promotion" performed (see section - * 3.2.1.1 of the C89 draft standard). A cast to the smaller container type - * is then required to avoid the compiler warning */ - master->complete_mask = (gpr_uint16)(master->complete_mask | (1u << op)); - if (!success) { - master->success = 0; - } - if (master->complete_mask == master->need_mask) { - for (i = 0; i < GRPC_IOREQ_OP_COUNT; i++) { - if (call->request_set[i] != master_set) { - continue; - } - call->request_set[i] = REQSET_DONE; - switch ((grpc_ioreq_op)i) { - case GRPC_IOREQ_RECV_MESSAGE: - case GRPC_IOREQ_SEND_MESSAGE: - call->request_set[i] = REQSET_EMPTY; - if (!master->success) { - call->write_state = WRITE_STATE_WRITE_CLOSED; - } - break; - case GRPC_IOREQ_SEND_STATUS: - if (call->request_data[GRPC_IOREQ_SEND_STATUS].send_status.details != - NULL) { - GRPC_MDSTR_UNREF( - call->request_data[GRPC_IOREQ_SEND_STATUS].send_status.details); - call->request_data[GRPC_IOREQ_SEND_STATUS].send_status.details = - NULL; - } - break; - case GRPC_IOREQ_RECV_CLOSE: - case GRPC_IOREQ_SEND_INITIAL_METADATA: - case GRPC_IOREQ_SEND_TRAILING_METADATA: - case GRPC_IOREQ_SEND_CLOSE: - break; - case GRPC_IOREQ_RECV_STATUS: - get_final_status(call, call->request_data[GRPC_IOREQ_RECV_STATUS]); - break; - case GRPC_IOREQ_RECV_STATUS_DETAILS: - get_final_details(call, - call->request_data[GRPC_IOREQ_RECV_STATUS_DETAILS]); - break; - case GRPC_IOREQ_RECV_INITIAL_METADATA: - GPR_SWAP(grpc_metadata_array, call->buffered_metadata[0], - *call->request_data[GRPC_IOREQ_RECV_INITIAL_METADATA] - .recv_metadata); - break; - case GRPC_IOREQ_RECV_TRAILING_METADATA: - GPR_SWAP(grpc_metadata_array, call->buffered_metadata[1], - *call->request_data[GRPC_IOREQ_RECV_TRAILING_METADATA] - .recv_metadata); - break; - case GRPC_IOREQ_OP_COUNT: - abort(); - break; - } - } - cr = &call->completed_requests[call->num_completed_requests++]; - cr->success = master->success; - cr->on_complete = master->on_complete; - cr->user_data = master->user_data; - } -} - -static void finish_ioreq_op(grpc_call *call, grpc_ioreq_op op, int success) { - if (is_op_live(call, op)) { - finish_live_ioreq_op(call, op, success); + if (0 == *out_details_capacity) { + *out_details_capacity = 8; + *out_details = gpr_malloc(*out_details_capacity); } + **out_details = 0; } -static void early_out_write_ops(grpc_call *call) { - switch (call->write_state) { - case WRITE_STATE_WRITE_CLOSED: - finish_ioreq_op(call, GRPC_IOREQ_SEND_MESSAGE, 0); - finish_ioreq_op(call, GRPC_IOREQ_SEND_STATUS, 0); - finish_ioreq_op(call, GRPC_IOREQ_SEND_TRAILING_METADATA, 0); - finish_ioreq_op(call, GRPC_IOREQ_SEND_CLOSE, 1); - /* fallthrough */ - case WRITE_STATE_STARTED: - finish_ioreq_op(call, GRPC_IOREQ_SEND_INITIAL_METADATA, 0); - /* fallthrough */ - case WRITE_STATE_INITIAL: - /* do nothing */ - break; - } +static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) { + return (grpc_linked_mdelem *)&md->internal_data; } -static void call_on_done_send(grpc_exec_ctx *exec_ctx, void *pc, int success) { - grpc_call *call = pc; - GPR_TIMER_BEGIN("call_on_done_send", 0); - lock(call); - if (call->last_send_contains & (1 << GRPC_IOREQ_SEND_INITIAL_METADATA)) { - finish_ioreq_op(call, GRPC_IOREQ_SEND_INITIAL_METADATA, success); - call->write_state = WRITE_STATE_STARTED; - } - if (call->last_send_contains & (1 << GRPC_IOREQ_SEND_MESSAGE)) { - finish_ioreq_op(call, GRPC_IOREQ_SEND_MESSAGE, success); - } - if (call->last_send_contains & (1 << GRPC_IOREQ_SEND_CLOSE)) { - finish_ioreq_op(call, GRPC_IOREQ_SEND_TRAILING_METADATA, success); - finish_ioreq_op(call, GRPC_IOREQ_SEND_STATUS, success); - finish_ioreq_op(call, GRPC_IOREQ_SEND_CLOSE, 1); - call->write_state = WRITE_STATE_WRITE_CLOSED; - } - if (!success) { - call->write_state = WRITE_STATE_WRITE_CLOSED; - early_out_write_ops(call); - } - call->send_ops.nops = 0; - call->last_send_contains = 0; - call->sending = 0; - unlock(exec_ctx, call); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "sending"); - GPR_TIMER_END("call_on_done_send", 0); -} - -static void finish_message(grpc_call *call) { - GPR_TIMER_BEGIN("finish_message", 0); - if (call->error_status_set == 0) { - /* TODO(ctiller): this could be a lot faster if coded directly */ - grpc_byte_buffer *byte_buffer; - /* some aliases for readability */ - gpr_slice *slices = call->incoming_message.slices; - const size_t nslices = call->incoming_message.count; - - if ((call->incoming_message_flags & GRPC_WRITE_INTERNAL_COMPRESS) && - (call->compression_algorithm > GRPC_COMPRESS_NONE)) { - byte_buffer = grpc_raw_compressed_byte_buffer_create( - slices, nslices, call->compression_algorithm); +static int prepare_application_metadata(grpc_call *call, int count, + grpc_metadata *metadata, + int is_trailing, + int prepend_extra_metadata) { + int i; + grpc_metadata_batch *batch = + &call->metadata_batch[0 /* is_receiving */][is_trailing]; + if (prepend_extra_metadata) { + if (call->send_extra_metadata_count == 0) { + prepend_extra_metadata = 0; } else { - byte_buffer = grpc_raw_byte_buffer_create(slices, nslices); - } - grpc_bbq_push(&call->incoming_queue, byte_buffer); - } - gpr_slice_buffer_reset_and_unref(&call->incoming_message); - GPR_ASSERT(call->incoming_message.count == 0); - call->reading_message = 0; - GPR_TIMER_END("finish_message", 0); -} - -static int begin_message(grpc_call *call, grpc_begin_message msg) { - /* can't begin a message when we're still reading a message */ - if (call->reading_message) { - char *message = NULL; - gpr_asprintf( - &message, "Message terminated early; read %d bytes, expected %d", - (int)call->incoming_message.length, (int)call->incoming_message_length); - cancel_with_status(call, GRPC_STATUS_INVALID_ARGUMENT, message); - gpr_free(message); - return 0; - } - /* sanity check: if message flags indicate a compressed message, the - * compression level should already be present in the call, as parsed off its - * corresponding metadata. */ - if ((msg.flags & GRPC_WRITE_INTERNAL_COMPRESS) && - (call->compression_algorithm == GRPC_COMPRESS_NONE)) { - char *message = NULL; - char *alg_name; - if (!grpc_compression_algorithm_name(call->compression_algorithm, - &alg_name)) { - /* This shouldn't happen, other than due to data corruption */ - alg_name = "<unknown>"; - } - gpr_asprintf(&message, - "Invalid compression algorithm (%s) for compressed message.", - alg_name); - cancel_with_status(call, GRPC_STATUS_INTERNAL, message); - gpr_free(message); - return 0; - } - /* stash away parameters, and prepare for incoming slices */ - if (msg.length > grpc_channel_get_max_message_length(call->channel)) { - char *message = NULL; - gpr_asprintf( - &message, - "Maximum message length of %d exceeded by a message of length %d", - grpc_channel_get_max_message_length(call->channel), msg.length); - cancel_with_status(call, GRPC_STATUS_INVALID_ARGUMENT, message); - gpr_free(message); - return 0; - } else if (msg.length > 0) { - call->reading_message = 1; - call->incoming_message_length = msg.length; - call->incoming_message_flags = msg.flags; - return 1; - } else { - finish_message(call); - return 1; - } -} - -static int add_slice_to_message(grpc_call *call, gpr_slice slice) { - if (GPR_SLICE_LENGTH(slice) == 0) { - gpr_slice_unref(slice); - return 1; - } - /* we have to be reading a message to know what to do here */ - if (!call->reading_message) { - gpr_slice_unref(slice); - cancel_with_status(call, GRPC_STATUS_INVALID_ARGUMENT, - "Received payload data while not reading a message"); - return 0; - } - /* append the slice to the incoming buffer */ - gpr_slice_buffer_add(&call->incoming_message, slice); - if (call->incoming_message.length > call->incoming_message_length) { - /* if we got too many bytes, complain */ - char *message = NULL; - gpr_asprintf( - &message, "Receiving message overflow; read %d bytes, expected %d", - (int)call->incoming_message.length, (int)call->incoming_message_length); - cancel_with_status(call, GRPC_STATUS_INVALID_ARGUMENT, message); - gpr_free(message); - return 0; - } else if (call->incoming_message.length == call->incoming_message_length) { - finish_message(call); - return 1; - } else { - return 1; - } -} - -static void call_on_done_recv(grpc_exec_ctx *exec_ctx, void *pc, int success) { - grpc_call *call = pc; - grpc_call *child_call; - grpc_call *next_child_call; - size_t i; - GPR_TIMER_BEGIN("call_on_done_recv", 0); - lock(call); - call->receiving = 0; - if (success) { - for (i = 0; success && i < call->recv_ops.nops; i++) { - grpc_stream_op *op = &call->recv_ops.ops[i]; - switch (op->type) { - case GRPC_NO_OP: - break; - case GRPC_OP_METADATA: - GPR_TIMER_BEGIN("recv_metadata", 0); - recv_metadata(exec_ctx, call, &op->data.metadata); - GPR_TIMER_END("recv_metadata", 0); - break; - case GRPC_OP_BEGIN_MESSAGE: - GPR_TIMER_BEGIN("begin_message", 0); - success = begin_message(call, op->data.begin_message); - GPR_TIMER_END("begin_message", 0); - break; - case GRPC_OP_SLICE: - GPR_TIMER_BEGIN("add_slice_to_message", 0); - success = add_slice_to_message(call, op->data.slice); - GPR_TIMER_END("add_slice_to_message", 0); - break; + for (i = 0; i < call->send_extra_metadata_count; i++) { + GRPC_MDELEM_REF(call->send_extra_metadata[i].md); } - } - if (!success) { - grpc_stream_ops_unref_owned_objects(&call->recv_ops.ops[i], - call->recv_ops.nops - i); - } - if (call->recv_state == GRPC_STREAM_RECV_CLOSED) { - GPR_ASSERT(call->read_state <= READ_STATE_READ_CLOSED); - call->read_state = READ_STATE_READ_CLOSED; - } - if (call->recv_state == GRPC_STREAM_CLOSED) { - GPR_ASSERT(call->read_state <= READ_STATE_STREAM_CLOSED); - call->read_state = READ_STATE_STREAM_CLOSED; - if (call->have_alarm) { - grpc_timer_cancel(exec_ctx, &call->alarm); + for (i = 1; i < call->send_extra_metadata_count; i++) { + call->send_extra_metadata[i].prev = &call->send_extra_metadata[i - 1]; } - /* propagate cancellation to any interested children */ - child_call = call->first_child; - if (child_call != NULL) { - do { - next_child_call = child_call->sibling_next; - if (child_call->cancellation_is_inherited) { - GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - grpc_call_cancel(child_call, NULL); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); - } - child_call = next_child_call; - } while (child_call != call->first_child); + for (i = 0; i < call->send_extra_metadata_count - 1; i++) { + call->send_extra_metadata[i].next = &call->send_extra_metadata[i + 1]; } - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "closed"); } - finish_read_ops(call); - } else { - finish_ioreq_op(call, GRPC_IOREQ_RECV_MESSAGE, 0); - finish_ioreq_op(call, GRPC_IOREQ_RECV_STATUS, 0); - finish_ioreq_op(call, GRPC_IOREQ_RECV_CLOSE, 0); - finish_ioreq_op(call, GRPC_IOREQ_RECV_TRAILING_METADATA, 0); - finish_ioreq_op(call, GRPC_IOREQ_RECV_INITIAL_METADATA, 0); - finish_ioreq_op(call, GRPC_IOREQ_RECV_STATUS_DETAILS, 0); } - call->recv_ops.nops = 0; - unlock(exec_ctx, call); - - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "receiving"); - GPR_TIMER_END("call_on_done_recv", 0); -} - -static int prepare_application_metadata(grpc_call *call, size_t count, - grpc_metadata *metadata) { - size_t i; for (i = 0; i < count; i++) { grpc_metadata *md = &metadata[i]; - grpc_metadata *next_md = (i == count - 1) ? NULL : &metadata[i + 1]; - grpc_metadata *prev_md = (i == 0) ? NULL : &metadata[i - 1]; grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data; GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data)); l->md = grpc_mdelem_from_string_and_buffer(call->metadata_context, md->key, @@ -1099,243 +579,49 @@ static int prepare_application_metadata(grpc_call *call, size_t count, gpr_log(GPR_ERROR, "attempt to send invalid metadata value"); return 0; } - l->next = next_md ? (grpc_linked_mdelem *)&next_md->internal_data : NULL; - l->prev = prev_md ? (grpc_linked_mdelem *)&prev_md->internal_data : NULL; } - return 1; -} - -static grpc_mdelem_list chain_metadata_from_app(grpc_call *call, size_t count, - grpc_metadata *metadata) { - grpc_mdelem_list out; - if (count == 0) { - out.head = out.tail = NULL; - return out; + for (i = 1; i < count; i++) { + linked_from_md(&metadata[i])->prev = linked_from_md(&metadata[i - 1]); } - out.head = (grpc_linked_mdelem *)&(metadata[0].internal_data); - out.tail = (grpc_linked_mdelem *)&(metadata[count - 1].internal_data); - return out; -} - -/* Copy the contents of a byte buffer into stream ops */ -static void copy_byte_buffer_to_stream_ops(grpc_byte_buffer *byte_buffer, - grpc_stream_op_buffer *sopb) { - size_t i; - - switch (byte_buffer->type) { - case GRPC_BB_RAW: - for (i = 0; i < byte_buffer->data.raw.slice_buffer.count; i++) { - gpr_slice slice = byte_buffer->data.raw.slice_buffer.slices[i]; - gpr_slice_ref(slice); - grpc_sopb_add_slice(sopb, slice); - } - break; + for (i = 0; i < count - 1; i++) { + linked_from_md(&metadata[i])->next = linked_from_md(&metadata[i + 1]); } -} - -static int fill_send_ops(grpc_call *call, grpc_transport_stream_op *op) { - grpc_ioreq_data data; - gpr_uint32 flags; - grpc_metadata_batch mdb; - size_t i; - GPR_ASSERT(op->send_ops == NULL); - - switch (call->write_state) { - case WRITE_STATE_INITIAL: - if (!is_op_live(call, GRPC_IOREQ_SEND_INITIAL_METADATA)) { - break; - } - data = call->request_data[GRPC_IOREQ_SEND_INITIAL_METADATA]; - mdb.list = chain_metadata_from_app(call, data.send_metadata.count, - data.send_metadata.metadata); - mdb.garbage.head = mdb.garbage.tail = NULL; - mdb.deadline = call->send_deadline; - for (i = 0; i < call->send_initial_metadata_count; i++) { - grpc_metadata_batch_link_head(&mdb, &call->send_initial_metadata[i]); - } - grpc_sopb_add_metadata(&call->send_ops, mdb); - op->send_ops = &call->send_ops; - call->last_send_contains |= 1 << GRPC_IOREQ_SEND_INITIAL_METADATA; - call->send_initial_metadata_count = 0; - /* fall through intended */ - case WRITE_STATE_STARTED: - if (is_op_live(call, GRPC_IOREQ_SEND_MESSAGE)) { - size_t length; - data = call->request_data[GRPC_IOREQ_SEND_MESSAGE]; - flags = call->request_flags[GRPC_IOREQ_SEND_MESSAGE]; - length = grpc_byte_buffer_length(data.send_message); - GPR_ASSERT(length <= GPR_UINT32_MAX); - grpc_sopb_add_begin_message(&call->send_ops, (gpr_uint32)length, flags); - copy_byte_buffer_to_stream_ops(data.send_message, &call->send_ops); - op->send_ops = &call->send_ops; - call->last_send_contains |= 1 << GRPC_IOREQ_SEND_MESSAGE; - } - if (is_op_live(call, GRPC_IOREQ_SEND_CLOSE)) { - op->is_last_send = 1; - op->send_ops = &call->send_ops; - call->last_send_contains |= 1 << GRPC_IOREQ_SEND_CLOSE; - if (!call->is_client) { - /* send trailing metadata */ - data = call->request_data[GRPC_IOREQ_SEND_TRAILING_METADATA]; - mdb.list = chain_metadata_from_app(call, data.send_metadata.count, - data.send_metadata.metadata); - mdb.garbage.head = mdb.garbage.tail = NULL; - mdb.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); - /* send status */ - /* TODO(ctiller): cache common status values */ - data = call->request_data[GRPC_IOREQ_SEND_STATUS]; - grpc_metadata_batch_add_tail( - &mdb, &call->status_link, - grpc_channel_get_reffed_status_elem(call->channel, - data.send_status.code)); - if (data.send_status.details) { - grpc_metadata_batch_add_tail( - &mdb, &call->details_link, - grpc_mdelem_from_metadata_strings( - call->metadata_context, - GRPC_MDSTR_REF( - grpc_channel_get_message_string(call->channel)), - data.send_status.details)); - call->request_data[GRPC_IOREQ_SEND_STATUS].send_status.details = - NULL; - } - grpc_sopb_add_metadata(&call->send_ops, mdb); - } - } + switch (prepend_extra_metadata * 2 + (count != 0)) { + case 0: + /* no prepend, no metadata => nothing to do */ + batch->list.head = batch->list.tail = NULL; break; - case WRITE_STATE_WRITE_CLOSED: + case 1: + /* metadata, but no prepend */ + batch->list.head = linked_from_md(&metadata[0]); + batch->list.tail = linked_from_md(&metadata[count - 1]); + batch->list.head->prev = NULL; + batch->list.tail->next = NULL; break; - } - if (op->send_ops) { - op->on_done_send = &call->on_done_send; - } - return op->send_ops != NULL; -} - -static grpc_call_error start_ioreq_error(grpc_call *call, - gpr_uint32 mutated_ops, - grpc_call_error ret) { - size_t i; - for (i = 0; i < GRPC_IOREQ_OP_COUNT; i++) { - if (mutated_ops & (1u << i)) { - call->request_set[i] = REQSET_EMPTY; - } - } - return ret; -} - -static void finish_read_ops(grpc_call *call) { - int empty; - - if (is_op_live(call, GRPC_IOREQ_RECV_MESSAGE)) { - empty = - (NULL == (*call->request_data[GRPC_IOREQ_RECV_MESSAGE].recv_message = - grpc_bbq_pop(&call->incoming_queue))); - if (!empty) { - finish_live_ioreq_op(call, GRPC_IOREQ_RECV_MESSAGE, 1); - empty = grpc_bbq_empty(&call->incoming_queue); - } - } else { - empty = grpc_bbq_empty(&call->incoming_queue); - } - - switch (call->read_state) { - case READ_STATE_STREAM_CLOSED: - if (empty && !call->have_alarm) { - finish_ioreq_op(call, GRPC_IOREQ_RECV_CLOSE, 1); - } - /* fallthrough */ - case READ_STATE_READ_CLOSED: - if (empty) { - finish_ioreq_op(call, GRPC_IOREQ_RECV_MESSAGE, 1); - } - finish_ioreq_op(call, GRPC_IOREQ_RECV_STATUS, 1); - finish_ioreq_op(call, GRPC_IOREQ_RECV_STATUS_DETAILS, 1); - finish_ioreq_op(call, GRPC_IOREQ_RECV_TRAILING_METADATA, 1); - /* fallthrough */ - case READ_STATE_GOT_INITIAL_METADATA: - finish_ioreq_op(call, GRPC_IOREQ_RECV_INITIAL_METADATA, 1); - /* fallthrough */ - case READ_STATE_INITIAL: - /* do nothing */ + case 2: + /* prepend, but no md */ + batch->list.head = &call->send_extra_metadata[0]; + batch->list.tail = + &call->send_extra_metadata[call->send_extra_metadata_count - 1]; + batch->list.head->prev = NULL; + batch->list.tail->next = NULL; break; + case 3: + /* prepend AND md */ + batch->list.head = &call->send_extra_metadata[0]; + call->send_extra_metadata[call->send_extra_metadata_count - 1].next = + linked_from_md(&metadata[0]); + linked_from_md(&metadata[0])->prev = + &call->send_extra_metadata[call->send_extra_metadata_count - 1]; + batch->list.tail = linked_from_md(&metadata[count - 1]); + batch->list.head->prev = NULL; + batch->list.tail->next = NULL; + break; + default: + GPR_UNREACHABLE_CODE(return 0); } -} - -static grpc_call_error start_ioreq(grpc_call *call, const grpc_ioreq *reqs, - size_t nreqs, - grpc_ioreq_completion_func completion, - void *user_data) { - size_t i; - gpr_uint16 have_ops = 0; - grpc_ioreq_op op; - reqinfo_master *master; - grpc_ioreq_data data; - gpr_uint8 set; - - if (nreqs == 0) { - return GRPC_CALL_OK; - } - - set = reqs[0].op; - - for (i = 0; i < nreqs; i++) { - op = reqs[i].op; - if (call->request_set[op] < GRPC_IOREQ_OP_COUNT) { - return start_ioreq_error(call, have_ops, - GRPC_CALL_ERROR_TOO_MANY_OPERATIONS); - } else if (call->request_set[op] == REQSET_DONE) { - return start_ioreq_error(call, have_ops, GRPC_CALL_ERROR_ALREADY_INVOKED); - } - data = reqs[i].data; - if (op == GRPC_IOREQ_SEND_INITIAL_METADATA || - op == GRPC_IOREQ_SEND_TRAILING_METADATA) { - if (!prepare_application_metadata(call, data.send_metadata.count, - data.send_metadata.metadata)) { - return start_ioreq_error(call, have_ops, - GRPC_CALL_ERROR_INVALID_METADATA); - } - } - if (op == GRPC_IOREQ_SEND_STATUS) { - set_status_code(call, STATUS_FROM_SERVER_STATUS, - (gpr_uint32)reqs[i].data.send_status.code); - if (reqs[i].data.send_status.details) { - set_status_details(call, STATUS_FROM_SERVER_STATUS, - GRPC_MDSTR_REF(reqs[i].data.send_status.details)); - } - } - /* NB: the following integer arithmetic operation needs to be in its - * expanded form due to the "integral promotion" performed (see section - * 3.2.1.1 of the C89 draft standard). A cast to the smaller container type - * is then required to avoid the compiler warning */ - have_ops = (gpr_uint16)(have_ops | (1u << op)); - - call->request_data[op] = data; - call->request_flags[op] = reqs[i].flags; - call->request_set[op] = set; - } - - master = &call->masters[set]; - master->success = 1; - master->need_mask = have_ops; - master->complete_mask = 0; - master->on_complete = completion; - master->user_data = user_data; - finish_read_ops(call); - early_out_write_ops(call); - - return GRPC_CALL_OK; -} - -grpc_call_error grpc_call_start_ioreq_and_call_back( - grpc_exec_ctx *exec_ctx, grpc_call *call, const grpc_ioreq *reqs, - size_t nreqs, grpc_ioreq_completion_func on_complete, void *user_data) { - grpc_call_error err; - lock(call); - err = start_ioreq(call, reqs, nreqs, on_complete, user_data); - unlock(exec_ctx, call); - return err; + return 1; } void grpc_call_destroy(grpc_call *c) { @@ -1343,6 +629,7 @@ void grpc_call_destroy(grpc_call *c) { grpc_call *parent = c->parent; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_TIMER_BEGIN("grpc_call_destroy", 0); GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); if (parent) { @@ -1359,17 +646,18 @@ void grpc_call_destroy(grpc_call *c) { GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); } - lock(c); + gpr_mu_lock(&c->mu); GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; if (c->have_alarm) { grpc_timer_cancel(&exec_ctx, &c->alarm); } - cancel = c->read_state != READ_STATE_STREAM_CLOSED; - unlock(&exec_ctx, c); + cancel = !c->received_final_op; + gpr_mu_unlock(&c->mu); if (cancel) grpc_call_cancel(c, NULL); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, c, "destroy"); grpc_exec_ctx_finish(&exec_ctx); + GPR_TIMER_END("grpc_call_destroy", 0); } grpc_call_error grpc_call_cancel(grpc_call *call, void *reserved) { @@ -1390,66 +678,67 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, "c=%p, status=%d, description=%s, reserved=%p)", 4, (c, (int)status, description, reserved)); GPR_ASSERT(reserved == NULL); - lock(c); - r = cancel_with_status(c, status, description); - unlock(&exec_ctx, c); + gpr_mu_lock(&c->mu); + r = cancel_with_status(&exec_ctx, c, status, description); + gpr_mu_unlock(&c->mu); grpc_exec_ctx_finish(&exec_ctx); return r; } -static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status, +typedef struct cancel_closure { + grpc_closure closure; + grpc_call *call; + grpc_status_code status; +} cancel_closure; + +static void done_cancel(grpc_exec_ctx *exec_ctx, void *ccp, int success) { + cancel_closure *cc = ccp; + GRPC_CALL_INTERNAL_UNREF(exec_ctx, cc->call, "cancel"); + gpr_free(cc); +} + +static void send_cancel(grpc_exec_ctx *exec_ctx, void *ccp, int success) { + grpc_transport_stream_op op; + cancel_closure *cc = ccp; + memset(&op, 0, sizeof(op)); + op.cancel_with_status = cc->status; + /* reuse closure to catch completion */ + grpc_closure_init(&cc->closure, done_cancel, cc); + op.on_complete = &cc->closure; + execute_op(exec_ctx, cc->call, &op); +} + +static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + grpc_status_code status, const char *description) { grpc_mdstr *details = description ? grpc_mdstr_from_string(c->metadata_context, description) : NULL; + cancel_closure *cc = gpr_malloc(sizeof(*cc)); GPR_ASSERT(status != GRPC_STATUS_OK); set_status_code(c, STATUS_FROM_API_OVERRIDE, (gpr_uint32)status); set_status_details(c, STATUS_FROM_API_OVERRIDE, details); - c->cancel_with_status = status; + grpc_closure_init(&cc->closure, send_cancel, cc); + cc->call = c; + cc->status = status; + GRPC_CALL_INTERNAL_REF(c, "cancel"); + grpc_exec_ctx_enqueue(exec_ctx, &cc->closure, 1); return GRPC_CALL_OK; } -static void finished_loose_op(grpc_exec_ctx *exec_ctx, void *call, - int success_ignored) { - GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "loose-op"); -} - -typedef struct { - grpc_call *call; - grpc_closure closure; -} finished_loose_op_allocated_args; - -static void finished_loose_op_allocated(grpc_exec_ctx *exec_ctx, void *alloc, - int success) { - finished_loose_op_allocated_args *args = alloc; - finished_loose_op(exec_ctx, args->call, success); - gpr_free(args); -} - static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_transport_stream_op *op) { grpc_call_element *elem; - GPR_ASSERT(op->on_consumed == NULL); - if (op->cancel_with_status != GRPC_STATUS_OK || op->bind_pollset) { - GRPC_CALL_INTERNAL_REF(call, "loose-op"); - if (op->bind_pollset) { - op->on_consumed = &call->on_done_bind; - } else { - finished_loose_op_allocated_args *args = gpr_malloc(sizeof(*args)); - args->call = call; - grpc_closure_init(&args->closure, finished_loose_op_allocated, args); - op->on_consumed = &args->closure; - } - } - + GPR_TIMER_BEGIN("execute_op", 0); elem = CALL_ELEM_FROM_CALL(call, 0); op->context = call->context; elem->filter->start_transport_stream_op(exec_ctx, elem, op); + GPR_TIMER_END("execute_op", 0); } char *grpc_call_get_peer(grpc_call *call) { @@ -1467,14 +756,13 @@ grpc_call *grpc_call_from_top_element(grpc_call_element *elem) { static void call_alarm(grpc_exec_ctx *exec_ctx, void *arg, int success) { grpc_call *call = arg; - lock(call); + gpr_mu_lock(&call->mu); call->have_alarm = 0; if (success) { - cancel_with_status(call, GRPC_STATUS_DEADLINE_EXCEEDED, + cancel_with_status(exec_ctx, call, GRPC_STATUS_DEADLINE_EXCEEDED, "Deadline Exceeded"); } - finish_read_ops(call); - unlock(exec_ctx, call); + gpr_mu_unlock(&call->mu); GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "alarm"); } @@ -1540,76 +828,69 @@ static gpr_uint32 decode_compression(grpc_mdelem *md) { return algorithm; } -static void recv_metadata(grpc_exec_ctx *exec_ctx, grpc_call *call, - grpc_metadata_batch *md) { - grpc_linked_mdelem *l; +static grpc_mdelem *recv_common_filter(grpc_call *call, grpc_mdelem *elem) { + if (elem->key == grpc_channel_get_status_string(call->channel)) { + GPR_TIMER_BEGIN("status", 0); + set_status_code(call, STATUS_FROM_WIRE, decode_status(elem)); + GPR_TIMER_END("status", 0); + return NULL; + } else if (elem->key == grpc_channel_get_message_string(call->channel)) { + GPR_TIMER_BEGIN("status-details", 0); + set_status_details(call, STATUS_FROM_WIRE, GRPC_MDSTR_REF(elem->value)); + GPR_TIMER_END("status-details", 0); + return NULL; + } + return elem; +} + +static grpc_mdelem *publish_app_metadata(grpc_call *call, grpc_mdelem *elem, + int is_trailing) { grpc_metadata_array *dest; grpc_metadata *mdusr; - int is_trailing; - - is_trailing = call->read_state >= READ_STATE_GOT_INITIAL_METADATA; - for (l = md->list.head; l != NULL; l = l->next) { - grpc_mdelem *mdel = l->md; - grpc_mdstr *key = mdel->key; - if (key == grpc_channel_get_status_string(call->channel)) { - GPR_TIMER_BEGIN("status", 0); - set_status_code(call, STATUS_FROM_WIRE, decode_status(mdel)); - GPR_TIMER_END("status", 0); - } else if (key == grpc_channel_get_message_string(call->channel)) { - GPR_TIMER_BEGIN("status-details", 0); - set_status_details(call, STATUS_FROM_WIRE, GRPC_MDSTR_REF(mdel->value)); - GPR_TIMER_END("status-details", 0); - } else if (key == - grpc_channel_get_compression_algorithm_string(call->channel)) { - GPR_TIMER_BEGIN("compression_algorithm", 0); - set_compression_algorithm(call, decode_compression(mdel)); - GPR_TIMER_END("compression_algorithm", 0); - } else if (key == grpc_channel_get_encodings_accepted_by_peer_string( - call->channel)) { - GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0); - set_encodings_accepted_by_peer(call, mdel); - GPR_TIMER_END("encodings_accepted_by_peer", 0); - } else { - GPR_TIMER_BEGIN("report_up", 0); - dest = &call->buffered_metadata[is_trailing]; - if (dest->count == dest->capacity) { - dest->capacity = GPR_MAX(dest->capacity + 8, dest->capacity * 2); - dest->metadata = - gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity); - } - mdusr = &dest->metadata[dest->count++]; - mdusr->key = grpc_mdstr_as_c_string(mdel->key); - mdusr->value = grpc_mdstr_as_c_string(mdel->value); - mdusr->value_length = GPR_SLICE_LENGTH(mdel->value->slice); - if (call->owned_metadata_count == call->owned_metadata_capacity) { - call->owned_metadata_capacity = - GPR_MAX(call->owned_metadata_capacity + 8, - call->owned_metadata_capacity * 2); - call->owned_metadata = - gpr_realloc(call->owned_metadata, - sizeof(grpc_mdelem *) * call->owned_metadata_capacity); - } - call->owned_metadata[call->owned_metadata_count++] = mdel; - l->md = NULL; - GPR_TIMER_END("report_up", 0); - } - } - if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) != - 0 && - !call->is_client) { - GPR_TIMER_BEGIN("set_deadline_alarm", 0); - set_deadline_alarm(exec_ctx, call, md->deadline); - GPR_TIMER_END("set_deadline_alarm", 0); - } - if (!is_trailing) { - call->read_state = READ_STATE_GOT_INITIAL_METADATA; - } + GPR_TIMER_BEGIN("publish_app_metadata", 0); + dest = call->buffered_metadata[is_trailing]; + if (dest->count == dest->capacity) { + dest->capacity = GPR_MAX(dest->capacity + 8, dest->capacity * 2); + dest->metadata = + gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity); + } + mdusr = &dest->metadata[dest->count++]; + mdusr->key = grpc_mdstr_as_c_string(elem->key); + mdusr->value = grpc_mdstr_as_c_string(elem->value); + mdusr->value_length = GPR_SLICE_LENGTH(elem->value->slice); + GPR_TIMER_END("publish_app_metadata", 0); + return elem; +} - for (l = md->list.head; l; l = l->next) { - if (l->md) GRPC_MDELEM_UNREF(l->md); +static grpc_mdelem *recv_initial_filter(void *callp, grpc_mdelem *elem) { + grpc_call *call = callp; + elem = recv_common_filter(call, elem); + if (elem == NULL) { + return NULL; + } else if (elem->key == + grpc_channel_get_compression_algorithm_string(call->channel)) { + GPR_TIMER_BEGIN("compression_algorithm", 0); + set_compression_algorithm(call, decode_compression(elem)); + GPR_TIMER_END("compression_algorithm", 0); + return NULL; + } else if (elem->key == grpc_channel_get_encodings_accepted_by_peer_string( + call->channel)) { + GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0); + set_encodings_accepted_by_peer(call, elem); + GPR_TIMER_END("encodings_accepted_by_peer", 0); + return NULL; + } else { + return publish_app_metadata(call, elem, 0); } - for (l = md->garbage.head; l; l = l->next) { - GRPC_MDELEM_UNREF(l->md); +} + +static grpc_mdelem *recv_trailing_filter(void *callp, grpc_mdelem *elem) { + grpc_call *call = callp; + elem = recv_common_filter(call, elem); + if (elem == NULL) { + return NULL; + } else { + return publish_app_metadata(call, elem, 1); } } @@ -1626,19 +907,7 @@ static void set_status_value_directly(grpc_status_code status, void *dest) { } static void set_cancelled_value(grpc_status_code status, void *dest) { - *(grpc_status_code *)dest = (status != GRPC_STATUS_OK); -} - -static void finish_batch(grpc_exec_ctx *exec_ctx, grpc_call *call, int success, - void *tag) { - grpc_cq_end_op(exec_ctx, call->cq, tag, success, done_completion, call, - allocate_completion(call)); -} - -static void finish_batch_with_close(grpc_exec_ctx *exec_ctx, grpc_call *call, - int success, void *tag) { - grpc_cq_end_op(exec_ctx, call->cq, tag, 1, done_completion, call, - allocate_completion(call)); + *(int *)dest = (status != GRPC_STATUS_OK); } static int are_write_flags_valid(gpr_uint32 flags) { @@ -1649,258 +918,504 @@ static int are_write_flags_valid(gpr_uint32 flags) { return !(flags & invalid_positions); } -grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops, - size_t nops, void *tag, void *reserved) { - grpc_ioreq reqs[GRPC_IOREQ_OP_COUNT]; - size_t in; - size_t out; +static batch_control *allocate_batch_control(grpc_call *call) { + size_t i; + for (i = 0; i < MAX_CONCURRENT_BATCHES; i++) { + if ((call->used_batches & (1 << i)) == 0) { + call->used_batches = + (gpr_uint8)(call->used_batches | (gpr_uint8)(1 << i)); + return &call->active_batches[i]; + } + } + GPR_UNREACHABLE_CODE(return NULL); +} + +static void finish_batch_completion(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_cq_completion *storage) { + batch_control *bctl = user_data; + grpc_call *call = bctl->call; + gpr_mu_lock(&call->mu); + call->used_batches = (gpr_uint8)( + call->used_batches & ~(gpr_uint8)(1 << (bctl - call->active_batches))); + gpr_mu_unlock(&call->mu); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); +} + +static void post_batch_completion(grpc_exec_ctx *exec_ctx, + batch_control *bctl) { + grpc_call *call = bctl->call; + if (bctl->is_notify_tag_closure) { + grpc_exec_ctx_enqueue(exec_ctx, bctl->notify_tag, bctl->success); + gpr_mu_lock(&call->mu); + bctl->call->used_batches = + (gpr_uint8)(bctl->call->used_batches & + ~(gpr_uint8)(1 << (bctl - bctl->call->active_batches))); + gpr_mu_unlock(&call->mu); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); + } else { + grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, bctl->success, + finish_batch_completion, bctl, &bctl->cq_completion); + } +} + +static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, + batch_control *bctl) { + grpc_call *call = bctl->call; + for (;;) { + size_t remaining = call->receiving_stream->length - + (*call->receiving_buffer)->data.raw.slice_buffer.length; + if (remaining == 0) { + call->receiving_message = 0; + grpc_byte_stream_destroy(call->receiving_stream); + call->receiving_stream = NULL; + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } + return; + } + if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, + &call->receiving_slice, remaining, + &call->receiving_slice_ready)) { + gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); + } else { + return; + } + } +} + +static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, + int success) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + + GPR_ASSERT(success); + gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); + + continue_receiving_slices(exec_ctx, bctl); +} + +static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, int success) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + grpc_call *child_call; + grpc_call *next_child_call; + + gpr_mu_lock(&call->mu); + if (bctl->send_initial_metadata) { + grpc_metadata_batch_destroy( + &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]); + } + if (bctl->send_message) { + call->sending_message = 0; + } + if (bctl->send_final_op) { + grpc_metadata_batch_destroy( + &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]); + } + if (bctl->recv_initial_metadata) { + grpc_metadata_batch *md = + &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; + grpc_metadata_batch_filter(md, recv_initial_filter, call); + + if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) != + 0 && + !call->is_client) { + GPR_TIMER_BEGIN("set_deadline_alarm", 0); + set_deadline_alarm(exec_ctx, call, md->deadline); + GPR_TIMER_END("set_deadline_alarm", 0); + } + } + if (bctl->recv_final_op) { + grpc_metadata_batch *md = + &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; + grpc_metadata_batch_filter(md, recv_trailing_filter, call); + + if (call->have_alarm) { + grpc_timer_cancel(exec_ctx, &call->alarm); + } + /* propagate cancellation to any interested children */ + child_call = call->first_child; + if (child_call != NULL) { + do { + next_child_call = child_call->sibling_next; + if (child_call->cancellation_is_inherited) { + GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); + grpc_call_cancel(child_call, NULL); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); + } + child_call = next_child_call; + } while (child_call != call->first_child); + } + + if (call->is_client) { + get_final_status(call, set_status_value_directly, + call->final_op.client.status); + get_final_details(call, call->final_op.client.status_details, + call->final_op.client.status_details_capacity); + } else { + get_final_status(call, set_cancelled_value, + call->final_op.server.cancelled); + } + + success = 1; + } + bctl->success = success != 0; + gpr_mu_unlock(&call->mu); + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } +} + +static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, + int success) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + + if (call->receiving_stream == NULL) { + *call->receiving_buffer = NULL; + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } + } else if (call->receiving_stream->length > + grpc_channel_get_max_message_length(call->channel)) { + cancel_with_status(exec_ctx, call, GRPC_STATUS_INTERNAL, + "Max message size exceeded"); + grpc_byte_stream_destroy(call->receiving_stream); + call->receiving_stream = NULL; + *call->receiving_buffer = NULL; + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } + } else { + call->test_only_last_message_flags = call->receiving_stream->flags; + if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) && + (call->compression_algorithm > GRPC_COMPRESS_NONE)) { + *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create( + NULL, 0, call->compression_algorithm); + } else { + *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0); + } + grpc_closure_init(&call->receiving_slice_ready, receiving_slice_ready, + bctl); + continue_receiving_slices(exec_ctx, bctl); + /* early out */ + return; + } +} + +static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, + grpc_call *call, const grpc_op *ops, + size_t nops, void *notify_tag, + int is_notify_tag_closure) { + grpc_transport_stream_op stream_op; + size_t i; const grpc_op *op; - grpc_ioreq *req; - void (*finish_func)(grpc_exec_ctx *, grpc_call *, int, void *) = finish_batch; - grpc_call_error error; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + batch_control *bctl; + int num_completion_callbacks_needed = 1; + grpc_call_error error = GRPC_CALL_OK; GPR_TIMER_BEGIN("grpc_call_start_batch", 0); - GRPC_API_TRACE( - "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, reserved=%p)", - 5, (call, ops, (unsigned long)nops, tag, reserved)); + GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag); - if (reserved != NULL) { - error = GRPC_CALL_ERROR; - goto done; - } + memset(&stream_op, 0, sizeof(stream_op)); - GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, tag); + /* TODO(ctiller): this feels like it could be made lock-free */ + gpr_mu_lock(&call->mu); + bctl = allocate_batch_control(call); + memset(bctl, 0, sizeof(*bctl)); + bctl->call = call; + bctl->notify_tag = notify_tag; + bctl->is_notify_tag_closure = (gpr_uint8)(is_notify_tag_closure != 0); if (nops == 0) { - grpc_cq_begin_op(call->cq); GRPC_CALL_INTERNAL_REF(call, "completion"); - grpc_cq_end_op(&exec_ctx, call->cq, tag, 1, done_completion, call, - allocate_completion(call)); - error = GRPC_CALL_OK; - goto done; + bctl->success = 1; + if (!is_notify_tag_closure) { + grpc_cq_begin_op(call->cq); + } + gpr_mu_unlock(&call->mu); + post_batch_completion(exec_ctx, bctl); + return GRPC_CALL_OK; } - /* rewrite batch ops into ioreq ops */ - for (in = 0, out = 0; in < nops; in++) { - op = &ops[in]; + /* rewrite batch ops into a transport op */ + for (i = 0; i < nops; i++) { + op = &ops[i]; if (op->reserved != NULL) { error = GRPC_CALL_ERROR; - goto done; + goto done_with_error; } switch (op->op) { case GRPC_OP_SEND_INITIAL_METADATA: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; + } + if (call->sent_initial_metadata) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; + } + if (op->data.send_initial_metadata.count > INT_MAX) { + error = GRPC_CALL_ERROR_INVALID_METADATA; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + bctl->send_initial_metadata = 1; + call->sent_initial_metadata = 1; + if (!prepare_application_metadata( + call, (int)op->data.send_initial_metadata.count, + op->data.send_initial_metadata.metadata, 0, call->is_client)) { + error = GRPC_CALL_ERROR_INVALID_METADATA; + goto done_with_error; } - req->op = GRPC_IOREQ_SEND_INITIAL_METADATA; - req->data.send_metadata.count = op->data.send_initial_metadata.count; - req->data.send_metadata.metadata = - op->data.send_initial_metadata.metadata; - req->flags = op->flags; + /* TODO(ctiller): just make these the same variable? */ + call->metadata_batch[0][0].deadline = call->send_deadline; + stream_op.send_initial_metadata = + &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */]; break; case GRPC_OP_SEND_MESSAGE: if (!are_write_flags_valid(op->flags)) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } if (op->data.send_message == NULL) { error = GRPC_CALL_ERROR_INVALID_MESSAGE; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->sending_message) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_SEND_MESSAGE; - req->data.send_message = op->data.send_message; - req->flags = op->flags; + bctl->send_message = 1; + call->sending_message = 1; + grpc_slice_buffer_stream_init( + &call->sending_stream, + &op->data.send_message->data.raw.slice_buffer, op->flags); + stream_op.send_message = &call->sending_stream.base; break; case GRPC_OP_SEND_CLOSE_FROM_CLIENT: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } if (!call->is_client) { error = GRPC_CALL_ERROR_NOT_ON_SERVER; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->sent_final_op) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_SEND_CLOSE; - req->flags = op->flags; + bctl->send_final_op = 1; + call->sent_final_op = 1; + stream_op.send_trailing_metadata = + &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_SEND_STATUS_FROM_SERVER: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } if (call->is_client) { error = GRPC_CALL_ERROR_NOT_ON_CLIENT; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->sent_final_op) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_SEND_TRAILING_METADATA; - req->flags = op->flags; - req->data.send_metadata.count = - op->data.send_status_from_server.trailing_metadata_count; - req->data.send_metadata.metadata = - op->data.send_status_from_server.trailing_metadata; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (op->data.send_status_from_server.trailing_metadata_count > + INT_MAX) { + error = GRPC_CALL_ERROR_INVALID_METADATA; + goto done_with_error; } - req->op = GRPC_IOREQ_SEND_STATUS; - req->data.send_status.code = op->data.send_status_from_server.status; - req->data.send_status.details = - op->data.send_status_from_server.status_details != NULL - ? grpc_mdstr_from_string( - call->metadata_context, - op->data.send_status_from_server.status_details) - : NULL; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + bctl->send_final_op = 1; + call->sent_final_op = 1; + call->send_extra_metadata_count = 1; + call->send_extra_metadata[0].md = grpc_channel_get_reffed_status_elem( + call->channel, op->data.send_status_from_server.status); + if (op->data.send_status_from_server.status_details != NULL) { + call->send_extra_metadata[1].md = grpc_mdelem_from_metadata_strings( + call->metadata_context, + GRPC_MDSTR_REF(grpc_channel_get_message_string(call->channel)), + grpc_mdstr_from_string( + call->metadata_context, + op->data.send_status_from_server.status_details)); + call->send_extra_metadata_count++; + set_status_details( + call, STATUS_FROM_API_OVERRIDE, + GRPC_MDSTR_REF(call->send_extra_metadata[1].md->value)); } - req->op = GRPC_IOREQ_SEND_CLOSE; + set_status_code(call, STATUS_FROM_API_OVERRIDE, + (gpr_uint32)op->data.send_status_from_server.status); + if (!prepare_application_metadata( + call, + (int)op->data.send_status_from_server.trailing_metadata_count, + op->data.send_status_from_server.trailing_metadata, 1, 1)) { + error = GRPC_CALL_ERROR_INVALID_METADATA; + goto done_with_error; + } + stream_op.send_trailing_metadata = + &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_RECV_INITIAL_METADATA: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; - } - if (!call->is_client) { - error = GRPC_CALL_ERROR_NOT_ON_SERVER; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->received_initial_metadata) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_RECV_INITIAL_METADATA; - req->data.recv_metadata = op->data.recv_initial_metadata; - req->data.recv_metadata->count = 0; - req->flags = op->flags; + call->received_initial_metadata = 1; + call->buffered_metadata[0] = op->data.recv_initial_metadata; + bctl->recv_initial_metadata = 1; + stream_op.recv_initial_metadata = + &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; break; case GRPC_OP_RECV_MESSAGE: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->receiving_message) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; } - req->op = GRPC_IOREQ_RECV_MESSAGE; - req->data.recv_message = op->data.recv_message; - req->flags = op->flags; + call->receiving_message = 1; + bctl->recv_message = 1; + call->receiving_buffer = op->data.recv_message; + stream_op.recv_message = &call->receiving_stream; + grpc_closure_init(&call->receiving_stream_ready, receiving_stream_ready, + bctl); + stream_op.recv_message_ready = &call->receiving_stream_ready; + num_completion_callbacks_needed++; break; case GRPC_OP_RECV_STATUS_ON_CLIENT: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } if (!call->is_client) { error = GRPC_CALL_ERROR_NOT_ON_SERVER; - goto done; - } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + goto done_with_error; } - req->op = GRPC_IOREQ_RECV_STATUS; - req->flags = op->flags; - req->data.recv_status.set_value = set_status_value_directly; - req->data.recv_status.user_data = op->data.recv_status_on_client.status; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->received_final_op) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_RECV_STATUS_DETAILS; - req->data.recv_status_details.details = + call->received_final_op = 1; + call->buffered_metadata[1] = + op->data.recv_status_on_client.trailing_metadata; + call->final_op.client.status = op->data.recv_status_on_client.status; + call->final_op.client.status_details = op->data.recv_status_on_client.status_details; - req->data.recv_status_details.details_capacity = + call->final_op.client.status_details_capacity = op->data.recv_status_on_client.status_details_capacity; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; - } - req->op = GRPC_IOREQ_RECV_TRAILING_METADATA; - req->data.recv_metadata = - op->data.recv_status_on_client.trailing_metadata; - req->data.recv_metadata->count = 0; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; - } - req->op = GRPC_IOREQ_RECV_CLOSE; - finish_func = finish_batch_with_close; + bctl->recv_final_op = 1; + stream_op.recv_trailing_metadata = + &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_RECV_CLOSE_ON_SERVER: /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; - goto done; + goto done_with_error; } - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->is_client) { + error = GRPC_CALL_ERROR_NOT_ON_CLIENT; + goto done_with_error; } - req->op = GRPC_IOREQ_RECV_STATUS; - req->flags = op->flags; - req->data.recv_status.set_value = set_cancelled_value; - req->data.recv_status.user_data = - op->data.recv_close_on_server.cancelled; - req = &reqs[out++]; - if (out > GRPC_IOREQ_OP_COUNT) { - error = GRPC_CALL_ERROR_BATCH_TOO_BIG; - goto done; + if (call->received_final_op) { + error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + goto done_with_error; } - req->op = GRPC_IOREQ_RECV_CLOSE; - finish_func = finish_batch_with_close; + call->received_final_op = 1; + call->final_op.server.cancelled = + op->data.recv_close_on_server.cancelled; + bctl->recv_final_op = 1; + stream_op.recv_trailing_metadata = + &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; break; } } GRPC_CALL_INTERNAL_REF(call, "completion"); - grpc_cq_begin_op(call->cq); + if (!is_notify_tag_closure) { + grpc_cq_begin_op(call->cq); + } + gpr_ref_init(&bctl->steps_to_complete, num_completion_callbacks_needed); + + stream_op.context = call->context; + grpc_closure_init(&bctl->finish_batch, finish_batch, bctl); + stream_op.on_complete = &bctl->finish_batch; + gpr_mu_unlock(&call->mu); + + execute_op(exec_ctx, call, &stream_op); - error = grpc_call_start_ioreq_and_call_back(&exec_ctx, call, reqs, out, - finish_func, tag); done: - grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_call_start_batch", 0); return error; + +done_with_error: + /* reverse any mutations that occured */ + if (bctl->send_initial_metadata) { + call->sent_initial_metadata = 0; + grpc_metadata_batch_clear(&call->metadata_batch[0][0]); + } + if (bctl->send_message) { + call->sending_message = 0; + grpc_byte_stream_destroy(&call->sending_stream.base); + } + if (bctl->send_final_op) { + call->sent_final_op = 0; + grpc_metadata_batch_clear(&call->metadata_batch[0][1]); + } + if (bctl->recv_initial_metadata) { + call->received_initial_metadata = 0; + } + if (bctl->recv_message) { + call->receiving_message = 0; + } + if (bctl->recv_final_op) { + call->received_final_op = 0; + } + gpr_mu_unlock(&call->mu); + goto done; +} + +grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops, + size_t nops, void *tag, void *reserved) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_call_error err; + + GRPC_API_TRACE( + "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, reserved=%p)", + 5, (call, ops, (unsigned long)nops, tag, reserved)); + + if (reserved != NULL) { + err = GRPC_CALL_ERROR; + } else { + err = call_start_batch(&exec_ctx, call, ops, nops, tag, 0); + } + + grpc_exec_ctx_finish(&exec_ctx); + return err; +} + +grpc_call_error grpc_call_start_batch_and_execute(grpc_exec_ctx *exec_ctx, + grpc_call *call, + const grpc_op *ops, + size_t nops, + grpc_closure *closure) { + return call_start_batch(exec_ctx, call, ops, nops, closure, 1); } void grpc_call_context_set(grpc_call *call, grpc_context_index elem, diff --git a/src/core/surface/call.h b/src/core/surface/call.h index 9b7c6f9bfb..24ab9c33bb 100644 --- a/src/core/surface/call.h +++ b/src/core/surface/call.h @@ -44,51 +44,6 @@ extern "C" { #endif -/* Primitive operation types - grpc_op's get rewritten into these */ -typedef enum { - GRPC_IOREQ_RECV_INITIAL_METADATA, - GRPC_IOREQ_RECV_MESSAGE, - GRPC_IOREQ_RECV_TRAILING_METADATA, - GRPC_IOREQ_RECV_STATUS, - GRPC_IOREQ_RECV_STATUS_DETAILS, - GRPC_IOREQ_RECV_CLOSE, - GRPC_IOREQ_SEND_INITIAL_METADATA, - GRPC_IOREQ_SEND_MESSAGE, - GRPC_IOREQ_SEND_TRAILING_METADATA, - GRPC_IOREQ_SEND_STATUS, - GRPC_IOREQ_SEND_CLOSE, - GRPC_IOREQ_OP_COUNT -} grpc_ioreq_op; - -typedef union { - grpc_metadata_array *recv_metadata; - grpc_byte_buffer **recv_message; - struct { - void (*set_value)(grpc_status_code status, void *user_data); - void *user_data; - } recv_status; - struct { - char **details; - size_t *details_capacity; - } recv_status_details; - struct { - size_t count; - grpc_metadata *metadata; - } send_metadata; - grpc_byte_buffer *send_message; - struct { - grpc_status_code code; - grpc_mdstr *details; - } send_status; -} grpc_ioreq_data; - -typedef struct { - grpc_ioreq_op op; - gpr_uint32 flags; - /**< A copy of the write flags from grpc_op */ - grpc_ioreq_data data; -} grpc_ioreq; - typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx, grpc_call *call, int success, void *user_data); @@ -105,7 +60,7 @@ void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_completion_queue *cq); grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call); -#ifdef GRPC_CALL_REF_COUNT_DEBUG +#ifdef GRPC_STREAM_REFCOUNT_DEBUG void grpc_call_internal_ref(grpc_call *call, const char *reason); void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *call, const char *reason); @@ -121,12 +76,14 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *call); grpc_call_internal_unref(exec_ctx, call) #endif -grpc_call_error grpc_call_start_ioreq_and_call_back( - grpc_exec_ctx *exec_ctx, grpc_call *call, const grpc_ioreq *reqs, - size_t nreqs, grpc_ioreq_completion_func on_complete, void *user_data); - grpc_call_stack *grpc_call_get_call_stack(grpc_call *call); +grpc_call_error grpc_call_start_batch_and_execute(grpc_exec_ctx *exec_ctx, + grpc_call *call, + const grpc_op *ops, + size_t nops, + grpc_closure *closure); + /* Given the top call_element, get the call object. */ grpc_call *grpc_call_from_top_element(grpc_call_element *surface_element); @@ -157,16 +114,6 @@ void *grpc_call_context_get(grpc_call *call, grpc_context_index elem); #define GRPC_CALL_LOG_BATCH(sev, call, ops, nops, tag) \ if (grpc_api_trace) grpc_call_log_batch(sev, call, ops, nops, tag) -#define GRPC_SERVER_LOG_REQUEST_CALL(sev, server, call, details, \ - initial_metadata, cq_bound_to_call, \ - cq_for_notifications, tag) \ - if (grpc_api_trace) \ - grpc_server_log_request_call(sev, server, call, details, initial_metadata, \ - cq_bound_to_call, cq_for_notifications, tag) - -#define GRPC_SERVER_LOG_SHUTDOWN(sev, server, cq, tag) \ - if (grpc_api_trace) grpc_server_log_shutdown(sev, server, cq, tag) - gpr_uint8 grpc_call_is_client(grpc_call *call); #ifdef __cplusplus diff --git a/src/core/surface/call_log_batch.c b/src/core/surface/call_log_batch.c index 2dd9737cf8..16212f6386 100644 --- a/src/core/surface/call_log_batch.c +++ b/src/core/surface/call_log_batch.c @@ -110,9 +110,6 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity, void *tag) { char *tmp; size_t i; - gpr_log(file, line, severity, - "grpc_call_start_batch(call=%p, ops=%p, nops=%d, tag=%p)", call, ops, - nops, tag); for (i = 0; i < nops; i++) { tmp = grpc_op_string(&ops[i]); gpr_log(file, line, severity, "ops[%d]: %s", i, tmp); diff --git a/src/core/surface/call_test_only.h b/src/core/surface/call_test_only.h index df4be3248b..b3a2bbd6f1 100644 --- a/src/core/surface/call_test_only.h +++ b/src/core/surface/call_test_only.h @@ -57,7 +57,6 @@ gpr_uint32 grpc_call_test_only_get_message_flags(grpc_call *call); * To be indexed by grpc_compression_algorithm enum values. */ gpr_uint32 grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call); - #ifdef __cplusplus } #endif diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 51d9130b63..921f8c275e 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -37,6 +37,8 @@ #include <string.h> #include <grpc/support/alloc.h> +#include <grpc/support/slice.h> +#include <grpc/support/slice_buffer.h> #include "src/core/census/grpc_filter.h" #include "src/core/channel/channel_args.h" @@ -56,6 +58,8 @@ typedef struct { grpc_closure *notify; grpc_connect_in_args args; grpc_connect_out_args *result; + grpc_closure initial_string_sent; + gpr_slice_buffer initial_string_buffer; grpc_endpoint *tcp; @@ -73,15 +77,31 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) { connector *c = (connector *)con; if (gpr_unref(&c->refs)) { grpc_mdctx_unref(c->mdctx); + /* c->initial_string_buffer does not need to be destroyed */ gpr_free(c); } } +static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg, + int success) { + connector_unref(exec_ctx, arg); +} + static void connected(grpc_exec_ctx *exec_ctx, void *arg, int success) { connector *c = arg; grpc_closure *notify; grpc_endpoint *tcp = c->tcp; if (tcp != NULL) { + if (!GPR_SLICE_IS_EMPTY(c->args.initial_connect_string)) { + grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent, + c); + gpr_slice_buffer_init(&c->initial_string_buffer); + gpr_slice_buffer_add(&c->initial_string_buffer, + c->args.initial_connect_string); + connector_ref(arg); + grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer, + &c->initial_string_sent); + } c->result->transport = grpc_create_chttp2_transport( exec_ctx, c->args.channel_args, tcp, c->mdctx, 1); grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL, diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c index aa90b3f7f5..d56e5cbe84 100644 --- a/src/core/surface/completion_queue.c +++ b/src/core/surface/completion_queue.c @@ -71,9 +71,29 @@ struct grpc_completion_queue { int is_server_cq; int num_pluckers; plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS]; - grpc_closure pollset_destroy_done; + grpc_closure pollset_shutdown_done; + + grpc_completion_queue *next_free; }; +static gpr_mu g_freelist_mu; +grpc_completion_queue *g_freelist; + +static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, + int success); + +void grpc_cq_global_init(void) { gpr_mu_init(&g_freelist_mu); } + +void grpc_cq_global_shutdown(void) { + gpr_mu_destroy(&g_freelist_mu); + while (g_freelist) { + grpc_completion_queue *next = g_freelist->next_free; + grpc_pollset_destroy(&g_freelist->pollset); + gpr_free(g_freelist); + g_freelist = next; + } +} + struct grpc_cq_alarm { grpc_timer alarm; grpc_cq_completion completion; @@ -83,22 +103,41 @@ struct grpc_cq_alarm { void *tag; }; -static void on_pollset_destroy_done(grpc_exec_ctx *exec_ctx, void *cc, - int success); - grpc_completion_queue *grpc_completion_queue_create(void *reserved) { - grpc_completion_queue *cc = gpr_malloc(sizeof(grpc_completion_queue)); - GRPC_API_TRACE("grpc_completion_queue_create(reserved=%p)", 1, (reserved)); + grpc_completion_queue *cc; GPR_ASSERT(!reserved); - memset(cc, 0, sizeof(*cc)); + + GPR_TIMER_BEGIN("grpc_completion_queue_create", 0); + + GRPC_API_TRACE("grpc_completion_queue_create(reserved=%p)", 1, (reserved)); + + gpr_mu_lock(&g_freelist_mu); + if (g_freelist == NULL) { + gpr_mu_unlock(&g_freelist_mu); + + cc = gpr_malloc(sizeof(grpc_completion_queue)); + grpc_pollset_init(&cc->pollset); + } else { + cc = g_freelist; + g_freelist = g_freelist->next_free; + gpr_mu_unlock(&g_freelist_mu); + /* pollset already initialized */ + } + /* Initial ref is dropped by grpc_completion_queue_shutdown */ gpr_ref_init(&cc->pending_events, 1); /* One for destroy(), one for pollset_shutdown */ gpr_ref_init(&cc->owning_refs, 2); - grpc_pollset_init(&cc->pollset); cc->completed_tail = &cc->completed_head; cc->completed_head.next = (gpr_uintptr)cc->completed_tail; - grpc_closure_init(&cc->pollset_destroy_done, on_pollset_destroy_done, cc); + cc->shutdown = 0; + cc->shutdown_called = 0; + cc->is_server_cq = 0; + cc->num_pluckers = 0; + grpc_closure_init(&cc->pollset_shutdown_done, on_pollset_shutdown_done, cc); + + GPR_TIMER_END("grpc_completion_queue_create", 0); + return cc; } @@ -113,8 +152,8 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) { gpr_ref(&cc->owning_refs); } -static void on_pollset_destroy_done(grpc_exec_ctx *exec_ctx, void *arg, - int success) { +static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg, + int success) { grpc_completion_queue *cc = arg; GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy"); } @@ -129,8 +168,11 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (gpr_uintptr)&cc->completed_head); - grpc_pollset_destroy(&cc->pollset); - gpr_free(cc); + grpc_pollset_reset(&cc->pollset); + gpr_mu_lock(&g_freelist_mu); + cc->next_free = g_freelist; + g_freelist = cc; + gpr_mu_unlock(&g_freelist_mu); } } @@ -185,8 +227,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; + grpc_pollset_shutdown(exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); - grpc_pollset_shutdown(exec_ctx, &cc->pollset, &cc->pollset_destroy_done); } GPR_TIMER_END("grpc_cq_end_op", 0); @@ -365,29 +407,31 @@ done: to zero here, then enter shutdown mode and wake up any waiters */ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0); GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc)); gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); if (cc->shutdown_called) { gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); + GPR_TIMER_END("grpc_completion_queue_shutdown", 0); return; } cc->shutdown_called = 1; - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); - if (gpr_unref(&cc->pending_events)) { - gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset)); GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); - grpc_pollset_shutdown(&exec_ctx, &cc->pollset, &cc->pollset_destroy_done); + grpc_pollset_shutdown(&exec_ctx, &cc->pollset, &cc->pollset_shutdown_done); } + gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset)); grpc_exec_ctx_finish(&exec_ctx); + GPR_TIMER_END("grpc_completion_queue_shutdown", 0); } void grpc_completion_queue_destroy(grpc_completion_queue *cc) { GRPC_API_TRACE("grpc_completion_queue_destroy(cc=%p)", 1, (cc)); + GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0); grpc_completion_queue_shutdown(cc); GRPC_CQ_INTERNAL_UNREF(cc, "destroy"); + GPR_TIMER_END("grpc_completion_queue_destroy", 0); } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { diff --git a/src/core/surface/completion_queue.h b/src/core/surface/completion_queue.h index 5f8282e542..a40bb048ac 100644 --- a/src/core/surface/completion_queue.h +++ b/src/core/surface/completion_queue.h @@ -83,4 +83,7 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc); void grpc_cq_mark_server_cq(grpc_completion_queue *cc); int grpc_cq_is_server_cq(grpc_completion_queue *cc); +void grpc_cq_global_init(void); +void grpc_cq_global_shutdown(void); + #endif /* GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H */ diff --git a/src/core/surface/init.c b/src/core/surface/init.c index b2e66a830e..f8cba01cad 100644 --- a/src/core/surface/init.c +++ b/src/core/surface/init.c @@ -52,6 +52,7 @@ #include "src/core/profiling/timers.h" #include "src/core/surface/api_trace.h" #include "src/core/surface/call.h" +#include "src/core/surface/completion_queue.h" #include "src/core/surface/init.h" #include "src/core/surface/surface_trace.h" #include "src/core/transport/chttp2_transport.h" @@ -118,6 +119,7 @@ void grpc_init(void) { } } gpr_timers_global_init(); + grpc_cq_global_init(); for (i = 0; i < g_number_of_plugins; i++) { if (g_all_of_the_plugins[i].init != NULL) { g_all_of_the_plugins[i].init(); @@ -133,8 +135,9 @@ void grpc_shutdown(void) { GRPC_API_TRACE("grpc_shutdown(void)", 0, ()); gpr_mu_lock(&g_init_mu); if (--g_initializations == 0) { - grpc_iomgr_shutdown(); grpc_executor_shutdown(); + grpc_cq_global_shutdown(); + grpc_iomgr_shutdown(); census_shutdown(); gpr_timers_global_destroy(); grpc_tracer_shutdown(); diff --git a/src/core/surface/lame_client.c b/src/core/surface/lame_client.c index e72264fbcd..fc458a603d 100644 --- a/src/core/surface/lame_client.c +++ b/src/core/surface/lame_client.c @@ -55,38 +55,33 @@ typedef struct { const char *error_message; } channel_data; +static void fill_metadata(grpc_call_element *elem, grpc_metadata_batch *mdb) { + call_data *calld = elem->call_data; + channel_data *chand = elem->channel_data; + char tmp[GPR_LTOA_MIN_BUFSIZE]; + gpr_ltoa(chand->error_code, tmp); + calld->status.md = grpc_mdelem_from_strings(chand->mdctx, "grpc-status", tmp); + calld->details.md = grpc_mdelem_from_strings(chand->mdctx, "grpc-message", + chand->error_message); + calld->status.prev = calld->details.next = NULL; + calld->status.next = &calld->details; + calld->details.prev = &calld->status; + mdb->list.head = &calld->status; + mdb->list.tail = &calld->details; + mdb->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); +} + static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { - call_data *calld = elem->call_data; - channel_data *chand = elem->channel_data; GRPC_CALL_LOG_OP(GPR_INFO, elem, op); - if (op->send_ops != NULL) { - grpc_stream_ops_unref_owned_objects(op->send_ops->ops, op->send_ops->nops); - op->on_done_send->cb(exec_ctx, op->on_done_send->cb_arg, 0); - } - if (op->recv_ops != NULL) { - char tmp[GPR_LTOA_MIN_BUFSIZE]; - grpc_metadata_batch mdb; - gpr_ltoa(chand->error_code, tmp); - calld->status.md = - grpc_mdelem_from_strings(chand->mdctx, "grpc-status", tmp); - calld->details.md = grpc_mdelem_from_strings(chand->mdctx, "grpc-message", - chand->error_message); - calld->status.prev = calld->details.next = NULL; - calld->status.next = &calld->details; - calld->details.prev = &calld->status; - mdb.list.head = &calld->status; - mdb.list.tail = &calld->details; - mdb.garbage.head = mdb.garbage.tail = NULL; - mdb.deadline = gpr_inf_future(GPR_CLOCK_REALTIME); - grpc_sopb_add_metadata(op->recv_ops, mdb); - *op->recv_state = GRPC_STREAM_CLOSED; - op->on_done_recv->cb(exec_ctx, op->on_done_recv->cb_arg, 1); - } - if (op->on_consumed != NULL) { - op->on_consumed->cb(exec_ctx, op->on_consumed->cb_arg, 0); + if (op->recv_initial_metadata != NULL) { + fill_metadata(elem, op->recv_initial_metadata); + } else if (op->recv_trailing_metadata != NULL) { + fill_metadata(elem, op->recv_trailing_metadata); } + grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, 0); + grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, 0); } static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { @@ -109,25 +104,19 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, } static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *transport_server_data, - grpc_transport_stream_op *initial_op) { - if (initial_op) { - grpc_transport_stream_op_finish_with_failure(exec_ctx, initial_op); - } -} + grpc_call_element_args *args) {} static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {} static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, grpc_mdctx *mdctx, - int is_first, int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; - GPR_ASSERT(is_first); - GPR_ASSERT(is_last); - chand->mdctx = mdctx; - chand->master = master; + GPR_ASSERT(args->is_first); + GPR_ASSERT(args->is_last); + chand->mdctx = args->metadata_context; + chand->master = args->master; } static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, @@ -135,8 +124,9 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, static const grpc_channel_filter lame_filter = { lame_start_transport_stream_op, lame_start_transport_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, lame_get_peer, "lame-client", + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + lame_get_peer, "lame-client", }; #define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack *)((c) + 1)) diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c index 1282ee99ed..e8afe1a908 100644 --- a/src/core/surface/secure_channel_create.c +++ b/src/core/surface/secure_channel_create.c @@ -37,6 +37,8 @@ #include <string.h> #include <grpc/support/alloc.h> +#include <grpc/support/slice.h> +#include <grpc/support/slice_buffer.h> #include "src/core/census/grpc_filter.h" #include "src/core/channel/channel_args.h" @@ -61,6 +63,8 @@ typedef struct { grpc_closure *notify; grpc_connect_in_args args; grpc_connect_out_args *result; + grpc_closure initial_string_sent; + gpr_slice_buffer initial_string_buffer; gpr_mu mu; grpc_endpoint *connecting_endpoint; @@ -80,6 +84,7 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) { connector *c = (connector *)con; if (gpr_unref(&c->refs)) { grpc_mdctx_unref(c->mdctx); + /* c->initial_string_buffer does not need to be destroyed */ gpr_free(c); } } @@ -118,6 +123,14 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, notify->cb(exec_ctx, notify->cb_arg, 1); } +static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg, + int success) { + connector *c = arg; + grpc_security_connector_do_handshake(exec_ctx, &c->security_connector->base, + c->connecting_endpoint, + on_secure_handshake_done, c); +} + static void connected(grpc_exec_ctx *exec_ctx, void *arg, int success) { connector *c = arg; grpc_closure *notify; @@ -127,8 +140,19 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, int success) { GPR_ASSERT(c->connecting_endpoint == NULL); c->connecting_endpoint = tcp; gpr_mu_unlock(&c->mu); - grpc_security_connector_do_handshake(exec_ctx, &c->security_connector->base, - tcp, on_secure_handshake_done, c); + if (!GPR_SLICE_IS_EMPTY(c->args.initial_connect_string)) { + grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent, + c); + gpr_slice_buffer_init(&c->initial_string_buffer); + gpr_slice_buffer_add(&c->initial_string_buffer, + c->args.initial_connect_string); + grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer, + &c->initial_string_sent); + } else { + grpc_security_connector_do_handshake(exec_ctx, + &c->security_connector->base, tcp, + on_secure_handshake_done, c); + } } else { memset(c->result, 0, sizeof(*c->result)); notify = c->notify; @@ -230,7 +254,7 @@ static const grpc_subchannel_factory_vtable subchannel_factory_vtable = { Asynchronously: - resolve target - connect to it (trying alternatives as presented) - perform handshakes */ -grpc_channel *grpc_secure_channel_create(grpc_credentials *creds, +grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, void *reserved) { @@ -261,9 +285,9 @@ grpc_channel *grpc_secure_channel_create(grpc_credentials *creds, "Security connector exists in channel args."); } - if (grpc_credentials_create_security_connector( - creds, target, args, NULL, &security_connector, - &new_args_from_connector) != GRPC_SECURITY_OK) { + if (grpc_channel_credentials_create_security_connector( + creds, target, args, &security_connector, &new_args_from_connector) != + GRPC_SECURITY_OK) { grpc_exec_ctx_finish(&exec_ctx); return grpc_lame_client_channel_create( target, GRPC_STATUS_INVALID_ARGUMENT, diff --git a/src/core/surface/server.c b/src/core/surface/server.c index 819226278d..e9f380083f 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -84,18 +84,18 @@ typedef struct requested_call { grpc_completion_queue *cq_for_notification; grpc_call **call; grpc_cq_completion completion; + grpc_metadata_array *initial_metadata; union { struct { grpc_call_details *details; - grpc_metadata_array *initial_metadata; } batch; struct { registered_method *registered_method; gpr_timespec *deadline; - grpc_metadata_array *initial_metadata; grpc_byte_buffer **optional_payload; } registered; } data; + grpc_closure publish; } requested_call; typedef struct channel_registered_method { @@ -150,16 +150,16 @@ struct call_data { grpc_mdstr *path; grpc_mdstr *host; gpr_timespec deadline; - int got_initial_metadata; grpc_completion_queue *cq_new; - grpc_stream_op_buffer *recv_ops; - grpc_stream_state *recv_state; - grpc_closure *on_done_recv; + grpc_metadata_batch *recv_initial_metadata; + grpc_metadata_array initial_metadata; - grpc_closure server_on_recv; + grpc_closure got_initial_metadata; + grpc_closure server_on_recv_initial_metadata; grpc_closure kill_zombie_closure; + grpc_closure *on_done_recv_initial_metadata; call_data *pending_next; }; @@ -396,7 +396,6 @@ static void finish_destroy_channel(grpc_exec_ctx *exec_ctx, void *cd, int success) { channel_data *chand = cd; grpc_server *server = chand->server; - gpr_log(GPR_DEBUG, "finish_destroy_channel: %p", chand->channel); GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, chand->channel, "server"); server_unref(exec_ctx, server); } @@ -571,79 +570,35 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) { return md; } -static void server_on_recv(grpc_exec_ctx *exec_ctx, void *ptr, int success) { +static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr, + int success) { grpc_call_element *elem = ptr; call_data *calld = elem->call_data; gpr_timespec op_deadline; - if (success && !calld->got_initial_metadata) { - size_t i; - size_t nops = calld->recv_ops->nops; - grpc_stream_op *ops = calld->recv_ops->ops; - for (i = 0; i < nops; i++) { - grpc_stream_op *op = &ops[i]; - if (op->type != GRPC_OP_METADATA) continue; - grpc_metadata_batch_filter(&op->data.metadata, server_filter, elem); - op_deadline = op->data.metadata.deadline; - if (0 != - gpr_time_cmp(op_deadline, gpr_inf_future(op_deadline.clock_type))) { - calld->deadline = op->data.metadata.deadline; - } - if (calld->host && calld->path) { - calld->got_initial_metadata = 1; - start_new_rpc(exec_ctx, elem); - } - break; - } + grpc_metadata_batch_filter(calld->recv_initial_metadata, server_filter, elem); + op_deadline = calld->recv_initial_metadata->deadline; + if (0 != gpr_time_cmp(op_deadline, gpr_inf_future(op_deadline.clock_type))) { + calld->deadline = op_deadline; } - - switch (*calld->recv_state) { - case GRPC_STREAM_OPEN: - break; - case GRPC_STREAM_SEND_CLOSED: - break; - case GRPC_STREAM_RECV_CLOSED: - gpr_mu_lock(&calld->mu_state); - if (calld->state == NOT_STARTED) { - calld->state = ZOMBIED; - gpr_mu_unlock(&calld->mu_state); - grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem); - grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, 1); - } else { - gpr_mu_unlock(&calld->mu_state); - } - break; - case GRPC_STREAM_CLOSED: - gpr_mu_lock(&calld->mu_state); - if (calld->state == NOT_STARTED) { - calld->state = ZOMBIED; - gpr_mu_unlock(&calld->mu_state); - grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem); - grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, 1); - } else if (calld->state == PENDING) { - calld->state = ZOMBIED; - gpr_mu_unlock(&calld->mu_state); - /* zombied call will be destroyed when it's removed from the pending - queue... later */ - } else { - gpr_mu_unlock(&calld->mu_state); - } - break; + if (calld->host && calld->path) { + /* do nothing */ + } else { + success = 0; } - calld->on_done_recv->cb(exec_ctx, calld->on_done_recv->cb_arg, success); + calld->on_done_recv_initial_metadata->cb( + exec_ctx, calld->on_done_recv_initial_metadata->cb_arg, success); } static void server_mutate_op(grpc_call_element *elem, grpc_transport_stream_op *op) { call_data *calld = elem->call_data; - if (op->recv_ops) { - /* substitute our callback for the higher callback */ - calld->recv_ops = op->recv_ops; - calld->recv_state = op->recv_state; - calld->on_done_recv = op->on_done_recv; - op->on_done_recv = &calld->server_on_recv; + if (op->recv_initial_metadata != NULL) { + calld->recv_initial_metadata = op->recv_initial_metadata; + calld->on_done_recv_initial_metadata = op->on_complete; + op->on_complete = &calld->server_on_recv_initial_metadata; } } @@ -655,12 +610,48 @@ static void server_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_next_op(exec_ctx, elem, op); } -static void accept_stream(void *cd, grpc_transport *transport, +static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr, + int success) { + grpc_call_element *elem = ptr; + call_data *calld = elem->call_data; + if (success) { + start_new_rpc(exec_ctx, elem); + } else { + gpr_mu_lock(&calld->mu_state); + if (calld->state == NOT_STARTED) { + calld->state = ZOMBIED; + gpr_mu_unlock(&calld->mu_state); + grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem); + grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, 1); + } else if (calld->state == PENDING) { + calld->state = ZOMBIED; + gpr_mu_unlock(&calld->mu_state); + /* zombied call will be destroyed when it's removed from the pending + queue... later */ + } else { + gpr_mu_unlock(&calld->mu_state); + } + } +} + +static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd, + grpc_transport *transport, const void *transport_server_data) { channel_data *chand = cd; /* create a call */ - grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data, NULL, - 0, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_call *call = + grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data, + NULL, 0, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_call_element *elem = + grpc_call_stack_element(grpc_call_get_call_stack(call), 0); + call_data *calld = elem->call_data; + grpc_op op; + memset(&op, 0, sizeof(op)); + op.op = GRPC_OP_RECV_INITIAL_METADATA; + op.data.recv_initial_metadata = &calld->initial_metadata; + grpc_closure_init(&calld->got_initial_metadata, got_initial_metadata, elem); + grpc_call_start_batch_and_execute(exec_ctx, call, &op, 1, + &calld->got_initial_metadata); } static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd, @@ -685,8 +676,7 @@ static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd, } static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - const void *server_transport_data, - grpc_transport_stream_op *initial_op) { + grpc_call_element_args *args) { call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; memset(calld, 0, sizeof(call_data)); @@ -694,11 +684,10 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, calld->call = grpc_call_from_top_element(elem); gpr_mu_init(&calld->mu_state); - grpc_closure_init(&calld->server_on_recv, server_on_recv, elem); + grpc_closure_init(&calld->server_on_recv_initial_metadata, + server_on_recv_initial_metadata, elem); server_ref(chand->server); - - if (initial_op) server_mutate_op(elem, initial_op); } static void destroy_call_elem(grpc_exec_ctx *exec_ctx, @@ -714,6 +703,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, if (calld->path) { GRPC_MDSTR_UNREF(calld->path); } + grpc_metadata_array_destroy(&calld->initial_metadata); gpr_mu_destroy(&calld->mu_state); @@ -721,17 +711,16 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, } static void init_channel_elem(grpc_exec_ctx *exec_ctx, - grpc_channel_element *elem, grpc_channel *master, - const grpc_channel_args *args, - grpc_mdctx *metadata_context, int is_first, - int is_last) { + grpc_channel_element *elem, + grpc_channel_element_args *args) { channel_data *chand = elem->channel_data; - GPR_ASSERT(is_first); - GPR_ASSERT(!is_last); + GPR_ASSERT(args->is_first); + GPR_ASSERT(!args->is_last); chand->server = NULL; chand->channel = NULL; - chand->path_key = grpc_mdstr_from_string(metadata_context, ":path"); - chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority"); + chand->path_key = grpc_mdstr_from_string(args->metadata_context, ":path"); + chand->authority_key = + grpc_mdstr_from_string(args->metadata_context, ":authority"); chand->next = chand->prev = chand; chand->registered_methods = NULL; chand->connectivity_state = GRPC_CHANNEL_IDLE; @@ -769,8 +758,9 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, static const grpc_channel_filter server_surface_filter = { server_start_transport_stream_op, grpc_channel_next_op, sizeof(call_data), - init_call_elem, destroy_call_elem, sizeof(channel_data), init_channel_elem, - destroy_channel_elem, grpc_call_next_get_peer, "server", + init_call_elem, grpc_call_stack_ignore_set_pollset, destroy_call_elem, + sizeof(channel_data), init_channel_elem, destroy_channel_elem, + grpc_call_next_get_peer, "server", }; void grpc_server_register_completion_queue(grpc_server *server, @@ -1022,8 +1012,6 @@ void grpc_server_shutdown_and_notify(grpc_server *server, GRPC_API_TRACE("grpc_server_shutdown_and_notify(server=%p, cq=%p, tag=%p)", 3, (server, cq, tag)); - GRPC_SERVER_LOG_SHUTDOWN(GPR_INFO, server, cq, tag); - /* lock, and gather up some stuff to do */ gpr_mu_lock(&server->mu_global); grpc_cq_begin_op(cq); @@ -1187,12 +1175,9 @@ grpc_call_error grpc_server_request_call( GRPC_API_TRACE( "grpc_server_request_call(" "server=%p, call=%p, details=%p, initial_metadata=%p, " - "cq_bound_to_call=%p, cq_for_notification=%p, tag%p)", + "cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)", 7, (server, call, details, initial_metadata, cq_bound_to_call, cq_for_notification, tag)); - GRPC_SERVER_LOG_REQUEST_CALL(GPR_INFO, server, call, details, - initial_metadata, cq_bound_to_call, - cq_for_notification, tag); if (!grpc_cq_is_server_cq(cq_for_notification)) { gpr_free(rc); error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE; @@ -1207,7 +1192,7 @@ grpc_call_error grpc_server_request_call( rc->cq_for_notification = cq_for_notification; rc->call = call; rc->data.batch.details = details; - rc->data.batch.initial_metadata = initial_metadata; + rc->initial_metadata = initial_metadata; error = queue_call_request(&exec_ctx, server, rc); done: grpc_exec_ctx_finish(&exec_ctx); @@ -1244,7 +1229,7 @@ grpc_call_error grpc_server_request_registered_call( rc->call = call; rc->data.registered.registered_method = rm; rc->data.registered.deadline = deadline; - rc->data.registered.initial_metadata = initial_metadata; + rc->initial_metadata = initial_metadata; rc->data.registered.optional_payload = optional_payload; error = queue_call_request(&exec_ctx, server, rc); done: @@ -1253,12 +1238,7 @@ done: } static void publish_registered_or_batch(grpc_exec_ctx *exec_ctx, - grpc_call *call, int success, - void *tag); -static void publish_was_not_set(grpc_exec_ctx *exec_ctx, grpc_call *call, - int success, void *tag) { - abort(); -} + void *user_data, int success); static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) { gpr_slice slice = value->slice; @@ -1273,9 +1253,10 @@ static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) { static void begin_call(grpc_exec_ctx *exec_ctx, grpc_server *server, call_data *calld, requested_call *rc) { - grpc_ioreq_completion_func publish = publish_was_not_set; - grpc_ioreq req[2]; - grpc_ioreq *r = req; + grpc_op ops[1]; + grpc_op *op = ops; + + memset(ops, 0, sizeof(ops)); /* called once initial metadata has been read by the call, but BEFORE the ioreq to fetch it out of the call has been executed. @@ -1284,8 +1265,10 @@ static void begin_call(grpc_exec_ctx *exec_ctx, grpc_server *server, an ioreq op, that should complete immediately. */ grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call); + grpc_closure_init(&rc->publish, publish_registered_or_batch, rc); *rc->call = calld->call; calld->cq_new = rc->cq_for_notification; + GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata); switch (rc->type) { case BATCH_CALL: GPR_ASSERT(calld->host != NULL); @@ -1295,31 +1278,22 @@ static void begin_call(grpc_exec_ctx *exec_ctx, grpc_server *server, cpstr(&rc->data.batch.details->method, &rc->data.batch.details->method_capacity, calld->path); rc->data.batch.details->deadline = calld->deadline; - r->op = GRPC_IOREQ_RECV_INITIAL_METADATA; - r->data.recv_metadata = rc->data.batch.initial_metadata; - r->flags = 0; - r++; - publish = publish_registered_or_batch; break; case REGISTERED_CALL: *rc->data.registered.deadline = calld->deadline; - r->op = GRPC_IOREQ_RECV_INITIAL_METADATA; - r->data.recv_metadata = rc->data.registered.initial_metadata; - r->flags = 0; - r++; if (rc->data.registered.optional_payload) { - r->op = GRPC_IOREQ_RECV_MESSAGE; - r->data.recv_message = rc->data.registered.optional_payload; - r->flags = 0; - r++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message = rc->data.registered.optional_payload; + op++; } - publish = publish_registered_or_batch; break; + default: + GPR_UNREACHABLE_CODE(return ); } GRPC_CALL_INTERNAL_REF(calld->call, "server"); - grpc_call_start_ioreq_and_call_back(exec_ctx, calld->call, req, - (size_t)(r - req), publish, rc); + grpc_call_start_batch_and_execute(exec_ctx, calld->call, ops, + (size_t)(op - ops), &rc->publish); } static void done_request_event(grpc_exec_ctx *exec_ctx, void *req, @@ -1342,25 +1316,19 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req, static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server, requested_call *rc) { *rc->call = NULL; - switch (rc->type) { - case BATCH_CALL: - rc->data.batch.initial_metadata->count = 0; - break; - case REGISTERED_CALL: - rc->data.registered.initial_metadata->count = 0; - break; - } + rc->initial_metadata->count = 0; + server_ref(server); grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0, done_request_event, rc, &rc->completion); } -static void publish_registered_or_batch(grpc_exec_ctx *exec_ctx, - grpc_call *call, int success, - void *prc) { +static void publish_registered_or_batch(grpc_exec_ctx *exec_ctx, void *prc, + int success) { + requested_call *rc = prc; + grpc_call *call = *rc->call; grpc_call_element *elem = grpc_call_stack_element(grpc_call_get_call_stack(call), 0); - requested_call *rc = prc; call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; server_ref(chand->server); diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c index 580b91573c..1408f9c159 100644 --- a/src/core/surface/server_chttp2.c +++ b/src/core/surface/server_chttp2.c @@ -107,9 +107,11 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { } for (i = 0; i < resolved->naddrs; i++) { - port_temp = grpc_tcp_server_add_port( + grpc_tcp_listener *listener; + listener = grpc_tcp_server_add_port( tcp, (struct sockaddr *)&resolved->addrs[i].addr, resolved->addrs[i].len); + port_temp = grpc_tcp_listener_get_port(listener); if (port_temp >= 0) { if (port_num == -1) { port_num = port_temp; diff --git a/src/core/surface/version.c b/src/core/surface/version.c index e559d51448..962a72112a 100644 --- a/src/core/surface/version.c +++ b/src/core/surface/version.c @@ -36,4 +36,4 @@ #include <grpc/grpc.h> -const char *grpc_version_string(void) { return "0.11.0.0"; } +const char *grpc_version_string(void) { return "0.12.0.0"; } diff --git a/src/core/transport/byte_stream.c b/src/core/transport/byte_stream.c new file mode 100644 index 0000000000..81e8e77ccb --- /dev/null +++ b/src/core/transport/byte_stream.c @@ -0,0 +1,76 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/transport/byte_stream.h" + +#include <stdlib.h> + +#include <grpc/support/log.h> + +int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, gpr_slice *slice, + size_t max_size_hint, grpc_closure *on_complete) { + return byte_stream->next(exec_ctx, byte_stream, slice, max_size_hint, + on_complete); +} + +void grpc_byte_stream_destroy(grpc_byte_stream *byte_stream) { + byte_stream->destroy(byte_stream); +} + +/* slice_buffer_stream */ + +static int slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + gpr_slice *slice, size_t max_size_hint, + grpc_closure *on_complete) { + grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; + GPR_ASSERT(stream->cursor < stream->backing_buffer->count); + *slice = gpr_slice_ref(stream->backing_buffer->slices[stream->cursor]); + stream->cursor++; + return 1; +} + +static void slice_buffer_stream_destroy(grpc_byte_stream *byte_stream) {} + +void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, + gpr_slice_buffer *slice_buffer, + gpr_uint32 flags) { + GPR_ASSERT(slice_buffer->length <= GPR_UINT32_MAX); + stream->base.length = (gpr_uint32)slice_buffer->length; + stream->base.flags = flags; + stream->base.next = slice_buffer_stream_next; + stream->base.destroy = slice_buffer_stream_destroy; + stream->backing_buffer = slice_buffer; + stream->cursor = 0; +} diff --git a/src/core/transport/byte_stream.h b/src/core/transport/byte_stream.h new file mode 100644 index 0000000000..c94d8ff275 --- /dev/null +++ b/src/core/transport/byte_stream.h @@ -0,0 +1,88 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H +#define GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H + +#include "src/core/iomgr/exec_ctx.h" +#include <grpc/support/slice_buffer.h> + +/** Internal bit flag for grpc_begin_message's \a flags signaling the use of + * compression for the message */ +#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) +/** Mask of all valid internal flags. */ +#define GRPC_WRITE_INTERNAL_USED_MASK (GRPC_WRITE_INTERNAL_COMPRESS) + +struct grpc_byte_stream; +typedef struct grpc_byte_stream grpc_byte_stream; + +struct grpc_byte_stream { + gpr_uint32 length; + gpr_uint32 flags; + int (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, + gpr_slice *slice, size_t max_size_hint, + grpc_closure *on_complete); + void (*destroy)(grpc_byte_stream *byte_stream); +}; + +/* returns 1 if the bytes are available immediately (in which case + * on_complete will not be called), 0 if the bytes will be available + * asynchronously. + * + * on entry, *remaining can be set as a hint as to the maximum number + * of bytes that would be acceptable to read. + * + * fills *buffer, *length, *remaining with the bytes, length of bytes + * and length of data remaining to be read before either returning 1 + * or calling on_complete. + * + * once a slice is returned into *slice, it is owned by the caller. + */ +int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, gpr_slice *slice, + size_t max_size_hint, grpc_closure *on_complete); + +void grpc_byte_stream_destroy(grpc_byte_stream *byte_stream); + +/* grpc_byte_stream that wraps a slice buffer */ +typedef struct grpc_slice_buffer_stream { + grpc_byte_stream base; + gpr_slice_buffer *backing_buffer; + size_t cursor; +} grpc_slice_buffer_stream; + +void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, + gpr_slice_buffer *slice_buffer, + gpr_uint32 flags); + +#endif /* GRPC_INTERNAL_CORE_TRANSPORT_BYTE_STREAM_H */ diff --git a/src/core/transport/chttp2/frame_data.c b/src/core/transport/chttp2/frame_data.c index 07179a4571..e07fbb2cc7 100644 --- a/src/core/transport/chttp2/frame_data.c +++ b/src/core/transport/chttp2/frame_data.c @@ -45,12 +45,20 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_init( grpc_chttp2_data_parser *parser) { parser->state = GRPC_CHTTP2_DATA_FH_0; - grpc_sopb_init(&parser->incoming_sopb); + parser->parsing_frame = NULL; return GRPC_CHTTP2_PARSE_OK; } -void grpc_chttp2_data_parser_destroy(grpc_chttp2_data_parser *parser) { - grpc_sopb_destroy(&parser->incoming_sopb); +void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx, + grpc_chttp2_data_parser *parser) { + grpc_byte_stream *bs; + if (parser->parsing_frame) { + grpc_chttp2_incoming_byte_stream_finished(exec_ctx, parser->parsing_frame); + } + while ( + (bs = grpc_chttp2_incoming_frame_queue_pop(&parser->incoming_frames))) { + grpc_byte_stream_destroy(bs); + } } grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame( @@ -69,6 +77,62 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame( return GRPC_CHTTP2_PARSE_OK; } +void grpc_chttp2_incoming_frame_queue_merge( + grpc_chttp2_incoming_frame_queue *head_dst, + grpc_chttp2_incoming_frame_queue *tail_src) { + if (tail_src->head == NULL) { + return; + } + + if (head_dst->head == NULL) { + *head_dst = *tail_src; + memset(tail_src, 0, sizeof(*tail_src)); + return; + } + + head_dst->tail->next_message = tail_src->head; + head_dst->tail = tail_src->tail; + memset(tail_src, 0, sizeof(*tail_src)); +} + +grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop( + grpc_chttp2_incoming_frame_queue *q) { + grpc_byte_stream *out; + if (q->head == NULL) { + return NULL; + } + out = &q->head->base; + if (q->head == q->tail) { + memset(q, 0, sizeof(*q)); + } else { + q->head = q->head->next_message; + } + return out; +} + +void grpc_chttp2_encode_data(gpr_uint32 id, gpr_slice_buffer *inbuf, + gpr_uint32 write_bytes, int is_eof, + gpr_slice_buffer *outbuf) { + gpr_slice hdr; + gpr_uint8 *p; + + hdr = gpr_slice_malloc(9); + p = GPR_SLICE_START_PTR(hdr); + GPR_ASSERT(write_bytes < 16777316); + *p++ = (gpr_uint8)(write_bytes >> 16); + *p++ = (gpr_uint8)(write_bytes >> 8); + *p++ = (gpr_uint8)(write_bytes); + *p++ = GRPC_CHTTP2_FRAME_DATA; + *p++ = is_eof ? GRPC_CHTTP2_DATA_FLAG_END_STREAM : 0; + *p++ = (gpr_uint8)(id >> 24); + *p++ = (gpr_uint8)(id >> 16); + *p++ = (gpr_uint8)(id >> 8); + *p++ = (gpr_uint8)(id); + gpr_slice_buffer_add(outbuf, hdr); + + gpr_slice_buffer_move_first(inbuf, write_bytes, outbuf); +} + grpc_chttp2_parse_error grpc_chttp2_data_parser_parse( grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_transport_parsing *transport_parsing, @@ -77,7 +141,8 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse( gpr_uint8 *const end = GPR_SLICE_END_PTR(slice); gpr_uint8 *cur = beg; grpc_chttp2_data_parser *p = parser; - gpr_uint32 message_flags = 0; + gpr_uint32 message_flags; + grpc_chttp2_incoming_byte_stream *incoming_byte_stream; if (is_last && p->is_last_frame) { stream_parsing->received_close = 1; @@ -132,11 +197,14 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse( p->frame_size |= ((gpr_uint32)*cur); p->state = GRPC_CHTTP2_DATA_FRAME; ++cur; + message_flags = 0; if (p->is_frame_compressed) { message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } - grpc_sopb_add_begin_message(&p->incoming_sopb, p->frame_size, - message_flags); + p->parsing_frame = incoming_byte_stream = + grpc_chttp2_incoming_byte_stream_create( + exec_ctx, transport_parsing, stream_parsing, p->frame_size, + message_flags, &p->incoming_frames); /* fallthrough */ case GRPC_CHTTP2_DATA_FRAME: if (cur == end) { @@ -147,20 +215,25 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse( grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); if ((gpr_uint32)(end - cur) == p->frame_size) { - grpc_sopb_add_slice( - &p->incoming_sopb, + grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); + grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame); + p->parsing_frame = NULL; p->state = GRPC_CHTTP2_DATA_FH_0; return GRPC_CHTTP2_PARSE_OK; } else if ((gpr_uint32)(end - cur) > p->frame_size) { - grpc_sopb_add_slice(&p->incoming_sopb, - gpr_slice_sub(slice, (size_t)(cur - beg), - (size_t)(cur + p->frame_size - beg))); + grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, + gpr_slice_sub(slice, (size_t)(cur - beg), + (size_t)(cur + p->frame_size - beg))); + grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame); + p->parsing_frame = NULL; cur += p->frame_size; goto fh_0; /* loop */ } else { - grpc_sopb_add_slice( - &p->incoming_sopb, + grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); GPR_ASSERT((size_t)(end - cur) <= p->frame_size); p->frame_size -= (gpr_uint32)(end - cur); diff --git a/src/core/transport/chttp2/frame_data.h b/src/core/transport/chttp2/frame_data.h index 6762484e5b..472f9cebdb 100644 --- a/src/core/transport/chttp2/frame_data.h +++ b/src/core/transport/chttp2/frame_data.h @@ -39,7 +39,7 @@ #include "src/core/iomgr/exec_ctx.h" #include <grpc/support/slice.h> #include <grpc/support/slice_buffer.h> -#include "src/core/transport/stream_op.h" +#include "src/core/transport/byte_stream.h" #include "src/core/transport/chttp2/frame.h" typedef enum { @@ -51,6 +51,14 @@ typedef enum { GRPC_CHTTP2_DATA_FRAME } grpc_chttp2_stream_state; +typedef struct grpc_chttp2_incoming_byte_stream + grpc_chttp2_incoming_byte_stream; + +typedef struct grpc_chttp2_incoming_frame_queue { + grpc_chttp2_incoming_byte_stream *head; + grpc_chttp2_incoming_byte_stream *tail; +} grpc_chttp2_incoming_frame_queue; + typedef struct { grpc_chttp2_stream_state state; gpr_uint8 is_last_frame; @@ -58,14 +66,22 @@ typedef struct { gpr_uint32 frame_size; int is_frame_compressed; - grpc_stream_op_buffer incoming_sopb; + grpc_chttp2_incoming_frame_queue incoming_frames; + grpc_chttp2_incoming_byte_stream *parsing_frame; } grpc_chttp2_data_parser; +void grpc_chttp2_incoming_frame_queue_merge( + grpc_chttp2_incoming_frame_queue *head_dst, + grpc_chttp2_incoming_frame_queue *tail_src); +grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop( + grpc_chttp2_incoming_frame_queue *q); + /* initialize per-stream state for data frame parsing */ grpc_chttp2_parse_error grpc_chttp2_data_parser_init( grpc_chttp2_data_parser *parser); -void grpc_chttp2_data_parser_destroy(grpc_chttp2_data_parser *parser); +void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx, + grpc_chttp2_data_parser *parser); /* start processing a new data frame */ grpc_chttp2_parse_error grpc_chttp2_data_parser_begin_frame( @@ -81,4 +97,8 @@ grpc_chttp2_parse_error grpc_chttp2_data_parser_parse( /* create a slice with an empty data frame and is_last set */ gpr_slice grpc_chttp2_data_frame_create_empty_close(gpr_uint32 id); +void grpc_chttp2_encode_data(gpr_uint32 id, gpr_slice_buffer *inbuf, + gpr_uint32 write_bytes, int is_eof, + gpr_slice_buffer *outbuf); + #endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_FRAME_DATA_H */ diff --git a/src/core/transport/chttp2/frame_window_update.c b/src/core/transport/chttp2/frame_window_update.c index 91bbcfe2c1..74ca29baf9 100644 --- a/src/core/transport/chttp2/frame_window_update.c +++ b/src/core/transport/chttp2/frame_window_update.c @@ -89,7 +89,8 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse( } if (p->byte == 4) { - if (p->amount == 0 || (p->amount & 0x80000000u)) { + gpr_uint32 received_update = p->amount; + if (received_update == 0 || (received_update & 0x80000000u)) { gpr_log(GPR_ERROR, "invalid window update bytes: %d", p->amount); return GRPC_CHTTP2_CONNECTION_ERROR; } @@ -97,17 +98,15 @@ grpc_chttp2_parse_error grpc_chttp2_window_update_parser_parse( if (transport_parsing->incoming_stream_id != 0) { if (stream_parsing != NULL) { - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("update", transport_parsing, - stream_parsing, outgoing_window_update, - p->amount); - stream_parsing->outgoing_window_update += p->amount; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", transport_parsing, + stream_parsing, outgoing_window, + received_update); grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); } } else { - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT("update", transport_parsing, - outgoing_window_update, p->amount); - transport_parsing->outgoing_window_update += p->amount; + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parse", transport_parsing, + outgoing_window, received_update); } } diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/hpack_encoder.c index 24a5d958b8..62b60abaf5 100644 --- a/src/core/transport/chttp2/stream_encoder.c +++ b/src/core/transport/chttp2/hpack_encoder.c @@ -31,13 +31,15 @@ * */ -#include "src/core/transport/chttp2/stream_encoder.h" +#include "src/core/transport/chttp2/hpack_encoder.h" #include <assert.h> #include <string.h> +#include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/useful.h> + #include "src/core/transport/chttp2/bin_encoder.h" #include "src/core/transport/chttp2/hpack_table.h" #include "src/core/transport/chttp2/timeout_encoding.h" @@ -54,18 +56,13 @@ /* don't consider adding anything bigger than this to the hpack table */ #define MAX_DECODER_SPACE_USAGE 512 -/* what kind of frame our we encoding? */ -typedef enum { HEADER, DATA, NONE } frame_type; - typedef struct { - frame_type cur_frame_type; + int is_first_frame; /* number of bytes in 'output' when we started the frame - used to calculate frame length */ size_t output_length_at_start_of_frame; /* index (in output) of the header for the current frame */ size_t header_idx; - /* was the last frame emitted a header? (if yes, we'll need a CONTINUATION */ - gpr_uint8 last_was_header; /* have we seen a regular (non-colon-prefixed) header yet? */ gpr_uint8 seen_regular_header; /* output stream id */ @@ -92,58 +89,35 @@ static void fill_header(gpr_uint8 *p, gpr_uint8 type, gpr_uint32 id, size_t len, static void finish_frame(framer_state *st, int is_header_boundary, int is_last_in_stream) { gpr_uint8 type = 0xff; - switch (st->cur_frame_type) { - case HEADER: - type = st->last_was_header ? GRPC_CHTTP2_FRAME_CONTINUATION - : GRPC_CHTTP2_FRAME_HEADER; - st->last_was_header = 1; - break; - case DATA: - type = GRPC_CHTTP2_FRAME_DATA; - st->last_was_header = 0; - is_header_boundary = 0; - break; - case NONE: - return; - } + type = st->is_first_frame ? GRPC_CHTTP2_FRAME_HEADER + : GRPC_CHTTP2_FRAME_CONTINUATION; fill_header( GPR_SLICE_START_PTR(st->output->slices[st->header_idx]), type, st->stream_id, st->output->length - st->output_length_at_start_of_frame, (gpr_uint8)( (is_last_in_stream ? GRPC_CHTTP2_DATA_FLAG_END_STREAM : 0) | (is_header_boundary ? GRPC_CHTTP2_DATA_FLAG_END_HEADERS : 0))); - st->cur_frame_type = NONE; + st->is_first_frame = 0; } /* begin a new frame: reserve off header space, remember how many bytes we'd output before beginning */ -static void begin_frame(framer_state *st, frame_type type) { - GPR_ASSERT(type != NONE); - GPR_ASSERT(st->cur_frame_type == NONE); - st->cur_frame_type = type; +static void begin_frame(framer_state *st) { st->header_idx = gpr_slice_buffer_add_indexed(st->output, gpr_slice_malloc(9)); st->output_length_at_start_of_frame = st->output->length; } -static void begin_new_frame(framer_state *st, frame_type type) { - finish_frame(st, 1, 0); - st->last_was_header = 0; - begin_frame(st, type); -} - /* make sure that the current frame is of the type desired, and has sufficient space to add at least about_to_add bytes -- finishes the current frame if needed */ -static void ensure_frame_type(framer_state *st, frame_type type, - size_t need_bytes) { - if (st->cur_frame_type == type && - st->output->length - st->output_length_at_start_of_frame + need_bytes <= - GRPC_CHTTP2_MAX_PAYLOAD_LENGTH) { +static void ensure_space(framer_state *st, size_t need_bytes) { + if (st->output->length - st->output_length_at_start_of_frame + need_bytes <= + GRPC_CHTTP2_MAX_PAYLOAD_LENGTH) { return; } - finish_frame(st, type != HEADER, 0); - begin_frame(st, type); + finish_frame(st, 0, 0); + begin_frame(st); } /* increment a filter count, halve all counts if one element reaches max */ @@ -165,54 +139,60 @@ static void add_header_data(framer_state *st, gpr_slice slice) { size_t len = GPR_SLICE_LENGTH(slice); size_t remaining; if (len == 0) return; - ensure_frame_type(st, HEADER, 1); remaining = GRPC_CHTTP2_MAX_PAYLOAD_LENGTH + st->output_length_at_start_of_frame - st->output->length; if (len <= remaining) { gpr_slice_buffer_add(st->output, slice); } else { gpr_slice_buffer_add(st->output, gpr_slice_split_head(&slice, remaining)); + finish_frame(st, 0, 0); + begin_frame(st); add_header_data(st, slice); } } static gpr_uint8 *add_tiny_header_data(framer_state *st, size_t len) { - ensure_frame_type(st, HEADER, len); + ensure_space(st, len); return gpr_slice_buffer_tiny_add(st->output, len); } -/* add an element to the decoder table: returns metadata element to unref */ -static grpc_mdelem *add_elem(grpc_chttp2_hpack_compressor *c, - grpc_mdelem *elem) { +static void evict_entry(grpc_chttp2_hpack_compressor *c) { + c->tail_remote_index++; + GPR_ASSERT(c->tail_remote_index > 0); + GPR_ASSERT(c->table_size >= + c->table_elem_size[c->tail_remote_index % c->cap_table_elems]); + GPR_ASSERT(c->table_elems > 0); + c->table_size = (gpr_uint16)( + c->table_size - + c->table_elem_size[c->tail_remote_index % c->cap_table_elems]); + c->table_elems--; +} + +/* add an element to the decoder table */ +static void add_elem(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem) { gpr_uint32 key_hash = elem->key->hash; gpr_uint32 elem_hash = GRPC_MDSTR_KV_HASH(key_hash, elem->value->hash); gpr_uint32 new_index = c->tail_remote_index + c->table_elems + 1; size_t elem_size = 32 + GPR_SLICE_LENGTH(elem->key->slice) + GPR_SLICE_LENGTH(elem->value->slice); - grpc_mdelem *elem_to_unref; GPR_ASSERT(elem_size < 65536); + if (elem_size > c->max_table_size) { + while (c->table_size > 0) { + evict_entry(c); + } + return; + } + /* Reserve space for this element in the remote table: if this overflows the current table, drop elements until it fits, matching the decompressor algorithm */ - /* TODO(ctiller): constant */ - while (c->table_size + elem_size > 4096) { - c->tail_remote_index++; - GPR_ASSERT(c->tail_remote_index > 0); - GPR_ASSERT(c->table_size >= - c->table_elem_size[c->tail_remote_index % - GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS]); - GPR_ASSERT(c->table_elems > 0); - c->table_size = - (gpr_uint16)(c->table_size - - c->table_elem_size[c->tail_remote_index % - GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS]); - c->table_elems--; + while (c->table_size + elem_size > c->max_table_size) { + evict_entry(c); } - GPR_ASSERT(c->table_elems < GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS); - c->table_elem_size[new_index % GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS] = - (gpr_uint16)elem_size; + GPR_ASSERT(c->table_elems < c->max_table_size); + c->table_elem_size[new_index % c->cap_table_elems] = (gpr_uint16)elem_size; c->table_size = (gpr_uint16)(c->table_size + elem_size); c->table_elems++; @@ -220,31 +200,27 @@ static grpc_mdelem *add_elem(grpc_chttp2_hpack_compressor *c, if (c->entries_elems[HASH_FRAGMENT_2(elem_hash)] == elem) { /* already there: update with new index */ c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index; - elem_to_unref = elem; } else if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == elem) { /* already there (cuckoo): update with new index */ c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index; - elem_to_unref = elem; } else if (c->entries_elems[HASH_FRAGMENT_2(elem_hash)] == NULL) { /* not there, but a free element: add */ - c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = elem; + c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem); c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index; - elem_to_unref = NULL; } else if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == NULL) { /* not there (cuckoo), but a free element: add */ - c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = elem; + c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem); c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index; - elem_to_unref = NULL; } else if (c->indices_elems[HASH_FRAGMENT_2(elem_hash)] < c->indices_elems[HASH_FRAGMENT_3(elem_hash)]) { /* not there: replace oldest */ - elem_to_unref = c->entries_elems[HASH_FRAGMENT_2(elem_hash)]; - c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = elem; + GRPC_MDELEM_UNREF(c->entries_elems[HASH_FRAGMENT_2(elem_hash)]); + c->entries_elems[HASH_FRAGMENT_2(elem_hash)] = GRPC_MDELEM_REF(elem); c->indices_elems[HASH_FRAGMENT_2(elem_hash)] = new_index; } else { /* not there: replace oldest */ - elem_to_unref = c->entries_elems[HASH_FRAGMENT_3(elem_hash)]; - c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = elem; + GRPC_MDELEM_UNREF(c->entries_elems[HASH_FRAGMENT_3(elem_hash)]); + c->entries_elems[HASH_FRAGMENT_3(elem_hash)] = GRPC_MDELEM_REF(elem); c->indices_elems[HASH_FRAGMENT_3(elem_hash)] = new_index; } @@ -270,8 +246,6 @@ static grpc_mdelem *add_elem(grpc_chttp2_hpack_compressor *c, c->entries_keys[HASH_FRAGMENT_3(key_hash)] = GRPC_MDSTR_REF(elem->key); c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index; } - - return elem_to_unref; } static void emit_indexed(grpc_chttp2_hpack_compressor *c, gpr_uint32 elem_index, @@ -364,15 +338,23 @@ static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c, add_header_data(st, gpr_slice_ref(value_slice)); } +static void emit_advertise_table_size_change(grpc_chttp2_hpack_compressor *c, + framer_state *st) { + gpr_uint32 len = GRPC_CHTTP2_VARINT_LENGTH(c->max_table_size, 3); + GRPC_CHTTP2_WRITE_VARINT(c->max_table_size, 3, 0x20, + add_tiny_header_data(st, len), len); + c->advertise_table_size_change = 0; +} + static gpr_uint32 dynidx(grpc_chttp2_hpack_compressor *c, gpr_uint32 elem_index) { return 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY + c->tail_remote_index + c->table_elems - elem_index; } -/* encode an mdelem; returns metadata element to unref */ -static grpc_mdelem *hpack_enc(grpc_chttp2_hpack_compressor *c, - grpc_mdelem *elem, framer_state *st) { +/* encode an mdelem */ +static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem, + framer_state *st) { gpr_uint32 key_hash = elem->key->hash; gpr_uint32 elem_hash = GRPC_MDSTR_KV_HASH(key_hash, elem->value->hash); size_t decoder_space_usage; @@ -397,7 +379,7 @@ static grpc_mdelem *hpack_enc(grpc_chttp2_hpack_compressor *c, /* HIT: complete element (first cuckoo hash) */ emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]), st); - return elem; + return; } if (c->entries_elems[HASH_FRAGMENT_3(elem_hash)] == elem && @@ -405,7 +387,7 @@ static grpc_mdelem *hpack_enc(grpc_chttp2_hpack_compressor *c, /* HIT: complete element (second cuckoo hash) */ emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]), st); - return elem; + return; } /* should this elem be in the table? */ @@ -423,12 +405,13 @@ static grpc_mdelem *hpack_enc(grpc_chttp2_hpack_compressor *c, /* HIT: key (first cuckoo hash) */ if (should_add_elem) { emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); - return add_elem(c, elem); + add_elem(c, elem); + return; } else { emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); - return elem; + return; } - GPR_UNREACHABLE_CODE(return NULL); + GPR_UNREACHABLE_CODE(return ); } indices_key = c->indices_keys[HASH_FRAGMENT_3(key_hash)]; @@ -437,24 +420,26 @@ static grpc_mdelem *hpack_enc(grpc_chttp2_hpack_compressor *c, /* HIT: key (first cuckoo hash) */ if (should_add_elem) { emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); - return add_elem(c, elem); + add_elem(c, elem); + return; } else { emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); - return elem; + return; } - GPR_UNREACHABLE_CODE(return NULL); + GPR_UNREACHABLE_CODE(return ); } /* no elem, key in the table... fall back to literal emission */ if (should_add_elem) { emit_lithdr_incidx_v(c, elem, st); - return add_elem(c, elem); + add_elem(c, elem); + return; } else { emit_lithdr_noidx_v(c, elem, st); - return elem; + return; } - GPR_UNREACHABLE_CODE(return NULL); + GPR_UNREACHABLE_CODE(return ); } #define STRLEN_LIT(x) (sizeof(x) - 1) @@ -469,8 +454,8 @@ static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline, mdelem = grpc_mdelem_from_metadata_strings( c->mdctx, GRPC_MDSTR_REF(c->timeout_key_str), grpc_mdstr_from_string(c->mdctx, timeout_str)); - mdelem = hpack_enc(c, mdelem, st); - if (mdelem) GRPC_MDELEM_UNREF(mdelem); + hpack_enc(c, mdelem, st); + GRPC_MDELEM_UNREF(mdelem); } gpr_slice grpc_chttp2_data_frame_create_empty_close(gpr_uint32 id) { @@ -479,11 +464,23 @@ gpr_slice grpc_chttp2_data_frame_create_empty_close(gpr_uint32 id) { return slice; } +static gpr_uint32 elems_for_bytes(gpr_uint32 bytes) { + return (bytes + 31) / 32; +} + void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c, grpc_mdctx *ctx) { memset(c, 0, sizeof(*c)); c->mdctx = ctx; c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout"); + c->max_table_size = GRPC_CHTTP2_HPACKC_INITIAL_TABLE_SIZE; + c->cap_table_elems = elems_for_bytes(c->max_table_size); + c->max_table_elems = c->cap_table_elems; + c->max_usable_size = GRPC_CHTTP2_HPACKC_INITIAL_TABLE_SIZE; + c->table_elem_size = + gpr_malloc(sizeof(*c->table_elem_size) * c->cap_table_elems); + memset(c->table_elem_size, 0, + sizeof(*c->table_elem_size) * c->cap_table_elems); } void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) { @@ -493,171 +490,88 @@ void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) { if (c->entries_elems[i]) GRPC_MDELEM_UNREF(c->entries_elems[i]); } GRPC_MDSTR_UNREF(c->timeout_key_str); + gpr_free(c->table_elem_size); } -gpr_uint32 grpc_chttp2_preencode(grpc_stream_op *inops, size_t *inops_count, - gpr_uint32 max_flow_controlled_bytes, - grpc_stream_op_buffer *outops) { - gpr_slice slice; - grpc_stream_op *op; - gpr_uint32 max_take_size; - gpr_uint32 flow_controlled_bytes_taken = 0; - gpr_uint32 curop = 0; - gpr_uint8 *p; - gpr_uint8 compressed_flag_set = 0; - - while (curop < *inops_count) { - GPR_ASSERT(flow_controlled_bytes_taken <= max_flow_controlled_bytes); - op = &inops[curop]; - switch (op->type) { - case GRPC_NO_OP: - /* skip */ - curop++; - break; - case GRPC_OP_METADATA: - grpc_metadata_batch_assert_ok(&op->data.metadata); - /* these just get copied as they don't impact the number of flow - controlled bytes */ - grpc_sopb_append(outops, op, 1); - curop++; - break; - case GRPC_OP_BEGIN_MESSAGE: - /* begin op: for now we just convert the op to a slice and fall - through - this lets us reuse the slice framing code below */ - compressed_flag_set = - (op->data.begin_message.flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0; - slice = gpr_slice_malloc(5); - - p = GPR_SLICE_START_PTR(slice); - p[0] = compressed_flag_set; - p[1] = (gpr_uint8)(op->data.begin_message.length >> 24); - p[2] = (gpr_uint8)(op->data.begin_message.length >> 16); - p[3] = (gpr_uint8)(op->data.begin_message.length >> 8); - p[4] = (gpr_uint8)(op->data.begin_message.length); - op->type = GRPC_OP_SLICE; - op->data.slice = slice; - /* fallthrough */ - case GRPC_OP_SLICE: - slice = op->data.slice; - if (!GPR_SLICE_LENGTH(slice)) { - /* skip zero length slices */ - gpr_slice_unref(slice); - curop++; - break; - } - max_take_size = max_flow_controlled_bytes - flow_controlled_bytes_taken; - if (max_take_size == 0) { - goto exit_loop; - } - if (GPR_SLICE_LENGTH(slice) > max_take_size) { - slice = gpr_slice_split_head(&op->data.slice, max_take_size); - grpc_sopb_add_slice(outops, slice); - } else { - /* consume this op immediately */ - grpc_sopb_append(outops, op, 1); - curop++; - } - flow_controlled_bytes_taken += (gpr_uint32)GPR_SLICE_LENGTH(slice); - break; - } +void grpc_chttp2_hpack_compressor_set_max_usable_size( + grpc_chttp2_hpack_compressor *c, gpr_uint32 max_table_size) { + c->max_usable_size = max_table_size; + grpc_chttp2_hpack_compressor_set_max_table_size( + c, GPR_MIN(c->max_table_size, max_table_size)); +} + +static void rebuild_elems(grpc_chttp2_hpack_compressor *c, gpr_uint32 new_cap) { + gpr_uint16 *table_elem_size = gpr_malloc(sizeof(*table_elem_size) * new_cap); + gpr_uint32 i; + + memset(table_elem_size, 0, sizeof(*table_elem_size) * new_cap); + GPR_ASSERT(c->table_elems <= new_cap); + + for (i = 0; i < c->table_elems; i++) { + gpr_uint32 ofs = c->tail_remote_index + i + 1; + table_elem_size[ofs % new_cap] = + c->table_elem_size[ofs % c->cap_table_elems]; } -exit_loop: - *inops_count -= curop; - memmove(inops, inops + curop, *inops_count * sizeof(grpc_stream_op)); - for (curop = 0; curop < *inops_count; curop++) { - if (inops[curop].type == GRPC_OP_METADATA) { - grpc_metadata_batch_assert_ok(&inops[curop].data.metadata); + c->cap_table_elems = new_cap; + gpr_free(c->table_elem_size); + c->table_elem_size = table_elem_size; +} + +void grpc_chttp2_hpack_compressor_set_max_table_size( + grpc_chttp2_hpack_compressor *c, gpr_uint32 max_table_size) { + max_table_size = GPR_MIN(max_table_size, c->max_usable_size); + if (max_table_size == c->max_table_size) { + return; + } + while (c->table_size > 0 && c->table_size > max_table_size) { + evict_entry(c); + } + c->max_table_size = max_table_size; + c->max_table_elems = elems_for_bytes(max_table_size); + if (c->max_table_elems > c->cap_table_elems) { + rebuild_elems(c, GPR_MAX(c->max_table_elems, 2 * c->cap_table_elems)); + } else if (c->max_table_elems < c->cap_table_elems / 3) { + gpr_uint32 new_cap = GPR_MAX(c->max_table_elems, 16); + if (new_cap != c->cap_table_elems) { + rebuild_elems(c, new_cap); } } - - return flow_controlled_bytes_taken; + c->advertise_table_size_change = 1; + gpr_log(GPR_DEBUG, "set max table size from encoder to %d", max_table_size); } -void grpc_chttp2_encode(grpc_stream_op *ops, size_t ops_count, int eof, - gpr_uint32 stream_id, - grpc_chttp2_hpack_compressor *compressor, - gpr_slice_buffer *output) { +void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, + gpr_uint32 stream_id, + grpc_metadata_batch *metadata, int is_eof, + gpr_slice_buffer *outbuf) { framer_state st; - gpr_slice slice; - grpc_stream_op *op; - size_t max_take_size; - gpr_uint32 curop = 0; - gpr_uint32 unref_op; grpc_linked_mdelem *l; - int need_unref = 0; gpr_timespec deadline; GPR_ASSERT(stream_id != 0); - st.cur_frame_type = NONE; - st.last_was_header = 0; st.seen_regular_header = 0; st.stream_id = stream_id; - st.output = output; - - while (curop < ops_count) { - op = &ops[curop]; - switch (op->type) { - case GRPC_NO_OP: - case GRPC_OP_BEGIN_MESSAGE: - gpr_log( - GPR_ERROR, - "These stream ops should be filtered out by grpc_chttp2_preencode"); - abort(); - case GRPC_OP_METADATA: - /* Encode a metadata batch; store the returned values, representing - a metadata element that needs to be unreffed back into the metadata - slot. THIS MAY NOT BE THE SAME ELEMENT (if a decoder table slot got - updated). After this loop, we'll do a batch unref of elements. */ - begin_new_frame(&st, HEADER); - need_unref |= op->data.metadata.garbage.head != NULL; - grpc_metadata_batch_assert_ok(&op->data.metadata); - for (l = op->data.metadata.list.head; l; l = l->next) { - l->md = hpack_enc(compressor, l->md, &st); - need_unref |= l->md != NULL; - } - deadline = op->data.metadata.deadline; - if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) != 0) { - deadline_enc(compressor, deadline, &st); - } - curop++; - break; - case GRPC_OP_SLICE: - slice = op->data.slice; - if (st.cur_frame_type == DATA && - st.output->length - st.output_length_at_start_of_frame == - GRPC_CHTTP2_MAX_PAYLOAD_LENGTH) { - finish_frame(&st, 0, 0); - } - ensure_frame_type(&st, DATA, 1); - max_take_size = GRPC_CHTTP2_MAX_PAYLOAD_LENGTH + - st.output_length_at_start_of_frame - st.output->length; - if (GPR_SLICE_LENGTH(slice) > max_take_size) { - slice = gpr_slice_split_head(&op->data.slice, max_take_size); - } else { - /* consume this op immediately */ - curop++; - } - gpr_slice_buffer_add(output, slice); - break; - } + st.output = outbuf; + st.is_first_frame = 1; + + /* Encode a metadata batch; store the returned values, representing + a metadata element that needs to be unreffed back into the metadata + slot. THIS MAY NOT BE THE SAME ELEMENT (if a decoder table slot got + updated). After this loop, we'll do a batch unref of elements. */ + begin_frame(&st); + if (c->advertise_table_size_change != 0) { + emit_advertise_table_size_change(c, &st); } - if (eof && st.cur_frame_type == NONE) { - begin_frame(&st, DATA); + grpc_metadata_batch_assert_ok(metadata); + for (l = metadata->list.head; l; l = l->next) { + hpack_enc(c, l->md, &st); } - finish_frame(&st, 1, eof); - - if (need_unref) { - for (unref_op = 0; unref_op < curop; unref_op++) { - op = &ops[unref_op]; - if (op->type != GRPC_OP_METADATA) continue; - for (l = op->data.metadata.list.head; l; l = l->next) { - if (l->md) GRPC_MDELEM_UNREF(l->md); - } - for (l = op->data.metadata.garbage.head; l; l = l->next) { - GRPC_MDELEM_UNREF(l->md); - } - } + deadline = metadata->deadline; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) != 0) { + deadline_enc(c, deadline, &st); } + + finish_frame(&st, 1, is_eof); } diff --git a/src/core/transport/chttp2/stream_encoder.h b/src/core/transport/chttp2/hpack_encoder.h index db52f2a0f6..ce83d101f2 100644 --- a/src/core/transport/chttp2/stream_encoder.h +++ b/src/core/transport/chttp2/hpack_encoder.h @@ -31,26 +31,38 @@ * */ -#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_ENCODER_H -#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_ENCODER_H +#ifndef GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H +#define GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H #include "src/core/transport/chttp2/frame.h" #include "src/core/transport/metadata.h" -#include "src/core/transport/stream_op.h" +#include "src/core/transport/metadata_batch.h" #include <grpc/support/port_platform.h> #include <grpc/support/slice.h> #include <grpc/support/slice_buffer.h> #define GRPC_CHTTP2_HPACKC_NUM_FILTERS 256 #define GRPC_CHTTP2_HPACKC_NUM_VALUES 256 -#define GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS (4096 / 32) +/* initial table size, per spec */ +#define GRPC_CHTTP2_HPACKC_INITIAL_TABLE_SIZE 4096 +/* maximum table size we'll actually use */ +#define GRPC_CHTTP2_HPACKC_MAX_TABLE_SIZE (1024 * 1024) typedef struct { gpr_uint32 filter_elems_sum; + gpr_uint32 max_table_size; + gpr_uint32 max_table_elems; + gpr_uint32 cap_table_elems; + /** if non-zero, advertise to the decoder that we'll start using a table + of this size */ + gpr_uint8 advertise_table_size_change; + /** maximum number of bytes we'll use for the decode table (to guard against + peers ooming us by setting decode table size high) */ + gpr_uint32 max_usable_size; /* one before the lowest usable table index */ gpr_uint32 tail_remote_index; - gpr_uint16 table_size; - gpr_uint16 table_elems; + gpr_uint32 table_size; + gpr_uint32 table_elems; /* filter tables for elems: this tables provides an approximate popularity count for particular hashes, and are used to determine whether @@ -71,23 +83,19 @@ typedef struct { gpr_uint32 indices_keys[GRPC_CHTTP2_HPACKC_NUM_VALUES]; gpr_uint32 indices_elems[GRPC_CHTTP2_HPACKC_NUM_VALUES]; - gpr_uint16 table_elem_size[GRPC_CHTTP2_HPACKC_MAX_TABLE_ELEMS]; + gpr_uint16 *table_elem_size; } grpc_chttp2_hpack_compressor; void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c, grpc_mdctx *mdctx); void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c); +void grpc_chttp2_hpack_compressor_set_max_table_size( + grpc_chttp2_hpack_compressor *c, gpr_uint32 max_table_size); +void grpc_chttp2_hpack_compressor_set_max_usable_size( + grpc_chttp2_hpack_compressor *c, gpr_uint32 max_table_size); -/* select stream ops to be encoded, moving them from inops to outops, and - moving subsequent ops in inops forward in the queue */ -gpr_uint32 grpc_chttp2_preencode(grpc_stream_op *inops, size_t *inops_count, - gpr_uint32 max_flow_controlled_bytes, - grpc_stream_op_buffer *outops); +void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, gpr_uint32 id, + grpc_metadata_batch *metadata, int is_eof, + gpr_slice_buffer *outbuf); -/* encode stream ops to output */ -void grpc_chttp2_encode(grpc_stream_op *ops, size_t ops_count, int eof, - gpr_uint32 stream_id, - grpc_chttp2_hpack_compressor *compressor, - gpr_slice_buffer *output); - -#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_STREAM_ENCODER_H */ +#endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_HPACK_ENCODER_H */ diff --git a/src/core/transport/chttp2/hpack_parser.c b/src/core/transport/chttp2/hpack_parser.c index 20d8312d54..57a3a20b1d 100644 --- a/src/core/transport/chttp2/hpack_parser.c +++ b/src/core/transport/chttp2/hpack_parser.c @@ -38,13 +38,15 @@ #include <string.h> #include <assert.h> -#include "src/core/transport/chttp2/bin_encoder.h" -#include "src/core/support/string.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/port_platform.h> #include <grpc/support/useful.h> +#include "src/core/profiling/timers.h" +#include "src/core/support/string.h" +#include "src/core/transport/chttp2/bin_encoder.h" + typedef enum { NOT_BINARY, B64_BYTE0, @@ -72,6 +74,8 @@ static int parse_begin(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end); static int parse_error(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end); +static int parse_illegal_op(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, + const gpr_uint8 *end); static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end); @@ -154,7 +158,7 @@ static const grpc_chttp2_hpack_parser_state first_byte_action[] = { parse_lithdr_incidx_x, parse_lithdr_incidx_v, parse_lithdr_notidx, parse_lithdr_notidx_x, parse_lithdr_notidx_v, parse_lithdr_nvridx, parse_lithdr_nvridx_x, parse_lithdr_nvridx_v, parse_max_tbl_size, - parse_max_tbl_size_x, parse_error}; + parse_max_tbl_size_x, parse_illegal_op}; /* indexes the first byte to a parse state function - generated by gen_hpack_tables.c */ @@ -167,7 +171,7 @@ static const gpr_uint8 first_byte_lut[256] = { LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX, LITHDR_NVRIDX_X, - ILLEGAL, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, + MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, MAX_TBL_SIZE, @@ -620,13 +624,15 @@ static const gpr_uint8 inverse_base64[256] = { }; /* emission helpers */ -static void on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md, - int add_to_table) { +static int on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md, + int add_to_table) { if (add_to_table) { - GRPC_MDELEM_REF(md); - grpc_chttp2_hptbl_add(&p->table, md); + if (!grpc_chttp2_hptbl_add(&p->table, md)) { + return 0; + } } p->on_header(p->on_header_user_data, md); + return 1; } static grpc_mdstr *take_string(grpc_chttp2_hpack_parser *p, @@ -712,9 +718,12 @@ static int parse_stream_dep0(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, static int finish_indexed_field(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); + if (md == NULL) { + gpr_log(GPR_ERROR, "Invalid HPACK index received: %d", p->index); + return 0; + } GRPC_MDELEM_REF(md); - on_hdr(p, md, 0); - return parse_begin(p, cur, end); + return on_hdr(p, md, 0) && parse_begin(p, cur, end); } /* parse an indexed field with index < 127 */ @@ -740,21 +749,21 @@ static int parse_indexed_field_x(grpc_chttp2_hpack_parser *p, static int finish_lithdr_incidx(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 1); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 1) && + parse_begin(p, cur, end); } /* finish a literal header with incremental indexing with no index */ static int finish_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - take_string(p, &p->key), - take_string(p, &p->value)), - 1); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + take_string(p, &p->key), + take_string(p, &p->value)), + 1) && + parse_begin(p, cur, end); } /* parse a literal header with incremental indexing; index < 63 */ @@ -793,21 +802,21 @@ static int parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, static int finish_lithdr_notidx(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 0); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 0) && + parse_begin(p, cur, end); } /* finish a literal header without incremental indexing with index = 0 */ static int finish_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - take_string(p, &p->key), - take_string(p, &p->value)), - 0); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + take_string(p, &p->key), + take_string(p, &p->value)), + 0) && + parse_begin(p, cur, end); } /* parse a literal header without incremental indexing; index < 15 */ @@ -846,21 +855,21 @@ static int parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, static int finish_lithdr_nvridx(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 0); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 0) && + parse_begin(p, cur, end); } /* finish a literal header that is never indexed with an extra value */ static int finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { - on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, - take_string(p, &p->key), - take_string(p, &p->value)), - 0); - return parse_begin(p, cur, end); + return on_hdr(p, grpc_mdelem_from_metadata_strings(p->table.mdctx, + take_string(p, &p->key), + take_string(p, &p->value)), + 0) && + parse_begin(p, cur, end); } /* parse a literal header that is never indexed; index < 15 */ @@ -899,14 +908,14 @@ static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, static int finish_max_tbl_size(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index); - abort(); /* not implemented */ - return parse_begin(p, cur, end); + return grpc_chttp2_hptbl_set_current_table_size(&p->table, p->index) && + parse_begin(p, cur, end); } /* parse a max table size change, max size < 15 */ static int parse_max_tbl_size(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, const gpr_uint8 *end) { - p->index = (*cur) & 0xf; + p->index = (*cur) & 0x1f; return finish_max_tbl_size(p, cur + 1, end); } @@ -916,7 +925,7 @@ static int parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, static const grpc_chttp2_hpack_parser_state and_then[] = { finish_max_tbl_size}; p->next_state = and_then; - p->index = 0xf; + p->index = 0x1f; p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } @@ -928,6 +937,13 @@ static int parse_error(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, return 0; } +static int parse_illegal_op(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, + const gpr_uint8 *end) { + GPR_ASSERT(cur != end); + gpr_log(GPR_DEBUG, "Illegal hpack op code %d", *cur); + return parse_error(p, cur, end); +} + /* parse the 1st byte of a varint into p->parsing.value no overflow is possible */ static int parse_value0(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur, @@ -1379,20 +1395,23 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse( grpc_chttp2_transport_parsing *transport_parsing, grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { grpc_chttp2_hpack_parser *parser = hpack_parser; + GPR_TIMER_BEGIN("grpc_chttp2_hpack_parser_parse", 0); if (!grpc_chttp2_hpack_parser_parse(parser, GPR_SLICE_START_PTR(slice), GPR_SLICE_END_PTR(slice))) { + GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0); return GRPC_CHTTP2_CONNECTION_ERROR; } if (is_last) { if (parser->is_boundary && parser->state != parse_begin) { gpr_log(GPR_ERROR, "end of header frame not aligned with a hpack record boundary"); + GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0); return GRPC_CHTTP2_CONNECTION_ERROR; } if (parser->is_boundary) { - grpc_chttp2_incoming_metadata_buffer_place_metadata_batch_into( - &stream_parsing->incoming_metadata, - &stream_parsing->data_parser.incoming_sopb); + stream_parsing + ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1; + stream_parsing->header_frames_received++; grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); } @@ -1404,5 +1423,6 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse( parser->is_boundary = 0xde; parser->is_eof = 0xde; } + GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0); return GRPC_CHTTP2_PARSE_OK; } diff --git a/src/core/transport/chttp2/hpack_table.c b/src/core/transport/chttp2/hpack_table.c index c442c2c341..f2362b056c 100644 --- a/src/core/transport/chttp2/hpack_table.c +++ b/src/core/transport/chttp2/hpack_table.c @@ -36,7 +36,9 @@ #include <assert.h> #include <string.h> +#include <grpc/support/alloc.h> #include <grpc/support/log.h> + #include "src/core/support/murmur_hash.h" static struct { @@ -169,12 +171,22 @@ static struct { {"www-authenticate", ""}, }; +static gpr_uint32 entries_for_bytes(gpr_uint32 bytes) { + return (bytes + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; +} + void grpc_chttp2_hptbl_init(grpc_chttp2_hptbl *tbl, grpc_mdctx *mdctx) { size_t i; memset(tbl, 0, sizeof(*tbl)); tbl->mdctx = mdctx; - tbl->max_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; + tbl->current_table_bytes = tbl->max_bytes = + GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; + tbl->max_entries = tbl->cap_entries = + entries_for_bytes(tbl->current_table_bytes); + tbl->ents = gpr_malloc(sizeof(*tbl->ents) * tbl->cap_entries); + memset(tbl->ents, 0, sizeof(*tbl->ents) * tbl->cap_entries); for (i = 1; i <= GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { tbl->static_ents[i - 1] = grpc_mdelem_from_strings( mdctx, static_table[i].key, static_table[i].value); @@ -187,9 +199,9 @@ void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl *tbl) { GRPC_MDELEM_UNREF(tbl->static_ents[i]); } for (i = 0; i < tbl->num_ents; i++) { - GRPC_MDELEM_UNREF( - tbl->ents[(tbl->first_ent + i) % GRPC_CHTTP2_MAX_TABLE_COUNT]); + GRPC_MDELEM_UNREF(tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]); } + gpr_free(tbl->ents); } grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl, @@ -201,8 +213,8 @@ grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl, /* Otherwise, find the value in the list of valid entries */ tbl_index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); if (tbl_index < tbl->num_ents) { - gpr_uint32 offset = (tbl->num_ents - 1u - tbl_index + tbl->first_ent) % - GRPC_CHTTP2_MAX_TABLE_COUNT; + gpr_uint32 offset = + (tbl->num_ents - 1u - tbl_index + tbl->first_ent) % tbl->cap_entries; return tbl->ents[offset]; } /* Invalid entry: return error */ @@ -216,21 +228,81 @@ static void evict1(grpc_chttp2_hptbl *tbl) { GPR_SLICE_LENGTH(first_ent->value->slice) + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; GPR_ASSERT(elem_bytes <= tbl->mem_used); - tbl->mem_used = (gpr_uint16)(tbl->mem_used - elem_bytes); - tbl->first_ent = - (gpr_uint16)((tbl->first_ent + 1) % GRPC_CHTTP2_MAX_TABLE_COUNT); + tbl->mem_used -= (gpr_uint32)elem_bytes; + tbl->first_ent = ((tbl->first_ent + 1) % tbl->cap_entries); tbl->num_ents--; GRPC_MDELEM_UNREF(first_ent); } -void grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) { +static void rebuild_ents(grpc_chttp2_hptbl *tbl, gpr_uint32 new_cap) { + grpc_mdelem **ents = gpr_malloc(sizeof(*ents) * new_cap); + gpr_uint32 i; + + for (i = 0; i < tbl->num_ents; i++) { + ents[i] = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; + } + gpr_free(tbl->ents); + tbl->ents = ents; + tbl->cap_entries = new_cap; + tbl->first_ent = 0; +} + +void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl *tbl, + gpr_uint32 max_bytes) { + if (tbl->max_bytes == max_bytes) { + return; + } + gpr_log(GPR_DEBUG, "Update hpack parser max size to %d", max_bytes); + while (tbl->mem_used > max_bytes) { + evict1(tbl); + } + tbl->max_bytes = max_bytes; +} + +int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl, + gpr_uint32 bytes) { + if (tbl->current_table_bytes == bytes) { + return 1; + } + if (bytes > tbl->max_bytes) { + gpr_log(GPR_ERROR, + "Attempt to make hpack table %d bytes when max is %d bytes", bytes, + tbl->max_bytes); + return 0; + } + gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes); + while (tbl->mem_used > bytes) { + evict1(tbl); + } + tbl->current_table_bytes = bytes; + tbl->max_entries = entries_for_bytes(bytes); + if (tbl->max_entries > tbl->cap_entries) { + rebuild_ents(tbl, GPR_MAX(tbl->max_entries, 2 * tbl->cap_entries)); + } else if (tbl->max_entries < tbl->cap_entries / 3) { + gpr_uint32 new_cap = GPR_MAX(tbl->max_entries, 16u); + if (new_cap != tbl->cap_entries) { + rebuild_ents(tbl, new_cap); + } + } + return 1; +} + +int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) { /* determine how many bytes of buffer this entry represents */ size_t elem_bytes = GPR_SLICE_LENGTH(md->key->slice) + GPR_SLICE_LENGTH(md->value->slice) + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; + if (tbl->current_table_bytes > tbl->max_bytes) { + gpr_log(GPR_ERROR, + "HPACK max table size reduced to %d but not reflected by hpack " + "stream (still at %d)", + tbl->max_bytes, tbl->current_table_bytes); + return 0; + } + /* we can't add elements bigger than the max table size */ - if (elem_bytes > tbl->max_bytes) { + if (elem_bytes > tbl->current_table_bytes) { /* HPACK draft 10 section 4.4 states: * If the size of the new entry is less than or equal to the maximum * size, that entry is added to the table. It is not an error to @@ -243,44 +315,43 @@ void grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) { while (tbl->num_ents) { evict1(tbl); } - return; + return 1; } /* evict entries to ensure no overflow */ - while (elem_bytes > (size_t)tbl->max_bytes - tbl->mem_used) { + while (elem_bytes > (size_t)tbl->current_table_bytes - tbl->mem_used) { evict1(tbl); } /* copy the finalized entry in */ - tbl->ents[tbl->last_ent] = md; + tbl->ents[(tbl->first_ent + tbl->num_ents) % tbl->cap_entries] = + GRPC_MDELEM_REF(md); /* update accounting values */ - tbl->last_ent = - (gpr_uint16)((tbl->last_ent + 1) % GRPC_CHTTP2_MAX_TABLE_COUNT); tbl->num_ents++; - tbl->mem_used = (gpr_uint16)(tbl->mem_used + elem_bytes); + tbl->mem_used += (gpr_uint32)elem_bytes; + return 1; } grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( const grpc_chttp2_hptbl *tbl, grpc_mdelem *md) { grpc_chttp2_hptbl_find_result r = {0, 0}; - gpr_uint16 i; + gpr_uint32 i; /* See if the string is in the static table */ for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { grpc_mdelem *ent = tbl->static_ents[i]; if (md->key != ent->key) continue; - r.index = (gpr_uint16)(i + 1); + r.index = i + 1u; r.has_value = md->value == ent->value; if (r.has_value) return r; } /* Scan the dynamic table */ for (i = 0; i < tbl->num_ents; i++) { - gpr_uint16 idx = - (gpr_uint16)(tbl->num_ents - i + GRPC_CHTTP2_LAST_STATIC_ENTRY); - grpc_mdelem *ent = - tbl->ents[(tbl->first_ent + i) % GRPC_CHTTP2_MAX_TABLE_COUNT]; + gpr_uint32 idx = + (gpr_uint32)(tbl->num_ents - i + GRPC_CHTTP2_LAST_STATIC_ENTRY); + grpc_mdelem *ent = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; if (md->key != ent->key) continue; r.index = idx; r.has_value = md->value == ent->value; diff --git a/src/core/transport/chttp2/hpack_table.h b/src/core/transport/chttp2/hpack_table.h index 4f882e2e03..e03b8c19b3 100644 --- a/src/core/transport/chttp2/hpack_table.h +++ b/src/core/transport/chttp2/hpack_table.h @@ -49,47 +49,59 @@ #define GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE /* Per entry overhead bytes as per the spec */ #define GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD 32 +#if 0 /* Maximum number of entries we could possibly fit in the table, given defined overheads */ #define GRPC_CHTTP2_MAX_TABLE_COUNT \ ((GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / \ GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD) +#endif /* hpack decoder table */ typedef struct { grpc_mdctx *mdctx; /* the first used entry in ents */ - gpr_uint16 first_ent; - /* the last used entry in ents */ - gpr_uint16 last_ent; + gpr_uint32 first_ent; /* how many entries are in the table */ - gpr_uint16 num_ents; + gpr_uint32 num_ents; /* the amount of memory used by the table, according to the hpack algorithm */ - gpr_uint16 mem_used; + gpr_uint32 mem_used; /* the max memory allowed to be used by the table, according to the hpack algorithm */ - gpr_uint16 max_bytes; + gpr_uint32 max_bytes; + /* the currently agreed size of the table, according to the hpack algorithm */ + gpr_uint32 current_table_bytes; + /* Maximum number of entries we could possibly fit in the table, given defined + overheads */ + gpr_uint32 max_entries; + /* Number of entries allocated in ents */ + gpr_uint32 cap_entries; /* a circular buffer of headers - this is stored in the opposite order to what hpack specifies, in order to simplify table management a little... meaning lookups need to SUBTRACT from the end position */ - grpc_mdelem *ents[GRPC_CHTTP2_MAX_TABLE_COUNT]; + grpc_mdelem **ents; grpc_mdelem *static_ents[GRPC_CHTTP2_LAST_STATIC_ENTRY]; } grpc_chttp2_hptbl; /* initialize a hpack table */ void grpc_chttp2_hptbl_init(grpc_chttp2_hptbl *tbl, grpc_mdctx *mdctx); void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl *tbl); +void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl *tbl, + gpr_uint32 max_bytes); +int grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl, + gpr_uint32 bytes); /* lookup a table entry based on its hpack index */ grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl, gpr_uint32 index); /* add a table entry to the index */ -void grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md); +int grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, + grpc_mdelem *md) GRPC_MUST_USE_RESULT; /* Find a key/value pair in the table... returns the index in the table of the most similar entry, or 0 if the value was not found */ typedef struct { - gpr_uint16 index; - gpr_uint8 has_value; + gpr_uint32 index; + int has_value; } grpc_chttp2_hptbl_find_result; grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( const grpc_chttp2_hptbl *tbl, grpc_mdelem *md); diff --git a/src/core/transport/chttp2/incoming_metadata.c b/src/core/transport/chttp2/incoming_metadata.c index 10c64f3356..956afc8e9d 100644 --- a/src/core/transport/chttp2/incoming_metadata.c +++ b/src/core/transport/chttp2/incoming_metadata.c @@ -48,14 +48,27 @@ void grpc_chttp2_incoming_metadata_buffer_init( void grpc_chttp2_incoming_metadata_buffer_destroy( grpc_chttp2_incoming_metadata_buffer *buffer) { size_t i; + if (!buffer->published) { + for (i = 0; i < buffer->count; i++) { + GRPC_MDELEM_UNREF(buffer->elems[i].md); + } + } + gpr_free(buffer->elems); +} + +void grpc_chttp2_incoming_metadata_buffer_reset( + grpc_chttp2_incoming_metadata_buffer *buffer) { + size_t i; + GPR_ASSERT(!buffer->published); for (i = 0; i < buffer->count; i++) { GRPC_MDELEM_UNREF(buffer->elems[i].md); } - gpr_free(buffer->elems); + buffer->count = 0; } void grpc_chttp2_incoming_metadata_buffer_add( grpc_chttp2_incoming_metadata_buffer *buffer, grpc_mdelem *elem) { + GPR_ASSERT(!buffer->published); if (buffer->capacity == buffer->count) { buffer->capacity = GPR_MAX(8, 2 * buffer->capacity); buffer->elems = @@ -66,117 +79,36 @@ void grpc_chttp2_incoming_metadata_buffer_add( void grpc_chttp2_incoming_metadata_buffer_set_deadline( grpc_chttp2_incoming_metadata_buffer *buffer, gpr_timespec deadline) { + GPR_ASSERT(!buffer->published); buffer->deadline = deadline; } -void grpc_chttp2_incoming_metadata_live_op_buffer_end( - grpc_chttp2_incoming_metadata_live_op_buffer *buffer) { - gpr_free(buffer->elems); - buffer->elems = NULL; -} - -void grpc_chttp2_incoming_metadata_buffer_place_metadata_batch_into( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_stream_op_buffer *sopb) { - grpc_metadata_batch b; - - b.list.head = NULL; - /* Store away the last element of the list, so that in patch_metadata_ops - we can reconstitute the list. - We can't do list building here as later incoming metadata may reallocate - the underlying array. */ - b.list.tail = (void *)(gpr_intptr)buffer->count; - b.garbage.head = b.garbage.tail = NULL; - b.deadline = buffer->deadline; - buffer->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); - - grpc_sopb_add_metadata(sopb, b); -} - void grpc_chttp2_incoming_metadata_buffer_swap( grpc_chttp2_incoming_metadata_buffer *a, grpc_chttp2_incoming_metadata_buffer *b) { + GPR_ASSERT(!a->published); + GPR_ASSERT(!b->published); GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, *a, *b); } -void grpc_incoming_metadata_buffer_move_to_referencing_sopb( - grpc_chttp2_incoming_metadata_buffer *src, - grpc_chttp2_incoming_metadata_buffer *dst, grpc_stream_op_buffer *sopb) { - size_t delta; - size_t i; - dst->deadline = gpr_time_min(src->deadline, dst->deadline); - - if (src->count == 0) { - return; - } - if (dst->count == 0) { - grpc_chttp2_incoming_metadata_buffer_swap(src, dst); - return; - } - delta = dst->count; - if (dst->capacity < src->count + dst->count) { - dst->capacity = GPR_MAX(dst->capacity * 2, src->count + dst->count); - dst->elems = gpr_realloc(dst->elems, dst->capacity * sizeof(*dst->elems)); - } - memcpy(dst->elems + dst->count, src->elems, src->count * sizeof(*src->elems)); - dst->count += src->count; - for (i = 0; i < sopb->nops; i++) { - if (sopb->ops[i].type != GRPC_OP_METADATA) continue; - sopb->ops[i].data.metadata.list.tail = - (void *)(delta + (gpr_uintptr)sopb->ops[i].data.metadata.list.tail); - } - src->count = 0; -} - -void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_stream_op_buffer *sopb, - grpc_chttp2_incoming_metadata_live_op_buffer *live_op_buffer) { - grpc_stream_op *ops = sopb->ops; - size_t nops = sopb->nops; - size_t i; - size_t j; - size_t mdidx = 0; - size_t last_mdidx; - int found_metadata = 0; - - /* rework the array of metadata into a linked list, making use - of the breadcrumbs we left in metadata batches during - add_metadata_batch */ - for (i = 0; i < nops; i++) { - grpc_stream_op *op = &ops[i]; - if (op->type != GRPC_OP_METADATA) continue; - found_metadata = 1; - /* we left a breadcrumb indicating where the end of this list is, - and since we add sequentially, we know from the end of the last - segment where this segment begins */ - last_mdidx = (size_t)(gpr_intptr)(op->data.metadata.list.tail); - GPR_ASSERT(last_mdidx > mdidx); - GPR_ASSERT(last_mdidx <= buffer->count); - /* turn the array into a doubly linked list */ - op->data.metadata.list.head = &buffer->elems[mdidx]; - op->data.metadata.list.tail = &buffer->elems[last_mdidx - 1]; - for (j = mdidx + 1; j < last_mdidx; j++) { - buffer->elems[j].prev = &buffer->elems[j - 1]; - buffer->elems[j - 1].next = &buffer->elems[j]; +void grpc_chttp2_incoming_metadata_buffer_publish( + grpc_chttp2_incoming_metadata_buffer *buffer, grpc_metadata_batch *batch) { + GPR_ASSERT(!buffer->published); + buffer->published = 1; + if (buffer->count > 0) { + size_t i; + for (i = 1; i < buffer->count; i++) { + buffer->elems[i].prev = &buffer->elems[i - 1]; } - buffer->elems[mdidx].prev = NULL; - buffer->elems[last_mdidx - 1].next = NULL; - /* track where we're up to */ - mdidx = last_mdidx; - } - if (found_metadata) { - live_op_buffer->elems = buffer->elems; - if (mdidx != buffer->count) { - /* we have a partially read metadata batch still in incoming_metadata */ - size_t new_count = buffer->count - mdidx; - size_t copy_bytes = sizeof(*buffer->elems) * new_count; - GPR_ASSERT(mdidx < buffer->count); - buffer->elems = gpr_malloc(copy_bytes); - memcpy(buffer->elems, live_op_buffer->elems + mdidx, copy_bytes); - buffer->count = buffer->capacity = new_count; - } else { - buffer->elems = NULL; - buffer->count = 0; - buffer->capacity = 0; + for (i = 0; i < buffer->count - 1; i++) { + buffer->elems[i].next = &buffer->elems[i + 1]; } + buffer->elems[0].prev = NULL; + buffer->elems[buffer->count - 1].next = NULL; + batch->list.head = &buffer->elems[0]; + batch->list.tail = &buffer->elems[buffer->count - 1]; + } else { + batch->list.head = batch->list.tail = NULL; } + batch->deadline = buffer->deadline; } diff --git a/src/core/transport/chttp2/incoming_metadata.h b/src/core/transport/chttp2/incoming_metadata.h index 2f1de411ba..0e1dabe825 100644 --- a/src/core/transport/chttp2/incoming_metadata.h +++ b/src/core/transport/chttp2/incoming_metadata.h @@ -41,12 +41,9 @@ typedef struct { size_t count; size_t capacity; gpr_timespec deadline; + int published; } grpc_chttp2_incoming_metadata_buffer; -typedef struct { - grpc_linked_mdelem *elems; -} grpc_chttp2_incoming_metadata_live_op_buffer; - /** assumes everything initially zeroed */ void grpc_chttp2_incoming_metadata_buffer_init( grpc_chttp2_incoming_metadata_buffer *buffer); @@ -54,27 +51,12 @@ void grpc_chttp2_incoming_metadata_buffer_destroy( grpc_chttp2_incoming_metadata_buffer *buffer); void grpc_chttp2_incoming_metadata_buffer_reset( grpc_chttp2_incoming_metadata_buffer *buffer); +void grpc_chttp2_incoming_metadata_buffer_publish( + grpc_chttp2_incoming_metadata_buffer *buffer, grpc_metadata_batch *batch); void grpc_chttp2_incoming_metadata_buffer_add( grpc_chttp2_incoming_metadata_buffer *buffer, grpc_mdelem *elem); void grpc_chttp2_incoming_metadata_buffer_set_deadline( grpc_chttp2_incoming_metadata_buffer *buffer, gpr_timespec deadline); -/** extend sopb with a metadata batch; this must be post-processed by - grpc_chttp2_incoming_metadata_buffer_postprocess_sopb before being handed - out of the transport */ -void grpc_chttp2_incoming_metadata_buffer_place_metadata_batch_into( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_stream_op_buffer *sopb); - -void grpc_incoming_metadata_buffer_move_to_referencing_sopb( - grpc_chttp2_incoming_metadata_buffer *src, - grpc_chttp2_incoming_metadata_buffer *dst, grpc_stream_op_buffer *sopb); - -void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op( - grpc_chttp2_incoming_metadata_buffer *buffer, grpc_stream_op_buffer *sopb, - grpc_chttp2_incoming_metadata_live_op_buffer *live_op_buffer); - -void grpc_chttp2_incoming_metadata_live_op_buffer_end( - grpc_chttp2_incoming_metadata_live_op_buffer *live_op_buffer); - #endif /* GRPC_INTERNAL_CORE_CHTTP2_INCOMING_METADATA_H */ diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index b35f8b5d88..b2ac8ee965 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -34,6 +34,8 @@ #ifndef GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H #define GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H +#include <assert.h> + #include "src/core/iomgr/endpoint.h" #include "src/core/transport/chttp2/frame.h" #include "src/core/transport/chttp2/frame_data.h" @@ -42,9 +44,9 @@ #include "src/core/transport/chttp2/frame_rst_stream.h" #include "src/core/transport/chttp2/frame_settings.h" #include "src/core/transport/chttp2/frame_window_update.h" +#include "src/core/transport/chttp2/hpack_encoder.h" #include "src/core/transport/chttp2/hpack_parser.h" #include "src/core/transport/chttp2/incoming_metadata.h" -#include "src/core/transport/chttp2/stream_encoder.h" #include "src/core/transport/chttp2/stream_map.h" #include "src/core/transport/connectivity_state.h" #include "src/core/transport/transport_impl.h" @@ -56,14 +58,14 @@ typedef struct grpc_chttp2_stream grpc_chttp2_stream; happen to them... this enum labels each list */ typedef enum { GRPC_CHTTP2_LIST_ALL_STREAMS, - GRPC_CHTTP2_LIST_READ_WRITE_STATE_CHANGED, + GRPC_CHTTP2_LIST_CHECK_READ_OPS, + GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE, GRPC_CHTTP2_LIST_WRITABLE, GRPC_CHTTP2_LIST_WRITING, GRPC_CHTTP2_LIST_WRITTEN, GRPC_CHTTP2_LIST_PARSING_SEEN, GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING, - GRPC_CHTTP2_LIST_CANCELLED_WAITING_FOR_WRITING, - GRPC_CHTTP2_LIST_INCOMING_WINDOW_UPDATED, + GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT, /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, @@ -113,22 +115,6 @@ typedef enum { GRPC_DTS_FRAME } grpc_chttp2_deframe_transport_state; -typedef enum { - GRPC_WRITE_STATE_OPEN, - GRPC_WRITE_STATE_QUEUED_CLOSE, - GRPC_WRITE_STATE_SENT_CLOSE -} grpc_chttp2_write_state; - -/* flags that can be or'd into stream_global::writing_now */ -#define GRPC_CHTTP2_WRITING_DATA 1 -#define GRPC_CHTTP2_WRITING_WINDOW 2 - -typedef enum { - GRPC_DONT_SEND_CLOSED = 0, - GRPC_SEND_CLOSED, - GRPC_SEND_CLOSED_WITH_RST_STREAM -} grpc_chttp2_send_closed; - typedef struct { grpc_chttp2_stream *head; grpc_chttp2_stream *tail; @@ -160,14 +146,28 @@ typedef struct grpc_chttp2_outstanding_ping { struct grpc_chttp2_outstanding_ping *prev; } grpc_chttp2_outstanding_ping; +/* forward declared in frame_data.h */ +struct grpc_chttp2_incoming_byte_stream { + grpc_byte_stream base; + gpr_refcount refs; + struct grpc_chttp2_incoming_byte_stream *next_message; + + grpc_chttp2_transport *transport; + grpc_chttp2_stream *stream; + int is_tail; + gpr_slice_buffer slices; + grpc_closure *on_next; + gpr_slice *next; +}; + typedef struct { /** data to write next write */ gpr_slice_buffer qbuf; /** window available for us to send to peer */ gpr_int64 outgoing_window; - /** window available for peer to send to us - updated after parse */ - gpr_uint32 incoming_window; + /** window available to announce to peer */ + gpr_int64 announce_incoming_window; /** how much window would we like to have for incoming_window */ gpr_uint32 connection_window_target; @@ -191,6 +191,9 @@ typedef struct { copied to next_stream_id in parsing when parsing commences */ gpr_uint32 next_stream_id; + /** how far to lookahead in a stream? */ + gpr_uint32 stream_lookahead; + /** last received stream id */ gpr_uint32 last_incoming_stream_id; @@ -209,6 +212,7 @@ typedef struct { gpr_slice_buffer outbuf; /** hpack encoding */ grpc_chttp2_hpack_compressor hpack_compressor; + gpr_int64 outgoing_window; /** is this a client? */ gpr_uint8 is_client; /** callback for when writing is done */ @@ -226,6 +230,9 @@ struct grpc_chttp2_transport_parsing { /** was a goaway frame received? */ gpr_uint8 goaway_received; + /** the last sent max_table_size setting */ + gpr_uint32 last_sent_max_table_size; + /** initial window change */ gpr_int64 initial_window_update; @@ -233,6 +240,7 @@ struct grpc_chttp2_transport_parsing { gpr_slice_buffer qbuf; /* metadata object cache */ grpc_mdstr *str_grpc_timeout; + grpc_mdelem *elem_grpc_status_ok; /** parser for headers */ grpc_chttp2_hpack_parser hpack_parser; /** simple one shot parsers */ @@ -246,8 +254,7 @@ struct grpc_chttp2_transport_parsing { grpc_chttp2_goaway_parser goaway_parser; /** window available for peer to send to us */ - gpr_uint32 incoming_window; - gpr_uint32 incoming_window_delta; + gpr_int64 incoming_window; /** next stream id available at the time of beginning parsing */ gpr_uint32 next_stream_id; @@ -278,7 +285,7 @@ struct grpc_chttp2_transport_parsing { gpr_uint32 goaway_last_stream_index; gpr_slice goaway_text; - gpr_int64 outgoing_window_update; + gpr_int64 outgoing_window; /** pings awaiting responses */ grpc_chttp2_outstanding_ping pings; @@ -345,8 +352,8 @@ struct grpc_chttp2_transport { struct { /* accept stream callback */ - void (*accept_stream)(void *user_data, grpc_transport *transport, - const void *server_data); + void (*accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_transport *transport, const void *server_data); void *accept_stream_user_data; /** connectivity tracking */ @@ -358,9 +365,6 @@ typedef struct { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ gpr_uint32 id; - grpc_closure *send_done_closure; - grpc_closure *recv_done_closure; - /** window available for us to send to peer */ gpr_int64 outgoing_window; /** The number of bytes the upper layers have offered to receive. @@ -371,54 +375,66 @@ typedef struct { not yet announced to HTTP2 flow control. As the upper layers offer to read more bytes, this value increases. As we advertise incoming flow control window, this value decreases. */ - gpr_uint32 unannounced_incoming_window; - /** The number of bytes of HTTP2 flow control we have advertised. - As we advertise incoming flow control window, this value increases. - As bytes are read, this value decreases. - Updated after parse. */ - gpr_uint32 incoming_window; - /** stream ops the transport user would like to send */ - grpc_stream_op_buffer *outgoing_sopb; + gpr_uint32 unannounced_incoming_window_for_parse; + gpr_uint32 unannounced_incoming_window_for_writing; + /** things the upper layers would like to send */ + grpc_metadata_batch *send_initial_metadata; + grpc_closure *send_initial_metadata_finished; + grpc_byte_stream *send_message; + grpc_closure *send_message_finished; + grpc_metadata_batch *send_trailing_metadata; + grpc_closure *send_trailing_metadata_finished; + + grpc_metadata_batch *recv_initial_metadata; + grpc_closure *recv_initial_metadata_finished; + grpc_byte_stream **recv_message; + grpc_closure *recv_message_ready; + grpc_metadata_batch *recv_trailing_metadata; + grpc_closure *recv_trailing_metadata_finished; + /** when the application requests writes be closed, the write_closed is 'queued'; when the close is flow controlled into the send path, we are 'sending' it; when the write has been performed it is 'sent' */ - grpc_chttp2_write_state write_state; - /** is this stream closed (boolean) */ + gpr_uint8 write_closed; + /** is this stream reading half-closed (boolean) */ gpr_uint8 read_closed; - /** has this stream been cancelled? (boolean) */ - gpr_uint8 cancelled; - grpc_status_code cancelled_status; - /** have we told the upper layer that this stream is cancelled? */ - gpr_uint8 published_cancelled; + /** is this stream finished closing (and reportably closed) */ + gpr_uint8 finished_close; /** is this stream in the stream map? (boolean) */ gpr_uint8 in_stream_map; - /** bitmask of GRPC_CHTTP2_WRITING_xxx above */ - gpr_uint8 writing_now; - /** has anything been written to this stream? */ - gpr_uint8 written_anything; - - /** stream state already published to the upper layer */ - grpc_stream_state published_state; - /** address to publish next stream state to */ - grpc_stream_state *publish_state; - /** pointer to sop buffer to fill in with new stream ops */ - grpc_stream_op_buffer *publish_sopb; - grpc_stream_op_buffer incoming_sopb; + /** has this stream seen an error? if 1, then pending incoming frames + can be thrown away */ + gpr_uint8 seen_error; - /** incoming metadata */ - grpc_chttp2_incoming_metadata_buffer incoming_metadata; - grpc_chttp2_incoming_metadata_live_op_buffer outstanding_metadata; + gpr_uint8 published_initial_metadata; + gpr_uint8 published_trailing_metadata; + gpr_uint8 faked_trailing_metadata; + + grpc_chttp2_incoming_metadata_buffer received_initial_metadata; + grpc_chttp2_incoming_metadata_buffer received_trailing_metadata; + + grpc_chttp2_incoming_frame_queue incoming_frames; } grpc_chttp2_stream_global; typedef struct { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ gpr_uint32 id; - /** sops that have passed flow control to be written */ - grpc_stream_op_buffer sopb; - /** how strongly should we indicate closure with the next write */ - grpc_chttp2_send_closed send_closed; + gpr_uint8 fetching; + gpr_uint8 sent_initial_metadata; + gpr_uint8 sent_message; + gpr_uint8 sent_trailing_metadata; + gpr_uint8 read_closed; + /** send this initial metadata */ + grpc_metadata_batch *send_initial_metadata; + grpc_byte_stream *send_message; + grpc_metadata_batch *send_trailing_metadata; + gpr_int64 outgoing_window; /** how much window should we announce? */ gpr_uint32 announce_window; + gpr_slice_buffer flow_controlled_buffer; + gpr_slice fetching_slice; + size_t stream_fetched; + grpc_closure finished_fetch; } grpc_chttp2_stream_writing; struct grpc_chttp2_stream_parsing { @@ -428,22 +444,29 @@ struct grpc_chttp2_stream_parsing { gpr_uint8 received_close; /** saw a rst_stream */ gpr_uint8 saw_rst_stream; - /** incoming_window has been reduced by this much during parsing */ - gpr_uint32 incoming_window_delta; + /** how many header frames have we received? */ + gpr_uint8 header_frames_received; + /** which metadata did we get (on this parse) */ + gpr_uint8 got_metadata_on_parse[2]; + /** should we raise the seen_error flag in transport_global */ + gpr_uint8 seen_error; /** window available for peer to send to us */ - gpr_uint32 incoming_window; + gpr_int64 incoming_window; /** parsing state for data frames */ grpc_chttp2_data_parser data_parser; /** reason give to rst_stream */ gpr_uint32 rst_stream_reason; - /* amount of window given */ - gpr_uint64 outgoing_window_update; + /** amount of window given */ + gpr_int64 outgoing_window; + /** number of bytes received - reset at end of parse thread execution */ + gpr_int64 received_bytes; /** incoming metadata */ - grpc_chttp2_incoming_metadata_buffer incoming_metadata; + grpc_chttp2_incoming_metadata_buffer metadata_buffer[2]; }; struct grpc_chttp2_stream { + grpc_stream_refcount *refcount; grpc_chttp2_stream_global global; grpc_chttp2_stream_writing writing; grpc_chttp2_stream_parsing parsing; @@ -466,7 +489,8 @@ struct grpc_chttp2_stream { /** Someone is unlocking the transport mutex: check to see if writes are required, and schedule them if so */ int grpc_chttp2_unlocking_check_writes(grpc_chttp2_transport_global *global, - grpc_chttp2_transport_writing *writing); + grpc_chttp2_transport_writing *writing, + int is_parsing); void grpc_chttp2_perform_writes( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, grpc_endpoint *endpoint); @@ -504,21 +528,10 @@ void grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); -void grpc_chttp2_list_add_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing); -void grpc_chttp2_list_remove_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); - -void grpc_chttp2_list_add_writing_stream( +/* returns 1 if stream added, 0 if it was already present */ +int grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing); + grpc_chttp2_stream_writing *stream_writing) GRPC_MUST_USE_RESULT; int grpc_chttp2_list_have_writing_streams( grpc_chttp2_transport_writing *transport_writing); int grpc_chttp2_list_pop_writing_stream( @@ -550,31 +563,44 @@ int grpc_chttp2_list_pop_waiting_for_concurrency( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); -void grpc_chttp2_list_add_closed_waiting_for_parsing( +void grpc_chttp2_list_add_check_read_ops( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_closed_waiting_for_parsing( +int grpc_chttp2_list_pop_check_read_ops( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); -void grpc_chttp2_list_add_cancelled_waiting_for_writing( +void grpc_chttp2_list_add_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, + grpc_chttp2_stream_writing *stream_writing); +int grpc_chttp2_list_pop_stalled_by_transport( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global **stream_global); + +void grpc_chttp2_list_add_unannounced_incoming_window_available( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_cancelled_waiting_for_writing( +void grpc_chttp2_list_remove_unannounced_incoming_window_available( grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); + grpc_chttp2_stream_global *stream_global); +int grpc_chttp2_list_pop_unannounced_incoming_window_available( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_transport_parsing *transport_parsing, + grpc_chttp2_stream_global **stream_global, + grpc_chttp2_stream_parsing **stream_parsing); -void grpc_chttp2_list_add_read_write_state_changed( +void grpc_chttp2_list_add_closed_waiting_for_parsing( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_read_write_state_changed( +int grpc_chttp2_list_pop_closed_waiting_for_parsing( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream( grpc_chttp2_transport_parsing *transport_parsing, gpr_uint32 id); grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( - grpc_chttp2_transport_parsing *transport_parsing, gpr_uint32 id); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + gpr_uint32 id); void grpc_chttp2_add_incoming_goaway( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, @@ -592,7 +618,10 @@ void grpc_chttp2_for_all_streams( grpc_chttp2_stream_global *stream_global)); void grpc_chttp2_parsing_become_skip_parser( - grpc_chttp2_transport_parsing *transport_parsing); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); + +void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, + grpc_closure **pclosure, int success); #define GRPC_CHTTP2_CLIENT_CONNECT_STRING "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \ @@ -607,26 +636,122 @@ extern int grpc_flowctl_trace; else \ stmt -#define GRPC_CHTTP2_FLOWCTL_TRACE_STREAM(reason, transport, context, var, \ - delta) \ - if (!(grpc_flowctl_trace)) { \ - } else { \ - grpc_chttp2_flowctl_trace(__FILE__, __LINE__, reason, #context, #var, \ - transport->is_client, context->id, \ - (gpr_int64)(context->var), (gpr_int64)(delta)); \ - } - -#define GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT(reason, context, var, delta) \ - if (!(grpc_flowctl_trace)) { \ - } else { \ - grpc_chttp2_flowctl_trace(__FILE__, __LINE__, reason, #context, #var, \ - context->is_client, 0, \ - (gpr_int64)(context->var), (gpr_int64)(delta)); \ - } - -void grpc_chttp2_flowctl_trace(const char *file, int line, const char *reason, - const char *context, const char *var, - int is_client, gpr_uint32 stream_id, - gpr_int64 current_value, gpr_int64 delta); +typedef enum { + GRPC_CHTTP2_FLOWCTL_MOVE, + GRPC_CHTTP2_FLOWCTL_CREDIT, + GRPC_CHTTP2_FLOWCTL_DEBIT +} grpc_chttp2_flowctl_op; + +#define GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, transport, id1, id2, dst_context, \ + dst_var, src_context, src_var) \ + do { \ + assert(id1 == id2); \ + if (grpc_flowctl_trace) { \ + grpc_chttp2_flowctl_trace( \ + __FILE__, __LINE__, phase, GRPC_CHTTP2_FLOWCTL_MOVE, #dst_context, \ + #dst_var, #src_context, #src_var, transport->is_client, id1, \ + dst_context->dst_var, src_context->src_var); \ + } \ + dst_context->dst_var += src_context->src_var; \ + src_context->src_var = 0; \ + } while (0) + +#define GRPC_CHTTP2_FLOW_MOVE_STREAM(phase, transport, dst_context, dst_var, \ + src_context, src_var) \ + GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, transport, dst_context->id, \ + src_context->id, dst_context, dst_var, \ + src_context, src_var) +#define GRPC_CHTTP2_FLOW_MOVE_TRANSPORT(phase, dst_context, dst_var, \ + src_context, src_var) \ + GRPC_CHTTP2_FLOW_MOVE_COMMON(phase, dst_context, 0, 0, dst_context, dst_var, \ + src_context, src_var) + +#define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context, \ + dst_var, amount) \ + do { \ + if (grpc_flowctl_trace) { \ + grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ + GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context, \ + #dst_var, NULL, #amount, transport->is_client, \ + id, dst_context->dst_var, amount); \ + } \ + dst_context->dst_var += amount; \ + } while (0) + +#define GRPC_CHTTP2_FLOW_CREDIT_STREAM(phase, transport, dst_context, dst_var, \ + amount) \ + GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, dst_context->id, \ + dst_context, dst_var, amount) +#define GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT(phase, dst_context, dst_var, amount) \ + GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, dst_context, 0, dst_context, dst_var, \ + amount) + +#define GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, id, dst_context, \ + dst_var, amount) \ + do { \ + if (grpc_flowctl_trace) { \ + grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ + GRPC_CHTTP2_FLOWCTL_DEBIT, #dst_context, \ + #dst_var, NULL, #amount, transport->is_client, \ + id, dst_context->dst_var, amount); \ + } \ + dst_context->dst_var -= amount; \ + } while (0) + +#define GRPC_CHTTP2_FLOW_DEBIT_STREAM(phase, transport, dst_context, dst_var, \ + amount) \ + GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, dst_context->id, \ + dst_context, dst_var, amount) +#define GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT(phase, dst_context, dst_var, amount) \ + GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, dst_context, 0, dst_context, dst_var, \ + amount) + +void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, + grpc_chttp2_flowctl_op op, const char *context1, + const char *var1, const char *context2, + const char *var2, int is_client, + gpr_uint32 stream_id, gpr_int64 val1, + gpr_int64 val2); + +void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream, + grpc_status_code status, gpr_slice *details); +void grpc_chttp2_mark_stream_closed( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global, int close_reads, + int close_writes); +void grpc_chttp2_start_writing(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global); + +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +#define GRPC_CHTTP2_STREAM_REF(stream_global, reason) \ + grpc_chttp2_stream_ref(stream_global, reason) +#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, reason) \ + grpc_chttp2_stream_unref(exec_ctx, stream_global, reason) +void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global, + const char *reason); +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_chttp2_stream_global *stream_global, + const char *reason); +#else +#define GRPC_CHTTP2_STREAM_REF(stream_global, reason) \ + grpc_chttp2_stream_ref(stream_global) +#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, reason) \ + grpc_chttp2_stream_unref(exec_ctx, stream_global) +void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global); +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_chttp2_stream_global *stream_global); +#endif + +grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + grpc_chttp2_stream_parsing *stream_parsing, gpr_uint32 frame_size, + gpr_uint32 flags, grpc_chttp2_incoming_frame_queue *add_to_queue); +void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, + grpc_chttp2_incoming_byte_stream *bs, + gpr_slice slice); +void grpc_chttp2_incoming_byte_stream_finished( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs); #endif diff --git a/src/core/transport/chttp2/parsing.c b/src/core/transport/chttp2/parsing.c index 5d4d8e70c4..1e06532c03 100644 --- a/src/core/transport/chttp2/parsing.c +++ b/src/core/transport/chttp2/parsing.c @@ -42,22 +42,28 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> -static int init_frame_parser(grpc_chttp2_transport_parsing *transport_parsing); +static int init_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_parsing *transport_parsing); static int init_header_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing, int is_continuation); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + int is_continuation); static int init_data_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); static int init_rst_stream_parser( - grpc_chttp2_transport_parsing *transport_parsing); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); static int init_settings_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); static int init_window_update_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing); -static int init_ping_parser(grpc_chttp2_transport_parsing *transport_parsing); -static int init_goaway_parser(grpc_chttp2_transport_parsing *transport_parsing); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); +static int init_ping_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_parsing *transport_parsing); +static int init_goaway_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_parsing *transport_parsing); static int init_skip_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing, int is_header); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + int is_header); static int parse_frame_slice(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, @@ -72,25 +78,16 @@ void grpc_chttp2_prepare_to_read( GPR_TIMER_BEGIN("grpc_chttp2_prepare_to_read", 0); transport_parsing->next_stream_id = transport_global->next_stream_id; + transport_parsing->last_sent_max_table_size = + transport_global->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]; /* update the parsing view of incoming window */ - if (transport_parsing->incoming_window != transport_global->incoming_window) { - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "parse", transport_parsing, incoming_window, - (gpr_int64)transport_global->incoming_window - - (gpr_int64)transport_parsing->incoming_window); - transport_parsing->incoming_window = transport_global->incoming_window; - } - while (grpc_chttp2_list_pop_incoming_window_updated( + while (grpc_chttp2_list_pop_unannounced_incoming_window_available( transport_global, transport_parsing, &stream_global, &stream_parsing)) { - stream_parsing->id = stream_global->id; - if (stream_parsing->incoming_window != stream_global->incoming_window) { - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "parse", transport_parsing, stream_parsing, incoming_window, - (gpr_int64)stream_global->incoming_window - - (gpr_int64)stream_parsing->incoming_window); - stream_parsing->incoming_window = stream_global->incoming_window; - } + GRPC_CHTTP2_FLOW_MOVE_STREAM("parse", transport_parsing, stream_parsing, + incoming_window, stream_global, + unannounced_incoming_window_for_parse); } GPR_TIMER_END("grpc_chttp2_prepare_to_read", 0); @@ -101,6 +98,8 @@ void grpc_chttp2_publish_reads( grpc_chttp2_transport_parsing *transport_parsing) { grpc_chttp2_stream_global *stream_global; grpc_chttp2_stream_parsing *stream_parsing; + int was_zero; + int is_zero; /* transport_parsing->last_incoming_stream_id is used as last-grpc_chttp2_stream-id when @@ -131,6 +130,7 @@ void grpc_chttp2_publish_reads( transport_global->settings[GRPC_SENT_SETTINGS], GRPC_CHTTP2_NUM_SETTINGS * sizeof(gpr_uint32)); transport_parsing->settings_ack_received = 0; + transport_global->sent_local_settings = 0; } /* move goaway to the global state if we received one (it will be @@ -144,98 +144,102 @@ void grpc_chttp2_publish_reads( } /* propagate flow control tokens to global state */ - if (transport_parsing->outgoing_window_update) { - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "parsed", transport_global, outgoing_window, - transport_parsing->outgoing_window_update); - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "parsed", transport_parsing, outgoing_window_update, - -(gpr_int64)transport_parsing->outgoing_window_update); - transport_global->outgoing_window += - transport_parsing->outgoing_window_update; - transport_parsing->outgoing_window_update = 0; - } - - if (transport_parsing->incoming_window_delta) { - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "parsed", transport_global, incoming_window, - -(gpr_int64)transport_parsing->incoming_window_delta); - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "parsed", transport_parsing, incoming_window_delta, - -(gpr_int64)transport_parsing->incoming_window_delta); - transport_global->incoming_window -= - transport_parsing->incoming_window_delta; - transport_parsing->incoming_window_delta = 0; + was_zero = transport_global->outgoing_window <= 0; + GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("parsed", transport_global, outgoing_window, + transport_parsing, outgoing_window); + is_zero = transport_global->outgoing_window <= 0; + if (was_zero && !is_zero) { + while (grpc_chttp2_list_pop_stalled_by_transport(transport_global, + &stream_global)) { + grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + } + } + + if (transport_parsing->incoming_window < + transport_global->connection_window_target * 3 / 4) { + gpr_int64 announce_bytes = transport_global->connection_window_target - + transport_parsing->incoming_window; + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_global, + announce_incoming_window, announce_bytes); + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_parsing, + incoming_window, announce_bytes); } /* for each stream that saw an update, fixup global state */ while (grpc_chttp2_list_pop_parsing_seen_stream( transport_global, transport_parsing, &stream_global, &stream_parsing)) { - /* update incoming flow control window */ - if (stream_parsing->incoming_window_delta) { - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "parsed", transport_parsing, stream_global, incoming_window, - -(gpr_int64)stream_parsing->incoming_window_delta); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "parsed", transport_parsing, stream_parsing, incoming_window_delta, - -(gpr_int64)stream_parsing->incoming_window_delta); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "parsed", transport_parsing, stream_global, max_recv_bytes, - -(gpr_int64)stream_parsing->incoming_window_delta); - stream_global->incoming_window -= stream_parsing->incoming_window_delta; - GPR_ASSERT(stream_global->max_recv_bytes >= - stream_parsing->incoming_window_delta); - stream_global->max_recv_bytes -= stream_parsing->incoming_window_delta; - stream_parsing->incoming_window_delta = 0; - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + if (stream_parsing->seen_error) { + stream_global->seen_error = 1; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } /* update outgoing flow control window */ - if (stream_parsing->outgoing_window_update) { - int was_zero = stream_global->outgoing_window <= 0; - int is_zero; - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("parsed", transport_parsing, - stream_global, outgoing_window, - stream_parsing->outgoing_window_update); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "parsed", transport_parsing, stream_parsing, outgoing_window_update, - -(gpr_int64)stream_parsing->outgoing_window_update); - GPR_ASSERT(stream_parsing->outgoing_window_update <= GPR_UINT32_MAX); - stream_global->outgoing_window += - (gpr_uint32)stream_parsing->outgoing_window_update; - stream_parsing->outgoing_window_update = 0; - is_zero = stream_global->outgoing_window <= 0; - if (was_zero && !is_zero) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); - } + was_zero = stream_global->outgoing_window <= 0; + GRPC_CHTTP2_FLOW_MOVE_STREAM("parsed", transport_global, stream_global, + outgoing_window, stream_parsing, + outgoing_window); + is_zero = stream_global->outgoing_window <= 0; + if (was_zero && !is_zero) { + grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } - /* updating closed status */ - if (stream_parsing->received_close) { - stream_global->read_closed = 1; - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + stream_global->max_recv_bytes -= (gpr_uint32)GPR_MIN( + stream_global->max_recv_bytes, stream_parsing->received_bytes); + stream_parsing->received_bytes = 0; + + /* publish incoming stream ops */ + if (stream_global->incoming_frames.tail != NULL) { + stream_global->incoming_frames.tail->is_tail = 0; + } + if (stream_parsing->data_parser.incoming_frames.head != NULL) { + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + grpc_chttp2_incoming_frame_queue_merge( + &stream_global->incoming_frames, + &stream_parsing->data_parser.incoming_frames); + if (stream_global->incoming_frames.tail != NULL) { + stream_global->incoming_frames.tail->is_tail = 1; } + + if (!stream_global->published_initial_metadata && + stream_parsing->got_metadata_on_parse[0]) { + stream_parsing->got_metadata_on_parse[0] = 0; + stream_global->published_initial_metadata = 1; + GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, + stream_parsing->metadata_buffer[0], + stream_global->received_initial_metadata); + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (!stream_global->published_trailing_metadata && + stream_parsing->got_metadata_on_parse[1]) { + stream_parsing->got_metadata_on_parse[1] = 0; + stream_global->published_trailing_metadata = 1; + GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, + stream_parsing->metadata_buffer[1], + stream_global->received_trailing_metadata); + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (stream_parsing->saw_rst_stream) { - stream_global->cancelled = 1; - stream_global->cancelled_status = grpc_chttp2_http2_error_to_grpc_status( - (grpc_chttp2_error_code)stream_parsing->rst_stream_reason); - if (stream_parsing->rst_stream_reason == GRPC_CHTTP2_NO_ERROR) { - stream_global->published_cancelled = 1; + if (stream_parsing->rst_stream_reason != GRPC_CHTTP2_NO_ERROR) { + grpc_status_code status_code = grpc_chttp2_http2_error_to_grpc_status( + (grpc_chttp2_error_code)stream_parsing->rst_stream_reason); + char *status_details; + gpr_slice slice_details; + gpr_asprintf(&status_details, "Received RST_STREAM err=%d", + stream_parsing->rst_stream_reason); + slice_details = gpr_slice_from_copied_string(status_details); + gpr_free(status_details); + grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, + status_code, &slice_details); } - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, + 1, 1); } - /* publish incoming stream ops */ - if (stream_parsing->data_parser.incoming_sopb.nops > 0) { - grpc_incoming_metadata_buffer_move_to_referencing_sopb( - &stream_parsing->incoming_metadata, &stream_global->incoming_metadata, - &stream_parsing->data_parser.incoming_sopb); - grpc_sopb_move_to(&stream_parsing->data_parser.incoming_sopb, - &stream_global->incoming_sopb); - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + if (stream_parsing->received_close) { + grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, + 1, 0); } } } @@ -363,7 +367,7 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx, GPR_ASSERT(cur < end); transport_parsing->incoming_stream_id |= ((gpr_uint32)*cur); transport_parsing->deframe_state = GRPC_DTS_FRAME; - if (!init_frame_parser(transport_parsing)) { + if (!init_frame_parser(exec_ctx, transport_parsing)) { return 0; } if (transport_parsing->incoming_stream_id) { @@ -428,7 +432,8 @@ int grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx, GPR_UNREACHABLE_CODE(return 0); } -static int init_frame_parser(grpc_chttp2_transport_parsing *transport_parsing) { +static int init_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_parsing *transport_parsing) { if (transport_parsing->expect_continuation_stream_id != 0) { if (transport_parsing->incoming_frame_type != GRPC_CHTTP2_FRAME_CONTINUATION) { @@ -445,30 +450,30 @@ static int init_frame_parser(grpc_chttp2_transport_parsing *transport_parsing) { transport_parsing->incoming_stream_id); return 0; } - return init_header_frame_parser(transport_parsing, 1); + return init_header_frame_parser(exec_ctx, transport_parsing, 1); } switch (transport_parsing->incoming_frame_type) { case GRPC_CHTTP2_FRAME_DATA: - return init_data_frame_parser(transport_parsing); + return init_data_frame_parser(exec_ctx, transport_parsing); case GRPC_CHTTP2_FRAME_HEADER: - return init_header_frame_parser(transport_parsing, 0); + return init_header_frame_parser(exec_ctx, transport_parsing, 0); case GRPC_CHTTP2_FRAME_CONTINUATION: gpr_log(GPR_ERROR, "Unexpected CONTINUATION frame"); return 0; case GRPC_CHTTP2_FRAME_RST_STREAM: - return init_rst_stream_parser(transport_parsing); + return init_rst_stream_parser(exec_ctx, transport_parsing); case GRPC_CHTTP2_FRAME_SETTINGS: - return init_settings_frame_parser(transport_parsing); + return init_settings_frame_parser(exec_ctx, transport_parsing); case GRPC_CHTTP2_FRAME_WINDOW_UPDATE: - return init_window_update_frame_parser(transport_parsing); + return init_window_update_frame_parser(exec_ctx, transport_parsing); case GRPC_CHTTP2_FRAME_PING: - return init_ping_parser(transport_parsing); + return init_ping_parser(exec_ctx, transport_parsing); case GRPC_CHTTP2_FRAME_GOAWAY: - return init_goaway_parser(transport_parsing); + return init_goaway_parser(exec_ctx, transport_parsing); default: gpr_log(GPR_ERROR, "Unknown frame type %02x", transport_parsing->incoming_frame_type); - return init_skip_frame_parser(transport_parsing, 0); + return init_skip_frame_parser(exec_ctx, transport_parsing, 0); } } @@ -482,7 +487,8 @@ static grpc_chttp2_parse_error skip_parser( static void skip_header(void *tp, grpc_mdelem *md) { GRPC_MDELEM_UNREF(md); } static int init_skip_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing, int is_header) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + int is_header) { if (is_header) { gpr_uint8 is_eoh = transport_parsing->expect_continuation_stream_id != 0; transport_parsing->parser = grpc_chttp2_header_parser_parse; @@ -499,65 +505,51 @@ static int init_skip_frame_parser( } void grpc_chttp2_parsing_become_skip_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { init_skip_frame_parser( - transport_parsing, + exec_ctx, transport_parsing, transport_parsing->parser == grpc_chttp2_header_parser_parse); } static grpc_chttp2_parse_error update_incoming_window( - grpc_chttp2_transport_parsing *transport_parsing, + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, grpc_chttp2_stream_parsing *stream_parsing) { - if (transport_parsing->incoming_frame_size > - transport_parsing->incoming_window) { + gpr_uint32 incoming_frame_size = transport_parsing->incoming_frame_size; + if (incoming_frame_size > transport_parsing->incoming_window) { gpr_log(GPR_ERROR, "frame of size %d overflows incoming window of %d", transport_parsing->incoming_frame_size, transport_parsing->incoming_window); return GRPC_CHTTP2_CONNECTION_ERROR; } - if (transport_parsing->incoming_frame_size > - stream_parsing->incoming_window) { + if (incoming_frame_size > stream_parsing->incoming_window) { gpr_log(GPR_ERROR, "frame of size %d overflows incoming window of %d", transport_parsing->incoming_frame_size, stream_parsing->incoming_window); return GRPC_CHTTP2_CONNECTION_ERROR; } - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "data", transport_parsing, incoming_window, - -(gpr_int64)transport_parsing->incoming_frame_size); - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT("data", transport_parsing, - incoming_window_delta, - transport_parsing->incoming_frame_size); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "data", transport_parsing, stream_parsing, incoming_window, - -(gpr_int64)transport_parsing->incoming_frame_size); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("data", transport_parsing, stream_parsing, - incoming_window_delta, - transport_parsing->incoming_frame_size); - - transport_parsing->incoming_window -= transport_parsing->incoming_frame_size; - transport_parsing->incoming_window_delta += - transport_parsing->incoming_frame_size; - stream_parsing->incoming_window -= transport_parsing->incoming_frame_size; - stream_parsing->incoming_window_delta += - transport_parsing->incoming_frame_size; + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", transport_parsing, incoming_window, + incoming_frame_size); + GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", transport_parsing, stream_parsing, + incoming_window, incoming_frame_size); + stream_parsing->received_bytes += incoming_frame_size; + grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); return GRPC_CHTTP2_PARSE_OK; } static int init_data_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { grpc_chttp2_stream_parsing *stream_parsing = grpc_chttp2_parsing_lookup_stream(transport_parsing, transport_parsing->incoming_stream_id); grpc_chttp2_parse_error err = GRPC_CHTTP2_PARSE_OK; if (!stream_parsing || stream_parsing->received_close) - return init_skip_frame_parser(transport_parsing, 0); + return init_skip_frame_parser(exec_ctx, transport_parsing, 0); if (err == GRPC_CHTTP2_PARSE_OK) { - err = update_incoming_window(transport_parsing, stream_parsing); + err = update_incoming_window(exec_ctx, transport_parsing, stream_parsing); } if (err == GRPC_CHTTP2_PARSE_OK) { err = grpc_chttp2_data_parser_begin_frame( @@ -577,7 +569,7 @@ static int init_data_frame_parser( &transport_parsing->qbuf, grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id, GRPC_CHTTP2_PROTOCOL_ERROR)); - return init_skip_frame_parser(transport_parsing, 0); + return init_skip_frame_parser(exec_ctx, transport_parsing, 0); case GRPC_CHTTP2_CONNECTION_ERROR: return 0; } @@ -586,11 +578,13 @@ static int init_data_frame_parser( static void free_timeout(void *p) { gpr_free(p); } -static void on_header(void *tp, grpc_mdelem *md) { +static void on_initial_header(void *tp, grpc_mdelem *md) { grpc_chttp2_transport_parsing *transport_parsing = tp; grpc_chttp2_stream_parsing *stream_parsing = transport_parsing->incoming_stream; + GPR_TIMER_BEGIN("on_initial_header", 0); + GPR_ASSERT(stream_parsing); GRPC_CHTTP2_IF_TRACING(gpr_log( @@ -598,6 +592,12 @@ static void on_header(void *tp, grpc_mdelem *md) { transport_parsing->is_client ? "CLI" : "SVR", grpc_mdstr_as_c_string(md->key), grpc_mdstr_as_c_string(md->value))); + if (md->key == transport_parsing->elem_grpc_status_ok->key && + md != transport_parsing->elem_grpc_status_ok) { + /* TODO(ctiller): check for a status like " 0" */ + stream_parsing->seen_error = 1; + } + if (md->key == transport_parsing->str_grpc_timeout) { gpr_timespec *cached_timeout = grpc_mdelem_get_user_data(md, free_timeout); if (!cached_timeout) { @@ -612,24 +612,57 @@ static void on_header(void *tp, grpc_mdelem *md) { grpc_mdelem_set_user_data(md, free_timeout, cached_timeout); } grpc_chttp2_incoming_metadata_buffer_set_deadline( - &stream_parsing->incoming_metadata, + &stream_parsing->metadata_buffer[0], gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout)); GRPC_MDELEM_UNREF(md); } else { - grpc_chttp2_incoming_metadata_buffer_add(&stream_parsing->incoming_metadata, - md); + grpc_chttp2_incoming_metadata_buffer_add( + &stream_parsing->metadata_buffer[0], md); } grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); + + GPR_TIMER_END("on_initial_header", 0); +} + +static void on_trailing_header(void *tp, grpc_mdelem *md) { + grpc_chttp2_transport_parsing *transport_parsing = tp; + grpc_chttp2_stream_parsing *stream_parsing = + transport_parsing->incoming_stream; + + GPR_TIMER_BEGIN("on_trailing_header", 0); + + GPR_ASSERT(stream_parsing); + + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_INFO, "HTTP:%d:TRL:%s: %s: %s", stream_parsing->id, + transport_parsing->is_client ? "CLI" : "SVR", + grpc_mdstr_as_c_string(md->key), grpc_mdstr_as_c_string(md->value))); + + if (md->key == transport_parsing->elem_grpc_status_ok->key && + md != transport_parsing->elem_grpc_status_ok) { + /* TODO(ctiller): check for a status like " 0" */ + stream_parsing->seen_error = 1; + } + + grpc_chttp2_incoming_metadata_buffer_add(&stream_parsing->metadata_buffer[1], + md); + + grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); + + GPR_TIMER_END("on_trailing_header", 0); } static int init_header_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing, int is_continuation) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + int is_continuation) { gpr_uint8 is_eoh = (transport_parsing->incoming_frame_flags & GRPC_CHTTP2_DATA_FLAG_END_HEADERS) != 0; int via_accept = 0; grpc_chttp2_stream_parsing *stream_parsing; + /* TODO(ctiller): when to increment header_frames_received? */ + if (is_eoh) { transport_parsing->expect_continuation_stream_id = 0; } else { @@ -649,7 +682,7 @@ static int init_header_frame_parser( if (is_continuation) { gpr_log(GPR_ERROR, "grpc_chttp2_stream disbanded before CONTINUATION received"); - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } if (transport_parsing->is_client) { if ((transport_parsing->incoming_stream_id & 1) && @@ -660,7 +693,7 @@ static int init_header_frame_parser( gpr_log(GPR_ERROR, "ignoring new grpc_chttp2_stream creation on client"); } - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } else if (transport_parsing->last_incoming_stream_id > transport_parsing->incoming_stream_id) { gpr_log(GPR_ERROR, @@ -669,19 +702,19 @@ static int init_header_frame_parser( "id=%d, new grpc_chttp2_stream id=%d", transport_parsing->last_incoming_stream_id, transport_parsing->incoming_stream_id); - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } else if ((transport_parsing->incoming_stream_id & 1) == 0) { gpr_log(GPR_ERROR, "ignoring grpc_chttp2_stream with non-client generated index %d", transport_parsing->incoming_stream_id); - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } stream_parsing = transport_parsing->incoming_stream = grpc_chttp2_parsing_accept_stream( - transport_parsing, transport_parsing->incoming_stream_id); + exec_ctx, transport_parsing, transport_parsing->incoming_stream_id); if (stream_parsing == NULL) { gpr_log(GPR_ERROR, "grpc_chttp2_stream not accepted"); - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } via_accept = 1; } else { @@ -691,11 +724,21 @@ static int init_header_frame_parser( if (stream_parsing->received_close) { gpr_log(GPR_ERROR, "skipping already closed grpc_chttp2_stream header"); transport_parsing->incoming_stream = NULL; - return init_skip_frame_parser(transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); } transport_parsing->parser = grpc_chttp2_header_parser_parse; transport_parsing->parser_data = &transport_parsing->hpack_parser; - transport_parsing->hpack_parser.on_header = on_header; + switch (stream_parsing->header_frames_received) { + case 0: + transport_parsing->hpack_parser.on_header = on_initial_header; + break; + case 1: + transport_parsing->hpack_parser.on_header = on_trailing_header; + break; + case 2: + gpr_log(GPR_ERROR, "too many header frames received"); + return init_skip_frame_parser(exec_ctx, transport_parsing, 1); + } transport_parsing->hpack_parser.on_header_user_data = transport_parsing; transport_parsing->hpack_parser.is_boundary = is_eoh; transport_parsing->hpack_parser.is_eof = @@ -708,7 +751,7 @@ static int init_header_frame_parser( } static int init_window_update_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_window_update_parser_begin_frame( &transport_parsing->simple.window_update, transport_parsing->incoming_frame_size, @@ -722,7 +765,8 @@ static int init_window_update_frame_parser( return ok; } -static int init_ping_parser(grpc_chttp2_transport_parsing *transport_parsing) { +static int init_ping_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_parsing *transport_parsing) { int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_ping_parser_begin_frame( &transport_parsing->simple.ping, transport_parsing->incoming_frame_size, @@ -733,7 +777,7 @@ static int init_ping_parser(grpc_chttp2_transport_parsing *transport_parsing) { } static int init_rst_stream_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_rst_stream_parser_begin_frame( &transport_parsing->simple.rst_stream, transport_parsing->incoming_frame_size, @@ -741,7 +785,7 @@ static int init_rst_stream_parser( transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream( transport_parsing, transport_parsing->incoming_stream_id); if (!transport_parsing->incoming_stream) { - return init_skip_frame_parser(transport_parsing, 0); + return init_skip_frame_parser(exec_ctx, transport_parsing, 0); } transport_parsing->parser = grpc_chttp2_rst_stream_parser_parse; transport_parsing->parser_data = &transport_parsing->simple.rst_stream; @@ -749,7 +793,7 @@ static int init_rst_stream_parser( } static int init_goaway_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { int ok = GRPC_CHTTP2_PARSE_OK == grpc_chttp2_goaway_parser_begin_frame( &transport_parsing->goaway_parser, transport_parsing->incoming_frame_size, @@ -760,7 +804,7 @@ static int init_goaway_parser( } static int init_settings_frame_parser( - grpc_chttp2_transport_parsing *transport_parsing) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { int ok; if (transport_parsing->incoming_stream_id != 0) { @@ -779,6 +823,9 @@ static int init_settings_frame_parser( } if (transport_parsing->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) { transport_parsing->settings_ack_received = 1; + grpc_chttp2_hptbl_set_max_bytes( + &transport_parsing->hpack_parser.table, + transport_parsing->last_sent_max_table_size); } transport_parsing->parser = grpc_chttp2_settings_parser_parse; transport_parsing->parser_data = &transport_parsing->simple.settings; @@ -806,7 +853,7 @@ static int parse_frame_slice(grpc_exec_ctx *exec_ctx, } return 1; case GRPC_CHTTP2_STREAM_ERROR: - grpc_chttp2_parsing_become_skip_parser(transport_parsing); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, transport_parsing); if (stream_parsing) { stream_parsing->saw_rst_stream = 1; stream_parsing->rst_stream_reason = GRPC_CHTTP2_PROTOCOL_ERROR; diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c index 781db7b0d6..a81ffcce79 100644 --- a/src/core/transport/chttp2/stream_lists.c +++ b/src/core/transport/chttp2/stream_lists.c @@ -142,12 +142,13 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, s->included[id] = 1; } -static void stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, - grpc_chttp2_stream_list_id id) { +static int stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_chttp2_stream_list_id id) { if (s->included[id]) { - return; + return 0; } stream_list_add_tail(t, s, id); + return 1; } /* wrappers for specializations */ @@ -192,12 +193,12 @@ void grpc_chttp2_list_remove_writable_stream( GRPC_CHTTP2_LIST_WRITABLE); } -void grpc_chttp2_list_add_writing_stream( +int grpc_chttp2_list_add_writing_stream( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing) { - stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_WRITING); + return stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), + STREAM_FROM_WRITING(stream_writing), + GRPC_CHTTP2_LIST_WRITING); } int grpc_chttp2_list_have_writing_streams( @@ -241,6 +242,40 @@ int grpc_chttp2_list_pop_written_stream( return r; } +void grpc_chttp2_list_add_unannounced_incoming_window_available( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global) { + GPR_ASSERT(stream_global->id != 0); + stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); +} + +void grpc_chttp2_list_remove_unannounced_incoming_window_available( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global) { + stream_list_maybe_remove( + TRANSPORT_FROM_GLOBAL(transport_global), + STREAM_FROM_GLOBAL(stream_global), + GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); +} + +int grpc_chttp2_list_pop_unannounced_incoming_window_available( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_transport_parsing *transport_parsing, + grpc_chttp2_stream_global **stream_global, + grpc_chttp2_stream_parsing **stream_parsing) { + grpc_chttp2_stream *stream; + int r = + stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, + GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); + if (r != 0) { + *stream_global = &stream->global; + *stream_parsing = &stream->parsing; + } + return r; +} + void grpc_chttp2_list_add_parsing_seen_stream( grpc_chttp2_transport_parsing *transport_parsing, grpc_chttp2_stream_parsing *stream_parsing) { @@ -284,91 +319,60 @@ int grpc_chttp2_list_pop_waiting_for_concurrency( return r; } -void grpc_chttp2_list_add_closed_waiting_for_parsing( +void grpc_chttp2_list_add_check_read_ops( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); + GRPC_CHTTP2_LIST_CHECK_READ_OPS); } -int grpc_chttp2_list_pop_closed_waiting_for_parsing( +int grpc_chttp2_list_pop_check_read_ops( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global) { grpc_chttp2_stream *stream; int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); + GRPC_CHTTP2_LIST_CHECK_READ_OPS); if (r != 0) { *stream_global = &stream->global; } return r; } -void grpc_chttp2_list_add_cancelled_waiting_for_writing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CANCELLED_WAITING_FOR_WRITING); +void grpc_chttp2_list_add_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, + grpc_chttp2_stream_writing *stream_writing) { + stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), + STREAM_FROM_WRITING(stream_writing), + GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } -int grpc_chttp2_list_pop_cancelled_waiting_for_writing( +int grpc_chttp2_list_pop_stalled_by_transport( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global) { grpc_chttp2_stream *stream; int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_CANCELLED_WAITING_FOR_WRITING); + GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); if (r != 0) { *stream_global = &stream->global; } return r; } -void grpc_chttp2_list_add_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_INCOMING_WINDOW_UPDATED); -} - -int grpc_chttp2_list_pop_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_INCOMING_WINDOW_UPDATED); - if (r != 0) { - *stream_global = &stream->global; - *stream_parsing = &stream->parsing; - } - return r; -} - -void grpc_chttp2_list_remove_incoming_window_updated( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_INCOMING_WINDOW_UPDATED); -} - -void grpc_chttp2_list_add_read_write_state_changed( +void grpc_chttp2_list_add_closed_waiting_for_parsing( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global) { stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_READ_WRITE_STATE_CHANGED); + GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); } -int grpc_chttp2_list_pop_read_write_state_changed( +int grpc_chttp2_list_pop_closed_waiting_for_parsing( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global) { grpc_chttp2_stream *stream; int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_READ_WRITE_STATE_CHANGED); + GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); if (r != 0) { *stream_global = &stream->global; } diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index 69ad8854ba..805d05222d 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -40,22 +40,28 @@ #include "src/core/profiling/timers.h" #include "src/core/transport/chttp2/http2_errors.h" -static void finalize_outbuf(grpc_chttp2_transport_writing *transport_writing); +static void finalize_outbuf(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_writing *transport_writing); int grpc_chttp2_unlocking_check_writes( grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing) { + grpc_chttp2_transport_writing *transport_writing, int is_parsing) { grpc_chttp2_stream_global *stream_global; grpc_chttp2_stream_writing *stream_writing; - grpc_chttp2_stream_global *first_reinserted_stream = NULL; - gpr_uint32 window_delta; + + GPR_TIMER_BEGIN("grpc_chttp2_unlocking_check_writes", 0); /* simple writes are queued to qbuf, and flushed here */ gpr_slice_buffer_swap(&transport_global->qbuf, &transport_writing->outbuf); GPR_ASSERT(transport_global->qbuf.count == 0); + grpc_chttp2_hpack_compressor_set_max_table_size( + &transport_writing->hpack_compressor, + transport_global->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); + if (transport_global->dirtied_local_settings && - !transport_global->sent_local_settings) { + !transport_global->sent_local_settings && !is_parsing) { gpr_slice_buffer_add( &transport_writing->outbuf, grpc_chttp2_settings_create( @@ -67,98 +73,103 @@ int grpc_chttp2_unlocking_check_writes( transport_global->sent_local_settings = 1; } + GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window, + transport_global, outgoing_window); + /* for each grpc_chttp2_stream that's become writable, frame it's data (according to available window sizes) and add to the output buffer */ while (grpc_chttp2_list_pop_writable_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { - if (stream_global == first_reinserted_stream) { - /* prevent infinite loop */ - grpc_chttp2_list_add_first_writable_stream(transport_global, - stream_global); - break; - } + gpr_uint8 sent_initial_metadata; stream_writing->id = stream_global->id; - stream_writing->send_closed = GRPC_DONT_SEND_CLOSED; - - if (stream_global->outgoing_sopb) { - window_delta = grpc_chttp2_preencode( - stream_global->outgoing_sopb->ops, - &stream_global->outgoing_sopb->nops, - (gpr_uint32)GPR_MIN(GPR_MIN(transport_global->outgoing_window, - stream_global->outgoing_window), - GPR_UINT32_MAX), - &stream_writing->sopb); - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT( - "write", transport_global, outgoing_window, -(gpr_int64)window_delta); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("write", transport_global, stream_global, - outgoing_window, - -(gpr_int64)window_delta); - transport_global->outgoing_window -= window_delta; - stream_global->outgoing_window -= window_delta; - - if (stream_global->write_state == GRPC_WRITE_STATE_QUEUED_CLOSE && - stream_global->outgoing_sopb->nops == 0) { - if (!transport_global->is_client && !stream_global->read_closed) { - stream_writing->send_closed = GRPC_SEND_CLOSED_WITH_RST_STREAM; + stream_writing->read_closed = stream_global->read_closed; + + GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_writing, stream_writing, + outgoing_window, stream_global, + outgoing_window); + + sent_initial_metadata = stream_writing->sent_initial_metadata; + if (!sent_initial_metadata && stream_global->send_initial_metadata) { + stream_writing->send_initial_metadata = + stream_global->send_initial_metadata; + stream_global->send_initial_metadata = NULL; + if (grpc_chttp2_list_add_writing_stream(transport_writing, + stream_writing)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); + } + sent_initial_metadata = 1; + } + if (sent_initial_metadata) { + if (stream_global->send_message != NULL) { + gpr_slice hdr = gpr_slice_malloc(5); + gpr_uint8 *p = GPR_SLICE_START_PTR(hdr); + gpr_uint32 len = stream_global->send_message->length; + GPR_ASSERT(stream_writing->send_message == NULL); + p[0] = (stream_global->send_message->flags & + GRPC_WRITE_INTERNAL_COMPRESS) != 0; + p[1] = (gpr_uint8)(len >> 24); + p[2] = (gpr_uint8)(len >> 16); + p[3] = (gpr_uint8)(len >> 8); + p[4] = (gpr_uint8)(len); + gpr_slice_buffer_add(&stream_writing->flow_controlled_buffer, hdr); + if (stream_global->send_message->length > 0) { + stream_writing->send_message = stream_global->send_message; } else { - stream_writing->send_closed = GRPC_SEND_CLOSED; + stream_writing->send_message = NULL; } + stream_writing->stream_fetched = 0; + stream_global->send_message = NULL; } - - if (stream_global->outgoing_window > 0 && - stream_global->outgoing_sopb->nops != 0) { - grpc_chttp2_list_add_writable_stream(transport_global, stream_global); - if (first_reinserted_stream == NULL && - transport_global->outgoing_window == 0) { - first_reinserted_stream = stream_global; + if ((stream_writing->send_message != NULL || + stream_writing->flow_controlled_buffer.length > 0) && + stream_writing->outgoing_window > 0) { + if (transport_writing->outgoing_window > 0) { + if (grpc_chttp2_list_add_writing_stream(transport_writing, + stream_writing)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); + } + } else { + grpc_chttp2_list_add_stalled_by_transport(transport_writing, + stream_writing); + } + } + if (stream_global->send_trailing_metadata) { + stream_writing->send_trailing_metadata = + stream_global->send_trailing_metadata; + stream_global->send_trailing_metadata = NULL; + if (grpc_chttp2_list_add_writing_stream(transport_writing, + stream_writing)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } } } if (!stream_global->read_closed && - stream_global->unannounced_incoming_window > 0) { - GPR_ASSERT(stream_writing->announce_window == 0); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "write", transport_writing, stream_writing, announce_window, - stream_global->unannounced_incoming_window); - stream_writing->announce_window = - stream_global->unannounced_incoming_window; - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "write", transport_global, stream_global, incoming_window, - stream_global->unannounced_incoming_window); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "write", transport_global, stream_global, unannounced_incoming_window, - -(gpr_int64)stream_global->unannounced_incoming_window); - stream_global->incoming_window += - stream_global->unannounced_incoming_window; - stream_global->unannounced_incoming_window = 0; - grpc_chttp2_list_add_incoming_window_updated(transport_global, - stream_global); - stream_global->writing_now |= GRPC_CHTTP2_WRITING_WINDOW; - } - if (stream_writing->sopb.nops > 0 || - stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) { - stream_global->writing_now |= GRPC_CHTTP2_WRITING_DATA; - } - if (stream_global->writing_now != 0) { - grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); + stream_global->unannounced_incoming_window_for_writing > 1024) { + GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_global, stream_writing, + announce_window, stream_global, + unannounced_incoming_window_for_writing); + if (grpc_chttp2_list_add_writing_stream(transport_writing, + stream_writing)) { + GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); + } } } /* if the grpc_chttp2_transport is ready to send a window update, do so here also; 3/4 is a magic number that will likely get tuned soon */ - if (transport_global->incoming_window < - transport_global->connection_window_target * 3 / 4) { - window_delta = transport_global->connection_window_target - - transport_global->incoming_window; + if (transport_global->announce_incoming_window > 0) { + gpr_uint32 announced = (gpr_uint32)GPR_MIN( + transport_global->announce_incoming_window, GPR_UINT32_MAX); + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", transport_global, + announce_incoming_window, announced); gpr_slice_buffer_add(&transport_writing->outbuf, - grpc_chttp2_window_update_create(0, window_delta)); - GRPC_CHTTP2_FLOWCTL_TRACE_TRANSPORT("write", transport_global, - incoming_window, window_delta); - transport_global->incoming_window += window_delta; + grpc_chttp2_window_update_create(0, announced)); } + GPR_TIMER_END("grpc_chttp2_unlocking_check_writes", 0); + return transport_writing->outbuf.count > 0 || grpc_chttp2_list_have_writing_streams(transport_writing); } @@ -169,47 +180,145 @@ void grpc_chttp2_perform_writes( GPR_ASSERT(transport_writing->outbuf.count > 0 || grpc_chttp2_list_have_writing_streams(transport_writing)); - finalize_outbuf(transport_writing); + finalize_outbuf(exec_ctx, transport_writing); - GPR_ASSERT(transport_writing->outbuf.count > 0); GPR_ASSERT(endpoint); - grpc_endpoint_write(exec_ctx, endpoint, &transport_writing->outbuf, - &transport_writing->done_cb); + if (transport_writing->outbuf.count > 0) { + grpc_endpoint_write(exec_ctx, endpoint, &transport_writing->outbuf, + &transport_writing->done_cb); + } else { + grpc_exec_ctx_enqueue(exec_ctx, &transport_writing->done_cb, 1); + } } -static void finalize_outbuf(grpc_chttp2_transport_writing *transport_writing) { +static void finalize_outbuf(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_writing *transport_writing) { grpc_chttp2_stream_writing *stream_writing; GPR_TIMER_BEGIN("finalize_outbuf", 0); while ( grpc_chttp2_list_pop_writing_stream(transport_writing, &stream_writing)) { - if (stream_writing->sopb.nops > 0 || - stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) { - grpc_chttp2_encode(stream_writing->sopb.ops, stream_writing->sopb.nops, - stream_writing->send_closed != GRPC_DONT_SEND_CLOSED, - stream_writing->id, - &transport_writing->hpack_compressor, - &transport_writing->outbuf); - stream_writing->sopb.nops = 0; + gpr_uint32 max_outgoing = + (gpr_uint32)GPR_MIN(GRPC_CHTTP2_MAX_PAYLOAD_LENGTH, + GPR_MIN(stream_writing->outgoing_window, + transport_writing->outgoing_window)); + /* send initial metadata if it's available */ + if (stream_writing->send_initial_metadata != NULL) { + grpc_chttp2_encode_header( + &transport_writing->hpack_compressor, stream_writing->id, + stream_writing->send_initial_metadata, 0, &transport_writing->outbuf); + stream_writing->send_initial_metadata = NULL; + stream_writing->sent_initial_metadata = 1; } - if (stream_writing->announce_window > 0) { + /* send any window updates */ + if (stream_writing->announce_window > 0 && + stream_writing->send_initial_metadata == NULL) { + gpr_uint32 announce = stream_writing->announce_window; gpr_slice_buffer_add( &transport_writing->outbuf, grpc_chttp2_window_update_create(stream_writing->id, stream_writing->announce_window)); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "write", transport_writing, stream_writing, announce_window, - -(gpr_int64)stream_writing->announce_window); + GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing, stream_writing, + announce_window, announce); stream_writing->announce_window = 0; } - if (stream_writing->send_closed == GRPC_SEND_CLOSED_WITH_RST_STREAM) { - gpr_slice_buffer_add(&transport_writing->outbuf, - grpc_chttp2_rst_stream_create(stream_writing->id, - GRPC_CHTTP2_NO_ERROR)); + /* fetch any body bytes */ + while (!stream_writing->fetching && stream_writing->send_message && + stream_writing->flow_controlled_buffer.length < max_outgoing && + stream_writing->stream_fetched < + stream_writing->send_message->length) { + if (grpc_byte_stream_next(exec_ctx, stream_writing->send_message, + &stream_writing->fetching_slice, max_outgoing, + &stream_writing->finished_fetch)) { + stream_writing->stream_fetched += + GPR_SLICE_LENGTH(stream_writing->fetching_slice); + if (stream_writing->stream_fetched == + stream_writing->send_message->length) { + stream_writing->send_message = NULL; + } + gpr_slice_buffer_add(&stream_writing->flow_controlled_buffer, + stream_writing->fetching_slice); + } else { + stream_writing->fetching = 1; + } + } + /* send any body bytes */ + if (stream_writing->flow_controlled_buffer.length > 0) { + if (max_outgoing > 0) { + gpr_uint32 send_bytes = (gpr_uint32)GPR_MIN( + max_outgoing, stream_writing->flow_controlled_buffer.length); + int is_last_data_frame = + stream_writing->send_message == NULL && + send_bytes == stream_writing->flow_controlled_buffer.length; + int is_last_frame = is_last_data_frame && + stream_writing->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty( + stream_writing->send_trailing_metadata); + grpc_chttp2_encode_data( + stream_writing->id, &stream_writing->flow_controlled_buffer, + send_bytes, is_last_frame, &transport_writing->outbuf); + GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing, + stream_writing, outgoing_window, + send_bytes); + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", transport_writing, + outgoing_window, send_bytes); + if (is_last_frame) { + stream_writing->send_trailing_metadata = NULL; + stream_writing->sent_trailing_metadata = 1; + } + if (is_last_data_frame) { + GPR_ASSERT(stream_writing->send_message == NULL); + stream_writing->sent_message = 1; + } + } else if (transport_writing->outgoing_window == 0) { + grpc_chttp2_list_add_stalled_by_transport(transport_writing, + stream_writing); + grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); + } + } + /* send trailing metadata if it's available and we're ready for it */ + if (stream_writing->send_message == NULL && + stream_writing->flow_controlled_buffer.length == 0 && + stream_writing->send_trailing_metadata != NULL) { + if (grpc_metadata_batch_is_empty( + stream_writing->send_trailing_metadata)) { + grpc_chttp2_encode_data(stream_writing->id, + &stream_writing->flow_controlled_buffer, 0, 1, + &transport_writing->outbuf); + } else { + grpc_chttp2_encode_header(&transport_writing->hpack_compressor, + stream_writing->id, + stream_writing->send_trailing_metadata, 1, + &transport_writing->outbuf); + } + if (!transport_writing->is_client && !stream_writing->read_closed) { + gpr_slice_buffer_add(&transport_writing->outbuf, + grpc_chttp2_rst_stream_create( + stream_writing->id, GRPC_CHTTP2_NO_ERROR)); + } + stream_writing->send_trailing_metadata = NULL; + stream_writing->sent_trailing_metadata = 1; + } + /* if there's more to write, then loop, otherwise prepare to finish the + * write */ + if ((stream_writing->flow_controlled_buffer.length > 0 || + (stream_writing->send_message && !stream_writing->fetching)) && + stream_writing->outgoing_window > 0) { + if (transport_writing->outgoing_window > 0) { + if (grpc_chttp2_list_add_writing_stream(transport_writing, + stream_writing)) { + /* do nothing - already reffed */ + } + } else { + grpc_chttp2_list_add_stalled_by_transport(transport_writing, + stream_writing); + grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); + } + } else { + grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); } - grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); } GPR_TIMER_END("finalize_outbuf", 0); @@ -223,24 +332,26 @@ void grpc_chttp2_cleanup_writing( while (grpc_chttp2_list_pop_written_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { - GPR_ASSERT(stream_global->writing_now != 0); - if (stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) { - stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE; - if (!transport_global->is_client) { - stream_global->read_closed = 1; - } + if (stream_writing->sent_trailing_metadata) { + grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, + !transport_global->is_client, 1); } - if (stream_global->writing_now & GRPC_CHTTP2_WRITING_DATA) { - if (stream_global->outgoing_sopb != NULL && - stream_global->outgoing_sopb->nops == 0) { - GPR_ASSERT(stream_global->write_state != GRPC_WRITE_STATE_QUEUED_CLOSE); - stream_global->outgoing_sopb = NULL; - grpc_exec_ctx_enqueue(exec_ctx, stream_global->send_done_closure, 1); - } + if (stream_writing->sent_initial_metadata) { + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_initial_metadata_finished, 1); + } + if (stream_writing->sent_message) { + GPR_ASSERT(stream_writing->send_message == NULL); + GPR_ASSERT(stream_global->send_message_finished); + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_message_finished, 1); + stream_writing->sent_message = 0; + } + if (stream_writing->sent_trailing_metadata) { + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_trailing_metadata_finished, 1); } - stream_global->writing_now = 0; - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); } gpr_slice_buffer_reset_and_unref(&transport_writing->outbuf); } diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index effc3c4b3b..77c68c9df0 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -75,14 +75,14 @@ int grpc_flowctl_trace = 0; #define STREAM_FROM_GLOBAL(sg) \ ((grpc_chttp2_stream *)((char *)(sg)-offsetof(grpc_chttp2_stream, global))) +#define STREAM_FROM_PARSING(sg) \ + ((grpc_chttp2_stream *)((char *)(sg)-offsetof(grpc_chttp2_stream, parsing))) + static const grpc_transport_vtable vtable; static void lock(grpc_chttp2_transport *t); static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); -static void unlock_check_read_write_state(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t); - /* forward declarations of various callbacks that we'll build closures around */ static void writing_action(grpc_exec_ctx *exec_ctx, void *t, int iomgr_success_ignored); @@ -103,11 +103,13 @@ static void perform_stream_op_locked( grpc_chttp2_stream_global *stream_global, grpc_transport_stream_op *op); /** Cancel a stream: coming from the transport API */ -static void cancel_from_api(grpc_chttp2_transport_global *transport_global, +static void cancel_from_api(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global, grpc_status_code status); -static void close_from_api(grpc_chttp2_transport_global *transport_global, +static void close_from_api(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global, grpc_status_code status, gpr_slice *optional_message); @@ -128,6 +130,9 @@ static void connectivity_state_set( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, grpc_connectivity_state state, const char *reason); +static void check_read_ops(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global); + /* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -151,6 +156,7 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_goaway_parser_destroy(&t->parsing.goaway_parser); GRPC_MDSTR_UNREF(t->parsing.str_grpc_timeout); + GRPC_MDELEM_UNREF(t->parsing.elem_grpc_status_ok); for (i = 0; i < STREAM_LIST_COUNT; i++) { GPR_ASSERT(t->lists[i].head == NULL); @@ -236,14 +242,17 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->endpoint_reading = 1; t->global.next_stream_id = is_client ? 1 : 2; t->global.is_client = is_client; - t->global.outgoing_window = DEFAULT_WINDOW; - t->global.incoming_window = DEFAULT_WINDOW; + t->writing.outgoing_window = DEFAULT_WINDOW; + t->parsing.incoming_window = DEFAULT_WINDOW; + t->global.stream_lookahead = DEFAULT_WINDOW; t->global.connection_window_target = DEFAULT_CONNECTION_WINDOW_TARGET; t->global.ping_counter = 1; t->global.pings.next = t->global.pings.prev = &t->global.pings; t->parsing.is_client = is_client; t->parsing.str_grpc_timeout = grpc_mdstr_from_string(t->metadata_context, "grpc-timeout"); + t->parsing.elem_grpc_status_ok = + grpc_mdelem_from_strings(t->metadata_context, "grpc-status", "0"); t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->writing.is_client = is_client; @@ -329,6 +338,43 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->global.next_stream_id = (gpr_uint32)channel_args->args[i].value.integer; } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES)) { + if (channel_args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s: must be an integer", + GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES); + } else if (channel_args->args[i].value.integer <= 5) { + gpr_log(GPR_ERROR, "%s: must be at least 5", + GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES); + } else { + t->global.stream_lookahead = + (gpr_uint32)channel_args->args[i].value.integer; + } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER)) { + if (channel_args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s: must be an integer", + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER); + } else if (channel_args->args[i].value.integer < 0) { + gpr_log(GPR_DEBUG, "%s: must be non-negative", + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER); + } else { + push_setting(t, GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, + (gpr_uint32)channel_args->args[i].value.integer); + } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER)) { + if (channel_args->args[i].type != GRPC_ARG_INTEGER) { + gpr_log(GPR_ERROR, "%s: must be an integer", + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER); + } else if (channel_args->args[i].value.integer < 0) { + gpr_log(GPR_DEBUG, "%s: must be non-negative", + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER); + } else { + grpc_chttp2_hpack_compressor_set_max_usable_size( + &t->writing.hpack_compressor, + (gpr_uint32)channel_args->args[i].value.integer); + } } } } @@ -392,19 +438,46 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, } } +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global, + const char *reason) { + grpc_stream_ref(STREAM_FROM_GLOBAL(stream_global)->refcount, reason); +} +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_chttp2_stream_global *stream_global, + const char *reason) { + grpc_stream_unref(exec_ctx, STREAM_FROM_GLOBAL(stream_global)->refcount, + reason); +} +#else +void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global) { + grpc_stream_ref(STREAM_FROM_GLOBAL(stream_global)->refcount); +} +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_chttp2_stream_global *stream_global) { + grpc_stream_unref(exec_ctx, STREAM_FROM_GLOBAL(stream_global)->refcount); +} +#endif + static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, const void *server_data, - grpc_transport_stream_op *initial_op) { + grpc_stream *gs, grpc_stream_refcount *refcount, + const void *server_data) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; memset(s, 0, sizeof(*s)); - grpc_chttp2_incoming_metadata_buffer_init(&s->parsing.incoming_metadata); - grpc_chttp2_incoming_metadata_buffer_init(&s->global.incoming_metadata); - grpc_sopb_init(&s->writing.sopb); - grpc_sopb_init(&s->global.incoming_sopb); + s->refcount = refcount; + GRPC_CHTTP2_STREAM_REF(&s->global, "chttp2"); + + grpc_chttp2_incoming_metadata_buffer_init(&s->parsing.metadata_buffer[0]); + grpc_chttp2_incoming_metadata_buffer_init(&s->parsing.metadata_buffer[1]); + grpc_chttp2_incoming_metadata_buffer_init( + &s->global.received_initial_metadata); + grpc_chttp2_incoming_metadata_buffer_init( + &s->global.received_trailing_metadata); grpc_chttp2_data_parser_init(&s->parsing.data_parser); + gpr_slice_buffer_init(&s->writing.flow_controlled_buffer); REF_TRANSPORT(t, "stream"); @@ -413,20 +486,17 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (server_data) { GPR_ASSERT(t->parsing_active); s->global.id = (gpr_uint32)(gpr_uintptr)server_data; + s->parsing.id = s->global.id; s->global.outgoing_window = t->global.settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - s->global.max_recv_bytes = s->parsing.incoming_window = - s->global.incoming_window = - t->global.settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + s->parsing.incoming_window = s->global.max_recv_bytes = + t->global.settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; *t->accepting_stream = s; grpc_chttp2_stream_map_add(&t->parsing_stream_map, s->global.id, s); s->global.in_stream_map = 1; } - - if (initial_op) - perform_stream_op_locked(exec_ctx, &t->global, &s->global, initial_op); unlock(exec_ctx, t); return 0; @@ -437,10 +507,13 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; int i; + grpc_byte_stream *bs; + + GPR_TIMER_BEGIN("destroy_stream", 0); gpr_mu_lock(&t->mu); - GPR_ASSERT(s->global.published_state == GRPC_STREAM_CLOSED || + GPR_ASSERT((s->global.write_closed && s->global.read_closed) || s->global.id == 0); GPR_ASSERT(!s->global.in_stream_map); if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { @@ -451,8 +524,9 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->global.id) == NULL); } - grpc_chttp2_list_remove_incoming_window_updated(&t->global, &s->global); grpc_chttp2_list_remove_writable_stream(&t->global, &s->global); + grpc_chttp2_list_remove_unannounced_incoming_window_available(&t->global, + &s->global); gpr_mu_unlock(&t->mu); @@ -464,17 +538,29 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } } - GPR_ASSERT(s->global.outgoing_sopb == NULL); - GPR_ASSERT(s->global.publish_sopb == NULL); - grpc_sopb_destroy(&s->writing.sopb); - grpc_sopb_destroy(&s->global.incoming_sopb); - grpc_chttp2_data_parser_destroy(&s->parsing.data_parser); - grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.incoming_metadata); - grpc_chttp2_incoming_metadata_buffer_destroy(&s->global.incoming_metadata); - grpc_chttp2_incoming_metadata_live_op_buffer_end( - &s->global.outstanding_metadata); + while ( + (bs = grpc_chttp2_incoming_frame_queue_pop(&s->global.incoming_frames))) { + grpc_byte_stream_destroy(bs); + } + + GPR_ASSERT(s->global.send_initial_metadata_finished == NULL); + GPR_ASSERT(s->global.send_message_finished == NULL); + GPR_ASSERT(s->global.send_trailing_metadata_finished == NULL); + GPR_ASSERT(s->global.recv_initial_metadata_finished == NULL); + GPR_ASSERT(s->global.recv_message_ready == NULL); + GPR_ASSERT(s->global.recv_trailing_metadata_finished == NULL); + grpc_chttp2_data_parser_destroy(exec_ctx, &s->parsing.data_parser); + grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.metadata_buffer[0]); + grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.metadata_buffer[1]); + grpc_chttp2_incoming_metadata_buffer_destroy( + &s->global.received_initial_metadata); + grpc_chttp2_incoming_metadata_buffer_destroy( + &s->global.received_trailing_metadata); + gpr_slice_buffer_destroy(&s->writing.flow_controlled_buffer); UNREF_TRANSPORT(exec_ctx, t, "stream"); + + GPR_TIMER_END("destroy_stream", 0); } grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream( @@ -486,12 +572,14 @@ grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream( } grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( - grpc_chttp2_transport_parsing *transport_parsing, gpr_uint32 id) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + gpr_uint32 id) { grpc_chttp2_stream *accepting; grpc_chttp2_transport *t = TRANSPORT_FROM_PARSING(transport_parsing); GPR_ASSERT(t->accepting_stream == NULL); t->accepting_stream = &accepting; - t->channel_callback.accept_stream(t->channel_callback.accept_stream_user_data, + t->channel_callback.accept_stream(exec_ctx, + t->channel_callback.accept_stream_user_data, &t->base, (void *)(gpr_uintptr)id); t->accepting_stream = NULL; return &accepting->parsing; @@ -511,14 +599,15 @@ static void lock(grpc_chttp2_transport *t) { gpr_mu_lock(&t->mu); } static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { GPR_TIMER_BEGIN("unlock", 0); - unlock_check_read_write_state(exec_ctx, t); if (!t->writing_active && !t->closed && - grpc_chttp2_unlocking_check_writes(&t->global, &t->writing)) { + grpc_chttp2_unlocking_check_writes(&t->global, &t->writing, + t->parsing_active)) { t->writing_active = 1; REF_TRANSPORT(t, "writing"); grpc_exec_ctx_enqueue(exec_ctx, &t->writing_action, 1); prevent_endpoint_shutdown(t); } + check_read_ops(exec_ctx, &t->global); gpr_mu_unlock(&t->mu); GPR_TIMER_END("unlock", 0); @@ -558,7 +647,6 @@ void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx, drop_connection(exec_ctx, t); } - /* cleanup writing related jazz */ grpc_chttp2_cleanup_writing(exec_ctx, &t->global, &t->writing); /* leave the writing flag up on shutdown to prevent further writes in unlock() @@ -598,6 +686,7 @@ void grpc_chttp2_add_incoming_goaway( static void maybe_start_some_streams( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global) { grpc_chttp2_stream_global *stream_global; + gpr_uint32 stream_incoming_window; /* start streams where we have free grpc_chttp2_stream ids and free * concurrency */ while (transport_global->next_stream_id <= MAX_CLIENT_STREAM_ID && @@ -607,13 +696,16 @@ static void maybe_start_some_streams( [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS] && grpc_chttp2_list_pop_waiting_for_concurrency(transport_global, &stream_global)) { + /* safe since we can't (legally) be parsing this stream yet */ + grpc_chttp2_stream_parsing *stream_parsing = + &STREAM_FROM_GLOBAL(stream_global)->parsing; GRPC_CHTTP2_IF_TRACING(gpr_log( GPR_DEBUG, "HTTP:%s: Allocating new grpc_chttp2_stream %p to id %d", transport_global->is_client ? "CLI" : "SVR", stream_global, transport_global->next_stream_id)); GPR_ASSERT(stream_global->id == 0); - stream_global->id = transport_global->next_stream_id; + stream_global->id = stream_parsing->id = transport_global->next_stream_id; transport_global->next_stream_id += 2; if (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID) { @@ -625,103 +717,172 @@ static void maybe_start_some_streams( stream_global->outgoing_window = transport_global->settings[GRPC_PEER_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - stream_global->incoming_window = + stream_parsing->incoming_window = stream_incoming_window = transport_global->settings[GRPC_SENT_SETTINGS] [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; stream_global->max_recv_bytes = - GPR_MAX(stream_global->incoming_window, stream_global->max_recv_bytes); + GPR_MAX(stream_incoming_window, stream_global->max_recv_bytes); grpc_chttp2_stream_map_add( &TRANSPORT_FROM_GLOBAL(transport_global)->new_stream_map, stream_global->id, STREAM_FROM_GLOBAL(stream_global)); stream_global->in_stream_map = 1; transport_global->concurrent_stream_count++; - grpc_chttp2_list_add_incoming_window_updated(transport_global, - stream_global); grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } /* cancel out streams that will never be started */ while (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID && grpc_chttp2_list_pop_waiting_for_concurrency(transport_global, &stream_global)) { - cancel_from_api(transport_global, stream_global, GRPC_STATUS_UNAVAILABLE); + cancel_from_api(exec_ctx, transport_global, stream_global, + GRPC_STATUS_UNAVAILABLE); + } +} + +static grpc_closure *add_closure_barrier(grpc_closure *closure) { + closure->final_data += 2; + return closure; +} + +void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, + grpc_closure **pclosure, int success) { + grpc_closure *closure = *pclosure; + if (closure == NULL) { + return; + } + closure->final_data -= 2; + if (!success) { + closure->final_data |= 1; + } + if (closure->final_data < 2) { + grpc_exec_ctx_enqueue(exec_ctx, closure, closure->final_data == 0); + } + *pclosure = NULL; +} + +static int contains_non_ok_status( + grpc_chttp2_transport_global *transport_global, + grpc_metadata_batch *batch) { + grpc_mdelem *ok_elem = + TRANSPORT_FROM_GLOBAL(transport_global)->parsing.elem_grpc_status_ok; + grpc_linked_mdelem *l; + for (l = batch->list.head; l; l = l->next) { + if (l->md->key == ok_elem->key && l->md != ok_elem) { + return 1; + } } + return 0; } +static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, int success) {} + static void perform_stream_op_locked( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global, grpc_transport_stream_op *op) { + grpc_closure *on_complete; + GPR_TIMER_BEGIN("perform_stream_op_locked", 0); + + on_complete = op->on_complete; + if (on_complete == NULL) { + on_complete = grpc_closure_create(do_nothing, NULL); + } + /* use final_data as a barrier until enqueue time; the inital counter is + dropped at the end of this function */ + on_complete->final_data = 2; + if (op->cancel_with_status != GRPC_STATUS_OK) { - cancel_from_api(transport_global, stream_global, op->cancel_with_status); + cancel_from_api(exec_ctx, transport_global, stream_global, + op->cancel_with_status); } if (op->close_with_status != GRPC_STATUS_OK) { - close_from_api(transport_global, stream_global, op->close_with_status, - op->optional_close_message); - } - - if (op->send_ops) { - GPR_ASSERT(stream_global->outgoing_sopb == NULL); - stream_global->send_done_closure = op->on_done_send; - if (!stream_global->cancelled) { - stream_global->written_anything = 1; - stream_global->outgoing_sopb = op->send_ops; - if (op->is_last_send && - stream_global->write_state == GRPC_WRITE_STATE_OPEN) { - stream_global->write_state = GRPC_WRITE_STATE_QUEUED_CLOSE; - } - if (stream_global->id == 0) { - GRPC_CHTTP2_IF_TRACING(gpr_log( - GPR_DEBUG, - "HTTP:%s: New grpc_chttp2_stream %p waiting for concurrency", - transport_global->is_client ? "CLI" : "SVR", stream_global)); + close_from_api(exec_ctx, transport_global, stream_global, + op->close_with_status, op->optional_close_message); + } + + if (op->send_initial_metadata != NULL) { + GPR_ASSERT(stream_global->send_initial_metadata_finished == NULL); + stream_global->send_initial_metadata_finished = + add_closure_barrier(on_complete); + stream_global->send_initial_metadata = op->send_initial_metadata; + if (contains_non_ok_status(transport_global, op->send_initial_metadata)) { + stream_global->seen_error = 1; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (!stream_global->write_closed) { + if (transport_global->is_client) { + GPR_ASSERT(stream_global->id == 0); grpc_chttp2_list_add_waiting_for_concurrency(transport_global, stream_global); maybe_start_some_streams(exec_ctx, transport_global); - } else if (stream_global->outgoing_window > 0) { + } else { + GPR_ASSERT(stream_global->id != 0); grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } } else { - grpc_sopb_reset(op->send_ops); - grpc_exec_ctx_enqueue(exec_ctx, stream_global->send_done_closure, 0); + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_initial_metadata_finished, 0); + } + } + + if (op->send_message != NULL) { + GPR_ASSERT(stream_global->send_message_finished == NULL); + GPR_ASSERT(stream_global->send_message == NULL); + stream_global->send_message_finished = add_closure_barrier(on_complete); + if (stream_global->write_closed) { + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_message_finished, 0); + } else if (stream_global->id != 0) { + stream_global->send_message = op->send_message; + grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } } - if (op->recv_ops) { - GPR_ASSERT(stream_global->publish_sopb == NULL); - GPR_ASSERT(stream_global->published_state != GRPC_STREAM_CLOSED); - stream_global->recv_done_closure = op->on_done_recv; - stream_global->publish_sopb = op->recv_ops; - stream_global->publish_sopb->nops = 0; - stream_global->publish_state = op->recv_state; - /* clamp max recv bytes */ - op->max_recv_bytes = GPR_MIN(op->max_recv_bytes, GPR_UINT32_MAX); - if (stream_global->max_recv_bytes < op->max_recv_bytes) { - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "op", transport_global, stream_global, max_recv_bytes, - op->max_recv_bytes - stream_global->max_recv_bytes); - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM( - "op", transport_global, stream_global, unannounced_incoming_window, - op->max_recv_bytes - stream_global->max_recv_bytes); - stream_global->unannounced_incoming_window += - (gpr_uint32)op->max_recv_bytes - stream_global->max_recv_bytes; - stream_global->max_recv_bytes = (gpr_uint32)op->max_recv_bytes; + if (op->send_trailing_metadata != NULL) { + GPR_ASSERT(stream_global->send_trailing_metadata_finished == NULL); + stream_global->send_trailing_metadata_finished = + add_closure_barrier(on_complete); + stream_global->send_trailing_metadata = op->send_trailing_metadata; + if (contains_non_ok_status(transport_global, op->send_trailing_metadata)) { + stream_global->seen_error = 1; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } - grpc_chttp2_incoming_metadata_live_op_buffer_end( - &stream_global->outstanding_metadata); - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); - if (stream_global->id != 0) { + if (stream_global->write_closed) { + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->send_trailing_metadata_finished, + grpc_metadata_batch_is_empty(op->send_trailing_metadata)); + } else if (stream_global->id != 0) { + /* TODO(ctiller): check if there's flow control for any outstanding + bytes before going writable */ grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } } - if (op->bind_pollset) { - add_to_pollset_locked(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global), - op->bind_pollset); + if (op->recv_initial_metadata != NULL) { + GPR_ASSERT(stream_global->recv_initial_metadata_finished == NULL); + stream_global->recv_initial_metadata_finished = + add_closure_barrier(on_complete); + stream_global->recv_initial_metadata = op->recv_initial_metadata; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } - grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, 1); + if (op->recv_message != NULL) { + GPR_ASSERT(stream_global->recv_message_ready == NULL); + stream_global->recv_message_ready = op->recv_message_ready; + stream_global->recv_message = op->recv_message; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + + if (op->recv_trailing_metadata != NULL) { + GPR_ASSERT(stream_global->recv_trailing_metadata_finished == NULL); + stream_global->recv_trailing_metadata_finished = + add_closure_barrier(on_complete); + stream_global->recv_trailing_metadata = op->recv_trailing_metadata; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + + grpc_chttp2_complete_closure_step(exec_ctx, &on_complete, 1); + GPR_TIMER_END("perform_stream_op_locked", 0); } @@ -811,12 +972,49 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, * INPUT PROCESSING */ -static grpc_stream_state compute_state(gpr_uint8 write_closed, - gpr_uint8 read_closed) { - if (write_closed && read_closed) return GRPC_STREAM_CLOSED; - if (write_closed) return GRPC_STREAM_SEND_CLOSED; - if (read_closed) return GRPC_STREAM_RECV_CLOSED; - return GRPC_STREAM_OPEN; +static void check_read_ops(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global) { + grpc_chttp2_stream_global *stream_global; + grpc_byte_stream *bs; + while ( + grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) { + if (stream_global->recv_initial_metadata_finished != NULL && + stream_global->published_initial_metadata) { + grpc_chttp2_incoming_metadata_buffer_publish( + &stream_global->received_initial_metadata, + stream_global->recv_initial_metadata); + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->recv_initial_metadata_finished, 1); + } + if (stream_global->recv_message_ready != NULL) { + if (stream_global->incoming_frames.head != NULL) { + *stream_global->recv_message = grpc_chttp2_incoming_frame_queue_pop( + &stream_global->incoming_frames); + GPR_ASSERT(*stream_global->recv_message != NULL); + grpc_exec_ctx_enqueue(exec_ctx, stream_global->recv_message_ready, 1); + stream_global->recv_message_ready = NULL; + } else if (stream_global->published_trailing_metadata) { + *stream_global->recv_message = NULL; + grpc_exec_ctx_enqueue(exec_ctx, stream_global->recv_message_ready, 1); + stream_global->recv_message_ready = NULL; + } + } + if (stream_global->recv_trailing_metadata_finished != NULL && + stream_global->read_closed && stream_global->write_closed) { + while (stream_global->seen_error && + (bs = grpc_chttp2_incoming_frame_queue_pop( + &stream_global->incoming_frames)) != NULL) { + grpc_byte_stream_destroy(bs); + } + if (stream_global->incoming_frames.head == NULL) { + grpc_chttp2_incoming_metadata_buffer_publish( + &stream_global->received_trailing_metadata, + stream_global->recv_trailing_metadata); + grpc_chttp2_complete_closure_step( + exec_ctx, &stream_global->recv_trailing_metadata_finished, 1); + } + } + } } static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -832,7 +1030,7 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, s->global.in_stream_map = 0; if (t->parsing.incoming_stream == &s->parsing) { t->parsing.incoming_stream = NULL; - grpc_chttp2_parsing_become_skip_parser(&t->parsing); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, &t->parsing); } if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { close_transport_locked(exec_ctx, t); @@ -847,109 +1045,10 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } -static void unlock_check_read_write_state(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t) { - grpc_chttp2_transport_global *transport_global = &t->global; - grpc_chttp2_stream_global *stream_global; - grpc_stream_state state; - - if (!t->parsing_active) { - /* if a stream is in the stream map, and gets cancelled, we need to ensure - we are not parsing before continuing the cancellation to keep things in - a sane state */ - while (grpc_chttp2_list_pop_closed_waiting_for_parsing(transport_global, - &stream_global)) { - GPR_ASSERT(stream_global->in_stream_map); - GPR_ASSERT(stream_global->write_state != GRPC_WRITE_STATE_OPEN); - GPR_ASSERT(stream_global->read_closed); - remove_stream(exec_ctx, t, stream_global->id); - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); - } - } - - if (!t->writing_active) { - while (grpc_chttp2_list_pop_cancelled_waiting_for_writing(transport_global, - &stream_global)) { - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); - } - } - - while (grpc_chttp2_list_pop_read_write_state_changed(transport_global, - &stream_global)) { - if (stream_global->cancelled) { - if (t->writing_active && - stream_global->write_state != GRPC_WRITE_STATE_SENT_CLOSE) { - grpc_chttp2_list_add_cancelled_waiting_for_writing(transport_global, - stream_global); - } else { - stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE; - if (stream_global->outgoing_sopb != NULL) { - grpc_sopb_reset(stream_global->outgoing_sopb); - stream_global->outgoing_sopb = NULL; - grpc_exec_ctx_enqueue(exec_ctx, stream_global->send_done_closure, 1); - } - stream_global->read_closed = 1; - if (!stream_global->published_cancelled) { - char buffer[GPR_LTOA_MIN_BUFSIZE]; - gpr_ltoa(stream_global->cancelled_status, buffer); - grpc_chttp2_incoming_metadata_buffer_add( - &stream_global->incoming_metadata, - grpc_mdelem_from_strings(t->metadata_context, "grpc-status", - buffer)); - grpc_chttp2_incoming_metadata_buffer_place_metadata_batch_into( - &stream_global->incoming_metadata, &stream_global->incoming_sopb); - stream_global->published_cancelled = 1; - } - } - } - if (stream_global->write_state == GRPC_WRITE_STATE_SENT_CLOSE && - stream_global->read_closed && stream_global->in_stream_map) { - if (t->parsing_active) { - grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global, - stream_global); - } else { - remove_stream(exec_ctx, t, stream_global->id); - } - } - if (!stream_global->publish_sopb) { - continue; - } - if (stream_global->writing_now != 0) { - continue; - } - /* FIXME(ctiller): we include in_stream_map in our computation of - whether the stream is write-closed. This is completely bogus, - but has the effect of delaying stream-closed until the stream - is indeed evicted from the stream map, making it safe to delete. - To fix this will require having an edge after stream-closed - indicating that the stream is closed AND safe to delete. */ - state = compute_state( - stream_global->write_state == GRPC_WRITE_STATE_SENT_CLOSE && - !stream_global->in_stream_map, - stream_global->read_closed); - if (stream_global->incoming_sopb.nops == 0 && - state == stream_global->published_state) { - continue; - } - grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op( - &stream_global->incoming_metadata, &stream_global->incoming_sopb, - &stream_global->outstanding_metadata); - grpc_sopb_swap(stream_global->publish_sopb, &stream_global->incoming_sopb); - stream_global->published_state = *stream_global->publish_state = state; - grpc_exec_ctx_enqueue(exec_ctx, stream_global->recv_done_closure, 1); - stream_global->recv_done_closure = NULL; - stream_global->publish_sopb = NULL; - stream_global->publish_state = NULL; - } -} - -static void cancel_from_api(grpc_chttp2_transport_global *transport_global, +static void cancel_from_api(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global, grpc_status_code status) { - stream_global->cancelled = 1; - stream_global->cancelled_status = status; if (stream_global->id != 0) { gpr_slice_buffer_add( &transport_global->qbuf, @@ -957,11 +1056,85 @@ static void cancel_from_api(grpc_chttp2_transport_global *transport_global, stream_global->id, (gpr_uint32)grpc_chttp2_grpc_status_to_http2_error(status))); } - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status, + NULL); + grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1, + 1); } -static void close_from_api(grpc_chttp2_transport_global *transport_global, +void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global, + grpc_status_code status, gpr_slice *slice) { + if (status != GRPC_STATUS_OK) { + stream_global->seen_error = 1; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + /* stream_global->recv_trailing_metadata_finished gives us a + last chance replacement: we've received trailing metadata, + but something more important has become available to signal + to the upper layers - drop what we've got, and then publish + what we want - which is safe because we haven't told anyone + about the metadata yet */ + if (!stream_global->published_trailing_metadata || + stream_global->recv_trailing_metadata_finished != NULL) { + grpc_mdctx *mdctx = + TRANSPORT_FROM_GLOBAL(transport_global)->metadata_context; + char status_string[GPR_LTOA_MIN_BUFSIZE]; + gpr_ltoa(status, status_string); + grpc_chttp2_incoming_metadata_buffer_add( + &stream_global->received_trailing_metadata, + grpc_mdelem_from_strings(mdctx, "grpc-status", status_string)); + if (slice) { + grpc_chttp2_incoming_metadata_buffer_add( + &stream_global->received_trailing_metadata, + grpc_mdelem_from_metadata_strings( + mdctx, grpc_mdstr_from_string(mdctx, "grpc-message"), + grpc_mdstr_from_slice(mdctx, gpr_slice_ref(*slice)))); + } + stream_global->published_trailing_metadata = 1; + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + } + if (slice) { + gpr_slice_unref(*slice); + } +} + +void grpc_chttp2_mark_stream_closed( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global, int close_reads, + int close_writes) { + if (stream_global->read_closed && stream_global->write_closed) { + /* already closed */ + return; + } + grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); + if (close_reads && !stream_global->read_closed) { + stream_global->read_closed = 1; + stream_global->published_initial_metadata = 1; + stream_global->published_trailing_metadata = 1; + } + if (close_writes && !stream_global->write_closed) { + stream_global->write_closed = 1; + } + if (stream_global->read_closed && stream_global->write_closed) { + if (stream_global->id != 0 && + TRANSPORT_FROM_GLOBAL(transport_global)->parsing_active) { + grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global, + stream_global); + } else { + if (stream_global->id != 0) { + remove_stream(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global), + stream_global->id); + } + stream_global->finished_close = 1; + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2"); + } + } +} + +static void close_from_api(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global, grpc_status_code status, gpr_slice *optional_message) { @@ -973,10 +1146,7 @@ static void close_from_api(grpc_chttp2_transport_global *transport_global, GPR_ASSERT(status >= 0 && (int)status < 100); - stream_global->cancelled = 1; - stream_global->cancelled_status = status; GPR_ASSERT(stream_global->id != 0); - GPR_ASSERT(!stream_global->written_anything); /* Hand roll a header block. This is unnecessarily ugly - at some point we should find a more elegant @@ -1059,23 +1229,30 @@ static void close_from_api(grpc_chttp2_transport_global *transport_global, &transport_global->qbuf, grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR)); - grpc_chttp2_list_add_read_write_state_changed(transport_global, - stream_global); + if (optional_message) { + gpr_slice_ref(*optional_message); + } + grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, status, + optional_message); + grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1, + 1); } static void cancel_stream_cb(grpc_chttp2_transport_global *transport_global, void *user_data, grpc_chttp2_stream_global *stream_global) { - cancel_from_api(transport_global, stream_global, GRPC_STATUS_UNAVAILABLE); + cancel_from_api(user_data, transport_global, stream_global, + GRPC_STATUS_UNAVAILABLE); } -static void end_all_the_calls(grpc_chttp2_transport *t) { - grpc_chttp2_for_all_streams(&t->global, NULL, cancel_stream_cb); +static void end_all_the_calls(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + grpc_chttp2_for_all_streams(&t->global, exec_ctx, cancel_stream_cb); } static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { close_transport_locked(exec_ctx, t); - end_all_the_calls(t); + end_all_the_calls(exec_ctx, t); } /** update window from a settings change */ @@ -1086,12 +1263,11 @@ static void update_global_window(void *args, gpr_uint32 id, void *stream) { grpc_chttp2_stream_global *stream_global = &s->global; int was_zero; int is_zero; + gpr_int64 initial_window_update = t->parsing.initial_window_update; - GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("settings", transport_global, stream_global, - outgoing_window, - t->parsing.initial_window_update); was_zero = stream_global->outgoing_window <= 0; - stream_global->outgoing_window += t->parsing.initial_window_update; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("settings", transport_global, stream_global, + outgoing_window, initial_window_update); is_zero = stream_global->outgoing_window <= 0; if (was_zero && !is_zero) { @@ -1112,6 +1288,9 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, int success) { size_t i; int keep_reading = 0; grpc_chttp2_transport *t = tp; + grpc_chttp2_transport_global *transport_global = &t->global; + grpc_chttp2_transport_parsing *transport_parsing = &t->parsing; + grpc_chttp2_stream_global *stream_global; GPR_TIMER_BEGIN("recv_data", 0); @@ -1123,11 +1302,11 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, int success) { /* merge stream lists */ grpc_chttp2_stream_map_move_into(&t->new_stream_map, &t->parsing_stream_map); - grpc_chttp2_prepare_to_read(&t->global, &t->parsing); + grpc_chttp2_prepare_to_read(transport_global, transport_parsing); gpr_mu_unlock(&t->mu); GPR_TIMER_BEGIN("recv_data.parse", 0); for (; i < t->read_buffer.count && - grpc_chttp2_perform_read(exec_ctx, &t->parsing, + grpc_chttp2_perform_read(exec_ctx, transport_parsing, t->read_buffer.slices[i]); i++) ; @@ -1139,16 +1318,28 @@ static void recv_data(grpc_exec_ctx *exec_ctx, void *tp, int success) { /* merge stream lists */ grpc_chttp2_stream_map_move_into(&t->new_stream_map, &t->parsing_stream_map); - t->global.concurrent_stream_count = + transport_global->concurrent_stream_count = (gpr_uint32)grpc_chttp2_stream_map_size(&t->parsing_stream_map); - if (t->parsing.initial_window_update != 0) { + if (transport_parsing->initial_window_update != 0) { grpc_chttp2_stream_map_for_each(&t->parsing_stream_map, update_global_window, t); - t->parsing.initial_window_update = 0; + transport_parsing->initial_window_update = 0; } /* handle higher level things */ - grpc_chttp2_publish_reads(exec_ctx, &t->global, &t->parsing); + grpc_chttp2_publish_reads(exec_ctx, transport_global, transport_parsing); t->parsing_active = 0; + /* if a stream is in the stream map, and gets cancelled, we need to ensure + * we are not parsing before continuing the cancellation to keep things in + * a sane state */ + while (grpc_chttp2_list_pop_closed_waiting_for_parsing(transport_global, + &stream_global)) { + GPR_ASSERT(stream_global->in_stream_map); + GPR_ASSERT(stream_global->write_closed); + GPR_ASSERT(stream_global->read_closed); + remove_stream(exec_ctx, t, stream_global->id); + stream_global->finished_close = 1; + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2"); + } } if (!success || i != t->read_buffer.count || t->closed) { drop_connection(exec_ctx, t); @@ -1206,35 +1397,234 @@ static void add_to_pollset_set_locked(grpc_exec_ctx *exec_ctx, } } +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) { + grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; + lock(t); + add_to_pollset_locked(exec_ctx, t, pollset); + unlock(exec_ctx, t); +} + /* - * TRACING + * BYTE STREAM */ -void grpc_chttp2_flowctl_trace(const char *file, int line, const char *reason, - const char *context, const char *var, - int is_client, gpr_uint32 stream_id, - gpr_int64 current_value, gpr_int64 delta) { - char *identifier; - char *context_scope; - char *context_thread; - char *underscore_pos = strchr(context, '_'); - GPR_ASSERT(underscore_pos); - context_thread = gpr_strdup(underscore_pos + 1); - context_scope = gpr_strdup(context); - context_scope[underscore_pos - context] = 0; - if (stream_id) { - gpr_asprintf(&identifier, "%s[%d]", context_scope, stream_id); +static void incoming_byte_stream_update_flow_control( + grpc_chttp2_transport_global *transport_global, + grpc_chttp2_stream_global *stream_global, size_t max_size_hint, + size_t have_already) { + gpr_uint32 max_recv_bytes; + + /* clamp max recv hint to an allowable size */ + if (max_size_hint >= GPR_UINT32_MAX - transport_global->stream_lookahead) { + max_recv_bytes = GPR_UINT32_MAX - transport_global->stream_lookahead; } else { - identifier = gpr_strdup(context_scope); - } - gpr_log(GPR_INFO, - "FLOWCTL: %s %-10s %8s %-27s %8lld %c %8lld = %8lld %-10s [%s:%d]", - is_client ? "client" : "server", identifier, context_thread, var, - current_value, delta < 0 ? '-' : '+', delta < 0 ? -delta : delta, - current_value + delta, reason, file, line); - gpr_free(identifier); - gpr_free(context_thread); - gpr_free(context_scope); + max_recv_bytes = (gpr_uint32)max_size_hint; + } + + /* account for bytes already received but unknown to higher layers */ + if (max_recv_bytes >= have_already) { + max_recv_bytes -= (gpr_uint32)have_already; + } else { + max_recv_bytes = 0; + } + + /* add some small lookahead to keep pipelines flowing */ + GPR_ASSERT(max_recv_bytes <= + GPR_UINT32_MAX - transport_global->stream_lookahead); + max_recv_bytes += transport_global->stream_lookahead; + if (stream_global->max_recv_bytes < max_recv_bytes) { + gpr_uint32 add_max_recv_bytes = + max_recv_bytes - stream_global->max_recv_bytes; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, + max_recv_bytes, add_max_recv_bytes); + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, + unannounced_incoming_window_for_parse, + add_max_recv_bytes); + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, + unannounced_incoming_window_for_writing, + add_max_recv_bytes); + grpc_chttp2_list_add_unannounced_incoming_window_available(transport_global, + stream_global); + grpc_chttp2_list_add_writable_stream(transport_global, stream_global); + } +} + +static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + gpr_slice *slice, size_t max_size_hint, + grpc_closure *on_complete) { + grpc_chttp2_incoming_byte_stream *bs = + (grpc_chttp2_incoming_byte_stream *)byte_stream; + grpc_chttp2_transport_global *transport_global = &bs->transport->global; + grpc_chttp2_stream_global *stream_global = &bs->stream->global; + + lock(bs->transport); + if (bs->is_tail) { + incoming_byte_stream_update_flow_control(transport_global, stream_global, + max_size_hint, bs->slices.length); + } + if (bs->slices.count > 0) { + *slice = gpr_slice_buffer_take_first(&bs->slices); + unlock(exec_ctx, bs->transport); + return 1; + } else { + bs->on_next = on_complete; + bs->next = slice; + unlock(exec_ctx, bs->transport); + return 0; + } +} + +static void incoming_byte_stream_unref(grpc_chttp2_incoming_byte_stream *bs) { + if (gpr_unref(&bs->refs)) { + gpr_slice_buffer_destroy(&bs->slices); + gpr_free(bs); + } +} + +static void incoming_byte_stream_destroy(grpc_byte_stream *byte_stream) { + incoming_byte_stream_unref((grpc_chttp2_incoming_byte_stream *)byte_stream); +} + +void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, + grpc_chttp2_incoming_byte_stream *bs, + gpr_slice slice) { + gpr_mu_lock(&bs->transport->mu); + if (bs->on_next != NULL) { + *bs->next = slice; + grpc_exec_ctx_enqueue(exec_ctx, bs->on_next, 1); + bs->on_next = NULL; + } else { + gpr_slice_buffer_add(&bs->slices, slice); + } + gpr_mu_unlock(&bs->transport->mu); +} + +void grpc_chttp2_incoming_byte_stream_finished( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs) { + incoming_byte_stream_unref(bs); +} + +grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, + grpc_chttp2_stream_parsing *stream_parsing, gpr_uint32 frame_size, + gpr_uint32 flags, grpc_chttp2_incoming_frame_queue *add_to_queue) { + grpc_chttp2_incoming_byte_stream *incoming_byte_stream = + gpr_malloc(sizeof(*incoming_byte_stream)); + incoming_byte_stream->base.length = frame_size; + incoming_byte_stream->base.flags = flags; + incoming_byte_stream->base.next = incoming_byte_stream_next; + incoming_byte_stream->base.destroy = incoming_byte_stream_destroy; + gpr_ref_init(&incoming_byte_stream->refs, 2); + incoming_byte_stream->next_message = NULL; + incoming_byte_stream->transport = TRANSPORT_FROM_PARSING(transport_parsing); + incoming_byte_stream->stream = STREAM_FROM_PARSING(stream_parsing); + gpr_slice_buffer_init(&incoming_byte_stream->slices); + incoming_byte_stream->on_next = NULL; + incoming_byte_stream->is_tail = 1; + if (add_to_queue->head == NULL) { + add_to_queue->head = incoming_byte_stream; + } else { + add_to_queue->tail->is_tail = 0; + add_to_queue->tail->next_message = incoming_byte_stream; + } + add_to_queue->tail = incoming_byte_stream; + if (frame_size == 0) { + lock(TRANSPORT_FROM_PARSING(transport_parsing)); + incoming_byte_stream_update_flow_control( + &TRANSPORT_FROM_PARSING(transport_parsing)->global, + &STREAM_FROM_PARSING(stream_parsing)->global, 0, 0); + unlock(exec_ctx, TRANSPORT_FROM_PARSING(transport_parsing)); + } + return incoming_byte_stream; +} + +/* + * TRACING + */ + +static char *format_flowctl_context_var(const char *context, const char *var, + gpr_int64 val, gpr_uint32 id, + char **scope) { + char *underscore_pos; + char *result; + if (context == NULL) { + *scope = NULL; + gpr_asprintf(&result, "%s(%lld)", var, val); + return result; + } + underscore_pos = strchr(context, '_'); + *scope = gpr_strdup(context); + (*scope)[underscore_pos - context] = 0; + if (id != 0) { + char *tmp = *scope; + gpr_asprintf(scope, "%s[%d]", tmp, id); + gpr_free(tmp); + } + gpr_asprintf(&result, "%s.%s(%lld)", underscore_pos + 1, var, val); + return result; +} + +static int samestr(char *a, char *b) { + if (a == NULL) { + return b == NULL; + } + if (b == NULL) { + return 0; + } + return 0 == strcmp(a, b); +} + +void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, + grpc_chttp2_flowctl_op op, const char *context1, + const char *var1, const char *context2, + const char *var2, int is_client, + gpr_uint32 stream_id, gpr_int64 val1, + gpr_int64 val2) { + char *scope1; + char *scope2; + char *label1 = + format_flowctl_context_var(context1, var1, val1, stream_id, &scope1); + char *label2 = + format_flowctl_context_var(context2, var2, val2, stream_id, &scope2); + char *clisvr = is_client ? "client" : "server"; + char *prefix; + + gpr_asprintf(&prefix, "FLOW % 8s: %s % 11s ", phase, clisvr, scope1); + + switch (op) { + case GRPC_CHTTP2_FLOWCTL_MOVE: + GPR_ASSERT(samestr(scope1, scope2)); + if (val2 != 0) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "%sMOVE % 40s <- % 40s giving %d", prefix, label1, label2, + val1 + val2); + } + break; + case GRPC_CHTTP2_FLOWCTL_CREDIT: + GPR_ASSERT(val2 >= 0); + if (val2 != 0) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "%sCREDIT % 40s by % 40s giving %d", prefix, label1, label2, + val1 + val2); + } + break; + case GRPC_CHTTP2_FLOWCTL_DEBIT: + GPR_ASSERT(val2 >= 0); + if (val2 != 0) { + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, + "%sDEBIT % 40s by % 40s giving %d", prefix, label1, label2, + val1 - val2); + } + break; + } + + gpr_free(scope1); + gpr_free(scope2); + gpr_free(label1); + gpr_free(label2); + gpr_free(prefix); } /* @@ -1246,7 +1636,7 @@ static char *chttp2_get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) { } static const grpc_transport_vtable vtable = { - sizeof(grpc_chttp2_stream), init_stream, perform_stream_op, + sizeof(grpc_chttp2_stream), init_stream, set_pollset, perform_stream_op, perform_transport_op, destroy_stream, destroy_transport, chttp2_get_peer}; grpc_transport *grpc_create_chttp2_transport( diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index 68f23177eb..a72dee4399 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -41,9 +41,10 @@ #include <grpc/support/alloc.h> #include <grpc/support/atm.h> #include <grpc/support/log.h> +#include <grpc/support/time.h> +#include "src/core/profiling/timers.h" #include "src/core/support/murmur_hash.h" #include "src/core/transport/chttp2/bin_encoder.h" -#include <grpc/support/time.h> #define INITIAL_STRTAB_CAPACITY 4 #define INITIAL_MDTAB_CAPACITY 4 @@ -232,24 +233,32 @@ static void metadata_context_destroy_locked(grpc_mdctx *ctx) { } void grpc_mdctx_ref(grpc_mdctx *ctx) { + GPR_TIMER_BEGIN("grpc_mdctx_ref", 0); lock(ctx); GPR_ASSERT(ctx->refs > 0); ctx->refs++; unlock(ctx); + GPR_TIMER_END("grpc_mdctx_ref", 0); } void grpc_mdctx_unref(grpc_mdctx *ctx) { + GPR_TIMER_BEGIN("grpc_mdctx_unref", 0); lock(ctx); GPR_ASSERT(ctx->refs > 0); ctx->refs--; unlock(ctx); + GPR_TIMER_END("grpc_mdctx_unref", 0); } static void grow_strtab(grpc_mdctx *ctx) { size_t capacity = ctx->strtab_capacity * 2; size_t i; - internal_string **strtab = gpr_malloc(sizeof(internal_string *) * capacity); + internal_string **strtab; internal_string *s, *next; + + GPR_TIMER_BEGIN("grow_strtab", 0); + + strtab = gpr_malloc(sizeof(internal_string *) * capacity); memset(strtab, 0, sizeof(internal_string *) * capacity); for (i = 0; i < ctx->strtab_capacity; i++) { @@ -263,12 +272,15 @@ static void grow_strtab(grpc_mdctx *ctx) { gpr_free(ctx->strtab); ctx->strtab = strtab; ctx->strtab_capacity = capacity; + + GPR_TIMER_END("grow_strtab", 0); } static void internal_destroy_string(internal_string *is) { internal_string **prev_next; internal_string *cur; grpc_mdctx *ctx = is->context; + GPR_TIMER_BEGIN("internal_destroy_string", 0); if (is->has_base64_and_huffman_encoded) { gpr_slice_unref(is->base64_and_huffman); } @@ -279,6 +291,7 @@ static void internal_destroy_string(internal_string *is) { *prev_next = cur->bucket_next; ctx->strtab_count--; gpr_free(is); + GPR_TIMER_END("internal_destroy_string", 0); } static void internal_string_ref(internal_string *s DEBUG_ARGS) { @@ -304,18 +317,22 @@ static void slice_ref(void *p) { internal_string *is = (internal_string *)((char *)p - offsetof(internal_string, refcount)); grpc_mdctx *ctx = is->context; + GPR_TIMER_BEGIN("slice_ref", 0); lock(ctx); INTERNAL_STRING_REF(is); unlock(ctx); + GPR_TIMER_END("slice_ref", 0); } static void slice_unref(void *p) { internal_string *is = (internal_string *)((char *)p - offsetof(internal_string, refcount)); grpc_mdctx *ctx = is->context; + GPR_TIMER_BEGIN("slice_unref", 0); lock(ctx); INTERNAL_STRING_UNREF(is); unlock(ctx); + GPR_TIMER_END("slice_unref", 0); } grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str) { @@ -334,6 +351,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *buf, gpr_uint32 hash = gpr_murmur_hash3(buf, length, ctx->hash_seed); internal_string *s; + GPR_TIMER_BEGIN("grpc_mdstr_from_buffer", 0); lock(ctx); /* search for an existing string */ @@ -342,6 +360,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *buf, 0 == memcmp(buf, GPR_SLICE_START_PTR(s->slice), length)) { INTERNAL_STRING_REF(s); unlock(ctx); + GPR_TIMER_END("grpc_mdstr_from_buffer", 0); return (grpc_mdstr *)s; } } @@ -382,6 +401,7 @@ grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *buf, } unlock(ctx); + GPR_TIMER_END("grpc_mdstr_from_buffer", 0); return (grpc_mdstr *)s; } @@ -391,6 +411,7 @@ static void gc_mdtab(grpc_mdctx *ctx) { internal_metadata **prev_next; internal_metadata *md, *next; + GPR_TIMER_BEGIN("gc_mdtab", 0); for (i = 0; i < ctx->mdtab_capacity; i++) { prev_next = &ctx->mdtab[i]; for (md = ctx->mdtab[i]; md; md = next) { @@ -412,17 +433,19 @@ static void gc_mdtab(grpc_mdctx *ctx) { } } } - - GPR_ASSERT(ctx->mdtab_free == 0); + GPR_TIMER_END("gc_mdtab", 0); } static void grow_mdtab(grpc_mdctx *ctx) { size_t capacity = ctx->mdtab_capacity * 2; size_t i; - internal_metadata **mdtab = - gpr_malloc(sizeof(internal_metadata *) * capacity); + internal_metadata **mdtab; internal_metadata *md, *next; gpr_uint32 hash; + + GPR_TIMER_BEGIN("grow_mdtab", 0); + + mdtab = gpr_malloc(sizeof(internal_metadata *) * capacity); memset(mdtab, 0, sizeof(internal_metadata *) * capacity); for (i = 0; i < ctx->mdtab_capacity; i++) { @@ -437,6 +460,8 @@ static void grow_mdtab(grpc_mdctx *ctx) { gpr_free(ctx->mdtab); ctx->mdtab = mdtab; ctx->mdtab_capacity = capacity; + + GPR_TIMER_END("grow_mdtab", 0); } static void rehash_mdtab(grpc_mdctx *ctx) { @@ -458,6 +483,8 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx, GPR_ASSERT(key->context == ctx); GPR_ASSERT(value->context == ctx); + GPR_TIMER_BEGIN("grpc_mdelem_from_metadata_strings", 0); + lock(ctx); /* search for an existing pair */ @@ -467,6 +494,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx, INTERNAL_STRING_UNREF(key); INTERNAL_STRING_UNREF(value); unlock(ctx); + GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); return (grpc_mdelem *)md; } } @@ -496,6 +524,8 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx, unlock(ctx); + GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); + return (grpc_mdelem *)md; } @@ -542,6 +572,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) { void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { internal_metadata *md = (internal_metadata *)gmd; + if (!md) return; #ifdef GRPC_METADATA_REFCOUNT_DEBUG gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "ELM UNREF:%p:%d->%d: '%s' = '%s'", md, @@ -552,12 +583,14 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { #endif if (2 == gpr_atm_full_fetch_add(&md->refcnt, -1)) { grpc_mdctx *ctx = md->context; + GPR_TIMER_BEGIN("grpc_mdelem_unref.to_zero", 0); lock(ctx); if (1 == gpr_atm_no_barrier_load(&md->refcnt)) { ctx->mdtab_free++; gpr_atm_no_barrier_store(&md->refcnt, 0); } unlock(ctx); + GPR_TIMER_END("grpc_mdelem_unref.to_zero", 0); } } diff --git a/src/core/transport/stream_op.c b/src/core/transport/metadata_batch.c index 6493e77bc5..c5d39e0c9f 100644 --- a/src/core/transport/stream_op.c +++ b/src/core/transport/metadata_batch.c @@ -31,7 +31,7 @@ * */ -#include "src/core/transport/stream_op.h" +#include "src/core/transport/metadata_batch.h" #include <string.h> @@ -40,143 +40,6 @@ #include "src/core/profiling/timers.h" -/* Exponential growth function: Given x, return a larger x. - Currently we grow by 1.5 times upon reallocation. */ -#define GROW(x) (3 * (x) / 2) - -void grpc_sopb_init(grpc_stream_op_buffer *sopb) { - sopb->ops = sopb->inlined_ops; - sopb->nops = 0; - sopb->capacity = GRPC_SOPB_INLINE_ELEMENTS; -} - -void grpc_sopb_destroy(grpc_stream_op_buffer *sopb) { - grpc_stream_ops_unref_owned_objects(sopb->ops, sopb->nops); - if (sopb->ops != sopb->inlined_ops) gpr_free(sopb->ops); -} - -void grpc_sopb_reset(grpc_stream_op_buffer *sopb) { - grpc_stream_ops_unref_owned_objects(sopb->ops, sopb->nops); - sopb->nops = 0; -} - -void grpc_sopb_swap(grpc_stream_op_buffer *a, grpc_stream_op_buffer *b) { - GPR_SWAP(size_t, a->nops, b->nops); - GPR_SWAP(size_t, a->capacity, b->capacity); - - if (a->ops == a->inlined_ops) { - if (b->ops == b->inlined_ops) { - /* swap contents of inlined buffer */ - grpc_stream_op temp[GRPC_SOPB_INLINE_ELEMENTS]; - memcpy(temp, a->ops, b->nops * sizeof(grpc_stream_op)); - memcpy(a->ops, b->ops, a->nops * sizeof(grpc_stream_op)); - memcpy(b->ops, temp, b->nops * sizeof(grpc_stream_op)); - } else { - /* a is inlined, b is not - copy a inlined into b, fix pointers */ - a->ops = b->ops; - b->ops = b->inlined_ops; - memcpy(b->ops, a->inlined_ops, b->nops * sizeof(grpc_stream_op)); - } - } else if (b->ops == b->inlined_ops) { - /* b is inlined, a is not - copy b inlined int a, fix pointers */ - b->ops = a->ops; - a->ops = a->inlined_ops; - memcpy(a->ops, b->inlined_ops, a->nops * sizeof(grpc_stream_op)); - } else { - /* no inlining: easy swap */ - GPR_SWAP(grpc_stream_op *, a->ops, b->ops); - } -} - -void grpc_stream_ops_unref_owned_objects(grpc_stream_op *ops, size_t nops) { - size_t i; - for (i = 0; i < nops; i++) { - switch (ops[i].type) { - case GRPC_OP_SLICE: - gpr_slice_unref(ops[i].data.slice); - break; - case GRPC_OP_METADATA: - grpc_metadata_batch_destroy(&ops[i].data.metadata); - break; - case GRPC_NO_OP: - case GRPC_OP_BEGIN_MESSAGE: - break; - } - } -} - -static void expandto(grpc_stream_op_buffer *sopb, size_t new_capacity) { - sopb->capacity = new_capacity; - if (sopb->ops == sopb->inlined_ops) { - sopb->ops = gpr_malloc(sizeof(grpc_stream_op) * new_capacity); - memcpy(sopb->ops, sopb->inlined_ops, sopb->nops * sizeof(grpc_stream_op)); - } else { - sopb->ops = gpr_realloc(sopb->ops, sizeof(grpc_stream_op) * new_capacity); - } -} - -static grpc_stream_op *add(grpc_stream_op_buffer *sopb) { - grpc_stream_op *out; - - GPR_ASSERT(sopb->nops <= sopb->capacity); - if (sopb->nops == sopb->capacity) { - expandto(sopb, GROW(sopb->capacity)); - } - out = sopb->ops + sopb->nops; - sopb->nops++; - return out; -} - -void grpc_sopb_add_no_op(grpc_stream_op_buffer *sopb) { - add(sopb)->type = GRPC_NO_OP; -} - -void grpc_sopb_add_begin_message(grpc_stream_op_buffer *sopb, gpr_uint32 length, - gpr_uint32 flags) { - grpc_stream_op *op = add(sopb); - op->type = GRPC_OP_BEGIN_MESSAGE; - op->data.begin_message.length = length; - op->data.begin_message.flags = flags; -} - -void grpc_sopb_add_metadata(grpc_stream_op_buffer *sopb, - grpc_metadata_batch b) { - grpc_stream_op *op = add(sopb); - op->type = GRPC_OP_METADATA; - op->data.metadata = b; -} - -void grpc_sopb_add_slice(grpc_stream_op_buffer *sopb, gpr_slice slice) { - grpc_stream_op *op = add(sopb); - op->type = GRPC_OP_SLICE; - op->data.slice = slice; -} - -void grpc_sopb_append(grpc_stream_op_buffer *sopb, grpc_stream_op *ops, - size_t nops) { - size_t orig_nops = sopb->nops; - size_t new_nops = orig_nops + nops; - - if (new_nops > sopb->capacity) { - expandto(sopb, GPR_MAX(GROW(sopb->capacity), new_nops)); - } - - memcpy(sopb->ops + orig_nops, ops, sizeof(grpc_stream_op) * nops); - sopb->nops = new_nops; -} - -void grpc_sopb_move_to(grpc_stream_op_buffer *src, grpc_stream_op_buffer *dst) { - if (src->nops == 0) { - return; - } - if (dst->nops == 0) { - grpc_sopb_swap(src, dst); - return; - } - grpc_sopb_append(dst, src->ops, src->nops); - src->nops = 0; -} - static void assert_valid_list(grpc_mdelem_list *list) { #ifndef NDEBUG grpc_linked_mdelem *l; @@ -200,13 +63,11 @@ static void assert_valid_list(grpc_mdelem_list *list) { #ifndef NDEBUG void grpc_metadata_batch_assert_ok(grpc_metadata_batch *batch) { assert_valid_list(&batch->list); - assert_valid_list(&batch->garbage); } #endif /* NDEBUG */ void grpc_metadata_batch_init(grpc_metadata_batch *batch) { - batch->list.head = batch->list.tail = batch->garbage.head = - batch->garbage.tail = NULL; + batch->list.head = batch->list.tail = NULL; batch->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); } @@ -215,9 +76,6 @@ void grpc_metadata_batch_destroy(grpc_metadata_batch *batch) { for (l = batch->list.head; l; l = l->next) { GRPC_MDELEM_UNREF(l->md); } - for (l = batch->garbage.head; l; l = l->next) { - GRPC_MDELEM_UNREF(l->md); - } } void grpc_metadata_batch_add_head(grpc_metadata_batch *batch, @@ -283,10 +141,6 @@ void grpc_metadata_batch_merge(grpc_metadata_batch *target, next = l->next; link_tail(&target->list, l); } - for (l = to_add->garbage.head; l; l = next) { - next = l->next; - link_tail(&target->garbage, l); - } } void grpc_metadata_batch_move(grpc_metadata_batch *dst, @@ -305,7 +159,6 @@ void grpc_metadata_batch_filter(grpc_metadata_batch *batch, GPR_TIMER_BEGIN("grpc_metadata_batch_filter", 0); assert_valid_list(&batch->list); - assert_valid_list(&batch->garbage); for (l = batch->list.head; l; l = next) { grpc_mdelem *orig = l->md; grpc_mdelem *filt = filter(user_data, orig); @@ -324,14 +177,28 @@ void grpc_metadata_batch_filter(grpc_metadata_batch *batch, batch->list.tail = l->prev; } assert_valid_list(&batch->list); - link_head(&batch->garbage, l); + GRPC_MDELEM_UNREF(l->md); } else if (filt != orig) { GRPC_MDELEM_UNREF(orig); l->md = filt; } } assert_valid_list(&batch->list); - assert_valid_list(&batch->garbage); GPR_TIMER_END("grpc_metadata_batch_filter", 0); } + +static grpc_mdelem *no_metadata_for_you(void *user_data, grpc_mdelem *elem) { + return NULL; +} + +void grpc_metadata_batch_clear(grpc_metadata_batch *batch) { + batch->deadline = gpr_inf_future(GPR_CLOCK_REALTIME); + grpc_metadata_batch_filter(batch, no_metadata_for_you, NULL); +} + +int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch) { + return batch->list.head == NULL && + gpr_time_cmp(gpr_inf_future(batch->deadline.clock_type), + batch->deadline) == 0; +} diff --git a/src/core/transport/stream_op.h b/src/core/transport/metadata_batch.h index 37f18b02d9..fc3a46004f 100644 --- a/src/core/transport/stream_op.h +++ b/src/core/transport/metadata_batch.h @@ -40,39 +40,6 @@ #include <grpc/support/time.h> #include "src/core/transport/metadata.h" -/* this many stream ops are inlined into a sopb before allocating */ -#define GRPC_SOPB_INLINE_ELEMENTS 4 - -/* Operations that can be performed on a stream. - Used by grpc_stream_op. */ -typedef enum grpc_stream_op_code { - /* Do nothing code. Useful if rewriting a batch to exclude some operations. - Must be ignored by receivers */ - GRPC_NO_OP, - GRPC_OP_METADATA, - /* Begin a message/metadata element/status - as defined by - grpc_message_type. */ - GRPC_OP_BEGIN_MESSAGE, - /* Add a slice of data to the current message/metadata element/status. - Must not overflow the forward declared length. */ - GRPC_OP_SLICE -} grpc_stream_op_code; - -/** Internal bit flag for grpc_begin_message's \a flags signaling the use of - * compression for the message */ -#define GRPC_WRITE_INTERNAL_COMPRESS (0x80000000u) -/** Mask of all valid internal flags. */ -#define GRPC_WRITE_INTERNAL_USED_MASK (GRPC_WRITE_INTERNAL_COMPRESS) - -/* Arguments for GRPC_OP_BEGIN_MESSAGE */ -typedef struct grpc_begin_message { - /* How many bytes of data will this message contain */ - gpr_uint32 length; - /* Write flags for the message: see grpc.h GRPC_WRITE_* for the public bits, - * GRPC_WRITE_INTERNAL_* for the internal ones. */ - gpr_uint32 flags; -} grpc_begin_message; - typedef struct grpc_linked_mdelem { grpc_mdelem *md; struct grpc_linked_mdelem *next; @@ -88,10 +55,6 @@ typedef struct grpc_mdelem_list { typedef struct grpc_metadata_batch { /** Metadata elements in this batch */ grpc_mdelem_list list; - /** Elements that have been removed from the batch, but have - not yet been unreffed - used to allow collecting garbage - under a single metadata context lock */ - grpc_mdelem_list garbage; /** Used to calculate grpc-timeout at the point of sending, or gpr_inf_future if this batch does not need to send a grpc-timeout */ @@ -102,6 +65,8 @@ void grpc_metadata_batch_init(grpc_metadata_batch *batch); void grpc_metadata_batch_destroy(grpc_metadata_batch *batch); void grpc_metadata_batch_merge(grpc_metadata_batch *target, grpc_metadata_batch *add); +void grpc_metadata_batch_clear(grpc_metadata_batch *batch); +int grpc_metadata_batch_is_empty(grpc_metadata_batch *batch); /** Moves the metadata information from \a src to \a dst. Upon return, \a src is * zeroed. */ @@ -159,54 +124,4 @@ void grpc_metadata_batch_assert_ok(grpc_metadata_batch *comd); } while (0) #endif -/* Represents a single operation performed on a stream/transport */ -typedef struct grpc_stream_op { - /* the operation to be applied */ - enum grpc_stream_op_code type; - /* the arguments to this operation. union fields are named according to the - associated op-code */ - union { - grpc_begin_message begin_message; - grpc_metadata_batch metadata; - gpr_slice slice; - } data; -} grpc_stream_op; - -/** A stream op buffer is a wrapper around stream operations that is - * dynamically extendable. */ -typedef struct grpc_stream_op_buffer { - grpc_stream_op *ops; - size_t nops; - size_t capacity; - grpc_stream_op inlined_ops[GRPC_SOPB_INLINE_ELEMENTS]; -} grpc_stream_op_buffer; - -/* Initialize a stream op buffer */ -void grpc_sopb_init(grpc_stream_op_buffer *sopb); -/* Destroy a stream op buffer */ -void grpc_sopb_destroy(grpc_stream_op_buffer *sopb); -/* Reset a sopb to no elements */ -void grpc_sopb_reset(grpc_stream_op_buffer *sopb); -/* Swap two sopbs */ -void grpc_sopb_swap(grpc_stream_op_buffer *a, grpc_stream_op_buffer *b); - -void grpc_stream_ops_unref_owned_objects(grpc_stream_op *ops, size_t nops); - -/* Append a GRPC_NO_OP to a buffer */ -void grpc_sopb_add_no_op(grpc_stream_op_buffer *sopb); -/* Append a GRPC_OP_BEGIN to a buffer */ -void grpc_sopb_add_begin_message(grpc_stream_op_buffer *sopb, gpr_uint32 length, - gpr_uint32 flags); -void grpc_sopb_add_metadata(grpc_stream_op_buffer *sopb, - grpc_metadata_batch metadata); -/* Append a GRPC_SLICE to a buffer - does not ref/unref the slice */ -void grpc_sopb_add_slice(grpc_stream_op_buffer *sopb, gpr_slice slice); -/* Append a buffer to a buffer - does not ref/unref any internal objects */ -void grpc_sopb_append(grpc_stream_op_buffer *sopb, grpc_stream_op *ops, - size_t nops); - -void grpc_sopb_move_to(grpc_stream_op_buffer *src, grpc_stream_op_buffer *dst); - -char *grpc_sopb_string(grpc_stream_op_buffer *sopb); - #endif /* GRPC_INTERNAL_CORE_TRANSPORT_STREAM_OP_H */ diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c index 828d212cfe..f2bebc62f3 100644 --- a/src/core/transport/transport.c +++ b/src/core/transport/transport.c @@ -33,9 +33,36 @@ #include "src/core/transport/transport.h" #include <grpc/support/alloc.h> +#include <grpc/support/atm.h> #include <grpc/support/log.h> #include "src/core/transport/transport_impl.h" +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason) { + gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count); + gpr_log(GPR_DEBUG, "STREAM %p:%p REF %d->%d %s", refcount, + refcount->destroy.cb_arg, val, val + 1, reason); +#else +void grpc_stream_ref(grpc_stream_refcount *refcount) { +#endif + gpr_ref(&refcount->refs); +} + +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount, + const char *reason) { + gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count); + gpr_log(GPR_DEBUG, "STREAM %p:%p UNREF %d->%d %s", refcount, + refcount->destroy.cb_arg, val, val - 1, reason); +#else +void grpc_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_stream_refcount *refcount) { +#endif + if (gpr_unref(&refcount->refs)) { + grpc_exec_ctx_enqueue(exec_ctx, &refcount->destroy, 1); + } +} + size_t grpc_transport_stream_size(grpc_transport *transport) { return transport->vtable->sizeof_stream; } @@ -47,10 +74,10 @@ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, - const void *server_data, - grpc_transport_stream_op *initial_op) { - return transport->vtable->init_stream(exec_ctx, transport, stream, - server_data, initial_op); + grpc_stream_refcount *refcount, + const void *server_data) { + return transport->vtable->init_stream(exec_ctx, transport, stream, refcount, + server_data); } void grpc_transport_perform_stream_op(grpc_exec_ctx *exec_ctx, @@ -66,6 +93,12 @@ void grpc_transport_perform_op(grpc_exec_ctx *exec_ctx, transport->vtable->perform_op(exec_ctx, transport, op); } +void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_transport *transport, grpc_stream *stream, + grpc_pollset *pollset) { + transport->vtable->set_pollset(exec_ctx, transport, stream, pollset); +} + void grpc_transport_destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream) { @@ -79,9 +112,8 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, void grpc_transport_stream_op_finish_with_failure( grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op) { - grpc_exec_ctx_enqueue(exec_ctx, op->on_done_recv, 0); - grpc_exec_ctx_enqueue(exec_ctx, op->on_done_send, 0); - grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, 0); + grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, 0); + grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, 0); } void grpc_transport_stream_op_add_cancellation(grpc_transport_stream_op *op, @@ -129,9 +161,9 @@ void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op, if (optional_message) { cmd = gpr_malloc(sizeof(*cmd)); cmd->message = *optional_message; - cmd->then_call = op->on_consumed; + cmd->then_call = op->on_complete; grpc_closure_init(&cmd->closure, free_message, cmd); - op->on_consumed = &cmd->closure; + op->on_complete = &cmd->closure; op->optional_close_message = &cmd->message; } op->close_with_status = status; diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index d4cee03862..f296ce8251 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -38,7 +38,8 @@ #include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_set.h" -#include "src/core/transport/stream_op.h" +#include "src/core/transport/metadata_batch.h" +#include "src/core/transport/byte_stream.h" #include "src/core/channel/context.h" /* forward declarations */ @@ -49,36 +50,35 @@ typedef struct grpc_transport grpc_transport; for a stream. */ typedef struct grpc_stream grpc_stream; -/* Represents the send/recv closed state of a stream. */ -typedef enum grpc_stream_state { - /* the stream is open for sends and receives */ - GRPC_STREAM_OPEN, - /* the stream is closed for sends, but may still receive data */ - GRPC_STREAM_SEND_CLOSED, - /* the stream is closed for receives, but may still send data */ - GRPC_STREAM_RECV_CLOSED, - /* the stream is closed for both sends and receives */ - GRPC_STREAM_CLOSED -} grpc_stream_state; +typedef struct grpc_stream_refcount { + gpr_refcount refs; + grpc_closure destroy; +} grpc_stream_refcount; + +/*#define GRPC_STREAM_REFCOUNT_DEBUG*/ +#ifdef GRPC_STREAM_REFCOUNT_DEBUG +void grpc_stream_ref(grpc_stream_refcount *refcount, const char *reason); +void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount, + const char *reason); +#else +void grpc_stream_ref(grpc_stream_refcount *refcount); +void grpc_stream_unref(grpc_exec_ctx *exec_ctx, grpc_stream_refcount *refcount); +#endif /* Transport stream op: a set of operations to perform on a transport against a single stream */ typedef struct grpc_transport_stream_op { - grpc_closure *on_consumed; + grpc_metadata_batch *send_initial_metadata; + grpc_metadata_batch *send_trailing_metadata; - grpc_stream_op_buffer *send_ops; - int is_last_send; - grpc_closure *on_done_send; + grpc_byte_stream *send_message; - grpc_stream_op_buffer *recv_ops; - grpc_stream_state *recv_state; - /** The number of bytes this peer is currently prepared to receive. - These bytes will be eventually used to replenish per-stream flow control - windows. */ - size_t max_recv_bytes; - grpc_closure *on_done_recv; + grpc_metadata_batch *recv_initial_metadata; + grpc_byte_stream **recv_message; + grpc_closure *recv_message_ready; + grpc_metadata_batch *recv_trailing_metadata; - grpc_pollset *bind_pollset; + grpc_closure *on_complete; /** If != GRPC_STATUS_OK, cancel this stream */ grpc_status_code cancel_with_status; @@ -110,8 +110,8 @@ typedef struct grpc_transport_op { gpr_slice *goaway_message; /** set the callback for accepting new streams; this is a permanent callback, unlike the other one-shot closures */ - void (*set_accept_stream)(void *user_data, grpc_transport *transport, - const void *server_data); + void (*set_accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_transport *transport, const void *server_data); void *set_accept_stream_user_data; /** add this transport to a pollset */ grpc_pollset *bind_pollset; @@ -136,8 +136,12 @@ size_t grpc_transport_stream_size(grpc_transport *transport); supplied from the accept_stream callback function */ int grpc_transport_init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *transport, grpc_stream *stream, - const void *server_data, - grpc_transport_stream_op *initial_op); + grpc_stream_refcount *refcount, + const void *server_data); + +void grpc_transport_set_pollset(grpc_exec_ctx *exec_ctx, + grpc_transport *transport, grpc_stream *stream, + grpc_pollset *pollset); /* Destroy transport data for a stream. diff --git a/src/core/transport/transport_impl.h b/src/core/transport/transport_impl.h index 900c6340ff..40bfb4b13a 100644 --- a/src/core/transport/transport_impl.h +++ b/src/core/transport/transport_impl.h @@ -43,8 +43,12 @@ typedef struct grpc_transport_vtable { /* implementation of grpc_transport_init_stream */ int (*init_stream)(grpc_exec_ctx *exec_ctx, grpc_transport *self, - grpc_stream *stream, const void *server_data, - grpc_transport_stream_op *initial_op); + grpc_stream *stream, grpc_stream_refcount *refcount, + const void *server_data); + + /* implementation of grpc_transport_set_pollset */ + void (*set_pollset)(grpc_exec_ctx *exec_ctx, grpc_transport *self, + grpc_stream *stream, grpc_pollset *pollset); /* implementation of grpc_transport_perform_stream_op */ void (*perform_stream_op)(grpc_exec_ctx *exec_ctx, grpc_transport *self, diff --git a/src/core/transport/transport_op_string.c b/src/core/transport/transport_op_string.c index f62c340e97..f3b6db29d6 100644 --- a/src/core/transport/transport_op_string.c +++ b/src/core/transport/transport_op_string.c @@ -69,42 +69,6 @@ static void put_metadata_list(gpr_strvec *b, grpc_metadata_batch md) { } } -char *grpc_sopb_string(grpc_stream_op_buffer *sopb) { - char *out; - char *tmp; - size_t i; - gpr_strvec b; - gpr_strvec_init(&b); - - for (i = 0; i < sopb->nops; i++) { - grpc_stream_op *op = &sopb->ops[i]; - if (i > 0) gpr_strvec_add(&b, gpr_strdup(", ")); - switch (op->type) { - case GRPC_NO_OP: - gpr_strvec_add(&b, gpr_strdup("NO_OP")); - break; - case GRPC_OP_BEGIN_MESSAGE: - gpr_asprintf(&tmp, "BEGIN_MESSAGE:%d", op->data.begin_message.length); - gpr_strvec_add(&b, tmp); - break; - case GRPC_OP_SLICE: - gpr_asprintf(&tmp, "SLICE:%d", GPR_SLICE_LENGTH(op->data.slice)); - gpr_strvec_add(&b, tmp); - break; - case GRPC_OP_METADATA: - gpr_strvec_add(&b, gpr_strdup("METADATA{")); - put_metadata_list(&b, op->data.metadata); - gpr_strvec_add(&b, gpr_strdup("}")); - break; - } - } - - out = gpr_strvec_flatten(&b, NULL); - gpr_strvec_destroy(&b); - - return out; -} - char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) { char *tmp; char *out; @@ -113,42 +77,52 @@ char *grpc_transport_stream_op_string(grpc_transport_stream_op *op) { gpr_strvec b; gpr_strvec_init(&b); - if (op->send_ops) { + if (op->send_initial_metadata != NULL) { if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); first = 0; - gpr_asprintf(&tmp, "SEND%s:%p", op->is_last_send ? "_LAST" : "", - op->on_done_send); - gpr_strvec_add(&b, tmp); - gpr_strvec_add(&b, gpr_strdup("[")); - gpr_strvec_add(&b, grpc_sopb_string(op->send_ops)); - gpr_strvec_add(&b, gpr_strdup("]")); + gpr_strvec_add(&b, gpr_strdup("SEND_INITIAL_METADATA{")); + put_metadata_list(&b, *op->send_initial_metadata); + gpr_strvec_add(&b, gpr_strdup("}")); } - if (op->recv_ops) { + if (op->send_message != NULL) { if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); first = 0; - gpr_asprintf(&tmp, "RECV:%p:max_recv_bytes=%d", op->on_done_recv, - op->max_recv_bytes); + gpr_asprintf(&tmp, "SEND_MESSAGE:flags=0x%08x:len=%d", + op->send_message->flags, op->send_message->length); gpr_strvec_add(&b, tmp); } - if (op->bind_pollset) { + if (op->send_trailing_metadata != NULL) { if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); first = 0; - gpr_strvec_add(&b, gpr_strdup("BIND")); + gpr_strvec_add(&b, gpr_strdup("SEND_TRAILING_METADATA{")); + put_metadata_list(&b, *op->send_trailing_metadata); + gpr_strvec_add(&b, gpr_strdup("}")); } - if (op->cancel_with_status != GRPC_STATUS_OK) { + if (op->recv_initial_metadata != NULL) { if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); first = 0; - gpr_asprintf(&tmp, "CANCEL:%d", op->cancel_with_status); - gpr_strvec_add(&b, tmp); + gpr_strvec_add(&b, gpr_strdup("RECV_INITIAL_METADATA")); + } + + if (op->recv_message != NULL) { + if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); + first = 0; + gpr_strvec_add(&b, gpr_strdup("RECV_MESSAGE")); } - if (op->on_consumed != NULL) { + if (op->recv_trailing_metadata != NULL) { if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); first = 0; - gpr_asprintf(&tmp, "ON_CONSUMED:%p", op->on_consumed); + gpr_strvec_add(&b, gpr_strdup("RECV_TRAILING_METADATA")); + } + + if (op->cancel_with_status != GRPC_STATUS_OK) { + if (!first) gpr_strvec_add(&b, gpr_strdup(" ")); + first = 0; + gpr_asprintf(&tmp, "CANCEL:%d", op->cancel_with_status); gpr_strvec_add(&b, tmp); } |