aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/core/lib/surface
diff options
context:
space:
mode:
Diffstat (limited to 'src/core/lib/surface')
-rw-r--r--src/core/lib/surface/alarm.c4
-rw-r--r--src/core/lib/surface/byte_buffer_reader.c19
-rw-r--r--src/core/lib/surface/call.c446
-rw-r--r--src/core/lib/surface/call.h2
-rw-r--r--src/core/lib/surface/call_log_batch.c2
-rw-r--r--src/core/lib/surface/channel.c69
-rw-r--r--src/core/lib/surface/channel.h9
-rw-r--r--src/core/lib/surface/channel_ping.c6
-rw-r--r--src/core/lib/surface/completion_queue.c86
-rw-r--r--src/core/lib/surface/completion_queue.h9
-rw-r--r--src/core/lib/surface/init.c2
-rw-r--r--src/core/lib/surface/init_secure.c8
-rw-r--r--src/core/lib/surface/lame_client.c20
-rw-r--r--src/core/lib/surface/server.c333
-rw-r--r--src/core/lib/surface/server.h1
-rw-r--r--src/core/lib/surface/version.c2
16 files changed, 723 insertions, 295 deletions
diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c
index 2cf2f00b31..aa9d60ee6a 100644
--- a/src/core/lib/surface/alarm.c
+++ b/src/core/lib/surface/alarm.c
@@ -48,9 +48,9 @@ struct grpc_alarm {
static void do_nothing_end_completion(grpc_exec_ctx *exec_ctx, void *arg,
grpc_cq_completion *c) {}
-static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void alarm_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
grpc_alarm *alarm = arg;
- grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, success,
+ grpc_cq_end_op(exec_ctx, alarm->cq, alarm->tag, error,
do_nothing_end_completion, NULL, &alarm->completion);
}
diff --git a/src/core/lib/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c
index 809fd5f1fa..c97079f638 100644
--- a/src/core/lib/surface/byte_buffer_reader.c
+++ b/src/core/lib/surface/byte_buffer_reader.c
@@ -62,12 +62,19 @@ void grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader,
case GRPC_BB_RAW:
gpr_slice_buffer_init(&decompressed_slices_buffer);
if (is_compressed(reader->buffer_in)) {
- grpc_msg_decompress(reader->buffer_in->data.raw.compression,
- &reader->buffer_in->data.raw.slice_buffer,
- &decompressed_slices_buffer);
- reader->buffer_out =
- grpc_raw_byte_buffer_create(decompressed_slices_buffer.slices,
- decompressed_slices_buffer.count);
+ if (grpc_msg_decompress(reader->buffer_in->data.raw.compression,
+ &reader->buffer_in->data.raw.slice_buffer,
+ &decompressed_slices_buffer) == 0) {
+ gpr_log(GPR_ERROR,
+ "Unexpected error decompressing data for algorithm with enum "
+ "value '%d'. Reading data as if it were uncompressed.",
+ reader->buffer_in->data.raw.compression);
+ reader->buffer_out = reader->buffer_in;
+ } else { /* all fine */
+ reader->buffer_out =
+ grpc_raw_byte_buffer_create(decompressed_slices_buffer.slices,
+ decompressed_slices_buffer.count);
+ }
gpr_slice_buffer_destroy(&decompressed_slices_buffer);
} else { /* not compressed, use the input buffer as output */
reader->buffer_out = reader->buffer_in;
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 9b2b94eedf..04291b0ee0 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -40,6 +40,7 @@
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
+#include <grpc/support/slice.h>
#include <grpc/support/string_util.h>
#include <grpc/support/useful.h>
@@ -52,7 +53,9 @@
#include "src/core/lib/surface/call.h"
#include "src/core/lib/surface/channel.h"
#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/transport/metadata.h"
#include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport.h"
/** The maximum number of concurrent batches possible.
Based upon the maximum number of individually queueable ops in the batch
@@ -65,12 +68,6 @@
- status/close recv (depending on client/server) */
#define MAX_CONCURRENT_BATCHES 6
-typedef struct {
- grpc_ioreq_completion_func on_complete;
- void *user_data;
- int success;
-} completed_request;
-
#define MAX_SEND_EXTRA_METADATA_COUNT 3
/* Status data for a request can come from several sources; this
@@ -97,31 +94,13 @@ typedef struct {
grpc_mdstr *details;
} received_status;
-/* How far through the GRPC stream have we read? */
-typedef enum {
- /* We are still waiting for initial metadata to complete */
- READ_STATE_INITIAL = 0,
- /* We have gotten initial metadata, and are reading either
- messages or trailing metadata */
- READ_STATE_GOT_INITIAL_METADATA,
- /* The stream is closed for reading */
- READ_STATE_READ_CLOSED,
- /* The stream is closed for reading & writing */
- READ_STATE_STREAM_CLOSED
-} read_state;
-
-typedef enum {
- WRITE_STATE_INITIAL = 0,
- WRITE_STATE_STARTED,
- WRITE_STATE_WRITE_CLOSED
-} write_state;
-
typedef struct batch_control {
grpc_call *call;
grpc_cq_completion cq_completion;
grpc_closure finish_batch;
void *notify_tag;
gpr_refcount steps_to_complete;
+ grpc_error *error;
uint8_t send_initial_metadata;
uint8_t send_message;
@@ -130,11 +109,11 @@ typedef struct batch_control {
uint8_t recv_message;
uint8_t recv_final_op;
uint8_t is_notify_tag_closure;
- uint8_t success;
} batch_control;
struct grpc_call {
grpc_completion_queue *cq;
+ grpc_polling_entity pollent;
grpc_channel *channel;
grpc_call *parent;
grpc_call *first_child;
@@ -176,10 +155,10 @@ struct grpc_call {
received_status status[STATUS_SOURCE_COUNT];
/* Call stats: only valid after trailing metadata received */
- grpc_transport_stream_stats stats;
+ grpc_call_stats stats;
- /* Compression algorithm for the call */
- grpc_compression_algorithm compression_algorithm;
+ /* Compression algorithm for *incoming* data */
+ grpc_compression_algorithm incoming_compression_algorithm;
/* Supported encodings (compression algorithms), a bitset */
uint32_t encodings_accepted_by_peer;
@@ -238,18 +217,19 @@ static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
grpc_status_code status,
const char *description);
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+ grpc_status_code status,
+ const char *description);
static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack,
- bool success);
+ grpc_error *error);
static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
- bool success);
-
-grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
- uint32_t propagation_mask,
- grpc_completion_queue *cq,
- const void *server_transport_data,
- grpc_mdelem **add_initial_metadata,
- size_t add_initial_metadata_count,
- gpr_timespec send_deadline) {
+ grpc_error *error);
+
+grpc_call *grpc_call_create(
+ grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+ grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+ const void *server_transport_data, grpc_mdelem **add_initial_metadata,
+ size_t add_initial_metadata_count, gpr_timespec send_deadline) {
size_t i, j;
grpc_channel_stack *channel_stack = grpc_channel_get_channel_stack(channel);
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -261,6 +241,8 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
call->channel = channel;
call->cq = cq;
call->parent = parent_call;
+ /* Always support no compression */
+ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE);
call->is_client = server_transport_data == NULL;
if (call->is_client) {
GPR_ASSERT(add_initial_metadata_count < MAX_SEND_EXTRA_METADATA_COUNT);
@@ -284,9 +266,20 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
call->context, server_transport_data,
CALL_STACK_FROM_CALL(call));
if (cq != NULL) {
+ GPR_ASSERT(
+ pollset_set_alternative == NULL &&
+ "Only one of 'cq' and 'pollset_set_alternative' should be non-NULL.");
GRPC_CQ_INTERNAL_REF(cq, "bind");
- grpc_call_stack_set_pollset(&exec_ctx, CALL_STACK_FROM_CALL(call),
- grpc_cq_pollset(cq));
+ call->pollent =
+ grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+ }
+ if (pollset_set_alternative != NULL) {
+ call->pollent =
+ grpc_polling_entity_create_from_pollset_set(pollset_set_alternative);
+ }
+ if (!grpc_polling_entity_is_empty(&call->pollent)) {
+ grpc_call_stack_set_pollset_or_pollset_set(
+ &exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
}
if (parent_call != NULL) {
GRPC_CALL_INTERNAL_REF(parent_call, "child");
@@ -341,10 +334,16 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
void grpc_call_set_completion_queue(grpc_exec_ctx *exec_ctx, grpc_call *call,
grpc_completion_queue *cq) {
GPR_ASSERT(cq);
+
+ if (grpc_polling_entity_pollset_set(&call->pollent) != NULL) {
+ gpr_log(GPR_ERROR, "A pollset_set is already registered for this call.");
+ abort();
+ }
call->cq = cq;
GRPC_CQ_INTERNAL_REF(cq, "bind");
- grpc_call_stack_set_pollset(exec_ctx, CALL_STACK_FROM_CALL(call),
- grpc_cq_pollset(cq));
+ call->pollent = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
+ grpc_call_stack_set_pollset_or_pollset_set(
+ exec_ctx, CALL_STACK_FROM_CALL(call), &call->pollent);
}
#ifdef GRPC_STREAM_REFCOUNT_DEBUG
@@ -361,7 +360,8 @@ void grpc_call_internal_unref(grpc_exec_ctx *exec_ctx, grpc_call *c REF_ARG) {
GRPC_CALL_STACK_UNREF(exec_ctx, CALL_STACK_FROM_CALL(c), REF_REASON);
}
-static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, bool success) {
+static void destroy_call(grpc_exec_ctx *exec_ctx, void *call,
+ grpc_error *error) {
size_t i;
int ii;
grpc_call *c = call;
@@ -391,7 +391,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, bool success) {
GRPC_CQ_INTERNAL_UNREF(c->cq, "bind");
}
grpc_channel *channel = c->channel;
- grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), c);
+ grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->stats, c);
GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "call");
GPR_TIMER_END("destroy_call", 0);
}
@@ -406,20 +406,27 @@ static void set_status_code(grpc_call *call, status_source source,
/* TODO(ctiller): what to do about the flush that was previously here */
}
-static void set_compression_algorithm(grpc_call *call,
- grpc_compression_algorithm algo) {
- call->compression_algorithm = algo;
+static void set_incoming_compression_algorithm(
+ grpc_call *call, grpc_compression_algorithm algo) {
+ GPR_ASSERT(algo < GRPC_COMPRESS_ALGORITHMS_COUNT);
+ call->incoming_compression_algorithm = algo;
}
grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
grpc_call *call) {
grpc_compression_algorithm algorithm;
gpr_mu_lock(&call->mu);
- algorithm = call->compression_algorithm;
+ algorithm = call->incoming_compression_algorithm;
gpr_mu_unlock(&call->mu);
return algorithm;
}
+static grpc_compression_algorithm compression_algorithm_for_level_locked(
+ grpc_call *call, grpc_compression_level level) {
+ return grpc_compression_algorithm_for_level(level,
+ call->encodings_accepted_by_peer);
+}
+
uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) {
uint32_t flags;
gpr_mu_lock(&call->mu);
@@ -545,15 +552,28 @@ static grpc_linked_mdelem *linked_from_md(grpc_metadata *md) {
return (grpc_linked_mdelem *)&md->internal_data;
}
+static grpc_metadata *get_md_elem(grpc_metadata *metadata,
+ grpc_metadata *additional_metadata, int i,
+ int count) {
+ grpc_metadata *res =
+ i < count ? &metadata[i] : &additional_metadata[i - count];
+ GPR_ASSERT(res);
+ return res;
+}
+
static int prepare_application_metadata(grpc_call *call, int count,
grpc_metadata *metadata,
int is_trailing,
- int prepend_extra_metadata) {
+ int prepend_extra_metadata,
+ grpc_metadata *additional_metadata,
+ int additional_metadata_count) {
+ int total_count = count + additional_metadata_count;
int i;
grpc_metadata_batch *batch =
&call->metadata_batch[0 /* is_receiving */][is_trailing];
- for (i = 0; i < count; i++) {
- grpc_metadata *md = &metadata[i];
+ for (i = 0; i < total_count; i++) {
+ const grpc_metadata *md =
+ get_md_elem(metadata, additional_metadata, i, count);
grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
l->md = grpc_mdelem_from_string_and_buffer(
@@ -572,9 +592,10 @@ static int prepare_application_metadata(grpc_call *call, int count,
break;
}
}
- if (i != count) {
+ if (i != total_count) {
for (int j = 0; j <= i; j++) {
- grpc_metadata *md = &metadata[j];
+ const grpc_metadata *md =
+ get_md_elem(metadata, additional_metadata, j, count);
grpc_linked_mdelem *l = (grpc_linked_mdelem *)&md->internal_data;
GRPC_MDELEM_UNREF(l->md);
}
@@ -595,24 +616,36 @@ static int prepare_application_metadata(grpc_call *call, int count,
}
}
}
- for (i = 1; i < count; i++) {
- linked_from_md(&metadata[i])->prev = linked_from_md(&metadata[i - 1]);
+ for (i = 1; i < total_count; i++) {
+ grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+ grpc_metadata *prev_md =
+ get_md_elem(metadata, additional_metadata, i - 1, count);
+ linked_from_md(md)->prev = linked_from_md(prev_md);
}
- for (i = 0; i < count - 1; i++) {
- linked_from_md(&metadata[i])->next = linked_from_md(&metadata[i + 1]);
+ for (i = 0; i < total_count - 1; i++) {
+ grpc_metadata *md = get_md_elem(metadata, additional_metadata, i, count);
+ grpc_metadata *next_md =
+ get_md_elem(metadata, additional_metadata, i + 1, count);
+ linked_from_md(md)->next = linked_from_md(next_md);
}
- switch (prepend_extra_metadata * 2 + (count != 0)) {
+
+ switch (prepend_extra_metadata * 2 + (total_count != 0)) {
case 0:
/* no prepend, no metadata => nothing to do */
batch->list.head = batch->list.tail = NULL;
break;
- case 1:
+ case 1: {
/* metadata, but no prepend */
- batch->list.head = linked_from_md(&metadata[0]);
- batch->list.tail = linked_from_md(&metadata[count - 1]);
+ grpc_metadata *first_md =
+ get_md_elem(metadata, additional_metadata, 0, count);
+ grpc_metadata *last_md =
+ get_md_elem(metadata, additional_metadata, total_count - 1, count);
+ batch->list.head = linked_from_md(first_md);
+ batch->list.tail = linked_from_md(last_md);
batch->list.head->prev = NULL;
batch->list.tail->next = NULL;
break;
+ }
case 2:
/* prepend, but no md */
batch->list.head = &call->send_extra_metadata[0];
@@ -621,17 +654,22 @@ static int prepare_application_metadata(grpc_call *call, int count,
batch->list.head->prev = NULL;
batch->list.tail->next = NULL;
break;
- case 3:
+ case 3: {
/* prepend AND md */
+ grpc_metadata *first_md =
+ get_md_elem(metadata, additional_metadata, 0, count);
+ grpc_metadata *last_md =
+ get_md_elem(metadata, additional_metadata, total_count - 1, count);
batch->list.head = &call->send_extra_metadata[0];
call->send_extra_metadata[call->send_extra_metadata_count - 1].next =
- linked_from_md(&metadata[0]);
- linked_from_md(&metadata[0])->prev =
+ linked_from_md(first_md);
+ linked_from_md(first_md)->prev =
&call->send_extra_metadata[call->send_extra_metadata_count - 1];
- batch->list.tail = linked_from_md(&metadata[count - 1]);
+ batch->list.tail = linked_from_md(last_md);
batch->list.head->prev = NULL;
batch->list.tail->next = NULL;
break;
+ }
default:
GPR_UNREACHABLE_CODE(return 0);
}
@@ -700,48 +738,103 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
return r;
}
-typedef struct cancel_closure {
+typedef struct termination_closure {
grpc_closure closure;
grpc_call *call;
grpc_status_code status;
-} cancel_closure;
+ gpr_slice optional_message;
+ grpc_closure *op_closure;
+ enum { TC_CANCEL, TC_CLOSE } type;
+} termination_closure;
+
+static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp,
+ grpc_error *error) {
+ termination_closure *tc = tcp;
+ switch (tc->type) {
+ case TC_CANCEL:
+ GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "cancel");
+ break;
+ case TC_CLOSE:
+ GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "close");
+ break;
+ }
+ gpr_slice_unref(tc->optional_message);
+ grpc_exec_ctx_sched(exec_ctx, tc->op_closure, GRPC_ERROR_NONE, NULL);
+ gpr_free(tc);
+}
-static void done_cancel(grpc_exec_ctx *exec_ctx, void *ccp, bool success) {
- cancel_closure *cc = ccp;
- GRPC_CALL_INTERNAL_UNREF(exec_ctx, cc->call, "cancel");
- gpr_free(cc);
+static void send_cancel(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
+ grpc_transport_stream_op op;
+ termination_closure *tc = tcp;
+ memset(&op, 0, sizeof(op));
+ op.cancel_with_status = tc->status;
+ /* reuse closure to catch completion */
+ grpc_closure_init(&tc->closure, done_termination, tc);
+ op.on_complete = &tc->closure;
+ execute_op(exec_ctx, tc->call, &op);
}
-static void send_cancel(grpc_exec_ctx *exec_ctx, void *ccp, bool success) {
+static void send_close(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
grpc_transport_stream_op op;
- cancel_closure *cc = ccp;
+ termination_closure *tc = tcp;
memset(&op, 0, sizeof(op));
- op.cancel_with_status = cc->status;
+ tc->optional_message = gpr_slice_ref(tc->optional_message);
+ grpc_transport_stream_op_add_close(&op, tc->status, &tc->optional_message);
/* reuse closure to catch completion */
- grpc_closure_init(&cc->closure, done_cancel, cc);
- op.on_complete = &cc->closure;
- execute_op(exec_ctx, cc->call, &op);
+ grpc_closure_init(&tc->closure, done_termination, tc);
+ tc->op_closure = op.on_complete;
+ op.on_complete = &tc->closure;
+ execute_op(exec_ctx, tc->call, &op);
+}
+
+static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx,
+ termination_closure *tc) {
+ grpc_mdstr *details = NULL;
+ if (GPR_SLICE_LENGTH(tc->optional_message) > 0) {
+ tc->optional_message = gpr_slice_ref(tc->optional_message);
+ details = grpc_mdstr_from_slice(tc->optional_message);
+ }
+
+ set_status_code(tc->call, STATUS_FROM_API_OVERRIDE, (uint32_t)tc->status);
+ set_status_details(tc->call, STATUS_FROM_API_OVERRIDE, details);
+
+ if (tc->type == TC_CANCEL) {
+ grpc_closure_init(&tc->closure, send_cancel, tc);
+ GRPC_CALL_INTERNAL_REF(tc->call, "cancel");
+ } else if (tc->type == TC_CLOSE) {
+ grpc_closure_init(&tc->closure, send_close, tc);
+ GRPC_CALL_INTERNAL_REF(tc->call, "close");
+ }
+ grpc_exec_ctx_sched(exec_ctx, &tc->closure, GRPC_ERROR_NONE, NULL);
+ return GRPC_CALL_OK;
}
static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
grpc_status_code status,
const char *description) {
- grpc_mdstr *details =
- description ? grpc_mdstr_from_string(description) : NULL;
- cancel_closure *cc = gpr_malloc(sizeof(*cc));
-
+ termination_closure *tc = gpr_malloc(sizeof(*tc));
+ memset(tc, 0, sizeof(termination_closure));
+ tc->type = TC_CANCEL;
+ tc->call = c;
+ tc->optional_message = gpr_slice_from_copied_string(description);
GPR_ASSERT(status != GRPC_STATUS_OK);
+ tc->status = status;
- set_status_code(c, STATUS_FROM_API_OVERRIDE, (uint32_t)status);
- set_status_details(c, STATUS_FROM_API_OVERRIDE, details);
+ return terminate_with_status(exec_ctx, tc);
+}
- grpc_closure_init(&cc->closure, send_cancel, cc);
- cc->call = c;
- cc->status = status;
- GRPC_CALL_INTERNAL_REF(c, "cancel");
- grpc_exec_ctx_enqueue(exec_ctx, &cc->closure, true, NULL);
+static grpc_call_error close_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c,
+ grpc_status_code status,
+ const char *description) {
+ termination_closure *tc = gpr_malloc(sizeof(*tc));
+ memset(tc, 0, sizeof(termination_closure));
+ tc->type = TC_CLOSE;
+ tc->call = c;
+ tc->optional_message = gpr_slice_from_copied_string(description);
+ GPR_ASSERT(status != GRPC_STATUS_OK);
+ tc->status = status;
- return GRPC_CALL_OK;
+ return terminate_with_status(exec_ctx, tc);
}
static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call,
@@ -775,11 +868,11 @@ grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
return CALL_FROM_TOP_ELEM(elem);
}
-static void call_alarm(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void call_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
grpc_call *call = arg;
gpr_mu_lock(&call->mu);
call->have_alarm = 0;
- if (success) {
+ if (error != GRPC_ERROR_CANCELLED) {
cancel_with_status(exec_ctx, call, GRPC_STATUS_DEADLINE_EXCEEDED,
"Deadline Exceeded");
}
@@ -828,12 +921,16 @@ static uint32_t decode_status(grpc_mdelem *md) {
return status;
}
-static uint32_t decode_compression(grpc_mdelem *md) {
+static grpc_compression_algorithm decode_compression(grpc_mdelem *md) {
grpc_compression_algorithm algorithm =
grpc_compression_algorithm_from_mdstr(md->value);
if (algorithm == GRPC_COMPRESS_ALGORITHMS_COUNT) {
const char *md_c_str = grpc_mdstr_as_c_string(md->value);
- gpr_log(GPR_ERROR, "Invalid compression algorithm: '%s'", md_c_str);
+ gpr_log(GPR_ERROR,
+ "Invalid incoming compression algorithm: '%s'. Interpreting "
+ "incoming data as uncompressed.",
+ md_c_str);
+ return GRPC_COMPRESS_NONE;
}
return algorithm;
}
@@ -878,9 +975,9 @@ static grpc_mdelem *recv_initial_filter(void *callp, grpc_mdelem *elem) {
if (elem == NULL) {
return NULL;
} else if (elem->key == GRPC_MDSTR_GRPC_ENCODING) {
- GPR_TIMER_BEGIN("compression_algorithm", 0);
- set_compression_algorithm(call, decode_compression(elem));
- GPR_TIMER_END("compression_algorithm", 0);
+ GPR_TIMER_BEGIN("incoming_compression_algorithm", 0);
+ set_incoming_compression_algorithm(call, decode_compression(elem));
+ GPR_TIMER_END("incoming_compression_algorithm", 0);
return NULL;
} else if (elem->key == GRPC_MDSTR_GRPC_ACCEPT_ENCODING) {
GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0);
@@ -961,7 +1058,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
batch_control *bctl) {
grpc_call *call = bctl->call;
if (bctl->is_notify_tag_closure) {
- grpc_exec_ctx_enqueue(exec_ctx, bctl->notify_tag, bctl->success, NULL);
+ /* unrefs bctl->error */
+ grpc_exec_ctx_sched(exec_ctx, bctl->notify_tag, bctl->error, NULL);
gpr_mu_lock(&call->mu);
bctl->call->used_batches =
(uint8_t)(bctl->call->used_batches &
@@ -969,7 +1067,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx,
gpr_mu_unlock(&call->mu);
GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion");
} else {
- grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, bctl->success,
+ /* unrefs bctl->error */
+ grpc_cq_end_op(exec_ctx, bctl->call->cq, bctl->notify_tag, bctl->error,
finish_batch_completion, bctl, &bctl->cq_completion);
}
}
@@ -1001,15 +1100,18 @@ static void continue_receiving_slices(grpc_exec_ctx *exec_ctx,
}
static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
- bool success) {
+ grpc_error *error) {
batch_control *bctl = bctlp;
grpc_call *call = bctl->call;
- if (success) {
+ if (error == GRPC_ERROR_NONE) {
gpr_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
call->receiving_slice);
continue_receiving_slices(exec_ctx, bctl);
} else {
+ if (grpc_trace_operation_failures) {
+ GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
+ }
grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
call->receiving_stream = NULL;
grpc_byte_buffer_destroy(*call->receiving_buffer);
@@ -1043,9 +1145,9 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
} else {
call->test_only_last_message_flags = call->receiving_stream->flags;
if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) &&
- (call->compression_algorithm > GRPC_COMPRESS_NONE)) {
+ (call->incoming_compression_algorithm > GRPC_COMPRESS_NONE)) {
*call->receiving_buffer = grpc_raw_compressed_byte_buffer_create(
- NULL, 0, call->compression_algorithm);
+ NULL, 0, call->incoming_compression_algorithm);
} else {
*call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0);
}
@@ -1058,35 +1160,89 @@ static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
}
static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
- bool success) {
+ grpc_error *error) {
batch_control *bctl = bctlp;
grpc_call *call = bctl->call;
gpr_mu_lock(&bctl->call->mu);
- if (bctl->call->has_initial_md_been_received || !success ||
+ if (bctl->call->has_initial_md_been_received || error != GRPC_ERROR_NONE ||
call->receiving_stream == NULL) {
gpr_mu_unlock(&bctl->call->mu);
- process_data_after_md(exec_ctx, bctlp, success);
+ process_data_after_md(exec_ctx, bctlp, error);
} else {
call->saved_receiving_stream_ready_bctlp = bctlp;
gpr_mu_unlock(&bctl->call->mu);
}
}
+static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx,
+ batch_control *bctl) {
+ grpc_call *call = bctl->call;
+ /* validate call->incoming_compression_algorithm */
+ if (call->incoming_compression_algorithm != GRPC_COMPRESS_NONE) {
+ const grpc_compression_algorithm algo =
+ call->incoming_compression_algorithm;
+ char *error_msg = NULL;
+ const grpc_compression_options compression_options =
+ grpc_channel_compression_options(call->channel);
+ /* check if algorithm is known */
+ if (algo >= GRPC_COMPRESS_ALGORITHMS_COUNT) {
+ gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.",
+ algo);
+ gpr_log(GPR_ERROR, "%s", error_msg);
+ close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+ } else if (grpc_compression_options_is_algorithm_enabled(
+ &compression_options, algo) == 0) {
+ /* check if algorithm is supported by current channel config */
+ char *algo_name;
+ grpc_compression_algorithm_name(algo, &algo_name);
+ gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.",
+ algo_name);
+ gpr_log(GPR_ERROR, "%s", error_msg);
+ close_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg);
+ } else {
+ call->incoming_compression_algorithm = algo;
+ }
+ gpr_free(error_msg);
+ }
+
+ /* make sure the received grpc-encoding is amongst the ones listed in
+ * grpc-accept-encoding */
+ GPR_ASSERT(call->encodings_accepted_by_peer != 0);
+ if (!GPR_BITGET(call->encodings_accepted_by_peer,
+ call->incoming_compression_algorithm)) {
+ extern int grpc_compression_trace;
+ if (grpc_compression_trace) {
+ char *algo_name;
+ grpc_compression_algorithm_name(call->incoming_compression_algorithm,
+ &algo_name);
+ gpr_log(GPR_ERROR,
+ "Compression algorithm (grpc-encoding = '%s') not present in "
+ "the bitset of accepted encodings (grpc-accept-encodings: "
+ "'0x%x')",
+ algo_name, call->encodings_accepted_by_peer);
+ }
+ }
+}
+
static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
- void *bctlp, bool success) {
+ void *bctlp, grpc_error *error) {
batch_control *bctl = bctlp;
grpc_call *call = bctl->call;
gpr_mu_lock(&call->mu);
- if (!success) {
- bctl->success = false;
+ if (error != GRPC_ERROR_NONE) {
+ bctl->error = GRPC_ERROR_REF(error);
} else {
grpc_metadata_batch *md =
&call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
grpc_metadata_batch_filter(md, recv_initial_filter, call);
+ GPR_TIMER_BEGIN("validate_filtered_metadata", 0);
+ validate_filtered_metadata(exec_ctx, bctl);
+ GPR_TIMER_END("validate_filtered_metadata", 0);
+
if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
0 &&
!call->is_client) {
@@ -1101,7 +1257,7 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
grpc_closure *saved_rsr_closure = grpc_closure_create(
receiving_stream_ready, call->saved_receiving_stream_ready_bctlp);
call->saved_receiving_stream_ready_bctlp = NULL;
- grpc_exec_ctx_enqueue(exec_ctx, saved_rsr_closure, success, NULL);
+ grpc_exec_ctx_sched(exec_ctx, saved_rsr_closure, error, NULL);
}
gpr_mu_unlock(&call->mu);
@@ -1111,15 +1267,18 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
}
}
-static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
+static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp,
+ grpc_error *error) {
batch_control *bctl = bctlp;
grpc_call *call = bctl->call;
grpc_call *child_call;
grpc_call *next_child_call;
+ GRPC_ERROR_REF(error);
+
gpr_mu_lock(&call->mu);
if (bctl->send_initial_metadata) {
- if (!success) {
+ if (error != GRPC_ERROR_NONE) {
set_status_code(call, STATUS_FROM_CORE, GRPC_STATUS_UNAVAILABLE);
}
grpc_metadata_batch_destroy(
@@ -1165,13 +1324,17 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
call->final_op.server.cancelled);
}
- success = 1;
+ GRPC_ERROR_UNREF(error);
+ error = GRPC_ERROR_NONE;
}
- bctl->success = success != 0;
+ GRPC_ERROR_UNREF(bctl->error);
+ bctl->error = GRPC_ERROR_REF(error);
gpr_mu_unlock(&call->mu);
if (gpr_unref(&bctl->steps_to_complete)) {
post_batch_completion(exec_ctx, bctl);
}
+
+ GRPC_ERROR_UNREF(error);
}
static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
@@ -1201,7 +1364,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
if (nops == 0) {
GRPC_CALL_INTERNAL_REF(call, "completion");
- bctl->success = 1;
+ bctl->error = GRPC_ERROR_NONE;
if (!is_notify_tag_closure) {
grpc_cq_begin_op(call->cq, notify_tag);
}
@@ -1229,7 +1392,40 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
goto done_with_error;
}
- if (op->data.send_initial_metadata.count > INT_MAX) {
+ /* process compression level */
+ grpc_metadata compression_md;
+ memset(&compression_md, 0, sizeof(grpc_metadata));
+ size_t additional_metadata_count = 0;
+ grpc_compression_level effective_compression_level;
+ bool level_set = false;
+ if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
+ effective_compression_level =
+ op->data.send_initial_metadata.maybe_compression_level.level;
+ level_set = true;
+ } else {
+ const grpc_compression_options copts =
+ grpc_channel_compression_options(call->channel);
+ level_set = copts.default_level.is_set;
+ if (level_set) {
+ effective_compression_level = copts.default_level.level;
+ }
+ }
+ if (level_set && !call->is_client) {
+ const grpc_compression_algorithm calgo =
+ compression_algorithm_for_level_locked(
+ call, effective_compression_level);
+ char *calgo_name;
+ grpc_compression_algorithm_name(calgo, &calgo_name);
+ // the following will be picked up by the compress filter and used as
+ // the call's compression algorithm.
+ compression_md.key = GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY;
+ compression_md.value = calgo_name;
+ compression_md.value_length = strlen(calgo_name);
+ additional_metadata_count++;
+ }
+
+ if (op->data.send_initial_metadata.count + additional_metadata_count >
+ INT_MAX) {
error = GRPC_CALL_ERROR_INVALID_METADATA;
goto done_with_error;
}
@@ -1237,7 +1433,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
call->sent_initial_metadata = 1;
if (!prepare_application_metadata(
call, (int)op->data.send_initial_metadata.count,
- op->data.send_initial_metadata.metadata, 0, call->is_client)) {
+ op->data.send_initial_metadata.metadata, 0, call->is_client,
+ &compression_md, (int)additional_metadata_count)) {
error = GRPC_CALL_ERROR_INVALID_METADATA;
goto done_with_error;
}
@@ -1325,7 +1522,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
if (!prepare_application_metadata(
call,
(int)op->data.send_status_from_server.trailing_metadata_count,
- op->data.send_status_from_server.trailing_metadata, 1, 1)) {
+ op->data.send_status_from_server.trailing_metadata, 1, 1, NULL,
+ 0)) {
error = GRPC_CALL_ERROR_INVALID_METADATA;
goto done_with_error;
}
@@ -1397,7 +1595,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
bctl->recv_final_op = 1;
stream_op.recv_trailing_metadata =
&call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
- stream_op.collect_stats = &call->stats;
+ stream_op.collect_stats = &call->stats.transport_stream_stats;
break;
case GRPC_OP_RECV_CLOSE_ON_SERVER:
/* Flag validation: currently allow no flags */
@@ -1419,7 +1617,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
bctl->recv_final_op = 1;
stream_op.recv_trailing_metadata =
&call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
- stream_op.collect_stats = &call->stats;
+ stream_op.collect_stats = &call->stats.transport_stream_stats;
break;
}
}
@@ -1474,7 +1672,8 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
grpc_call_error err;
GRPC_API_TRACE(
- "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, reserved=%p)",
+ "grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, "
+ "reserved=%p)",
5, (call, ops, (unsigned long)nops, tag, reserved));
if (reserved != NULL) {
@@ -1513,9 +1712,10 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; }
grpc_compression_algorithm grpc_call_compression_for_level(
grpc_call *call, grpc_compression_level level) {
gpr_mu_lock(&call->mu);
- const uint32_t accepted_encodings = call->encodings_accepted_by_peer;
+ grpc_compression_algorithm algo =
+ compression_algorithm_for_level_locked(call, level);
gpr_mu_unlock(&call->mu);
- return grpc_compression_algorithm_for_level(level, accepted_encodings);
+ return algo;
}
const char *grpc_call_error_to_string(grpc_call_error error) {
diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h
index 2725e060b8..b640345c21 100644
--- a/src/core/lib/surface/call.h
+++ b/src/core/lib/surface/call.h
@@ -53,6 +53,8 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx,
grpc_call *grpc_call_create(grpc_channel *channel, grpc_call *parent_call,
uint32_t propagation_mask,
grpc_completion_queue *cq,
+ /* if not NULL, it'll be used in lieu of \a cq */
+ grpc_pollset_set *pollset_set_alternative,
const void *server_transport_data,
grpc_mdelem **add_initial_metadata,
size_t add_initial_metadata_count,
diff --git a/src/core/lib/surface/call_log_batch.c b/src/core/lib/surface/call_log_batch.c
index a6d1d5149f..31c074f15d 100644
--- a/src/core/lib/surface/call_log_batch.c
+++ b/src/core/lib/surface/call_log_batch.c
@@ -112,7 +112,7 @@ void grpc_call_log_batch(char *file, int line, gpr_log_severity severity,
size_t i;
for (i = 0; i < nops; i++) {
tmp = grpc_op_string(&ops[i]);
- gpr_log(file, line, severity, "ops[%d]: %s", i, tmp);
+ gpr_log(file, line, severity, "ops[%" PRIuPTR "]: %s", i, tmp);
gpr_free(tmp);
}
}
diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c
index b6b760b5d8..8936e7164f 100644
--- a/src/core/lib/surface/channel.c
+++ b/src/core/lib/surface/channel.c
@@ -36,16 +36,17 @@
#include <stdlib.h>
#include <string.h>
+#include <grpc/compression.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/string_util.h>
+#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/iomgr/iomgr.h"
#include "src/core/lib/support/string.h"
#include "src/core/lib/surface/api_trace.h"
#include "src/core/lib/surface/call.h"
#include "src/core/lib/surface/channel_init.h"
-#include "src/core/lib/surface/init.h"
#include "src/core/lib/transport/static_metadata.h"
/** Cache grpc-status: X mdelems for X = 0..NUM_CACHED_STATUS_ELEMS.
@@ -64,10 +65,12 @@ typedef struct registered_call {
struct grpc_channel {
int is_client;
uint32_t max_message_length;
+ grpc_compression_options compression_options;
grpc_mdelem *default_authority;
gpr_mu registered_call_mu;
registered_call *registered_calls;
+
char *target;
};
@@ -80,7 +83,8 @@ struct grpc_channel {
/* the protobuf library will (by default) start warning at 100megs */
#define DEFAULT_MAX_MESSAGE_LENGTH (100 * 1024 * 1024)
-static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg, bool success);
+static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
+ grpc_error *error);
grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
const grpc_channel_args *input_args,
@@ -111,6 +115,7 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
channel->registered_calls = NULL;
channel->max_message_length = DEFAULT_MAX_MESSAGE_LENGTH;
+ grpc_compression_options_init(&channel->compression_options);
if (args) {
for (size_t i = 0; i < args->num_args; i++) {
if (0 == strcmp(args->args[i].key, GRPC_ARG_MAX_MESSAGE_LENGTH)) {
@@ -151,6 +156,27 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
":authority", args->args[i].value.string);
}
}
+ } else if (0 == strcmp(args->args[i].key,
+ GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL)) {
+ channel->compression_options.default_level.is_set = true;
+ GPR_ASSERT(args->args[i].value.integer >= 0 &&
+ args->args[i].value.integer < GRPC_COMPRESS_LEVEL_COUNT);
+ channel->compression_options.default_level.level =
+ (grpc_compression_level)args->args[i].value.integer;
+ } else if (0 == strcmp(args->args[i].key,
+ GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM)) {
+ channel->compression_options.default_algorithm.is_set = true;
+ GPR_ASSERT(args->args[i].value.integer >= 0 &&
+ args->args[i].value.integer <
+ GRPC_COMPRESS_ALGORITHMS_COUNT);
+ channel->compression_options.default_algorithm.algorithm =
+ (grpc_compression_algorithm)args->args[i].value.integer;
+ } else if (0 ==
+ strcmp(args->args[i].key,
+ GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET)) {
+ channel->compression_options.enabled_algorithms_bitset =
+ (uint32_t)args->args[i].value.integer |
+ 0x1; /* always support no compression */
}
}
grpc_channel_args_destroy(args);
@@ -166,12 +192,14 @@ char *grpc_channel_get_target(grpc_channel *channel) {
static grpc_call *grpc_channel_create_call_internal(
grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
- grpc_completion_queue *cq, grpc_mdelem *path_mdelem,
- grpc_mdelem *authority_mdelem, gpr_timespec deadline) {
+ grpc_completion_queue *cq, grpc_pollset_set *pollset_set_alternative,
+ grpc_mdelem *path_mdelem, grpc_mdelem *authority_mdelem,
+ gpr_timespec deadline) {
grpc_mdelem *send_metadata[2];
size_t num_metadata = 0;
GPR_ASSERT(channel->is_client);
+ GPR_ASSERT(!(cq != NULL && pollset_set_alternative != NULL));
send_metadata[num_metadata++] = path_mdelem;
if (authority_mdelem != NULL) {
@@ -180,8 +208,9 @@ static grpc_call *grpc_channel_create_call_internal(
send_metadata[num_metadata++] = GRPC_MDELEM_REF(channel->default_authority);
}
- return grpc_call_create(channel, parent_call, propagation_mask, cq, NULL,
- send_metadata, num_metadata, deadline);
+ return grpc_call_create(channel, parent_call, propagation_mask, cq,
+ pollset_set_alternative, NULL, send_metadata,
+ num_metadata, deadline);
}
grpc_call *grpc_channel_create_call(grpc_channel *channel,
@@ -201,7 +230,22 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
(int)deadline.clock_type, reserved));
GPR_ASSERT(!reserved);
return grpc_channel_create_call_internal(
- channel, parent_call, propagation_mask, cq,
+ channel, parent_call, propagation_mask, cq, NULL,
+ grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
+ grpc_mdstr_from_string(method)),
+ host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
+ grpc_mdstr_from_string(host))
+ : NULL,
+ deadline);
+}
+
+grpc_call *grpc_channel_create_pollset_set_call(
+ grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+ grpc_pollset_set *pollset_set, const char *method, const char *host,
+ gpr_timespec deadline, void *reserved) {
+ GPR_ASSERT(!reserved);
+ return grpc_channel_create_call_internal(
+ channel, parent_call, propagation_mask, NULL, pollset_set,
grpc_mdelem_from_metadata_strings(GRPC_MDSTR_PATH,
grpc_mdstr_from_string(method)),
host ? grpc_mdelem_from_metadata_strings(GRPC_MDSTR_AUTHORITY,
@@ -245,7 +289,7 @@ grpc_call *grpc_channel_create_registered_call(
(int)deadline.tv_nsec, (int)deadline.clock_type, reserved));
GPR_ASSERT(!reserved);
return grpc_channel_create_call_internal(
- channel, parent_call, propagation_mask, completion_queue,
+ channel, parent_call, propagation_mask, completion_queue, NULL,
GRPC_MDELEM_REF(rc->path),
rc->authority ? GRPC_MDELEM_REF(rc->authority) : NULL, deadline);
}
@@ -267,7 +311,7 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
}
static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
- bool iomgr_success) {
+ grpc_error *error) {
grpc_channel *channel = arg;
grpc_channel_stack_destroy(exec_ctx, CHANNEL_STACK_FROM_CHANNEL(channel));
while (channel->registered_calls) {
@@ -293,7 +337,7 @@ void grpc_channel_destroy(grpc_channel *channel) {
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (channel));
memset(&op, 0, sizeof(op));
- op.disconnect = 1;
+ op.disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed");
elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0);
elem->filter->start_transport_op(&exec_ctx, elem, &op);
@@ -306,6 +350,11 @@ grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel) {
return CHANNEL_STACK_FROM_CHANNEL(channel);
}
+grpc_compression_options grpc_channel_compression_options(
+ const grpc_channel *channel) {
+ return channel->compression_options;
+}
+
grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) {
char tmp[GPR_LTOA_MIN_BUFSIZE];
switch (i) {
diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h
index 22dae930e4..7eff7b8883 100644
--- a/src/core/lib/surface/channel.h
+++ b/src/core/lib/surface/channel.h
@@ -42,6 +42,11 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target,
grpc_channel_stack_type channel_stack_type,
grpc_transport *optional_transport);
+grpc_call *grpc_channel_create_pollset_set_call(
+ grpc_channel *channel, grpc_call *parent_call, uint32_t propagation_mask,
+ grpc_pollset_set *pollset_set, const char *method, const char *host,
+ gpr_timespec deadline, void *reserved);
+
/** Get a (borrowed) pointer to this channels underlying channel stack */
grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel);
@@ -71,4 +76,8 @@ void grpc_channel_internal_unref(grpc_exec_ctx *exec_ctx,
grpc_channel_internal_unref(exec_ctx, channel)
#endif
+/** Return the channel's compression options. */
+grpc_compression_options grpc_channel_compression_options(
+ const grpc_channel *channel);
+
#endif /* GRPC_CORE_LIB_SURFACE_CHANNEL_H */
diff --git a/src/core/lib/surface/channel_ping.c b/src/core/lib/surface/channel_ping.c
index 5a50698695..9818f9d2f2 100644
--- a/src/core/lib/surface/channel_ping.c
+++ b/src/core/lib/surface/channel_ping.c
@@ -53,10 +53,10 @@ static void ping_destroy(grpc_exec_ctx *exec_ctx, void *arg,
gpr_free(arg);
}
-static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
+static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
ping_result *pr = arg;
- grpc_cq_end_op(exec_ctx, pr->cq, pr->tag, success, ping_destroy, pr,
- &pr->completion_storage);
+ grpc_cq_end_op(exec_ctx, pr->cq, pr->tag, GRPC_ERROR_REF(error), ping_destroy,
+ pr, &pr->completion_storage);
}
void grpc_channel_ping(grpc_channel *channel, grpc_completion_queue *cq,
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 1f82c3bad2..2cc6aa74e0 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -1,6 +1,6 @@
/*
*
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -50,6 +50,8 @@
#include "src/core/lib/surface/event_string.h"
#include "src/core/lib/surface/surface_trace.h"
+int grpc_trace_operation_failures;
+
typedef struct {
grpc_pollset_worker **worker;
void *tag;
@@ -70,6 +72,8 @@ struct grpc_completion_queue {
int shutdown;
int shutdown_called;
int is_server_cq;
+ /** Can the server cq accept incoming channels */
+ int is_non_listening_server_cq;
int num_pluckers;
plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
grpc_closure pollset_shutdown_done;
@@ -84,12 +88,13 @@ struct grpc_completion_queue {
};
#define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
+#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
static gpr_mu g_freelist_mu;
static grpc_completion_queue *g_freelist;
static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc,
- bool success);
+ grpc_error *error);
void grpc_cq_global_init(void) { gpr_mu_init(&g_freelist_mu); }
@@ -149,6 +154,7 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
cc->shutdown = 0;
cc->shutdown_called = 0;
cc->is_server_cq = 0;
+ cc->is_non_listening_server_cq = 0;
cc->num_pluckers = 0;
#ifndef NDEBUG
cc->outstanding_tag_count = 0;
@@ -172,7 +178,7 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) {
}
static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg,
- bool success) {
+ grpc_error *error) {
grpc_completion_queue *cc = arg;
GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy");
}
@@ -215,7 +221,7 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) {
event, then enter shutdown mode */
/* Queue a GRPC_OP_COMPLETED operation */
void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
- void *tag, int success,
+ void *tag, grpc_error *error,
void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
grpc_cq_completion *storage),
void *done_arg, grpc_cq_completion *storage) {
@@ -227,16 +233,24 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
#endif
GPR_TIMER_BEGIN("grpc_cq_end_op", 0);
- GRPC_API_TRACE(
- "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, success=%d, done=%p, "
- "done_arg=%p, storage=%p)",
- 7, (exec_ctx, cc, tag, success, done, done_arg, storage));
+ if (grpc_api_trace ||
+ (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) {
+ const char *errmsg = grpc_error_string(error);
+ GRPC_API_TRACE(
+ "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, "
+ "done_arg=%p, storage=%p)",
+ 7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
+ if (grpc_trace_operation_failures) {
+ gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
+ }
+ grpc_error_free_string(errmsg);
+ }
storage->tag = tag;
storage->done = done;
storage->done_arg = done_arg;
- storage->next =
- ((uintptr_t)&cc->completed_head) | ((uintptr_t)(success != 0));
+ storage->next = ((uintptr_t)&cc->completed_head) |
+ ((uintptr_t)(error == GRPC_ERROR_NONE));
gpr_mu_lock(cc->mu);
#ifndef NDEBUG
@@ -263,8 +277,15 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
break;
}
}
- grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker);
+ grpc_error *kick_error =
+ grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker);
gpr_mu_unlock(cc->mu);
+ if (kick_error != GRPC_ERROR_NONE) {
+ const char *msg = grpc_error_string(kick_error);
+ gpr_log(GPR_ERROR, "Kick failed: %s", msg);
+ grpc_error_free_string(msg);
+ GRPC_ERROR_UNREF(kick_error);
+ }
} else {
cc->completed_tail->next =
((uintptr_t)storage) | (1u & (uintptr_t)cc->completed_tail->next);
@@ -278,6 +299,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
}
GPR_TIMER_END("grpc_cq_end_op", 0);
+
+ GRPC_ERROR_UNREF(error);
}
grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
@@ -343,8 +366,18 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
gpr_mu_lock(cc->mu);
continue;
} else {
- grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now,
- iteration_deadline);
+ grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc),
+ &worker, now, iteration_deadline);
+ if (err != GRPC_ERROR_NONE) {
+ gpr_mu_unlock(cc->mu);
+ const char *msg = grpc_error_string(err);
+ gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+ grpc_error_free_string(msg);
+ GRPC_ERROR_UNREF(err);
+ memset(&ret, 0, sizeof(ret));
+ ret.type = GRPC_QUEUE_TIMEOUT;
+ break;
+ }
}
}
GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret);
@@ -460,8 +493,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
grpc_exec_ctx_flush(&exec_ctx);
gpr_mu_lock(cc->mu);
} else {
- grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), &worker, now,
- iteration_deadline);
+ grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc),
+ &worker, now, iteration_deadline);
+ if (err != GRPC_ERROR_NONE) {
+ del_plucker(cc, tag, &worker);
+ gpr_mu_unlock(cc->mu);
+ const char *msg = grpc_error_string(err);
+ gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg);
+ grpc_error_free_string(msg);
+ GRPC_ERROR_UNREF(err);
+ memset(&ret, 0, sizeof(ret));
+ ret.type = GRPC_QUEUE_TIMEOUT;
+ break;
+ }
}
del_plucker(cc, tag, &worker);
}
@@ -511,6 +555,18 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
return POLLSET_FROM_CQ(cc);
}
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
+ return CQ_FROM_POLLSET(ps);
+}
+
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) {
+ cc->is_non_listening_server_cq = 1;
+}
+
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) {
+ return (cc->is_non_listening_server_cq == 1);
+}
+
void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index eef82cf014..b9dd1092b6 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
/*
*
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -39,6 +39,8 @@
#include <grpc/grpc.h>
#include "src/core/lib/iomgr/pollset.h"
+extern int grpc_trace_operation_failures;
+
typedef struct grpc_cq_completion {
/** user supplied tag */
void *tag;
@@ -75,13 +77,16 @@ void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag);
/* Queue a GRPC_OP_COMPLETED operation; tag must correspond to the tag passed to
grpc_cq_begin_op */
void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
- void *tag, int success,
+ void *tag, grpc_error *error,
void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
grpc_cq_completion *storage),
void *done_arg, grpc_cq_completion *storage);
grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc);
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc);
void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
int grpc_cq_is_server_cq(grpc_completion_queue *cc);
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index d7ee122f87..f07039cb94 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -164,6 +164,8 @@ void grpc_init(void) {
grpc_register_tracer("channel_stack_builder",
&grpc_trace_channel_stack_builder);
grpc_register_tracer("http1", &grpc_http1_trace);
+ grpc_register_tracer("compression", &grpc_compression_trace);
+ grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
grpc_security_pre_init();
grpc_iomgr_init();
grpc_executor_init();
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 3fda2c9e1e..7ee7b51568 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,10 +37,10 @@
#include <string.h>
#include "src/core/lib/debug/trace.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_connector.h"
#include "src/core/lib/surface/channel_init.h"
#include "src/core/lib/tsi/transport_security_interface.h"
diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.c
index f50ec54cea..5ea4cba5d1 100644
--- a/src/core/lib/surface/lame_client.c
+++ b/src/core/lib/surface/lame_client.c
@@ -80,7 +80,8 @@ static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
} else if (op->recv_trailing_metadata != NULL) {
fill_metadata(elem, op->recv_trailing_metadata);
}
- grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
+ grpc_transport_stream_op_finish_with_failure(
+ exec_ctx, op, GRPC_ERROR_CREATE("lame client channel"));
}
static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
@@ -91,23 +92,26 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
grpc_channel_element *elem,
grpc_transport_op *op) {
if (op->on_connectivity_state_change) {
- GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE);
- *op->connectivity_state = GRPC_CHANNEL_FATAL_FAILURE;
- op->on_connectivity_state_change->cb(
- exec_ctx, op->on_connectivity_state_change->cb_arg, 1);
+ GPR_ASSERT(*op->connectivity_state != GRPC_CHANNEL_SHUTDOWN);
+ *op->connectivity_state = GRPC_CHANNEL_SHUTDOWN;
+ grpc_exec_ctx_sched(exec_ctx, op->on_connectivity_state_change,
+ GRPC_ERROR_NONE, NULL);
}
if (op->on_consumed != NULL) {
- op->on_consumed->cb(exec_ctx, op->on_consumed->cb_arg, 1);
+ grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
}
if (op->send_ping != NULL) {
- op->send_ping->cb(exec_ctx, op->send_ping->cb_arg, 0);
+ grpc_exec_ctx_sched(exec_ctx, op->send_ping,
+ GRPC_ERROR_CREATE("lame client channel"), NULL);
}
+ GRPC_ERROR_UNREF(op->disconnect_with_error);
}
static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
grpc_call_element_args *args) {}
static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+ const grpc_call_stats *stats,
void *and_free_memory) {
gpr_free(and_free_memory);
}
@@ -127,7 +131,7 @@ const grpc_channel_filter grpc_lame_filter = {
lame_start_transport_op,
sizeof(call_data),
init_call_elem,
- grpc_call_stack_ignore_set_pollset,
+ grpc_call_stack_ignore_set_pollset_or_pollset_set,
destroy_call_elem,
sizeof(channel_data),
init_channel_elem,
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 2db95b9cdf..def6e5068b 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -1,6 +1,6 @@
/*
*
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,6 @@ typedef struct call_data call_data;
typedef struct channel_data channel_data;
typedef struct registered_method registered_method;
-typedef struct {
- call_data *next;
- call_data *prev;
-} call_link;
-
typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
typedef struct requested_call {
@@ -81,7 +76,6 @@ typedef struct requested_call {
void *tag;
grpc_server *server;
grpc_completion_queue *cq_bound_to_call;
- grpc_completion_queue *cq_for_notification;
grpc_call **call;
grpc_cq_completion completion;
grpc_metadata_array *initial_metadata;
@@ -108,6 +102,7 @@ struct channel_data {
grpc_server *server;
grpc_connectivity_state connectivity_state;
grpc_channel *channel;
+ size_t cq_idx;
/* linked list of all channels on a server */
channel_data *next;
channel_data *prev;
@@ -172,7 +167,7 @@ struct request_matcher {
grpc_server *server;
call_data *pending_head;
call_data *pending_tail;
- gpr_stack_lockfree *requests;
+ gpr_stack_lockfree **requests_per_cq;
};
struct registered_method {
@@ -180,6 +175,7 @@ struct registered_method {
char *host;
grpc_server_register_method_payload_handling payload_handling;
uint32_t flags;
+ /* one request matcher per method */
request_matcher request_matcher;
registered_method *next;
};
@@ -195,6 +191,7 @@ struct grpc_server {
grpc_completion_queue **cqs;
grpc_pollset **pollsets;
size_t cq_count;
+ bool started;
/* The two following mutexes control access to server-state
mu_global controls access to non-call-related state (e.g., channel state)
@@ -207,6 +204,7 @@ struct grpc_server {
gpr_mu mu_call; /* mutex for call-specific state */
registered_method *registered_methods;
+ /** one request matcher for unregistered methods */
request_matcher unregistered_request_matcher;
/** free list of available requested_calls indices */
gpr_stack_lockfree *request_freelist;
@@ -232,9 +230,10 @@ struct grpc_server {
#define SERVER_FROM_CALL_ELEM(elem) \
(((channel_data *)(elem)->channel_data)->server)
-static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld, bool success);
+static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld,
+ grpc_error *error);
static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
- requested_call *rc);
+ size_t cq_idx, requested_call *rc, grpc_error *error);
/* Before calling maybe_finish_shutdown, we must hold mu_global and not
hold mu_call */
static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_server *server);
@@ -265,14 +264,14 @@ struct shutdown_cleanup_args {
};
static void shutdown_cleanup(grpc_exec_ctx *exec_ctx, void *arg,
- bool iomgr_status_ignored) {
+ grpc_error *error) {
struct shutdown_cleanup_args *a = arg;
gpr_slice_unref(a->slice);
gpr_free(a);
}
static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
- int send_goaway, int send_disconnect) {
+ int send_goaway, grpc_error *send_disconnect) {
grpc_transport_op op;
struct shutdown_cleanup_args *sc;
grpc_channel_element *elem;
@@ -283,7 +282,7 @@ static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
sc->slice = gpr_slice_from_copied_string("Server shutdown");
op.goaway_message = &sc->slice;
op.goaway_status = GRPC_STATUS_OK;
- op.disconnect = send_disconnect;
+ op.disconnect_with_error = send_disconnect;
grpc_closure_init(&sc->closure, shutdown_cleanup, sc);
op.on_consumed = &sc->closure;
@@ -294,14 +293,16 @@ static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx,
channel_broadcaster *cb,
int send_goaway,
- int force_disconnect) {
+ grpc_error *force_disconnect) {
size_t i;
for (i = 0; i < cb->num_channels; i++) {
- send_shutdown(exec_ctx, cb->channels[i], send_goaway, force_disconnect);
+ send_shutdown(exec_ctx, cb->channels[i], send_goaway,
+ GRPC_ERROR_REF(force_disconnect));
GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, cb->channels[i], "broadcast");
}
gpr_free(cb->channels);
+ GRPC_ERROR_UNREF(force_disconnect);
}
/*
@@ -312,15 +313,23 @@ static void request_matcher_init(request_matcher *rm, size_t entries,
grpc_server *server) {
memset(rm, 0, sizeof(*rm));
rm->server = server;
- rm->requests = gpr_stack_lockfree_create(entries);
+ rm->requests_per_cq =
+ gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
+ for (size_t i = 0; i < server->cq_count; i++) {
+ rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
+ }
}
static void request_matcher_destroy(request_matcher *rm) {
- GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests) == -1);
- gpr_stack_lockfree_destroy(rm->requests);
+ for (size_t i = 0; i < rm->server->cq_count; i++) {
+ GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+ gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
+ }
+ gpr_free(rm->requests_per_cq);
}
-static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, bool success) {
+static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem,
+ grpc_error *error) {
grpc_call_destroy(grpc_call_from_top_element(elem));
}
@@ -335,17 +344,24 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx,
grpc_closure_init(
&calld->kill_zombie_closure, kill_zombie,
grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
- grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+ grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+ NULL);
}
}
static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
grpc_server *server,
- request_matcher *rm) {
+ request_matcher *rm,
+ grpc_error *error) {
int request_id;
- while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
- fail_call(exec_ctx, server, &server->requested_calls[request_id]);
+ for (size_t i = 0; i < server->cq_count; i++) {
+ while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+ -1) {
+ fail_call(exec_ctx, server, i, &server->requested_calls[request_id],
+ GRPC_ERROR_REF(error));
+ }
}
+ GRPC_ERROR_UNREF(error);
}
/*
@@ -364,15 +380,19 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
gpr_mu_destroy(&server->mu_call);
while ((rm = server->registered_methods) != NULL) {
server->registered_methods = rm->next;
- request_matcher_destroy(&rm->request_matcher);
+ if (server->started) {
+ request_matcher_destroy(&rm->request_matcher);
+ }
gpr_free(rm->method);
gpr_free(rm->host);
gpr_free(rm);
}
+ if (server->started) {
+ request_matcher_destroy(&server->unregistered_request_matcher);
+ }
for (i = 0; i < server->cq_count; i++) {
GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server");
}
- request_matcher_destroy(&server->unregistered_request_matcher);
gpr_stack_lockfree_destroy(server->request_freelist);
gpr_free(server->cqs);
gpr_free(server->pollsets);
@@ -398,7 +418,7 @@ static void orphan_channel(channel_data *chand) {
}
static void finish_destroy_channel(grpc_exec_ctx *exec_ctx, void *cd,
- bool success) {
+ grpc_error *error) {
channel_data *chand = cd;
grpc_server *server = chand->server;
GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, chand->channel, "server");
@@ -453,11 +473,11 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
}
static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
- call_data *calld, requested_call *rc) {
+ call_data *calld, size_t cq_idx, requested_call *rc) {
grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
grpc_call *call = calld->call;
*rc->call = call;
- calld->cq_new = rc->cq_for_notification;
+ calld->cq_new = server->cqs[cq_idx];
GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
switch (rc->type) {
case BATCH_CALL:
@@ -487,46 +507,57 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
channel_data *chand = elem->channel_data;
server_ref(chand->server);
- grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, true, done_request_event, rc,
- &rc->completion);
+ grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, GRPC_ERROR_NONE,
+ done_request_event, rc, &rc->completion);
}
-static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
- call_data *calld = arg;
+static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
+ grpc_error *error) {
+ grpc_call_element *call_elem = arg;
+ call_data *calld = call_elem->call_data;
+ channel_data *chand = call_elem->channel_data;
request_matcher *rm = calld->request_matcher;
grpc_server *server = rm->server;
- if (!success || gpr_atm_acq_load(&server->shutdown_flag)) {
+ if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) {
gpr_mu_lock(&calld->mu_state);
calld->state = ZOMBIED;
gpr_mu_unlock(&calld->mu_state);
grpc_closure_init(
&calld->kill_zombie_closure, kill_zombie,
grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
- grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+ grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, error, NULL);
return;
}
- int request_id = gpr_stack_lockfree_pop(rm->requests);
- if (request_id == -1) {
- gpr_mu_lock(&server->mu_call);
- gpr_mu_lock(&calld->mu_state);
- calld->state = PENDING;
- gpr_mu_unlock(&calld->mu_state);
- if (rm->pending_head == NULL) {
- rm->pending_tail = rm->pending_head = calld;
+ for (size_t i = 0; i < server->cq_count; i++) {
+ size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
+ int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+ if (request_id == -1) {
+ continue;
} else {
- rm->pending_tail->pending_next = calld;
- rm->pending_tail = calld;
+ gpr_mu_lock(&calld->mu_state);
+ calld->state = ACTIVATED;
+ gpr_mu_unlock(&calld->mu_state);
+ publish_call(exec_ctx, server, calld, cq_idx,
+ &server->requested_calls[request_id]);
+ return; /* early out */
}
- calld->pending_next = NULL;
- gpr_mu_unlock(&server->mu_call);
+ }
+
+ /* no cq to take the request found: queue it on the slow list */
+ gpr_mu_lock(&server->mu_call);
+ gpr_mu_lock(&calld->mu_state);
+ calld->state = PENDING;
+ gpr_mu_unlock(&calld->mu_state);
+ if (rm->pending_head == NULL) {
+ rm->pending_tail = rm->pending_head = calld;
} else {
- gpr_mu_lock(&calld->mu_state);
- calld->state = ACTIVATED;
- gpr_mu_unlock(&calld->mu_state);
- publish_call(exec_ctx, server, calld, &server->requested_calls[request_id]);
+ rm->pending_tail->pending_next = calld;
+ rm->pending_tail = calld;
}
+ calld->pending_next = NULL;
+ gpr_mu_unlock(&server->mu_call);
}
static void finish_start_new_rpc(
@@ -540,7 +571,8 @@ static void finish_start_new_rpc(
calld->state = ZOMBIED;
gpr_mu_unlock(&calld->mu_state);
grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
- grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+ grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure, GRPC_ERROR_NONE,
+ NULL);
return;
}
@@ -548,14 +580,14 @@ static void finish_start_new_rpc(
switch (payload_handling) {
case GRPC_SRM_PAYLOAD_NONE:
- publish_new_rpc(exec_ctx, calld, true);
+ publish_new_rpc(exec_ctx, elem, GRPC_ERROR_NONE);
break;
case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
grpc_op op;
memset(&op, 0, sizeof(op));
op.op = GRPC_OP_RECV_MESSAGE;
op.data.recv_message = &calld->payload;
- grpc_closure_init(&calld->publish, publish_new_rpc, calld);
+ grpc_closure_init(&calld->publish, publish_new_rpc, elem);
grpc_call_start_batch_and_execute(exec_ctx, calld->call, &op, 1,
&calld->publish);
break;
@@ -636,16 +668,21 @@ static int num_channels(grpc_server *server) {
}
static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx,
- grpc_server *server) {
- registered_method *rm;
- request_matcher_kill_requests(exec_ctx, server,
- &server->unregistered_request_matcher);
- request_matcher_zombify_all_pending_calls(
- exec_ctx, &server->unregistered_request_matcher);
- for (rm = server->registered_methods; rm; rm = rm->next) {
- request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
- request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+ grpc_server *server, grpc_error *error) {
+ if (server->started) {
+ request_matcher_kill_requests(exec_ctx, server,
+ &server->unregistered_request_matcher,
+ GRPC_ERROR_REF(error));
+ request_matcher_zombify_all_pending_calls(
+ exec_ctx, &server->unregistered_request_matcher);
+ for (registered_method *rm = server->registered_methods; rm;
+ rm = rm->next) {
+ request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher,
+ GRPC_ERROR_REF(error));
+ request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+ }
}
+ GRPC_ERROR_UNREF(error);
}
static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
@@ -655,7 +692,8 @@ static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
return;
}
- kill_pending_work_locked(exec_ctx, server);
+ kill_pending_work_locked(exec_ctx, server,
+ GRPC_ERROR_CREATE("Server Shutdown"));
if (server->root_channel_data.next != &server->root_channel_data ||
server->listeners_destroyed < num_listeners(server)) {
@@ -676,7 +714,8 @@ static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
for (i = 0; i < server->num_shutdown_tags; i++) {
server_ref(server);
grpc_cq_end_op(exec_ctx, server->shutdown_tags[i].cq,
- server->shutdown_tags[i].tag, 1, done_shutdown_event, server,
+ server->shutdown_tags[i].tag, GRPC_ERROR_NONE,
+ done_shutdown_event, server,
&server->shutdown_tags[i].completion);
}
}
@@ -699,11 +738,12 @@ static grpc_mdelem *server_filter(void *user_data, grpc_mdelem *md) {
}
static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
- bool success) {
+ grpc_error *error) {
grpc_call_element *elem = ptr;
call_data *calld = elem->call_data;
gpr_timespec op_deadline;
+ GRPC_ERROR_REF(error);
grpc_metadata_batch_filter(calld->recv_initial_metadata, server_filter, elem);
op_deadline = calld->recv_initial_metadata->deadline;
if (0 != gpr_time_cmp(op_deadline, gpr_inf_future(op_deadline.clock_type))) {
@@ -712,11 +752,13 @@ static void server_on_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
if (calld->host && calld->path) {
/* do nothing */
} else {
- success = 0;
+ GRPC_ERROR_UNREF(error);
+ error =
+ GRPC_ERROR_CREATE_REFERENCING("Missing :authority or :path", &error, 1);
}
- calld->on_done_recv_initial_metadata->cb(
- exec_ctx, calld->on_done_recv_initial_metadata->cb_arg, success);
+ grpc_exec_ctx_sched(exec_ctx, calld->on_done_recv_initial_metadata, error,
+ NULL);
}
static void server_mutate_op(grpc_call_element *elem,
@@ -741,10 +783,10 @@ static void server_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
}
static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
- bool success) {
+ grpc_error *error) {
grpc_call_element *elem = ptr;
call_data *calld = elem->call_data;
- if (success) {
+ if (error == GRPC_ERROR_NONE) {
start_new_rpc(exec_ctx, elem);
} else {
gpr_mu_lock(&calld->mu_state);
@@ -752,7 +794,8 @@ static void got_initial_metadata(grpc_exec_ctx *exec_ctx, void *ptr,
calld->state = ZOMBIED;
gpr_mu_unlock(&calld->mu_state);
grpc_closure_init(&calld->kill_zombie_closure, kill_zombie, elem);
- grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true, NULL);
+ grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+ GRPC_ERROR_NONE, NULL);
} else if (calld->state == PENDING) {
calld->state = ZOMBIED;
gpr_mu_unlock(&calld->mu_state);
@@ -769,9 +812,9 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd,
const void *transport_server_data) {
channel_data *chand = cd;
/* create a call */
- grpc_call *call =
- grpc_call_create(chand->channel, NULL, 0, NULL, transport_server_data,
- NULL, 0, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+ grpc_call *call = grpc_call_create(chand->channel, NULL, 0, NULL, NULL,
+ transport_server_data, NULL, 0,
+ gpr_inf_future(GPR_CLOCK_MONOTONIC));
grpc_call_element *elem =
grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
call_data *calld = elem->call_data;
@@ -785,10 +828,10 @@ static void accept_stream(grpc_exec_ctx *exec_ctx, void *cd,
}
static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd,
- bool iomgr_status_ignored) {
+ grpc_error *error) {
channel_data *chand = cd;
grpc_server *server = chand->server;
- if (chand->connectivity_state != GRPC_CHANNEL_FATAL_FAILURE) {
+ if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
grpc_transport_op op;
memset(&op, 0, sizeof(op));
op.on_connectivity_state_change = &chand->channel_connectivity_changed,
@@ -821,7 +864,7 @@ static void init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
}
static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
- void *ignored) {
+ const grpc_call_stats *stats, void *ignored) {
channel_data *chand = elem->channel_data;
call_data *calld = elem->call_data;
@@ -886,7 +929,7 @@ const grpc_channel_filter grpc_server_top_filter = {
grpc_channel_next_op,
sizeof(call_data),
init_call_elem,
- grpc_call_stack_ignore_set_pollset,
+ grpc_call_stack_ignore_set_pollset_or_pollset_set,
destroy_call_elem,
sizeof(channel_data),
init_channel_elem,
@@ -895,25 +938,46 @@ const grpc_channel_filter grpc_server_top_filter = {
"server",
};
-void grpc_server_register_completion_queue(grpc_server *server,
- grpc_completion_queue *cq,
- void *reserved) {
+static void register_completion_queue(grpc_server *server,
+ grpc_completion_queue *cq,
+ bool is_non_listening, void *reserved) {
size_t i, n;
- GRPC_API_TRACE(
- "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
- (server, cq, reserved));
GPR_ASSERT(!reserved);
for (i = 0; i < server->cq_count; i++) {
if (server->cqs[i] == cq) return;
}
- GRPC_CQ_INTERNAL_REF(cq, "server");
+
grpc_cq_mark_server_cq(cq);
+
+ if (is_non_listening) {
+ grpc_cq_mark_non_listening_server_cq(cq);
+ }
+
+ GRPC_CQ_INTERNAL_REF(cq, "server");
n = server->cq_count++;
server->cqs = gpr_realloc(server->cqs,
server->cq_count * sizeof(grpc_completion_queue *));
server->cqs[n] = cq;
}
+void grpc_server_register_completion_queue(grpc_server *server,
+ grpc_completion_queue *cq,
+ void *reserved) {
+ GRPC_API_TRACE(
+ "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
+ (server, cq, reserved));
+ register_completion_queue(server, cq, false, reserved);
+}
+
+void grpc_server_register_non_listening_completion_queue(
+ grpc_server *server, grpc_completion_queue *cq, void *reserved) {
+ GRPC_API_TRACE(
+ "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, "
+ "reserved=%p)",
+ 3, (server, cq, reserved));
+ register_completion_queue(server, cq, true, reserved);
+}
+
grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
size_t i;
@@ -940,8 +1004,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
for (i = 0; i < (size_t)server->max_requested_calls; i++) {
gpr_stack_lockfree_push(server->request_freelist, (int)i);
}
- request_matcher_init(&server->unregistered_request_matcher,
- server->max_requested_calls, server);
server->requested_calls = gpr_malloc(server->max_requested_calls *
sizeof(*server->requested_calls));
@@ -985,8 +1047,6 @@ void *grpc_server_register_method(
}
m = gpr_malloc(sizeof(registered_method));
memset(m, 0, sizeof(*m));
- request_matcher_init(&m->request_matcher, server->max_requested_calls,
- server);
m->method = gpr_strdup(method);
m->host = gpr_strdup(host);
m->next = server->registered_methods;
@@ -1003,13 +1063,23 @@ void grpc_server_start(grpc_server *server) {
GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
+ server->started = true;
+ size_t pollset_count = 0;
server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
for (i = 0; i < server->cq_count; i++) {
- server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+ if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) {
+ server->pollsets[pollset_count++] = grpc_cq_pollset(server->cqs[i]);
+ }
+ }
+ request_matcher_init(&server->unregistered_request_matcher,
+ server->max_requested_calls, server);
+ for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
+ request_matcher_init(&rm->request_matcher, server->max_requested_calls,
+ server);
}
for (l = server->listeners; l; l = l->next) {
- l->start(&exec_ctx, server, l->arg, server->pollsets, server->cq_count);
+ l->start(&exec_ctx, server, l->arg, server->pollsets, pollset_count);
}
grpc_exec_ctx_finish(&exec_ctx);
@@ -1017,8 +1087,8 @@ void grpc_server_start(grpc_server *server) {
void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
grpc_transport *transport,
+ grpc_pollset *accepting_pollset,
const grpc_channel_args *args) {
- size_t i;
size_t num_registered_methods;
size_t alloc;
registered_method *rm;
@@ -1033,12 +1103,6 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
uint32_t max_probes = 0;
grpc_transport_op op;
- for (i = 0; i < s->cq_count; i++) {
- memset(&op, 0, sizeof(op));
- op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
- grpc_transport_perform_op(exec_ctx, transport, &op);
- }
-
channel =
grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
chand = (channel_data *)grpc_channel_stack_element(
@@ -1048,6 +1112,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
server_ref(s);
chand->channel = channel;
+ size_t cq_idx;
+ grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
+ for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
+ if (s->cqs[cq_idx] == accepting_cq) break;
+ }
+ if (cq_idx == s->cq_count) {
+ /* completion queue not found: pick a random one to publish new calls to */
+ cq_idx = (size_t)rand() % s->cq_count;
+ }
+ chand->cq_idx = cq_idx;
+
num_registered_methods = 0;
for (rm = s->registered_methods; rm; rm = rm->next) {
num_registered_methods++;
@@ -1092,7 +1167,9 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
op.set_accept_stream_user_data = chand;
op.on_connectivity_state_change = &chand->channel_connectivity_changed;
op.connectivity_state = &chand->connectivity_state;
- op.disconnect = gpr_atm_acq_load(&s->shutdown_flag) != 0;
+ if (gpr_atm_acq_load(&s->shutdown_flag) != 0) {
+ op.disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
+ }
grpc_transport_perform_op(exec_ctx, transport, &op);
}
@@ -1103,7 +1180,7 @@ void done_published_shutdown(grpc_exec_ctx *exec_ctx, void *done_arg,
}
static void listener_destroy_done(grpc_exec_ctx *exec_ctx, void *s,
- bool success) {
+ grpc_error *error) {
grpc_server *server = s;
gpr_mu_lock(&server->mu_global);
server->listeners_destroyed++;
@@ -1125,8 +1202,8 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
gpr_mu_lock(&server->mu_global);
grpc_cq_begin_op(cq, tag);
if (server->shutdown_published) {
- grpc_cq_end_op(&exec_ctx, cq, tag, 1, done_published_shutdown, NULL,
- gpr_malloc(sizeof(grpc_cq_completion)));
+ grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown,
+ NULL, gpr_malloc(sizeof(grpc_cq_completion)));
gpr_mu_unlock(&server->mu_global);
goto done;
}
@@ -1149,7 +1226,8 @@ void grpc_server_shutdown_and_notify(grpc_server *server,
/* collect all unregistered then registered calls */
gpr_mu_lock(&server->mu_call);
- kill_pending_work_locked(&exec_ctx, server);
+ kill_pending_work_locked(&exec_ctx, server,
+ GRPC_ERROR_CREATE("Server Shutdown"));
gpr_mu_unlock(&server->mu_call);
maybe_finish_shutdown(&exec_ctx, server);
@@ -1177,7 +1255,8 @@ void grpc_server_cancel_all_calls(grpc_server *server) {
channel_broadcaster_init(server, &broadcaster);
gpr_mu_unlock(&server->mu_global);
- channel_broadcaster_shutdown(&exec_ctx, &broadcaster, 0, 1);
+ channel_broadcaster_shutdown(&exec_ctx, &broadcaster, 0,
+ GRPC_ERROR_CREATE("Cancelling all calls"));
grpc_exec_ctx_finish(&exec_ctx);
}
@@ -1218,19 +1297,21 @@ void grpc_server_add_listener(
}
static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
- grpc_server *server,
+ grpc_server *server, size_t cq_idx,
requested_call *rc) {
call_data *calld = NULL;
request_matcher *rm = NULL;
int request_id;
if (gpr_atm_acq_load(&server->shutdown_flag)) {
- fail_call(exec_ctx, server, rc);
+ fail_call(exec_ctx, server, cq_idx, rc,
+ GRPC_ERROR_CREATE("Server Shutdown"));
return GRPC_CALL_OK;
}
request_id = gpr_stack_lockfree_pop(server->request_freelist);
if (request_id == -1) {
/* out of request ids: just fail this one */
- fail_call(exec_ctx, server, rc);
+ fail_call(exec_ctx, server, cq_idx, rc,
+ GRPC_ERROR_CREATE("Server Shutdown"));
return GRPC_CALL_OK;
}
switch (rc->type) {
@@ -1243,12 +1324,12 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
}
server->requested_calls[request_id] = *rc;
gpr_free(rc);
- if (gpr_stack_lockfree_push(rm->requests, request_id)) {
+ if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
/* this was the first queued request: we need to lock and start
matching calls */
gpr_mu_lock(&server->mu_call);
while ((calld = rm->pending_head) != NULL) {
- request_id = gpr_stack_lockfree_pop(rm->requests);
+ request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
if (request_id == -1) break;
rm->pending_head = calld->pending_next;
gpr_mu_unlock(&server->mu_call);
@@ -1258,13 +1339,13 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
grpc_closure_init(
&calld->kill_zombie_closure, kill_zombie,
grpc_call_stack_element(grpc_call_get_call_stack(calld->call), 0));
- grpc_exec_ctx_enqueue(exec_ctx, &calld->kill_zombie_closure, true,
- NULL);
+ grpc_exec_ctx_sched(exec_ctx, &calld->kill_zombie_closure,
+ GRPC_ERROR_NONE, NULL);
} else {
GPR_ASSERT(calld->state == PENDING);
calld->state = ACTIVATED;
gpr_mu_unlock(&calld->mu_state);
- publish_call(exec_ctx, server, calld,
+ publish_call(exec_ctx, server, calld, cq_idx,
&server->requested_calls[request_id]);
}
gpr_mu_lock(&server->mu_call);
@@ -1288,7 +1369,13 @@ grpc_call_error grpc_server_request_call(
"cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)",
7, (server, call, details, initial_metadata, cq_bound_to_call,
cq_for_notification, tag));
- if (!grpc_cq_is_server_cq(cq_for_notification)) {
+ size_t cq_idx;
+ for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+ if (server->cqs[cq_idx] == cq_for_notification) {
+ break;
+ }
+ }
+ if (cq_idx == server->cq_count) {
gpr_free(rc);
error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
goto done;
@@ -1299,11 +1386,10 @@ grpc_call_error grpc_server_request_call(
rc->server = server;
rc->tag = tag;
rc->cq_bound_to_call = cq_bound_to_call;
- rc->cq_for_notification = cq_for_notification;
rc->call = call;
rc->data.batch.details = details;
rc->initial_metadata = initial_metadata;
- error = queue_call_request(&exec_ctx, server, rc);
+ error = queue_call_request(&exec_ctx, server, cq_idx, rc);
done:
grpc_exec_ctx_finish(&exec_ctx);
return error;
@@ -1325,7 +1411,14 @@ grpc_call_error grpc_server_request_registered_call(
"tag=%p)",
9, (server, rmp, call, deadline, initial_metadata, optional_payload,
cq_bound_to_call, cq_for_notification, tag));
- if (!grpc_cq_is_server_cq(cq_for_notification)) {
+
+ size_t cq_idx;
+ for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+ if (server->cqs[cq_idx] == cq_for_notification) {
+ break;
+ }
+ }
+ if (cq_idx == server->cq_count) {
gpr_free(rc);
error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
goto done;
@@ -1341,25 +1434,25 @@ grpc_call_error grpc_server_request_registered_call(
rc->server = server;
rc->tag = tag;
rc->cq_bound_to_call = cq_bound_to_call;
- rc->cq_for_notification = cq_for_notification;
rc->call = call;
rc->data.registered.registered_method = rm;
rc->data.registered.deadline = deadline;
rc->initial_metadata = initial_metadata;
rc->data.registered.optional_payload = optional_payload;
- error = queue_call_request(&exec_ctx, server, rc);
+ error = queue_call_request(&exec_ctx, server, cq_idx, rc);
done:
grpc_exec_ctx_finish(&exec_ctx);
return error;
}
static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
- requested_call *rc) {
+ size_t cq_idx, requested_call *rc, grpc_error *error) {
*rc->call = NULL;
rc->initial_metadata->count = 0;
+ GPR_ASSERT(error != GRPC_ERROR_NONE);
server_ref(server);
- grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0,
+ grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, error,
done_request_event, rc, &rc->completion);
}
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index 470ef23c69..fb6e4d60c5 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -53,6 +53,7 @@ void grpc_server_add_listener(
server */
void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
grpc_transport *transport,
+ grpc_pollset *accepting_pollset,
const grpc_channel_args *args);
const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
diff --git a/src/core/lib/surface/version.c b/src/core/lib/surface/version.c
index fe954cbefb..aca76d2bb7 100644
--- a/src/core/lib/surface/version.c
+++ b/src/core/lib/surface/version.c
@@ -36,4 +36,4 @@
#include <grpc/grpc.h>
-const char *grpc_version_string(void) { return "0.14.0-dev"; }
+const char *grpc_version_string(void) { return "0.15.0-dev"; }
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-14 13:43:26 +0000