diff options
Diffstat (limited to 'src')
159 files changed, 3693 insertions, 1541 deletions
diff --git a/src/core/ext/census/gen/trace_context.pb.c b/src/core/ext/census/gen/trace_context.pb.c index c8aea324ce..f4126d4d80 100644 --- a/src/core/ext/census/gen/trace_context.pb.c +++ b/src/core/ext/census/gen/trace_context.pb.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,51 +31,24 @@ * */ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.3.5-dev */ +/* Generated by nanopb-0.3.7-dev at Fri Jan 20 16:14:22 2017. */ #include "src/core/ext/census/gen/trace_context.pb.h" +/* @@protoc_insertion_point(includes) */ #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. #endif -const pb_field_t google_trace_TraceId_fields[3] = { - PB_FIELD( 1, FIXED64 , OPTIONAL, STATIC , FIRST, google_trace_TraceId, hi, hi, 0), - PB_FIELD( 2, FIXED64 , OPTIONAL, STATIC , OTHER, google_trace_TraceId, lo, hi, 0), +const pb_field_t google_trace_TraceContext_fields[5] = { + PB_FIELD( 1, FIXED64 , OPTIONAL, STATIC , FIRST, google_trace_TraceContext, trace_id_hi, trace_id_hi, 0), + PB_FIELD( 2, FIXED64 , OPTIONAL, STATIC , OTHER, google_trace_TraceContext, trace_id_lo, trace_id_hi, 0), + PB_FIELD( 3, FIXED64 , OPTIONAL, STATIC , OTHER, google_trace_TraceContext, span_id, trace_id_lo, 0), + PB_FIELD( 4, FIXED32 , OPTIONAL, STATIC , OTHER, google_trace_TraceContext, span_options, span_id, 0), PB_LAST_FIELD }; -const pb_field_t google_trace_TraceContext_fields[4] = { - PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, google_trace_TraceContext, trace_id, trace_id, &google_trace_TraceId_fields), - PB_FIELD( 2, FIXED64 , OPTIONAL, STATIC , OTHER, google_trace_TraceContext, span_id, trace_id, 0), - PB_FIELD( 3, BOOL , OPTIONAL, STATIC , OTHER, google_trace_TraceContext, is_sampled, span_id, 0), - PB_LAST_FIELD -}; - - -/* Check that field information fits in pb_field_t */ -#if !defined(PB_FIELD_32BIT) -/* If you get an error here, it means that you need to define PB_FIELD_32BIT - * compile-time option. You can do that in pb.h or on compiler command line. - * - * The reason you need to do this is that some of your messages contain tag - * numbers or field sizes that are larger than what can fit in 8 or 16 bit - * field descriptors. - */ -PB_STATIC_ASSERT((pb_membersize(google_trace_TraceContext, trace_id) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_google_trace_TraceId_google_trace_TraceContext) -#endif - -#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) -/* If you get an error here, it means that you need to define PB_FIELD_16BIT - * compile-time option. You can do that in pb.h or on compiler command line. - * - * The reason you need to do this is that some of your messages contain tag - * numbers or field sizes that are larger than what can fit in the default - * 8 bit descriptors. - */ -PB_STATIC_ASSERT((pb_membersize(google_trace_TraceContext, trace_id) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_google_trace_TraceId_google_trace_TraceContext) -#endif - +/* @@protoc_insertion_point(eof) */ diff --git a/src/core/ext/census/gen/trace_context.pb.h b/src/core/ext/census/gen/trace_context.pb.h index cfb2f04ccd..ea127bf70a 100644 --- a/src/core/ext/census/gen/trace_context.pb.h +++ b/src/core/ext/census/gen/trace_context.pb.h @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,11 +31,13 @@ * */ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.5-dev */ +/* Generated by nanopb-0.3.7-dev at Fri Jan 20 16:14:22 2017. */ #ifndef GRPC_CORE_EXT_CENSUS_GEN_TRACE_CONTEXT_PB_H #define GRPC_CORE_EXT_CENSUS_GEN_TRACE_CONTEXT_PB_H #include "third_party/nanopb/pb.h" + +/* @@protoc_insertion_point(includes) */ #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. #endif @@ -45,44 +47,35 @@ extern "C" { #endif /* Struct definitions */ -typedef struct _google_trace_TraceId { - bool has_hi; - uint64_t hi; - bool has_lo; - uint64_t lo; -} google_trace_TraceId; - typedef struct _google_trace_TraceContext { - bool has_trace_id; - google_trace_TraceId trace_id; + bool has_trace_id_hi; + uint64_t trace_id_hi; + bool has_trace_id_lo; + uint64_t trace_id_lo; bool has_span_id; uint64_t span_id; - bool has_is_sampled; - bool is_sampled; + bool has_span_options; + uint32_t span_options; +/* @@protoc_insertion_point(struct:google_trace_TraceContext) */ } google_trace_TraceContext; /* Default values for struct fields */ /* Initializer values for message structs */ -#define google_trace_TraceId_init_default {false, 0, false, 0} -#define google_trace_TraceContext_init_default {false, google_trace_TraceId_init_default, false, 0, false, 0} -#define google_trace_TraceId_init_zero {false, 0, false, 0} -#define google_trace_TraceContext_init_zero {false, google_trace_TraceId_init_zero, false, 0, false, 0} +#define google_trace_TraceContext_init_default {false, 0, false, 0, false, 0, false, 0} +#define google_trace_TraceContext_init_zero {false, 0, false, 0, false, 0, false, 0} /* Field tags (for use in manual encoding/decoding) */ -#define google_trace_TraceId_hi_tag 1 -#define google_trace_TraceId_lo_tag 2 -#define google_trace_TraceContext_trace_id_tag 1 -#define google_trace_TraceContext_span_id_tag 2 -#define google_trace_TraceContext_is_sampled_tag 3 +#define google_trace_TraceContext_trace_id_hi_tag 1 +#define google_trace_TraceContext_trace_id_lo_tag 2 +#define google_trace_TraceContext_span_id_tag 3 +#define google_trace_TraceContext_span_options_tag 4 /* Struct field encoding specification for nanopb */ -extern const pb_field_t google_trace_TraceId_fields[3]; -extern const pb_field_t google_trace_TraceContext_fields[4]; +extern const pb_field_t google_trace_TraceContext_fields[5]; /* Maximum encoded size of messages (where known) */ -#define google_trace_TraceId_size 18 -#define google_trace_TraceContext_size 31 +#define google_trace_TraceContext_size 32 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID @@ -95,5 +88,6 @@ extern const pb_field_t google_trace_TraceContext_fields[4]; #ifdef __cplusplus } /* extern "C" */ #endif +/* @@protoc_insertion_point(eof) */ -#endif /* GRPC_CORE_EXT_CENSUS_GEN_TRACE_CONTEXT_PB_H */ +#endif diff --git a/src/core/ext/census/trace_context.c b/src/core/ext/census/trace_context.c index fbb20d3448..47d0de19da 100644 --- a/src/core/ext/census/trace_context.c +++ b/src/core/ext/census/trace_context.c @@ -73,7 +73,7 @@ bool decode_trace_context(google_trace_TraceContext *ctxt, uint8_t *buffer, } // check fields - if (!ctxt->has_trace_id) { + if (!ctxt->has_trace_id_hi || !ctxt->has_trace_id_lo) { gpr_log(GPR_DEBUG, "Invalid TraceContext: missing trace_id"); return false; } diff --git a/src/core/ext/census/trace_context.h b/src/core/ext/census/trace_context.h index 1cb5e26ea7..f391a1b7c1 100644 --- a/src/core/ext/census/trace_context.h +++ b/src/core/ext/census/trace_context.h @@ -38,6 +38,9 @@ #include "src/core/ext/census/gen/trace_context.pb.h" +/* Span option flags. */ +#define SPAN_OPTIONS_IS_SAMPLED 0x01 + /* Maximum number of bytes required to encode a TraceContext (31) 1 byte for trace_id field 1 byte for trace_id length diff --git a/src/core/ext/census/trace_label.h b/src/core/ext/census/trace_label.h new file mode 100644 index 0000000000..0e4a8d885f --- /dev/null +++ b/src/core/ext/census/trace_label.h @@ -0,0 +1,61 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_LABEL_H +#define GRPC_CORE_EXT_CENSUS_TRACE_LABEL_H + +#include "src/core/ext/census/trace_string.h" + +/* Trace label (key/value pair) stores a label name and the label value. The + value can be one of trace_string/int64_t/bool. */ +typedef struct trace_label { + trace_string key; + enum label_type { + /* Unknown value for debugging/error purposes */ + LABEL_UNKNOWN = 0, + /* A string value */ + LABEL_STRING = 1, + /* An integer value. */ + LABEL_INT = 2, + /* A boolean value. */ + LABEL_BOOL = 3, + } value_type; + + union value { + trace_string label_str; + int64_t label_int; + bool label_bool; + } value; +} trace_label; + +#endif diff --git a/src/core/ext/census/trace_propagation.h b/src/core/ext/census/trace_propagation.h new file mode 100644 index 0000000000..75c4ebaa39 --- /dev/null +++ b/src/core/ext/census/trace_propagation.h @@ -0,0 +1,63 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_PROPAGATION_H +#define GRPC_CORE_EXT_CENSUS_TRACE_PROPAGATION_H + +#include "src/core/ext/census/tracing.h" + +/* Encoding and decoding functions for receiving and sending trace contexts + over the wire. Only RPC libraries should be calling these + functions. These functions return the number of bytes encoded/decoded + (0 if a failure has occurred). buf_size indicates the size of the + input/output buffer. trace_span_context is a struct that includes the + trace ID, span ID, and a set of option flags (is_sampled, etc.). */ + +/* Converts a span context to a binary byte buffer. */ +size_t trace_span_context_to_binary(const trace_span_context *ctxt, + uint8_t *buf, size_t buf_size); + +/* Reads a binary byte buffer and populates a span context structure. */ +size_t binary_to_trace_span_context(const uint8_t *buf, size_t buf_size, + trace_span_context *ctxt); + +/* Converts a span context to an http metadata compatible string. */ +size_t trace_span_context_to_http_format(const trace_span_context *ctxt, + char *buf, size_t buf_size); + +/* Reads an http metadata compatible string and populates a span context + structure. */ +size_t http_format_to_trace_span_context(const char *buf, size_t buf_size, + trace_span_context *ctxt); + +#endif diff --git a/src/core/ext/census/trace_status.h b/src/core/ext/census/trace_status.h new file mode 100644 index 0000000000..adc0ebd0c0 --- /dev/null +++ b/src/core/ext/census/trace_status.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_STATUS_H +#define GRPC_CORE_EXT_CENSUS_TRACE_STATUS_H + +#include "src/core/ext/census/trace_string.h" + +/* Stores a status code and status message for a trace. */ +typedef struct trace_status { + int64_t errorCode; + trace_string errorMessage; +} trace_status; + +#endif diff --git a/src/core/ext/census/trace_string.h b/src/core/ext/census/trace_string.h new file mode 100644 index 0000000000..8e77ee9f7e --- /dev/null +++ b/src/core/ext/census/trace_string.h @@ -0,0 +1,50 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACE_STRING_H +#define GRPC_CORE_EXT_CENSUS_TRACE_STRING_H + +#include <grpc/slice.h> + +/* String struct for tracing messages. Since this is a C API, we do not have + access to a string class. This is intended for use by higher level + languages which wrap around the C API, as most of them have a string class. + This will also be more efficient when copying, as we have an explicitly + specified length. Also, grpc_slice has reference counting which allows for + interning. */ +typedef struct trace_string { + char *string; + size_t length; +} trace_string; + +#endif diff --git a/src/core/ext/census/tracing.c b/src/core/ext/census/tracing.c index 9371fffc8d..8b74628f4f 100644 --- a/src/core/ext/census/tracing.c +++ b/src/core/ext/census/tracing.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,21 +31,41 @@ * */ -//#include "src/core/ext/census/tracing.h" +#include "src/core/ext/census/tracing.h" #include <grpc/census.h> -#include <stdlib.h> +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <openssl/rand.h> +#include "src/core/ext/census/mlog.h" -/* TODO(aveitch): These are all placeholder implementations. */ +void trace_start_span(const trace_span_context *span_ctxt, + const trace_string name, const start_span_options *opts, + trace_span_context *new_span_ctxt, + bool has_remote_parent) { + // Noop implementation. +} + +void trace_add_span_annotation(const trace_string description, + const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt) { + // Noop implementation. +} -int census_trace_mask(const census_context *context) { - abort(); - return CENSUS_TRACE_MASK_NONE; +void trace_add_span_network_event_annotation(const trace_string description, + const trace_label *labels, + const size_t n_labels, + const gpr_timespec timestamp, + bool sent, uint64_t id, + trace_span_context *span_ctxt) { + // Noop implementation. } -void census_set_trace_mask(int trace_mask) { abort(); } +void trace_add_span_labels(const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt) { + // Noop implementation. +} -void census_trace_print(census_context *context, uint32_t type, - const char *buffer, size_t n) { - abort(); +void trace_end_span(const trace_status *status, trace_span_context *span_ctxt) { + // Noop implementation. } diff --git a/src/core/ext/census/tracing.h b/src/core/ext/census/tracing.h new file mode 100644 index 0000000000..c2b947ae40 --- /dev/null +++ b/src/core/ext/census/tracing.h @@ -0,0 +1,124 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_CENSUS_TRACING_H +#define GRPC_CORE_EXT_CENSUS_TRACING_H + +#include <grpc/support/time.h> +#include <stdbool.h> +#include "src/core/ext/census/trace_context.h" +#include "src/core/ext/census/trace_label.h" +#include "src/core/ext/census/trace_status.h" + +/* This is the low level tracing API that other languages will interface with. + This is not intended to be accessed by the end-user, therefore it has been + designed with performance in mind rather than ease of use. */ + +/* The tracing level. */ +enum TraceLevel { + /* Annotations on this context will be silently discarded. */ + NO_TRACING = 0, + /* Annotations will not be saved to a persistent store. They will be + available via local APIs only. This setting is not propagated to child + spans. */ + TRANSIENT_TRACING = 1, + /* Annotations are recorded for the entire distributed trace and they are + saved to a persistent store. This setting is propagated to child spans. */ + PERSISTENT_TRACING = 2, +}; + +typedef struct trace_span_context { + /* Trace span context stores Span ID, Trace ID, and option flags. */ + /* Trace ID is 128 bits split into 2 64-bit chunks (hi and lo). */ + uint64_t trace_id_hi; + uint64_t trace_id_lo; + /* Span ID is 64 bits. */ + uint64_t span_id; + /* Span-options is 32-bit value which contains flag options. */ + uint32_t span_options; +} trace_span_context; + +typedef struct start_span_options { + /* If set, this will override the Span.local_start_time for the Span. */ + gpr_timespec local_start_timestamp; + + /* Linked spans can be used to identify spans that are linked to this span in + a different trace. This can be used (for example) in batching operations, + where a single batch handler processes multiple requests from different + traces. If set, points to a list of Spans are linked to the created Span.*/ + trace_span_context *linked_spans; + /* The number of linked spans. */ + size_t n_linked_spans; +} start_span_options; + +/* Create a new child Span (or root if parent is NULL), with parent being the + designated Span. The child span will have the provided name and starting + span options (optional). The bool has_remote_parent marks whether the + context refers to a remote parent span or not. */ +void trace_start_span(const trace_span_context *span_ctxt, + const trace_string name, const start_span_options *opts, + trace_span_context *new_span_ctxt, + bool has_remote_parent); + +/* Add a new Annotation to the Span. Annotations consist of a description + (trace_string) and a set of n labels (trace_label). This can be populated + with arbitrary user data. */ +void trace_add_span_annotation(const trace_string description, + const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt); + +/* Add a new NetworkEvent annotation to a Span. This function is only intended + to be used by RPC systems (either client or server), not by higher level + applications. The timestamp type will be system-defined, the sent argument + designates whether this is a network send event (client request, server + reply)or receive (server request, client reply). The id argument corresponds + to Span.Annotation.NetworkEvent.id from the data model, and serves to uniquely + identify each network message. */ +void trace_add_span_network_event(const trace_string description, + const trace_label *labels, + const size_t n_labels, + const gpr_timespec timestamp, bool sent, + uint64_t id, trace_span_context *span_ctxt); + +/* Add a set of labels to the Span. These will correspond to the field +Span.labels in the data model. */ +void trace_add_span_labels(const trace_label *labels, const size_t n_labels, + trace_span_context *span_ctxt); + +/* Mark the end of Span Execution with the given status. Only the timing of the +first EndSpan call for a given Span will be recorded, and implementations are +free to ignore all further calls using the Span. EndSpanOptions can +optionally be NULL. */ +void trace_end_span(const trace_status *status, trace_span_context *span_ctxt); + +#endif diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 07eb68a3eb..06038bb5ba 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -44,8 +44,8 @@ #include <grpc/support/useful.h> #include "src/core/ext/client_channel/http_connect_handshaker.h" -#include "src/core/ext/client_channel/http_proxy.h" #include "src/core/ext/client_channel/lb_policy_registry.h" +#include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/resolver_registry.h" #include "src/core/ext/client_channel/subchannel.h" #include "src/core/lib/channel/channel_args.h" @@ -153,10 +153,6 @@ static void *method_parameters_create_from_json(const grpc_json *json) { */ typedef struct client_channel_channel_data { - /** server name */ - char *server_name; - /** HTTP CONNECT proxy to use, if any */ - char *proxy_name; /** resolver for this channel */ grpc_resolver *resolver; /** have we started resolving this channel */ @@ -317,17 +313,6 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg, // Use pick_first if nothing was specified and we didn't select grpclb // above. if (lb_policy_name == NULL) lb_policy_name = "pick_first"; - // If using a proxy, add channel arg for server in HTTP CONNECT request. - if (chand->proxy_name != NULL) { - grpc_arg new_arg; - new_arg.key = GRPC_ARG_HTTP_CONNECT_SERVER; - new_arg.type = GRPC_ARG_STRING; - new_arg.value.string = chand->server_name; - grpc_channel_args *tmp_args = chand->resolver_result; - chand->resolver_result = - grpc_channel_args_copy_and_add(chand->resolver_result, &new_arg, 1); - grpc_channel_args_destroy(exec_ctx, tmp_args); - } // Instantiate LB policy. grpc_lb_policy_args lb_policy_args; lb_policy_args.args = chand->resolver_result; @@ -542,16 +527,21 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(arg->type == GRPC_ARG_POINTER); grpc_client_channel_factory_ref(arg->value.pointer.p); chand->client_channel_factory = arg->value.pointer.p; - // Instantiate resolver. + // Get server name to resolve, using proxy mapper if needed. arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVER_URI); GPR_ASSERT(arg != NULL); GPR_ASSERT(arg->type == GRPC_ARG_STRING); - chand->server_name = gpr_strdup(arg->value.string); - chand->proxy_name = grpc_get_http_proxy_server(); - char *name_to_resolve = - chand->proxy_name == NULL ? chand->server_name : chand->proxy_name; + char *proxy_name = NULL; + grpc_channel_args *new_args = NULL; + grpc_proxy_mappers_map_name(exec_ctx, arg->value.string, args->channel_args, + &proxy_name, &new_args); + // Instantiate resolver. chand->resolver = grpc_resolver_create( - exec_ctx, name_to_resolve, args->channel_args, chand->interested_parties); + exec_ctx, proxy_name != NULL ? proxy_name : arg->value.string, + new_args != NULL ? new_args : args->channel_args, + chand->interested_parties); + if (proxy_name != NULL) gpr_free(proxy_name); + if (new_args != NULL) grpc_channel_args_destroy(exec_ctx, new_args); if (chand->resolver == NULL) { return GRPC_ERROR_CREATE("resolver creation failed"); } @@ -562,8 +552,6 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem) { channel_data *chand = elem->channel_data; - gpr_free(chand->server_name); - gpr_free(chand->proxy_name); if (chand->resolver != NULL) { grpc_resolver_shutdown(exec_ctx, chand->resolver); GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel"); @@ -644,6 +632,12 @@ typedef struct client_channel_call_data { grpc_linked_mdelem lb_token_mdelem; } call_data; +grpc_subchannel_call *grpc_client_channel_get_subchannel_call( + grpc_call_element *call_elem) { + grpc_subchannel_call *scc = GET_CALL((call_data *)call_elem->call_data); + return scc == CANCELLED_CALL ? NULL : scc; +} + static void add_waiting_locked(call_data *calld, grpc_transport_stream_op *op) { GPR_TIMER_BEGIN("add_waiting_locked", 0); if (calld->waiting_ops_count == calld->waiting_ops_capacity) { diff --git a/src/core/ext/client_channel/client_channel.h b/src/core/ext/client_channel/client_channel.h index f02587d0c1..5e6e64e58b 100644 --- a/src/core/ext/client_channel/client_channel.h +++ b/src/core/ext/client_channel/client_channel.h @@ -57,4 +57,8 @@ void grpc_client_channel_watch_connectivity_state( grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_pollset *pollset, grpc_connectivity_state *state, grpc_closure *on_complete); +/* Debug helper: pull the subchannel call from a call stack element */ +grpc_subchannel_call *grpc_client_channel_get_subchannel_call( + grpc_call_element *elem); + #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_CLIENT_CHANNEL_H */ diff --git a/src/core/ext/client_channel/client_channel_plugin.c b/src/core/ext/client_channel/client_channel_plugin.c index 7f75233727..6f9df3e386 100644 --- a/src/core/ext/client_channel/client_channel_plugin.c +++ b/src/core/ext/client_channel/client_channel_plugin.c @@ -39,6 +39,7 @@ #include "src/core/ext/client_channel/client_channel.h" #include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/http_proxy.h" #include "src/core/ext/client_channel/lb_policy_registry.h" #include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/resolver_registry.h" @@ -82,6 +83,7 @@ void grpc_client_channel_init(void) { grpc_lb_policy_registry_init(); grpc_resolver_registry_init(); grpc_proxy_mapper_registry_init(); + grpc_register_http_proxy_mapper(); grpc_subchannel_index_init(); grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MIN, set_default_host_if_unset, NULL); diff --git a/src/core/ext/client_channel/http_proxy.c b/src/core/ext/client_channel/http_proxy.c index 9a6c818c4e..bbe4ff550c 100644 --- a/src/core/ext/client_channel/http_proxy.c +++ b/src/core/ext/client_channel/http_proxy.c @@ -40,10 +40,13 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/proxy_mapper_registry.h" #include "src/core/ext/client_channel/uri_parser.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/support/env.h" -char* grpc_get_http_proxy_server() { +static char* grpc_get_http_proxy_server() { char* uri_str = gpr_getenv("http_proxy"); if (uri_str == NULL) return NULL; grpc_uri* uri = grpc_uri_parse(uri_str, false /* suppress_errors */); @@ -66,3 +69,55 @@ done: grpc_uri_destroy(uri); return proxy_name; } + +static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + *name_to_resolve = grpc_get_http_proxy_server(); + if (*name_to_resolve == NULL) return false; + grpc_uri* uri = grpc_uri_parse(server_uri, false /* suppress_errors */); + if (uri == NULL || uri->path[0] == '\0') { + gpr_log(GPR_ERROR, + "'http_proxy' environment variable set, but cannot " + "parse server URI '%s' -- not using proxy", + server_uri); + if (uri != NULL) grpc_uri_destroy(uri); + return false; + } + if (strcmp(uri->scheme, "unix") == 0) { + gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'", + server_uri); + grpc_uri_destroy(uri); + return false; + } + grpc_arg new_arg; + new_arg.key = GRPC_ARG_HTTP_CONNECT_SERVER; + new_arg.type = GRPC_ARG_STRING; + new_arg.value.string = uri->path[0] == '/' ? uri->path + 1 : uri->path; + *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1); + grpc_uri_destroy(uri); + return true; +} + +static bool proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return false; +} + +static void proxy_mapper_destroy(grpc_proxy_mapper* mapper) {} + +static const grpc_proxy_mapper_vtable proxy_mapper_vtable = { + proxy_mapper_map_name, proxy_mapper_map_address, proxy_mapper_destroy}; + +static grpc_proxy_mapper proxy_mapper = {&proxy_mapper_vtable}; + +void grpc_register_http_proxy_mapper() { + grpc_proxy_mapper_register(true /* at_start */, &proxy_mapper); +} diff --git a/src/core/ext/client_channel/http_proxy.h b/src/core/ext/client_channel/http_proxy.h index 0d77ae253b..c8882b1ef1 100644 --- a/src/core/ext/client_channel/http_proxy.h +++ b/src/core/ext/client_channel/http_proxy.h @@ -34,8 +34,6 @@ #ifndef GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H #define GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H -/// Returns the name of the proxy to use, or NULL if no proxy is configured. -/// Caller takes ownership of result. -char* grpc_get_http_proxy_server(); +void grpc_register_http_proxy_mapper(); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_HTTP_PROXY_H */ diff --git a/src/core/ext/client_channel/proxy_mapper.c b/src/core/ext/client_channel/proxy_mapper.c index 6b6f328d3c..f92afe847b 100644 --- a/src/core/ext/client_channel/proxy_mapper.c +++ b/src/core/ext/client_channel/proxy_mapper.c @@ -38,13 +38,24 @@ void grpc_proxy_mapper_init(const grpc_proxy_mapper_vtable* vtable, mapper->vtable = vtable; } -bool grpc_proxy_mapper_map(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { - return mapper->vtable->map(exec_ctx, mapper, address, args, new_address, - new_args); +bool grpc_proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + return mapper->vtable->map_name(exec_ctx, mapper, server_uri, args, + name_to_resolve, new_args); +} + +bool grpc_proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return mapper->vtable->map_address(exec_ctx, mapper, address, args, + new_address, new_args); } void grpc_proxy_mapper_destroy(grpc_proxy_mapper* mapper) { diff --git a/src/core/ext/client_channel/proxy_mapper.h b/src/core/ext/client_channel/proxy_mapper.h index fa930379e7..6e4607fe4d 100644 --- a/src/core/ext/client_channel/proxy_mapper.h +++ b/src/core/ext/client_channel/proxy_mapper.h @@ -43,15 +43,22 @@ typedef struct grpc_proxy_mapper grpc_proxy_mapper; typedef struct { + /// Determines the proxy name to resolve for \a server_uri. + /// If no proxy is needed, returns false. + /// Otherwise, sets \a name_to_resolve, optionally sets \a new_args, + /// and returns true. + bool (*map_name)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, + const char* server_uri, const grpc_channel_args* args, + char** name_to_resolve, grpc_channel_args** new_args); /// Determines the proxy address to use to contact \a address. /// If no proxy is needed, returns false. /// Otherwise, sets \a new_address, optionally sets \a new_args, and /// returns true. - bool (*map)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); + bool (*map_address)(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); /// Destroys \a mapper. void (*destroy)(grpc_proxy_mapper* mapper); } grpc_proxy_mapper_vtable; @@ -63,11 +70,20 @@ struct grpc_proxy_mapper { void grpc_proxy_mapper_init(const grpc_proxy_mapper_vtable* vtable, grpc_proxy_mapper* mapper); -bool grpc_proxy_mapper_map(grpc_exec_ctx* exec_ctx, grpc_proxy_mapper* mapper, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); +bool grpc_proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args); + +bool grpc_proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper* mapper, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); + void grpc_proxy_mapper_destroy(grpc_proxy_mapper* mapper); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_PROXY_MAPPER_H */ diff --git a/src/core/ext/client_channel/proxy_mapper_registry.c b/src/core/ext/client_channel/proxy_mapper_registry.c index 0a156c8b1c..2c44b9d490 100644 --- a/src/core/ext/client_channel/proxy_mapper_registry.c +++ b/src/core/ext/client_channel/proxy_mapper_registry.c @@ -61,15 +61,28 @@ static void grpc_proxy_mapper_list_register(grpc_proxy_mapper_list* list, ++list->num_mappers; } -static bool grpc_proxy_mapper_list_map(grpc_exec_ctx* exec_ctx, - grpc_proxy_mapper_list* list, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { +static bool grpc_proxy_mapper_list_map_name(grpc_exec_ctx* exec_ctx, + grpc_proxy_mapper_list* list, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { for (size_t i = 0; i < list->num_mappers; ++i) { - if (grpc_proxy_mapper_map(exec_ctx, list->list[i], address, args, - new_address, new_args)) { + if (grpc_proxy_mapper_map_name(exec_ctx, list->list[i], server_uri, args, + name_to_resolve, new_args)) { + return true; + } + } + return false; +} + +static bool grpc_proxy_mapper_list_map_address( + grpc_exec_ctx* exec_ctx, grpc_proxy_mapper_list* list, + const grpc_resolved_address* address, const grpc_channel_args* args, + grpc_resolved_address** new_address, grpc_channel_args** new_args) { + for (size_t i = 0; i < list->num_mappers; ++i) { + if (grpc_proxy_mapper_map_address(exec_ctx, list->list[i], address, args, + new_address, new_args)) { return true; } } @@ -101,11 +114,20 @@ void grpc_proxy_mapper_register(bool at_start, grpc_proxy_mapper* mapper) { grpc_proxy_mapper_list_register(&g_proxy_mapper_list, at_start, mapper); } -bool grpc_proxy_mappers_map(grpc_exec_ctx* exec_ctx, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args) { - return grpc_proxy_mapper_list_map(exec_ctx, &g_proxy_mapper_list, address, - args, new_address, new_args); +bool grpc_proxy_mappers_map_name(grpc_exec_ctx* exec_ctx, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args) { + return grpc_proxy_mapper_list_map_name(exec_ctx, &g_proxy_mapper_list, + server_uri, args, name_to_resolve, + new_args); +} +bool grpc_proxy_mappers_map_address(grpc_exec_ctx* exec_ctx, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args) { + return grpc_proxy_mapper_list_map_address( + exec_ctx, &g_proxy_mapper_list, address, args, new_address, new_args); } diff --git a/src/core/ext/client_channel/proxy_mapper_registry.h b/src/core/ext/client_channel/proxy_mapper_registry.h index b76af8d456..742b57a2d4 100644 --- a/src/core/ext/client_channel/proxy_mapper_registry.h +++ b/src/core/ext/client_channel/proxy_mapper_registry.h @@ -44,10 +44,16 @@ void grpc_proxy_mapper_registry_shutdown(); /// the list. Otherwise, it will be added to the end. void grpc_proxy_mapper_register(bool at_start, grpc_proxy_mapper* mapper); -bool grpc_proxy_mappers_map(grpc_exec_ctx* exec_ctx, - const grpc_resolved_address* address, - const grpc_channel_args* args, - grpc_resolved_address** new_address, - grpc_channel_args** new_args); +bool grpc_proxy_mappers_map_name(grpc_exec_ctx* exec_ctx, + const char* server_uri, + const grpc_channel_args* args, + char** name_to_resolve, + grpc_channel_args** new_args); + +bool grpc_proxy_mappers_map_address(grpc_exec_ctx* exec_ctx, + const grpc_resolved_address* address, + const grpc_channel_args* args, + grpc_resolved_address** new_address, + grpc_channel_args** new_args); #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_PROXY_MAPPER_REGISTRY_H */ diff --git a/src/core/ext/client_channel/subchannel.c b/src/core/ext/client_channel/subchannel.c index f1e4e079e2..aa036e883b 100644 --- a/src/core/ext/client_channel/subchannel.c +++ b/src/core/ext/client_channel/subchannel.c @@ -336,8 +336,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, grpc_set_initial_connect_string(&addr, &c->initial_connect_string); grpc_resolved_address *new_address = NULL; grpc_channel_args *new_args = NULL; - if (grpc_proxy_mappers_map(exec_ctx, addr, args->args, &new_address, - &new_args)) { + if (grpc_proxy_mappers_map_address(exec_ctx, addr, args->args, &new_address, + &new_args)) { GPR_ASSERT(new_address != NULL); gpr_free(addr); addr = new_address; @@ -788,7 +788,8 @@ grpc_call_stack *grpc_subchannel_call_get_call_stack( return SUBCHANNEL_CALL_TO_CALL_STACK(subchannel_call); } -static void grpc_uri_to_sockaddr(char *uri_str, grpc_resolved_address *addr) { +static void grpc_uri_to_sockaddr(const char *uri_str, + grpc_resolved_address *addr) { grpc_uri *uri = grpc_uri_parse(uri_str, 0 /* suppress_errors */); GPR_ASSERT(uri != NULL); if (strcmp(uri->scheme, "ipv4") == 0) { @@ -803,14 +804,19 @@ static void grpc_uri_to_sockaddr(char *uri_str, grpc_resolved_address *addr) { void grpc_get_subchannel_address_arg(const grpc_channel_args *args, grpc_resolved_address *addr) { + const char *addr_uri_str = grpc_get_subchannel_address_uri_arg(args); + memset(addr, 0, sizeof(*addr)); + if (*addr_uri_str != '\0') { + grpc_uri_to_sockaddr(addr_uri_str, addr); + } +} + +const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args) { const grpc_arg *addr_arg = grpc_channel_args_find(args, GRPC_ARG_SUBCHANNEL_ADDRESS); GPR_ASSERT(addr_arg != NULL); // Should have been set by LB policy. GPR_ASSERT(addr_arg->type == GRPC_ARG_STRING); - memset(addr, 0, sizeof(*addr)); - if (*addr_arg->value.string != '\0') { - grpc_uri_to_sockaddr(addr_arg->value.string, addr); - } + return addr_arg->value.string; } grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr) { diff --git a/src/core/ext/client_channel/subchannel.h b/src/core/ext/client_channel/subchannel.h index 9bd35a7704..26ce954487 100644 --- a/src/core/ext/client_channel/subchannel.h +++ b/src/core/ext/client_channel/subchannel.h @@ -178,6 +178,9 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, void grpc_get_subchannel_address_arg(const grpc_channel_args *args, grpc_resolved_address *addr); +/// Returns the URI string for the address to connect to. +const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args); + /// Returns a new channel arg encoding the subchannel address as a string. /// Caller is responsible for freeing the string. grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr); diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 308facb7e7..ab62e5ed6a 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -112,11 +112,13 @@ #include "src/core/ext/client_channel/lb_policy_registry.h" #include "src/core/ext/client_channel/parse_address.h" #include "src/core/ext/lb_policy/grpclb/grpclb.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/slice/slice_hash_table.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/backoff.h" @@ -751,6 +753,96 @@ static void glb_rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_UNREF(error); } +static void destroy_balancer_name(grpc_exec_ctx *exec_ctx, + void *balancer_name) { + gpr_free(balancer_name); +} + +static void *copy_balancer_name(void *balancer_name) { + return gpr_strdup(balancer_name); +} + +static grpc_slice_hash_table_entry targets_info_entry_create( + const char *address, const char *balancer_name) { + static const grpc_slice_hash_table_vtable vtable = {destroy_balancer_name, + copy_balancer_name}; + grpc_slice_hash_table_entry entry; + entry.key = grpc_slice_from_copied_string(address); + entry.value = (void *)balancer_name; + entry.vtable = &vtable; + return entry; +} + +/* Returns the target URI for the LB service whose addresses are in \a + * addresses. Using this URI, a bidirectional streaming channel will be created + * for the reception of load balancing updates. + * + * The output argument \a targets_info will be updated to contain a mapping of + * "LB server address" to "balancer name", as reported by the naming system. + * This mapping will be propagated via the channel arguments of the + * aforementioned LB streaming channel, to be used by the security connector for + * secure naming checks. The user is responsible for freeing \a targets_info. */ +static char *get_lb_uri_target_addresses(grpc_exec_ctx *exec_ctx, + const grpc_lb_addresses *addresses, + grpc_slice_hash_table **targets_info) { + size_t num_grpclb_addrs = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; + } + /* All input addresses come from a resolver that claims they are LB services. + * It's the resolver's responsibility to make sure this policy is only + * instantiated and used in that case. Otherwise, something has gone wrong. */ + GPR_ASSERT(num_grpclb_addrs > 0); + + grpc_slice_hash_table_entry *targets_info_entries = + gpr_malloc(sizeof(*targets_info_entries) * num_grpclb_addrs); + + /* construct a target ipvX://ip1:port1,ip2:port2,... from the addresses in \a + * addresses */ + /* TODO(dgq): support mixed ip version */ + char **addr_strs = gpr_malloc(sizeof(char *) * num_grpclb_addrs); + size_t addr_index = 0; + + for (size_t i = 0; i < addresses->num_addresses; i++) { + if (addresses->addresses[i].user_data != NULL) { + gpr_log(GPR_ERROR, + "This LB policy doesn't support user data. It will be ignored"); + } + if (addresses->addresses[i].is_balancer) { + char *addr_str; + GPR_ASSERT(grpc_sockaddr_to_string( + &addr_str, &addresses->addresses[i].address, true) > 0); + targets_info_entries[addr_index] = targets_info_entry_create( + addr_str, addresses->addresses[i].balancer_name); + addr_strs[addr_index++] = addr_str; + } + } + GPR_ASSERT(addr_index == num_grpclb_addrs); + + size_t uri_path_len; + char *uri_path = gpr_strjoin_sep((const char **)addr_strs, num_grpclb_addrs, + ",", &uri_path_len); + for (size_t i = 0; i < num_grpclb_addrs; i++) gpr_free(addr_strs[i]); + gpr_free(addr_strs); + + char *target_uri_str = NULL; + /* TODO(dgq): Don't assume all addresses will share the scheme of the first + * one */ + gpr_asprintf(&target_uri_str, "%s:%s", + grpc_sockaddr_get_uri_scheme(&addresses->addresses[0].address), + uri_path); + gpr_free(uri_path); + + *targets_info = + grpc_slice_hash_table_create(num_grpclb_addrs, targets_info_entries); + for (size_t i = 0; i < num_grpclb_addrs; i++) { + grpc_slice_unref_internal(exec_ctx, targets_info_entries[i].key); + } + gpr_free(targets_info_entries); + + return target_uri_str; +} + static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { @@ -788,85 +880,30 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, } grpc_uri_destroy(uri); - /* All input addresses in addresses come from a resolver that claims - * they are LB services. It's the resolver's responsibility to make sure - * this policy is only instantiated and used in that case. - * - * Create a client channel over them to communicate with a LB service */ glb_policy->cc_factory = args->client_channel_factory; glb_policy->args = grpc_channel_args_copy(args->args); GPR_ASSERT(glb_policy->cc_factory != NULL); - /* construct a target from the addresses in args, given in the form - * ipvX://ip1:port1,ip2:port2,... - * TODO(dgq): support mixed ip version */ - char **addr_strs = gpr_malloc(sizeof(char *) * num_grpclb_addrs); - size_t addr_index = 0; - for (size_t i = 0; i < addresses->num_addresses; i++) { - if (addresses->addresses[i].user_data != NULL) { - gpr_log(GPR_ERROR, - "This LB policy doesn't support user data. It will be ignored"); - } - if (addresses->addresses[i].is_balancer) { - if (addr_index == 0) { - addr_strs[addr_index++] = - grpc_sockaddr_to_uri(&addresses->addresses[i].address); - } else { - GPR_ASSERT(grpc_sockaddr_to_string(&addr_strs[addr_index++], - &addresses->addresses[i].address, - true) > 0); - } - } - } - size_t uri_path_len; - char *target_uri_str = gpr_strjoin_sep((const char **)addr_strs, - num_grpclb_addrs, ",", &uri_path_len); - - /* Create a channel to talk to the LBs. - * - * We strip out the channel arg for the LB policy name, since we want - * to use the default (pick_first) in this case. - * - * We also strip out the channel arg for the resolved addresses, since - * that will be generated by the name resolver used in the LB channel. - * Note that the LB channel will use the sockaddr resolver, so this - * won't actually generate a query to DNS (or some other name service). - * However, the addresses returned by the sockaddr resolver will have - * is_balancer=false, whereas our own addresses have is_balancer=true. - * We need the LB channel to return addresses with is_balancer=false - * so that it does not wind up recursively using the grpclb LB policy, - * as per the special case logic in client_channel.c. - * - * Finally, we also strip out the channel arg for the server URI, - * since that will be different for the LB channel than for the parent - * channel. (The client channel factory will re-add this arg with - * the right value.) - */ - static const char *keys_to_remove[] = { - GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; - grpc_channel_args *new_args = grpc_channel_args_copy_and_remove( - args->args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove)); - glb_policy->lb_channel = grpc_client_channel_factory_create_channel( - exec_ctx, glb_policy->cc_factory, target_uri_str, - GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, new_args); - grpc_channel_args_destroy(exec_ctx, new_args); - - gpr_free(target_uri_str); - for (size_t i = 0; i < num_grpclb_addrs; i++) { - gpr_free(addr_strs[i]); - } - gpr_free(addr_strs); - + grpc_slice_hash_table *targets_info = NULL; + /* Create a client channel over them to communicate with a LB service */ + char *lb_service_target_addresses = + get_lb_uri_target_addresses(exec_ctx, addresses, &targets_info); + grpc_channel_args *lb_channel_args = + get_lb_channel_args(exec_ctx, targets_info, args->args); + glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( + exec_ctx, lb_service_target_addresses, args->client_channel_factory, + lb_channel_args); + grpc_slice_hash_table_unref(exec_ctx, targets_info); + grpc_channel_args_destroy(exec_ctx, lb_channel_args); + gpr_free(lb_service_target_addresses); if (glb_policy->lb_channel == NULL) { gpr_free(glb_policy); return NULL; } - grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable); gpr_mu_init(&glb_policy->mu); grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, "grpclb"); - return &glb_policy->base; } diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel.c b/src/core/ext/lb_policy/grpclb/grpclb_channel.c new file mode 100644 index 0000000000..1b8bbab1b6 --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel.c @@ -0,0 +1,77 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/alloc.h> +#include <grpc/support/string_util.h> + +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/string.h" + +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args) { + grpc_channel *lb_channel = grpc_client_channel_factory_create_channel( + exec_ctx, client_channel_factory, lb_service_target_addresses, + GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, args); + return lb_channel; +} + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args) { + /* We strip out the channel arg for the LB policy name, since we want + * to use the default (pick_first) in this case. + * + * We also strip out the channel arg for the resolved addresses, since + * that will be generated by the name resolver used in the LB channel. + * Note that the LB channel will use the sockaddr resolver, so this + * won't actually generate a query to DNS (or some other name service). + * However, the addresses returned by the sockaddr resolver will have + * is_balancer=false, whereas our own addresses have is_balancer=true. + * We need the LB channel to return addresses with is_balancer=false + * so that it does not wind up recursively using the grpclb LB policy, + * as per the special case logic in client_channel.c. + * + * Lastly, we also strip out the channel arg for the server URI, + * since that will be different for the LB channel than for the parent + * channel (the client channel factory will re-add this arg with + * the right value). */ + static const char *keys_to_remove[] = { + GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; + return grpc_channel_args_copy_and_remove(args, keys_to_remove, + GPR_ARRAY_SIZE(keys_to_remove)); +} diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel.h b/src/core/ext/lb_policy/grpclb/grpclb_channel.h new file mode 100644 index 0000000000..f66082d78e --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel.h @@ -0,0 +1,56 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H +#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H + +#include "src/core/ext/client_channel/lb_policy_factory.h" +#include "src/core/lib/slice/slice_hash_table.h" + +/** Create the channel used for communicating with an LB service. + * Note that an LB *service* may be comprised of several LB *servers*. + * + * \a lb_service_target_addresses is the target URI containing the addresses + * from resolving the LB service's name (eg, ipv4:10.0.0.1:1234,10.2.3.4:9876). + * \a client_channel_factory will be used for the creation of the LB channel, + * alongside the channel args passed in \a args. */ +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args); + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args); + +#endif /* GRPC_CORE_EXT_LB_POLICY_GRPCLB_GRPCLB_CHANNEL_H */ diff --git a/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c b/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c new file mode 100644 index 0000000000..2fee5f1b8e --- /dev/null +++ b/src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c @@ -0,0 +1,107 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/alloc.h> +#include <grpc/support/string_util.h> + +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/lb_policy/grpclb/grpclb_channel.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/support/string.h" + +grpc_channel *grpc_lb_policy_grpclb_create_lb_channel( + grpc_exec_ctx *exec_ctx, const char *lb_service_target_addresses, + grpc_client_channel_factory *client_channel_factory, + grpc_channel_args *args) { + grpc_channel_args *new_args = args; + grpc_channel_credentials *channel_credentials = + grpc_channel_credentials_find_in_args(args); + if (channel_credentials != NULL) { + /* Substitute the channel credentials with a version without call + * credentials: the load balancer is not necessarily trusted to handle + * bearer token credentials */ + static const char *keys_to_remove[] = {GRPC_ARG_CHANNEL_CREDENTIALS}; + grpc_channel_credentials *creds_sans_call_creds = + grpc_channel_credentials_duplicate_without_call_credentials( + channel_credentials); + GPR_ASSERT(creds_sans_call_creds != NULL); + grpc_arg args_to_add[] = { + grpc_channel_credentials_to_arg(creds_sans_call_creds)}; + /* Create the new set of channel args */ + new_args = grpc_channel_args_copy_and_add_and_remove( + args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), args_to_add, + GPR_ARRAY_SIZE(args_to_add)); + grpc_channel_credentials_unref(exec_ctx, creds_sans_call_creds); + } + grpc_channel *lb_channel = grpc_client_channel_factory_create_channel( + exec_ctx, client_channel_factory, lb_service_target_addresses, + GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, new_args); + if (channel_credentials != NULL) { + grpc_channel_args_destroy(exec_ctx, new_args); + } + return lb_channel; +} + +grpc_channel_args *get_lb_channel_args(grpc_exec_ctx *exec_ctx, + grpc_slice_hash_table *targets_info, + const grpc_channel_args *args) { + const grpc_arg targets_info_arg = + grpc_lb_targets_info_create_channel_arg(targets_info); + /* We strip out the channel arg for the LB policy name, since we want + * to use the default (pick_first) in this case. + * + * We also strip out the channel arg for the resolved addresses, since + * that will be generated by the name resolver used in the LB channel. + * Note that the LB channel will use the sockaddr resolver, so this + * won't actually generate a query to DNS (or some other name service). + * However, the addresses returned by the sockaddr resolver will have + * is_balancer=false, whereas our own addresses have is_balancer=true. + * We need the LB channel to return addresses with is_balancer=false + * so that it does not wind up recursively using the grpclb LB policy, + * as per the special case logic in client_channel.c. + * + * Lastly, we also strip out the channel arg for the server URI, + * since that will be different for the LB channel than for the parent + * channel (the client channel factory will re-add this arg with + * the right value). */ + static const char *keys_to_remove[] = { + GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI}; + /* Add the targets info table to be used for secure naming */ + return grpc_channel_args_copy_and_add_and_remove( + args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &targets_info_arg, + 1); +} diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index d17d8fa057..3e060d189a 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -739,6 +739,13 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sc_args.args = new_args; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); + if (grpc_lb_round_robin_trace) { + char *address_uri = + grpc_sockaddr_to_uri(&addresses->addresses[i].address); + gpr_log(GPR_DEBUG, "Created subchannel %p for address uri %s", + (void *)subchannel, address_uri); + gpr_free(address_uri); + } grpc_channel_args_destroy(exec_ctx, new_args); if (subchannel != NULL) { diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index c9f4021216..490a0c560e 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -39,6 +39,7 @@ #include <grpc/support/string_util.h> #include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/client_channel/resolver_registry.h" #include "src/core/ext/transport/chttp2/client/chttp2_connector.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/surface/api_trace.h" @@ -63,12 +64,17 @@ static grpc_channel *client_channel_factory_create_channel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const char *target, grpc_client_channel_type type, const grpc_channel_args *args) { + if (target == NULL) { + gpr_log(GPR_ERROR, "cannot create channel with NULL target name"); + return NULL; + } // Add channel arg containing the server URI. grpc_arg arg; arg.type = GRPC_ARG_STRING; arg.key = GRPC_ARG_SERVER_URI; - arg.value.string = (char *)target; + arg.value.string = grpc_resolver_factory_add_default_prefix_if_needed(target); grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1); + gpr_free(arg.value.string); grpc_channel *channel = grpc_channel_create(exec_ctx, target, new_args, GRPC_CLIENT_CHANNEL, NULL); grpc_channel_args_destroy(exec_ctx, new_args); diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index f979d9bad5..d8c18eb122 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -39,10 +39,16 @@ #include <grpc/support/string_util.h> #include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/client_channel/resolver_registry.h" +#include "src/core/ext/client_channel/uri_parser.h" #include "src/core/ext/transport/chttp2/client/chttp2_connector.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" #include "src/core/lib/security/transport/security_connector.h" +#include "src/core/lib/slice/slice_hash_table.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" @@ -52,12 +58,114 @@ static void client_channel_factory_ref( static void client_channel_factory_unref( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory) {} +static grpc_subchannel_args *get_secure_naming_subchannel_args( + grpc_exec_ctx *exec_ctx, const grpc_subchannel_args *args) { + grpc_channel_credentials *channel_credentials = + grpc_channel_credentials_find_in_args(args->args); + if (channel_credentials == NULL) { + gpr_log(GPR_ERROR, + "Can't create subchannel: channel credentials missing for secure " + "channel."); + return NULL; + } + // Make sure security connector does not already exist in args. + if (grpc_security_connector_find_in_args(args->args) != NULL) { + gpr_log(GPR_ERROR, + "Can't create subchannel: security connector already present in " + "channel args."); + return NULL; + } + // To which address are we connecting? By default, use the server URI. + const grpc_arg *server_uri_arg = + grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); + GPR_ASSERT(server_uri_arg != NULL); + GPR_ASSERT(server_uri_arg->type == GRPC_ARG_STRING); + const char *server_uri_str = server_uri_arg->value.string; + GPR_ASSERT(server_uri_str != NULL); + grpc_uri *server_uri = + grpc_uri_parse(server_uri_str, true /* supress errors */); + GPR_ASSERT(server_uri != NULL); + const char *server_uri_path; + server_uri_path = + server_uri->path[0] == '/' ? server_uri->path + 1 : server_uri->path; + const grpc_slice_hash_table *targets_info = + grpc_lb_targets_info_find_in_args(args->args); + char *target_name_to_check = NULL; + if (targets_info != NULL) { // LB channel + // Find the balancer name for the target. + const char *target_uri_str = + grpc_get_subchannel_address_uri_arg(args->args); + grpc_uri *target_uri = + grpc_uri_parse(target_uri_str, false /* suppress errors */); + GPR_ASSERT(target_uri != NULL); + if (target_uri->path[0] != '\0') { // "path" may be empty + const grpc_slice key = grpc_slice_from_static_string( + target_uri->path[0] == '/' ? target_uri->path + 1 : target_uri->path); + const char *value = grpc_slice_hash_table_get(targets_info, key); + if (value != NULL) target_name_to_check = gpr_strdup(value); + grpc_slice_unref_internal(exec_ctx, key); + } + if (target_name_to_check == NULL) { + // If the target name to check hasn't already been set, fall back to using + // SERVER_URI + target_name_to_check = gpr_strdup(server_uri_path); + } + grpc_uri_destroy(target_uri); + } else { // regular channel: the secure name is the original server URI. + target_name_to_check = gpr_strdup(server_uri_path); + } + grpc_uri_destroy(server_uri); + GPR_ASSERT(target_name_to_check != NULL); + grpc_channel_security_connector *subchannel_security_connector = NULL; + // Create the security connector using the credentials and target name. + grpc_channel_args *new_args_from_connector = NULL; + const grpc_security_status security_status = + grpc_channel_credentials_create_security_connector( + exec_ctx, channel_credentials, target_name_to_check, args->args, + &subchannel_security_connector, &new_args_from_connector); + if (security_status != GRPC_SECURITY_OK) { + gpr_log(GPR_ERROR, + "Failed to create secure subchannel for secure name '%s'", + target_name_to_check); + gpr_free(target_name_to_check); + return NULL; + } + gpr_free(target_name_to_check); + grpc_arg new_security_connector_arg = + grpc_security_connector_to_arg(&subchannel_security_connector->base); + + grpc_channel_args *new_args = grpc_channel_args_copy_and_add( + new_args_from_connector != NULL ? new_args_from_connector : args->args, + &new_security_connector_arg, 1); + GRPC_SECURITY_CONNECTOR_UNREF(exec_ctx, &subchannel_security_connector->base, + "lb_channel_create"); + if (new_args_from_connector != NULL) { + grpc_channel_args_destroy(exec_ctx, new_args_from_connector); + } + grpc_subchannel_args *final_sc_args = gpr_malloc(sizeof(*final_sc_args)); + memcpy(final_sc_args, args, sizeof(*args)); + final_sc_args->args = new_args; + return final_sc_args; +} + static grpc_subchannel *client_channel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const grpc_subchannel_args *args) { + grpc_subchannel_args *subchannel_args = + get_secure_naming_subchannel_args(exec_ctx, args); + if (subchannel_args == NULL) { + gpr_log( + GPR_ERROR, + "Failed to create subchannel arguments during subchannel creation."); + return NULL; + } grpc_connector *connector = grpc_chttp2_connector_create(); - grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args); + grpc_subchannel *s = + grpc_subchannel_create(exec_ctx, connector, subchannel_args); grpc_connector_unref(exec_ctx, connector); + grpc_channel_args_destroy(exec_ctx, + (grpc_channel_args *)subchannel_args->args); + gpr_free(subchannel_args); return s; } @@ -65,12 +173,17 @@ static grpc_channel *client_channel_factory_create_channel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const char *target, grpc_client_channel_type type, const grpc_channel_args *args) { + if (target == NULL) { + gpr_log(GPR_ERROR, "cannot create channel with NULL target name"); + return NULL; + } // Add channel arg containing the server URI. grpc_arg arg; arg.type = GRPC_ARG_STRING; arg.key = GRPC_ARG_SERVER_URI; - arg.value.string = (char *)target; + arg.value.string = grpc_resolver_factory_add_default_prefix_if_needed(target); grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1); + gpr_free(arg.value.string); grpc_channel *channel = grpc_channel_create(exec_ctx, target, new_args, GRPC_CLIENT_CHANNEL, NULL); grpc_channel_args_destroy(exec_ctx, new_args); @@ -85,10 +198,10 @@ static const grpc_client_channel_factory_vtable client_channel_factory_vtable = static grpc_client_channel_factory client_channel_factory = { &client_channel_factory_vtable}; -/* Create a secure client channel: - Asynchronously: - resolve target - - connect to it (trying alternatives as presented) - - perform handshakes */ +// Create a secure client channel: +// Asynchronously: - resolve target +// - connect to it (trying alternatives as presented) +// - perform handshakes grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, @@ -97,46 +210,27 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, GRPC_API_TRACE( "grpc_secure_channel_create(creds=%p, target=%s, args=%p, " "reserved=%p)", - 4, (creds, target, args, reserved)); + 4, ((void *)creds, target, (void *)args, (void *)reserved)); GPR_ASSERT(reserved == NULL); - // Make sure security connector does not already exist in args. - if (grpc_find_security_connector_in_args(args) != NULL) { - gpr_log(GPR_ERROR, "Cannot set security context in channel args."); + grpc_channel *channel = NULL; + if (creds != NULL) { + // Add channel args containing the client channel factory and channel + // credentials. + grpc_arg args_to_add[] = { + grpc_client_channel_factory_create_channel_arg(&client_channel_factory), + grpc_channel_credentials_to_arg(creds)}; + grpc_channel_args *new_args = grpc_channel_args_copy_and_add( + args, args_to_add, GPR_ARRAY_SIZE(args_to_add)); + // Create channel. + channel = client_channel_factory_create_channel( + &exec_ctx, &client_channel_factory, target, + GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args); + // Clean up. + grpc_channel_args_destroy(&exec_ctx, new_args); grpc_exec_ctx_finish(&exec_ctx); - return grpc_lame_client_channel_create( - target, GRPC_STATUS_INTERNAL, - "Security connector exists in channel args."); - } - // Create security connector and construct new channel args. - grpc_channel_security_connector *security_connector; - grpc_channel_args *new_args_from_connector; - if (grpc_channel_credentials_create_security_connector( - &exec_ctx, creds, target, args, &security_connector, - &new_args_from_connector) != GRPC_SECURITY_OK) { - grpc_exec_ctx_finish(&exec_ctx); - return grpc_lame_client_channel_create( - target, GRPC_STATUS_INTERNAL, "Failed to create security connector."); - } - // Add channel args containing the client channel factory and security - // connector. - grpc_arg args_to_add[2]; - args_to_add[0] = - grpc_client_channel_factory_create_channel_arg(&client_channel_factory); - args_to_add[1] = grpc_security_connector_to_arg(&security_connector->base); - grpc_channel_args *new_args = grpc_channel_args_copy_and_add( - new_args_from_connector != NULL ? new_args_from_connector : args, - args_to_add, GPR_ARRAY_SIZE(args_to_add)); - if (new_args_from_connector != NULL) { - grpc_channel_args_destroy(&exec_ctx, new_args_from_connector); } - // Create channel. - grpc_channel *channel = client_channel_factory_create_channel( - &exec_ctx, &client_channel_factory, target, - GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args); - // Clean up. - GRPC_SECURITY_CONNECTOR_UNREF(&exec_ctx, &security_connector->base, - "secure_client_channel_factory_create_channel"); - grpc_channel_args_destroy(&exec_ctx, new_args); - grpc_exec_ctx_finish(&exec_ctx); - return channel; /* may be NULL */ + return channel != NULL ? channel + : grpc_lame_client_channel_create( + target, GRPC_STATUS_INTERNAL, + "Failed to create secure client channel"); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 15f486d676..3ee5e976f8 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -124,6 +124,21 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error); +static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error); +static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error); + +static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_error *error); +static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, + grpc_closure *on_initiate, + grpc_closure *on_complete); + +#define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 + /******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -153,18 +168,9 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_destroy(&t->stream_map); grpc_connectivity_state_destroy(exec_ctx, &t->channel_callback.state_tracker); - grpc_combiner_destroy(exec_ctx, t->combiner); + GRPC_COMBINER_UNREF(exec_ctx, t->combiner, "chttp2_transport"); - /* callback remaining pings: they're not allowed to call into the transpot, - and maybe they hold resources that need to be freed */ - while (t->pings.next != &t->pings) { - grpc_chttp2_outstanding_ping *ping = t->pings.next; - grpc_closure_sched(exec_ctx, ping->on_recv, - GRPC_ERROR_CREATE("Transport closed")); - ping->next->prev = ping->prev; - ping->prev->next = ping->next; - gpr_free(ping); - } + cancel_pings(exec_ctx, t, GRPC_ERROR_CREATE("Transport destroyed")); while (t->write_cb_pool) { grpc_chttp2_write_cb *next = t->write_cb_pool->next; @@ -172,6 +178,7 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, t->write_cb_pool = next; } + gpr_free(t->ping_acks); gpr_free(t->peer_string); gpr_free(t); } @@ -224,10 +231,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->is_client = is_client; t->outgoing_window = DEFAULT_WINDOW; t->incoming_window = DEFAULT_WINDOW; - t->stream_lookahead = DEFAULT_WINDOW; - t->connection_window_target = DEFAULT_CONNECTION_WINDOW_TARGET; - t->ping_counter = 1; - t->pings.next = t->pings.prev = &t->pings; t->deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->is_first_frame = true; grpc_connectivity_state_init( @@ -248,6 +251,22 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_closure_init(&t->destructive_reclaimer_locked, destructive_reclaimer_locked, t, grpc_combiner_scheduler(t->combiner, false)); + grpc_closure_init(&t->start_bdp_ping_locked, start_bdp_ping_locked, t, + grpc_combiner_scheduler(t->combiner, false)); + grpc_closure_init(&t->finish_bdp_ping_locked, finish_bdp_ping_locked, t, + grpc_combiner_scheduler(t->combiner, false)); + + grpc_bdp_estimator_init(&t->bdp_estimator, t->peer_string); + t->last_pid_update = gpr_now(GPR_CLOCK_MONOTONIC); + grpc_pid_controller_init( + &t->pid_controller, + (grpc_pid_controller_args){.gain_p = 4, + .gain_i = 8, + .gain_d = 0, + .initial_control_value = log2(DEFAULT_WINDOW), + .min_control_value = -1, + .max_control_value = 22, + .integral_range = 10}); grpc_chttp2_goaway_parser_init(&t->goaway_parser); grpc_chttp2_hpack_parser_init(exec_ctx, &t->hpack_parser); @@ -273,6 +292,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; t->sent_local_settings = 0; t->write_buffer_size = DEFAULT_WINDOW; + t->enable_bdp_probe = true; if (is_client) { grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( @@ -290,6 +310,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, DEFAULT_MAX_HEADER_LIST_SIZE); + t->ping_policy = (grpc_chttp2_repeated_ping_policy){ + .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, + .min_time_between_pings = + gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), + }; + if (channel_args) { for (i = 0; i < channel_args->num_args; i++) { if (0 == strcmp(channel_args->args[i].key, @@ -307,14 +333,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES)) { - const grpc_integer_options options = {-1, 5, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - t->stream_lookahead = (uint32_t)value; - } - } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER)) { const grpc_integer_options options = {-1, 0, INT_MAX}; const int value = @@ -324,34 +342,53 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, (uint32_t)value); } } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { + t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS)) { + t->ping_policy.min_time_between_pings = gpr_time_from_millis( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, 0, + INT_MAX}), + GPR_TIMESPAN); + } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = (uint32_t)grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){0, 0, MAX_WRITE_BUFFER_SIZE}); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { + t->enable_bdp_probe = grpc_channel_arg_get_integer( + &channel_args->args[i], (grpc_integer_options){1, 0, 1}); } else { static const struct { const char *channel_arg_name; grpc_chttp2_setting_id setting_id; grpc_integer_options integer_options; bool availability[2] /* server, client */; - } settings_map[] = { - {GRPC_ARG_MAX_CONCURRENT_STREAMS, - GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, - {-1, 0, INT_MAX}, - {true, false}}, - {GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER, - GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, - {-1, 0, INT_MAX}, - {true, true}}, - {GRPC_ARG_MAX_METADATA_SIZE, - GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, - {-1, 0, INT_MAX}, - {true, true}}, - {GRPC_ARG_HTTP2_MAX_FRAME_SIZE, - GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, - {-1, 16384, 16777215}, - {true, true}}, - }; + } settings_map[] = {{GRPC_ARG_MAX_CONCURRENT_STREAMS, + GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, + {-1, 0, INT32_MAX}, + {true, false}}, + {GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER, + GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, + {-1, 0, INT32_MAX}, + {true, true}}, + {GRPC_ARG_MAX_METADATA_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, + {-1, 0, INT32_MAX}, + {true, true}}, + {GRPC_ARG_HTTP2_MAX_FRAME_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, + {-1, 16384, 16777215}, + {true, true}}, + {GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES, + GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, + {-1, 5, INT32_MAX}, + {true, true}}}; for (j = 0; j < (int)GPR_ARRAY_SIZE(settings_map); j++) { if (0 == strcmp(channel_args->args[i].key, settings_map[j].channel_arg_name)) { @@ -374,6 +411,9 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; + grpc_chttp2_initiate_write(exec_ctx, t, false, "init"); post_benign_reclaimer(exec_ctx, t); } @@ -425,6 +465,7 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:close"); } end_all_the_calls(exec_ctx, t, GRPC_ERROR_REF(error)); + cancel_pings(exec_ctx, t, GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); } @@ -475,11 +516,6 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (server_data) { s->id = (uint32_t)(uintptr_t)server_data; - s->outgoing_window = t->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - s->incoming_window = s->max_recv_bytes = - t->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; *t->accepting_stream = s; grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); post_destructive_reclaimer(exec_ctx, t); @@ -508,6 +544,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, } grpc_chttp2_list_remove_stalled_by_transport(t, s); + grpc_chttp2_list_remove_stalled_by_stream(t, s); for (int i = 0; i < STREAM_LIST_COUNT; i++) { if (s->included[i]) { @@ -647,13 +684,21 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("grpc_chttp2_initiate_write", 0); } -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, bool covered_by_poller, - const char *reason) { +void grpc_chttp2_become_writable( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_chttp2_stream_write_type stream_write_type, const char *reason) { if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:become"); - grpc_chttp2_initiate_write(exec_ctx, t, covered_by_poller, reason); + } + switch (stream_write_type) { + case GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK: + break; + case GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED: + grpc_chttp2_initiate_write(exec_ctx, t, true, reason); + break; + case GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED: + grpc_chttp2_initiate_write(exec_ctx, t, false, reason); + break; } } @@ -781,7 +826,6 @@ void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { grpc_chttp2_stream *s; - uint32_t stream_incoming_window; /* start streams where we have free grpc_chttp2_stream ids and free * concurrency */ while (t->next_stream_id <= MAX_CLIENT_STREAM_ID && @@ -804,15 +848,11 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, "no_more_stream_ids"); } - s->outgoing_window = t->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - s->incoming_window = stream_incoming_window = - t->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - s->max_recv_bytes = GPR_MAX(stream_incoming_window, s->max_recv_bytes); grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); post_destructive_reclaimer(exec_ctx, t); - grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); + grpc_chttp2_become_writable(exec_ctx, t, s, + GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED, + "new_stream"); } /* cancel out streams that will never be started */ while (t->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -907,7 +947,9 @@ static void maybe_become_writable_due_to_send_msg(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { if (s->id != 0 && (!s->write_buffering || s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); + grpc_chttp2_become_writable(exec_ctx, t, s, + GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED, + "op.send_message"); } } @@ -1069,7 +1111,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } } else { GPR_ASSERT(s->id != 0); - grpc_chttp2_become_writable(exec_ctx, t, s, true, + grpc_chttp2_become_writable(exec_ctx, t, s, + GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED, "op.send_initial_metadata"); } } else { @@ -1160,7 +1203,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } else if (s->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_become_writable(exec_ctx, t, s, true, + grpc_chttp2_become_writable(exec_ctx, t, s, + GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED, "op.send_trailing_metadata"); } } @@ -1179,8 +1223,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, s->recv_message = op->recv_message; if (s->id != 0 && (s->incoming_frames.head == NULL || s->incoming_frames.head->is_tail)) { - incoming_byte_stream_update_flow_control(exec_ctx, t, s, - t->stream_lookahead, 0); + incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, 0); } grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); } @@ -1224,43 +1267,46 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, GPR_TIMER_END("perform_stream_op", 0); } +static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_error *error) { + /* callback remaining pings: they're not allowed to call into the transpot, + and maybe they hold resources that need to be freed */ + for (size_t i = 0; i < GRPC_CHTTP2_PING_TYPE_COUNT; i++) { + grpc_chttp2_ping_queue *pq = &t->ping_queues[i]; + for (size_t j = 0; j < GRPC_CHTTP2_PCL_COUNT; j++) { + grpc_closure_list_fail_all(&pq->lists[j], GRPC_ERROR_REF(error)); + grpc_closure_list_sched(exec_ctx, &pq->lists[j]); + } + } + GRPC_ERROR_UNREF(error); +} + static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_closure *on_recv) { - grpc_chttp2_outstanding_ping *p = gpr_malloc(sizeof(*p)); - p->next = &t->pings; - p->prev = p->next->prev; - p->prev->next = p->next->prev = p; - p->id[0] = (uint8_t)((t->ping_counter >> 56) & 0xff); - p->id[1] = (uint8_t)((t->ping_counter >> 48) & 0xff); - p->id[2] = (uint8_t)((t->ping_counter >> 40) & 0xff); - p->id[3] = (uint8_t)((t->ping_counter >> 32) & 0xff); - p->id[4] = (uint8_t)((t->ping_counter >> 24) & 0xff); - p->id[5] = (uint8_t)((t->ping_counter >> 16) & 0xff); - p->id[6] = (uint8_t)((t->ping_counter >> 8) & 0xff); - p->id[7] = (uint8_t)(t->ping_counter & 0xff); - t->ping_counter++; - p->on_recv = on_recv; - grpc_slice_buffer_add(&t->qbuf, grpc_chttp2_ping_create(0, p->id)); - grpc_chttp2_initiate_write(exec_ctx, t, true, "send_ping"); + grpc_chttp2_ping_type ping_type, + grpc_closure *on_initiate, grpc_closure *on_ack) { + grpc_chttp2_ping_queue *pq = &t->ping_queues[ping_type]; + grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_INITIATE], on_initiate, + GRPC_ERROR_NONE); + if (grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_NEXT], on_ack, + GRPC_ERROR_NONE)) { + grpc_chttp2_initiate_write(exec_ctx, t, false, "send_ping"); + } } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - const uint8_t *opaque_8bytes) { - grpc_chttp2_outstanding_ping *ping; - for (ping = t->pings.next; ping != &t->pings; ping = ping->next) { - if (0 == memcmp(opaque_8bytes, ping->id, 8)) { - grpc_closure_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE); - ping->next->prev = ping->prev; - ping->prev->next = ping->next; - gpr_free(ping); - return; - } + uint64_t id) { + grpc_chttp2_ping_queue *pq = + &t->ping_queues[id % GRPC_CHTTP2_PING_TYPE_COUNT]; + if (pq->inflight_id != id) { + char *from = grpc_endpoint_get_peer(t->ep); + gpr_log(GPR_DEBUG, "Unknown ping response from %s: %" PRIx64, from, id); + gpr_free(from); + return; + } + grpc_closure_list_sched(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]); + if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) { + grpc_chttp2_initiate_write(exec_ctx, t, false, "continue_pings"); } - char *msg = gpr_dump((const char *)opaque_8bytes, 8, GPR_DUMP_HEX); - char *from = grpc_endpoint_get_peer(t->ep); - gpr_log(GPR_DEBUG, "Unknown ping response from %s: %s", from, msg); - gpr_free(from); - gpr_free(msg); } static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -1308,7 +1354,8 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, } if (op->send_ping) { - send_ping_locked(exec_ctx, t, op->send_ping); + send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, NULL, + op->send_ping); } if (close_transport != GRPC_ERROR_NONE) { @@ -1733,34 +1780,28 @@ static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } -/** update window from a settings change */ -typedef struct { - grpc_chttp2_transport *t; - grpc_exec_ctx *exec_ctx; -} update_global_window_args; +/******************************************************************************* + * INPUT PROCESSING - PARSING + */ -static void update_global_window(void *args, uint32_t id, void *stream) { - update_global_window_args *a = args; - grpc_chttp2_transport *t = a->t; - grpc_chttp2_stream *s = stream; - int was_zero; - int is_zero; - int64_t initial_window_update = t->initial_window_update; - - if (initial_window_update > 0) { - was_zero = s->outgoing_window <= 0; - GRPC_CHTTP2_FLOW_CREDIT_STREAM("settings", t, s, outgoing_window, - initial_window_update); - is_zero = s->outgoing_window <= 0; - - if (was_zero && !is_zero) { - grpc_chttp2_become_writable(a->exec_ctx, t, s, true, - "update_global_window"); - } +static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + double bdp_dbl) { + uint32_t bdp; + if (bdp_dbl <= 0) { + bdp = 0; + } else if (bdp_dbl > UINT32_MAX) { + bdp = UINT32_MAX; } else { - GRPC_CHTTP2_FLOW_DEBIT_STREAM("settings", t, s, outgoing_window, - -initial_window_update); + bdp = (uint32_t)(bdp_dbl); + } + int64_t delta = + (int64_t)bdp - + (int64_t)t->settings[GRPC_LOCAL_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + if (delta == 0 || (bdp != 0 && delta > -1024 && delta < 1024)) { + return; } + push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, bdp); } /******************************************************************************* @@ -1802,6 +1843,7 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, GPR_TIMER_BEGIN("reading_action_locked", 0); grpc_chttp2_transport *t = tp; + bool need_bdp_ping = false; GRPC_ERROR_REF(error); @@ -1819,9 +1861,14 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE, GRPC_ERROR_NONE}; for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) { + if (grpc_bdp_estimator_add_incoming_bytes( + &t->bdp_estimator, + (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i]))) { + need_bdp_ping = true; + } errors[1] = grpc_chttp2_perform_read(exec_ctx, t, t->read_buffer.slices[i]); - }; + } if (errors[1] != GRPC_ERROR_NONE) { errors[2] = try_http_parsing(exec_ctx, t); GRPC_ERROR_UNREF(error); @@ -1835,21 +1882,16 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, GPR_TIMER_BEGIN("post_parse_locked", 0); if (t->initial_window_update != 0) { - update_global_window_args args = {t, exec_ctx}; - grpc_chttp2_stream_map_for_each(&t->stream_map, update_global_window, - &args); + if (t->initial_window_update > 0) { + grpc_chttp2_stream *s; + while (grpc_chttp2_list_pop_stalled_by_stream(t, &s)) { + grpc_chttp2_become_writable( + exec_ctx, t, s, GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED, + "unstalled"); + } + } t->initial_window_update = 0; } - /* handle higher level things */ - if (t->incoming_window < t->connection_window_target * 3 / 4) { - int64_t announce_bytes = t->connection_window_target - t->incoming_window; - GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", t, announce_incoming_window, - announce_bytes); - GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", t, incoming_window, - announce_bytes); - grpc_chttp2_initiate_write(exec_ctx, t, false, "global incoming window"); - } - GPR_TIMER_END("post_parse_locked", 0); } @@ -1870,6 +1912,38 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, if (keep_reading) { grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->read_action_locked); + + if (t->enable_bdp_probe) { + if (need_bdp_ping) { + GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping"); + grpc_bdp_estimator_schedule_ping(&t->bdp_estimator); + send_ping_locked(exec_ctx, t, + GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, + &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); + } + + int64_t estimate = -1; + if (grpc_bdp_estimator_get_estimate(&t->bdp_estimator, &estimate)) { + double target = 1 + log2((double)estimate); + double memory_pressure = grpc_resource_quota_get_memory_pressure( + grpc_resource_user_quota(grpc_endpoint_get_resource_user(t->ep))); + if (memory_pressure > 0.8) { + target *= 1 - GPR_MIN(1, (memory_pressure - 0.8) / 0.1); + } + double bdp_error = + target - grpc_pid_controller_last(&t->pid_controller); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec dt_timespec = gpr_time_sub(now, t->last_pid_update); + double dt = (double)dt_timespec.tv_sec + dt_timespec.tv_nsec * 1e-9; + if (dt > 0.1) { + dt = 0.1; + } + double log2_bdp_guess = + grpc_pid_controller_update(&t->pid_controller, bdp_error, dt); + update_bdp(exec_ctx, t, pow(2, log2_bdp_guess)); + t->last_pid_update = now; + } + } GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "keep_reading"); } else { GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "reading_action"); @@ -1882,6 +1956,26 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, GPR_TIMER_END("reading_action_locked", 0); } +static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error) { + grpc_chttp2_transport *t = tp; + if (grpc_http_trace) { + gpr_log(GPR_DEBUG, "%s: Start BDP ping", t->peer_string); + } + grpc_bdp_estimator_start_ping(&t->bdp_estimator); +} + +static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error) { + grpc_chttp2_transport *t = tp; + if (grpc_http_trace) { + gpr_log(GPR_DEBUG, "%s: Complete BDP ping", t->peer_string); + } + grpc_bdp_estimator_complete_ping(&t->bdp_estimator); + + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "bdp_ping"); +} + /******************************************************************************* * CALLBACK LOOP */ @@ -1932,10 +2026,12 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, size_t max_size_hint, size_t have_already) { uint32_t max_recv_bytes; + uint32_t initial_window_size = + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; /* clamp max recv hint to an allowable size */ - if (max_size_hint >= UINT32_MAX - t->stream_lookahead) { - max_recv_bytes = UINT32_MAX - t->stream_lookahead; + if (max_size_hint >= UINT32_MAX - initial_window_size) { + max_recv_bytes = UINT32_MAX - initial_window_size; } else { max_recv_bytes = (uint32_t)max_size_hint; } @@ -1948,20 +2044,26 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, } /* add some small lookahead to keep pipelines flowing */ - GPR_ASSERT(max_recv_bytes <= UINT32_MAX - t->stream_lookahead); - max_recv_bytes += t->stream_lookahead; - if (s->max_recv_bytes < max_recv_bytes) { - uint32_t add_max_recv_bytes = max_recv_bytes - s->max_recv_bytes; - bool new_window_write_is_covered_by_poller = - s->max_recv_bytes < have_already; - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, max_recv_bytes, - add_max_recv_bytes); - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, incoming_window, + GPR_ASSERT(max_recv_bytes <= UINT32_MAX - initial_window_size); + if (s->incoming_window_delta < max_recv_bytes && !s->read_closed) { + uint32_t add_max_recv_bytes = + (uint32_t)(max_recv_bytes - s->incoming_window_delta); + grpc_chttp2_stream_write_type write_type = + GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED; + if (s->incoming_window_delta + initial_window_size < + (int64_t)have_already) { + write_type = GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED; + } + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, incoming_window_delta, add_max_recv_bytes); GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, announce_window, add_max_recv_bytes); - grpc_chttp2_become_writable(exec_ctx, t, s, - new_window_write_is_covered_by_poller, + if ((int64_t)s->incoming_window_delta + (int64_t)initial_window_size - + (int64_t)s->announce_window > + (int64_t)initial_window_size / 2) { + write_type = GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK; + } + grpc_chttp2_become_writable(exec_ctx, t, s, write_type, "read_incoming_stream"); } } diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 7de5f6362d..f487533c41 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -40,7 +40,7 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) { +grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint64_t opaque_8bytes) { grpc_slice slice = grpc_slice_malloc(9 + 8); uint8_t *p = GRPC_SLICE_START_PTR(slice); @@ -53,7 +53,14 @@ grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) { *p++ = 0; *p++ = 0; *p++ = 0; - memcpy(p, opaque_8bytes, 8); + *p++ = (uint8_t)(opaque_8bytes >> 56); + *p++ = (uint8_t)(opaque_8bytes >> 48); + *p++ = (uint8_t)(opaque_8bytes >> 40); + *p++ = (uint8_t)(opaque_8bytes >> 32); + *p++ = (uint8_t)(opaque_8bytes >> 24); + *p++ = (uint8_t)(opaque_8bytes >> 16); + *p++ = (uint8_t)(opaque_8bytes >> 8); + *p++ = (uint8_t)(opaque_8bytes); return slice; } @@ -70,6 +77,7 @@ grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser, } parser->byte = 0; parser->is_ack = flags; + parser->opaque_8bytes = 0; return GRPC_ERROR_NONE; } @@ -83,7 +91,7 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_ping_parser *p = parser; while (p->byte != 8 && cur != end) { - p->opaque_8bytes[p->byte] = *cur; + p->opaque_8bytes |= (((uint64_t)*cur) << (8 * p->byte)); cur++; p->byte++; } @@ -93,8 +101,12 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, if (p->is_ack) { grpc_chttp2_ack_ping(exec_ctx, t, p->opaque_8bytes); } else { - grpc_slice_buffer_add(&t->qbuf, - grpc_chttp2_ping_create(1, p->opaque_8bytes)); + if (t->ping_ack_count == t->ping_ack_capacity) { + t->ping_ack_capacity = GPR_MAX(t->ping_ack_capacity * 3 / 2, 3); + t->ping_acks = gpr_realloc( + t->ping_acks, t->ping_ack_capacity * sizeof(*t->ping_acks)); + } + t->ping_acks[t->ping_ack_count++] = p->opaque_8bytes; grpc_chttp2_initiate_write(exec_ctx, t, false, "ping response"); } } diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.h b/src/core/ext/transport/chttp2/transport/frame_ping.h index b9889e2d11..ef642465d7 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.h +++ b/src/core/ext/transport/chttp2/transport/frame_ping.h @@ -41,10 +41,10 @@ typedef struct { uint8_t byte; uint8_t is_ack; - uint8_t opaque_8bytes[8]; + uint64_t opaque_8bytes; } grpc_chttp2_ping_parser; -grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes); +grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint64_t opaque_8bytes); grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser, uint32_t length, uint8_t flags); diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 7d5beed09d..cb017e75f0 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -109,8 +109,13 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, (((uint32_t)p->reason_bytes[3])); grpc_error *error = GRPC_ERROR_NONE; if (reason != GRPC_HTTP2_NO_ERROR || s->header_frames_received < 2) { - error = grpc_error_set_int(GRPC_ERROR_CREATE("RST_STREAM"), - GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); + char *message; + gpr_asprintf(&message, "Received RST_STREAM with error code %d", reason); + error = grpc_error_set_int( + grpc_error_set_str(GRPC_ERROR_CREATE("RST_STREAM"), + GRPC_ERROR_STR_GRPC_MESSAGE, message), + GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); + gpr_free(message); } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index be9b663ac1..82290e34cd 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -236,7 +236,7 @@ grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p, } if (parser->id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE && parser->incoming_settings[parser->id] != parser->value) { - t->initial_window_update = + t->initial_window_update += (int64_t)parser->value - parser->incoming_settings[parser->id]; if (grpc_http_trace) { gpr_log(GPR_DEBUG, "adding %d for initial_window change", @@ -245,8 +245,9 @@ grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p, } parser->incoming_settings[parser->id] = parser->value; if (grpc_http_trace) { - gpr_log(GPR_DEBUG, "CHTTP2:%s: got setting %d = %d", - t->is_client ? "CLI" : "SVR", parser->id, parser->value); + gpr_log(GPR_DEBUG, "CHTTP2:%s:%s: got setting %d = %d", + t->is_client ? "CLI" : "SVR", t->peer_string, parser->id, + parser->value); } } else if (grpc_http_trace) { gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)", diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index 31a31c2871..8fa0bb471a 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -110,13 +110,12 @@ grpc_error *grpc_chttp2_window_update_parser_parse( if (t->incoming_stream_id != 0) { if (s != NULL) { - bool was_zero = s->outgoing_window <= 0; - GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", t, s, outgoing_window, + GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", t, s, outgoing_window_delta, received_update); - bool is_zero = s->outgoing_window <= 0; - if (was_zero && !is_zero) { - grpc_chttp2_become_writable(exec_ctx, t, s, false, - "stream.read_flow_control"); + if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) { + grpc_chttp2_become_writable( + exec_ctx, t, s, GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED, + "stream.read_flow_control"); } } } else { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index ee5edc92df..075d421dd4 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -50,7 +50,9 @@ #include "src/core/ext/transport/chttp2/transport/stream_map.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/connectivity_state.h" +#include "src/core/lib/transport/pid_controller.h" #include "src/core/lib/transport/transport_impl.h" /* streams are kept in various linked lists depending on what things need to @@ -59,6 +61,7 @@ typedef enum { GRPC_CHTTP2_LIST_WRITABLE, GRPC_CHTTP2_LIST_WRITING, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT, + GRPC_CHTTP2_LIST_STALLED_BY_STREAM, /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, @@ -72,6 +75,34 @@ typedef enum { GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER, } grpc_chttp2_write_state; +typedef enum { + GRPC_CHTTP2_PING_ON_NEXT_WRITE = 0, + GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, + GRPC_CHTTP2_PING_TYPE_COUNT /* must be last */ +} grpc_chttp2_ping_type; + +typedef enum { + GRPC_CHTTP2_PCL_INITIATE = 0, + GRPC_CHTTP2_PCL_NEXT, + GRPC_CHTTP2_PCL_INFLIGHT, + GRPC_CHTTP2_PCL_COUNT /* must be last */ +} grpc_chttp2_ping_closure_list; + +typedef struct { + grpc_closure_list lists[GRPC_CHTTP2_PCL_COUNT]; + uint64_t inflight_id; +} grpc_chttp2_ping_queue; + +typedef struct { + gpr_timespec min_time_between_pings; + int max_pings_without_data; +} grpc_chttp2_repeated_ping_policy; + +typedef struct { + gpr_timespec last_ping_sent_time; + int pings_before_data_required; +} grpc_chttp2_repeated_ping_state; + /* deframer state for the overall http2 stream of bytes */ typedef enum { /* prefix: one entry per http2 connection prefix byte */ @@ -144,14 +175,6 @@ typedef enum { GRPC_CHTTP2_GOAWAY_SENT, } grpc_chttp2_sent_goaway_state; -/* Outstanding ping request data */ -typedef struct grpc_chttp2_outstanding_ping { - uint8_t id[8]; - grpc_closure *on_recv; - struct grpc_chttp2_outstanding_ping *next; - struct grpc_chttp2_outstanding_ping *prev; -} grpc_chttp2_outstanding_ping; - typedef struct grpc_chttp2_write_cb { int64_t call_at_byte; grpc_closure *closure; @@ -204,6 +227,9 @@ struct grpc_chttp2_transport { /** is there a read request to the endpoint outstanding? */ uint8_t endpoint_reading; + /** should we probe bdp? */ + bool enable_bdp_probe; + /** various lists of streams */ grpc_chttp2_stream_list lists[STREAM_LIST_COUNT]; @@ -271,16 +297,19 @@ struct grpc_chttp2_transport { copied to next_stream_id in parsing when parsing commences */ uint32_t next_stream_id; - /** how far to lookahead in a stream? */ - uint32_t stream_lookahead; - /** last new stream id */ uint32_t last_new_stream_id; - /** pings awaiting responses */ - grpc_chttp2_outstanding_ping pings; - /** next payload for an outgoing ping */ - uint64_t ping_counter; + /** ping queues for various ping insertion points */ + grpc_chttp2_ping_queue ping_queues[GRPC_CHTTP2_PING_TYPE_COUNT]; + grpc_chttp2_repeated_ping_policy ping_policy; + grpc_chttp2_repeated_ping_state ping_state; + uint64_t ping_ctr; /* unique id for pings */ + + /** ping acks */ + size_t ping_ack_count; + size_t ping_ack_capacity; + uint64_t *ping_acks; /** parser for headers */ grpc_chttp2_hpack_parser hpack_parser; @@ -324,6 +353,13 @@ struct grpc_chttp2_transport { grpc_chttp2_write_cb *write_cb_pool; + /* bdp estimator */ + grpc_bdp_estimator bdp_estimator; + grpc_pid_controller pid_controller; + grpc_closure start_bdp_ping_locked; + grpc_closure finish_bdp_ping_locked; + gpr_timespec last_pid_update; + /* if non-NULL, close the transport with this error when writes are finished */ grpc_error *close_transport_on_writes_finished; @@ -362,12 +398,10 @@ struct grpc_chttp2_stream { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ uint32_t id; - /** window available for us to send to peer */ - int64_t outgoing_window; - /** The number of bytes the upper layers have offered to receive. - As the upper layer offers more bytes, this value increases. - As bytes are read, this value decreases. */ - uint32_t max_recv_bytes; + /** window available for us to send to peer, over or under the initial window + * size of the transport... ie: + * outgoing_window = outgoing_window_delta + transport.initial_window_size */ + int64_t outgoing_window_delta; /** things the upper layers would like to send */ grpc_metadata_batch *send_initial_metadata; grpc_closure *send_initial_metadata_finished; @@ -428,8 +462,10 @@ struct grpc_chttp2_stream { grpc_error *forced_close_error; /** how many header frames have we received? */ uint8_t header_frames_received; - /** window available for peer to send to us */ - int64_t incoming_window; + /** window available for peer to send to us (as a delta on + * transport.initial_window_size) + * incoming_window = incoming_window_delta + transport.initial_window_size */ + int64_t incoming_window_delta; /** parsing state for data frames */ grpc_chttp2_data_parser data_parser; /** number of bytes received - reset at end of parse thread execution */ @@ -478,36 +514,43 @@ bool grpc_chttp2_list_add_writable_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s); /** Get a writable stream returns non-zero if there was a stream available */ -int grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream **s); +bool grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); bool grpc_chttp2_list_remove_writable_stream( grpc_chttp2_transport *t, grpc_chttp2_stream *s) GRPC_MUST_USE_RESULT; bool grpc_chttp2_list_add_writing_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s); -int grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t); -int grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream **s); +bool grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t); +bool grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); void grpc_chttp2_list_add_written_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s); -int grpc_chttp2_list_pop_written_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream **s); +bool grpc_chttp2_list_pop_written_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t, grpc_chttp2_stream *s); -int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, - grpc_chttp2_stream **s); +bool grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t, grpc_chttp2_stream *s); void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s); -int grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, - grpc_chttp2_stream **s); +bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s); +void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); +bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); + grpc_chttp2_stream *grpc_chttp2_parsing_lookup_stream(grpc_chttp2_transport *t, uint32_t id); grpc_chttp2_stream *grpc_chttp2_parsing_accept_stream(grpc_exec_ctx *exec_ctx, @@ -672,13 +715,23 @@ void grpc_chttp2_incoming_byte_stream_finished( grpc_error *error); void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - const uint8_t *opaque_8bytes); + uint64_t id); + +typedef enum { + /* don't initiate a transport write, but piggyback on the next one */ + GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK, + /* initiate a covered write */ + GRPC_CHTTP2_STREAM_WRITE_INITIATE_COVERED, + /* initiate an uncovered write */ + GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED +} grpc_chttp2_stream_write_type; /** add a ref to the stream and add it to the writable list; ref will be dropped in writing.c */ void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_stream *s, bool covered_by_poller, + grpc_chttp2_stream *s, + grpc_chttp2_stream_write_type type, const char *reason); void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index f58cd696f9..24bd93067b 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -376,25 +376,45 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, return err; } + uint32_t target_incoming_window = GPR_MAX( + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], + 1024); GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", t, incoming_window, incoming_frame_size); + if (t->incoming_window <= target_incoming_window / 2) { + grpc_chttp2_initiate_write(exec_ctx, t, false, "flow_control"); + } if (s != NULL) { - if (incoming_frame_size > s->incoming_window) { + if (incoming_frame_size > + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]) { char *msg; gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64, - t->incoming_frame_size, s->incoming_window); + t->incoming_frame_size, + s->incoming_window_delta + + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } - GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", t, s, incoming_window, + GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", t, s, incoming_window_delta, incoming_frame_size); + if ((int64_t)t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] + + (int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= + (int64_t)t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] / + 2) { + grpc_chttp2_become_writable(exec_ctx, t, s, + GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED, + "window-update-required"); + } s->received_bytes += incoming_frame_size; - s->max_recv_bytes -= - (uint32_t)GPR_MIN(s->max_recv_bytes, incoming_frame_size); } return GRPC_ERROR_NONE; diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index a60264cc51..078818fb18 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -37,14 +37,14 @@ /* core list management */ -static int stream_list_empty(grpc_chttp2_transport *t, - grpc_chttp2_stream_list_id id) { +static bool stream_list_empty(grpc_chttp2_transport *t, + grpc_chttp2_stream_list_id id) { return t->lists[id].head == NULL; } -static int stream_list_pop(grpc_chttp2_transport *t, - grpc_chttp2_stream **stream, - grpc_chttp2_stream_list_id id) { +static bool stream_list_pop(grpc_chttp2_transport *t, + grpc_chttp2_stream **stream, + grpc_chttp2_stream_list_id id) { grpc_chttp2_stream *s = t->lists[id].head; if (s) { grpc_chttp2_stream *new_head = s->links[id].next; @@ -124,8 +124,8 @@ bool grpc_chttp2_list_add_writable_stream(grpc_chttp2_transport *t, return stream_list_add(t, s, GRPC_CHTTP2_LIST_WRITABLE); } -int grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream **s) { +bool grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WRITABLE); } @@ -139,12 +139,12 @@ bool grpc_chttp2_list_add_writing_stream(grpc_chttp2_transport *t, return stream_list_add(t, s, GRPC_CHTTP2_LIST_WRITING); } -int grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t) { +bool grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t) { return !stream_list_empty(t, GRPC_CHTTP2_LIST_WRITING); } -int grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream **s) { +bool grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WRITING); } @@ -153,8 +153,8 @@ void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t, stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); } -int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, - grpc_chttp2_stream **s) { +bool grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); } @@ -168,8 +168,8 @@ void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } -int grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, - grpc_chttp2_stream **s) { +bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } @@ -177,3 +177,18 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } + +void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); +} + +bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); +} + +bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); +} diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 148e3844b5..05e6f59947 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -56,6 +56,75 @@ static void finish_write_cb(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->write_cb_pool = cb; } +static void collapse_pings_from_into(grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, + grpc_chttp2_ping_queue *pq) { + for (size_t i = 0; i < GRPC_CHTTP2_PCL_COUNT; i++) { + grpc_closure_list_move(&t->ping_queues[ping_type].lists[i], &pq->lists[i]); + } +} + +static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type) { + grpc_chttp2_ping_queue *pq = &t->ping_queues[ping_type]; + if (grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) { + /* no ping needed: wait */ + return; + } + if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) { + /* ping already in-flight: wait */ + if (grpc_http_trace || grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "Ping delayed [%p]: already pinging", t->peer_string); + } + return; + } + if (t->ping_state.pings_before_data_required == 0 && + t->ping_policy.max_pings_without_data != 0) { + /* need to send something of substance before sending a ping again */ + if (grpc_http_trace || grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "Ping delayed [%p]: too many recent pings: %d/%d", + t->peer_string, t->ping_state.pings_before_data_required, + t->ping_policy.max_pings_without_data); + } + return; + } + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec elapsed = gpr_time_sub(now, t->ping_state.last_ping_sent_time); + /*gpr_log(GPR_DEBUG, "elapsed:%d.%09d min:%d.%09d", (int)elapsed.tv_sec, + elapsed.tv_nsec, (int)t->ping_policy.min_time_between_pings.tv_sec, + (int)t->ping_policy.min_time_between_pings.tv_nsec);*/ + if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < 0) { + /* not enough elapsed time between successive pings */ + if (grpc_http_trace || grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, + "Ping delayed [%p]: not enough time elapsed since last ping", + t->peer_string); + } + return; + } + /* coalesce equivalent pings into this one */ + switch (ping_type) { + case GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE: + collapse_pings_from_into(t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, pq); + break; + case GRPC_CHTTP2_PING_ON_NEXT_WRITE: + break; + case GRPC_CHTTP2_PING_TYPE_COUNT: + GPR_UNREACHABLE_CODE(break); + } + pq->inflight_id = t->ping_ctr * GRPC_CHTTP2_PING_TYPE_COUNT + ping_type; + t->ping_ctr++; + grpc_closure_list_sched(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INITIATE]); + grpc_closure_list_move(&pq->lists[GRPC_CHTTP2_PCL_NEXT], + &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]); + grpc_slice_buffer_add(&t->outbuf, + grpc_chttp2_ping_create(false, pq->inflight_id)); + t->ping_state.last_ping_sent_time = now; + t->ping_state.pings_before_data_required -= + (t->ping_state.pings_before_data_required != 0); +} + static void update_list(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, int64_t send_bytes, grpc_chttp2_write_cb **list, grpc_error *error) { @@ -139,6 +208,8 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, s->sent_initial_metadata = true; sent_initial_metadata = true; now_writing = true; + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; } /* send any window updates */ if (s->announce_window > 0) { @@ -146,15 +217,22 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_window_update_create( s->id, s->announce_window, &s->stats.outgoing)); + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, announce_window, announce); } if (sent_initial_metadata) { /* send any body bytes, if allowed by flow control */ if (s->flow_controlled_buffer.length > 0) { - uint32_t max_outgoing = - (uint32_t)GPR_MIN(t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], - GPR_MIN(s->outgoing_window, t->outgoing_window)); + uint32_t stream_outgoing_window = (uint32_t)GPR_MAX( + 0, + s->outgoing_window_delta + + (int64_t)t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]); + uint32_t max_outgoing = (uint32_t)GPR_MIN( + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], + GPR_MIN(stream_outgoing_window, t->outgoing_window)); if (max_outgoing > 0) { uint32_t send_bytes = (uint32_t)GPR_MIN(max_outgoing, s->flow_controlled_buffer.length); @@ -167,10 +245,12 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); - GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, outgoing_window, + GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, outgoing_window_delta, send_bytes); GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", t, outgoing_window, send_bytes); + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; @@ -189,6 +269,9 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, } else if (t->outgoing_window == 0) { grpc_chttp2_list_add_stalled_by_transport(t, s); now_writing = true; + } else if (stream_outgoing_window == 0) { + grpc_chttp2_list_add_stalled_by_stream(t, s); + now_writing = true; } } if (s->send_trailing_metadata != NULL && @@ -227,15 +310,32 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, /* if the grpc_chttp2_transport is ready to send a window update, do so here also; 3/4 is a magic number that will likely get tuned soon */ - if (t->announce_incoming_window > 0) { - uint32_t announced = - (uint32_t)GPR_MIN(t->announce_incoming_window, UINT32_MAX); - GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", t, announce_incoming_window, - announced); + uint32_t target_incoming_window = GPR_MAX( + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE], + 1024); + uint32_t threshold_to_send_transport_window_update = + t->outbuf.count > 0 ? 3 * target_incoming_window / 4 + : target_incoming_window / 2; + if (t->incoming_window <= threshold_to_send_transport_window_update) { + maybe_initiate_ping(exec_ctx, t, + GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE); + uint32_t announced = (uint32_t)GPR_CLAMP( + target_incoming_window - t->incoming_window, 0, UINT32_MAX); + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("write", t, incoming_window, announced); grpc_transport_one_way_stats throwaway_stats; grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_window_update_create( 0, announced, &throwaway_stats)); + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; + } + + for (size_t i = 0; i < t->ping_ack_count; i++) { + grpc_slice_buffer_add(&t->outbuf, + grpc_chttp2_ping_create(1, t->ping_acks[i])); } + t->ping_ack_count = 0; + + maybe_initiate_ping(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE); GPR_TIMER_END("grpc_chttp2_begin_write", 0); diff --git a/src/core/lib/channel/connected_channel.c b/src/core/lib/channel/connected_channel.c index ccc0619e1c..068c61c92a 100644 --- a/src/core/lib/channel/connected_channel.c +++ b/src/core/lib/channel/connected_channel.c @@ -140,7 +140,7 @@ static void con_get_channel_info(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, const grpc_channel_info *channel_info) {} -static const grpc_channel_filter connected_channel_filter = { +const grpc_channel_filter grpc_connected_filter = { con_start_transport_stream_op, con_start_transport_op, sizeof(call_data), @@ -158,7 +158,7 @@ static const grpc_channel_filter connected_channel_filter = { static void bind_transport(grpc_channel_stack *channel_stack, grpc_channel_element *elem, void *t) { channel_data *cd = (channel_data *)elem->channel_data; - GPR_ASSERT(elem->filter == &connected_channel_filter); + GPR_ASSERT(elem->filter == &grpc_connected_filter); GPR_ASSERT(cd->transport == NULL); cd->transport = t; @@ -178,7 +178,7 @@ bool grpc_add_connected_filter(grpc_exec_ctx *exec_ctx, grpc_transport *t = grpc_channel_stack_builder_get_transport(builder); GPR_ASSERT(t != NULL); return grpc_channel_stack_builder_append_filter( - builder, &connected_channel_filter, bind_transport, t); + builder, &grpc_connected_filter, bind_transport, t); } grpc_stream *grpc_connected_channel_get_stream(grpc_call_element *elem) { diff --git a/src/core/lib/channel/connected_channel.h b/src/core/lib/channel/connected_channel.h index 3585c0ecbc..5c7ea9ed26 100644 --- a/src/core/lib/channel/connected_channel.h +++ b/src/core/lib/channel/connected_channel.h @@ -36,8 +36,13 @@ #include "src/core/lib/channel/channel_stack_builder.h" +extern const grpc_channel_filter grpc_connected_filter; + bool grpc_add_connected_filter(grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder, void *arg_must_be_null); +/* Debug helper to dig the transport stream out of a call element */ +grpc_stream *grpc_connected_channel_get_stream(grpc_call_element *elem); + #endif /* GRPC_CORE_LIB_CHANNEL_CONNECTED_CHANNEL_H */ diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 8e4efd3370..509c1ff95d 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -51,20 +51,22 @@ void grpc_closure_list_init(grpc_closure_list *closure_list) { closure_list->head = closure_list->tail = NULL; } -void grpc_closure_list_append(grpc_closure_list *closure_list, +bool grpc_closure_list_append(grpc_closure_list *closure_list, grpc_closure *closure, grpc_error *error) { if (closure == NULL) { GRPC_ERROR_UNREF(error); - return; + return false; } closure->error_data.error = error; closure->next_data.next = NULL; - if (closure_list->head == NULL) { + bool was_empty = (closure_list->head == NULL); + if (was_empty) { closure_list->head = closure; } else { closure_list->tail->next_data.next = closure; } closure_list->tail = closure; + return was_empty; } void grpc_closure_list_fail_all(grpc_closure_list *list, diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 6748b21e59..2510d50b42 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -116,8 +116,9 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg, void grpc_closure_list_init(grpc_closure_list *list); /** add \a closure to the end of \a list - and set \a closure's result to \a error */ -void grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure, + and set \a closure's result to \a error + Returns true if \a list becomes non-empty */ +bool grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure, grpc_error *error); /** force all success bits in \a list to false */ diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index ba6c7087a9..fa9966c3a6 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -72,6 +72,7 @@ struct grpc_combiner { bool final_list_covered_by_poller; grpc_closure_list final_list; grpc_closure offload; + gpr_refcount refs; }; static void combiner_exec_uncovered(grpc_exec_ctx *exec_ctx, @@ -126,6 +127,7 @@ static bool is_covered_by_poller(grpc_combiner *lock) { grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue) { grpc_combiner *lock = gpr_malloc(sizeof(*lock)); + gpr_ref_init(&lock->refs, 1); lock->next_combiner_on_this_exec_ctx = NULL; lock->time_to_execute_final_list = false; lock->optional_workqueue = optional_workqueue; @@ -152,7 +154,7 @@ static void really_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { gpr_free(lock); } -void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { +static void start_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { gpr_atm old_state = gpr_atm_full_fetch_add(&lock->state, -STATE_UNORPHANED); GRPC_COMBINER_TRACE(gpr_log( GPR_DEBUG, "C:%p really_destroy old_state=%" PRIdPTR, lock, old_state)); @@ -161,6 +163,30 @@ void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { } } +#ifdef GRPC_COMBINER_REFCOUNT_DEBUG +#define GRPC_COMBINER_DEBUG_SPAM(op, delta) \ + gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, \ + "combiner[%p] %s %" PRIdPTR " --> %" PRIdPTR " %s", lock, (op), \ + gpr_atm_no_barrier_load(&lock->refs.count), \ + gpr_atm_no_barrier_load(&lock->refs.count) + (delta), reason); +#else +#define GRPC_COMBINER_DEBUG_SPAM(op, delta) +#endif + +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, + grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { + GRPC_COMBINER_DEBUG_SPAM("UNREF", -1); + if (gpr_unref(&lock->refs)) { + start_destroy(exec_ctx, lock); + } +} + +grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS) { + GRPC_COMBINER_DEBUG_SPAM(" REF", 1); + gpr_ref(&lock->refs); + return lock; +} + static void push_last_on_exec_ctx(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) { lock->next_combiner_on_this_exec_ctx = NULL; diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 81dff85d40..75dcb0b70a 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -48,8 +48,27 @@ // Initialize the lock, with an optional workqueue to shift load to when // necessary grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue); -// Destroy the lock -void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock); + +//#define GRPC_COMBINER_REFCOUNT_DEBUG +#ifdef GRPC_COMBINER_REFCOUNT_DEBUG +#define GRPC_COMBINER_DEBUG_ARGS \ + , const char *file, int line, const char *reason +#define GRPC_COMBINER_REF(combiner, reason) \ + grpc_combiner_ref((combiner), __FILE__, __LINE__, (reason)) +#define GRPC_COMBINER_UNREF(exec_ctx, combiner, reason) \ + grpc_combiner_unref((exec_ctx), (combiner), __FILE__, __LINE__, (reason)) +#else +#define GRPC_COMBINER_DEBUG_ARGS +#define GRPC_COMBINER_REF(combiner, reason) grpc_combiner_ref((combiner)) +#define GRPC_COMBINER_UNREF(exec_ctx, combiner, reason) \ + grpc_combiner_unref((exec_ctx), (combiner)) +#endif + +// Ref/unref the lock, for when we're sharing the lock ownership +// Prefer to use the macros above +grpc_combiner *grpc_combiner_ref(grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); +void grpc_combiner_unref(grpc_exec_ctx *exec_ctx, + grpc_combiner *lock GRPC_COMBINER_DEBUG_ARGS); // Fetch a scheduler to schedule closures against grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *lock, bool covered_by_poller); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index d5995a5ac6..511ffdcdf1 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -33,6 +33,8 @@ #include "src/core/lib/iomgr/resource_quota.h" +#include <limits.h> +#include <stdint.h> #include <string.h> #include <grpc/support/alloc.h> @@ -44,6 +46,8 @@ int grpc_resource_quota_trace = 0; +#define MEMORY_USAGE_ESTIMATION_MAX 65536 + /* Internal linked list pointers for a resource user */ typedef struct { grpc_resource_user *next; @@ -126,9 +130,12 @@ struct grpc_resource_quota { /* refcount */ gpr_refcount refs; + /* estimate of current memory usage + scaled to the range [0..RESOURCE_USAGE_ESTIMATION_MAX] */ + gpr_atm memory_usage_estimation; + /* Master combiner lock: all activity on a quota executes under this combiner - * (so no mutex is needed for this data structure) - */ + * (so no mutex is needed for this data structure) */ grpc_combiner *combiner; /* Size of the resource quota */ int64_t size; @@ -269,6 +276,16 @@ static void rq_step_sched(grpc_exec_ctx *exec_ctx, GRPC_ERROR_NONE); } +/* update the atomically available resource estimate - use no barriers since + timeliness of delivery really doesn't matter much */ +static void rq_update_estimate(grpc_resource_quota *resource_quota) { + gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, + (gpr_atm)((1.0 - + ((double)resource_quota->free_pool) / + ((double)resource_quota->size)) * + MEMORY_USAGE_ESTIMATION_MAX)); +} + /* returns true if all allocations are completed */ static bool rq_alloc(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { @@ -281,6 +298,7 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, int64_t amt = -resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool -= amt; + rq_update_estimate(resource_quota); if (grpc_resource_quota_trace) { gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64 " bytes; rq_free_pool -> %" PRId64, @@ -315,6 +333,7 @@ static bool rq_reclaim_from_per_user_free_pool( int64_t amt = resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool += amt; + rq_update_estimate(resource_quota); if (grpc_resource_quota_trace) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 " bytes; rq_free_pool -> %" PRId64, @@ -531,6 +550,7 @@ static void rq_resize(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) { int64_t delta = a->size - a->resource_quota->size; a->resource_quota->size += delta; a->resource_quota->free_pool += delta; + rq_update_estimate(a->resource_quota); rq_step_sched(exec_ctx, a->resource_quota); grpc_resource_quota_unref_internal(exec_ctx, a->resource_quota); gpr_free(a); @@ -557,6 +577,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { resource_quota->size = INT64_MAX; resource_quota->step_scheduled = false; resource_quota->reclaiming = false; + gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); if (name != NULL) { resource_quota->name = gpr_strdup(name); } else { @@ -578,7 +599,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { void grpc_resource_quota_unref_internal(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { if (gpr_unref(&resource_quota->refs)) { - grpc_combiner_destroy(exec_ctx, resource_quota->combiner); + GRPC_COMBINER_UNREF(exec_ctx, resource_quota->combiner, "resource_quota"); gpr_free(resource_quota->name); gpr_free(resource_quota); } @@ -602,6 +623,13 @@ void grpc_resource_quota_ref(grpc_resource_quota *resource_quota) { grpc_resource_quota_ref_internal(resource_quota); } +double grpc_resource_quota_get_memory_pressure( + grpc_resource_quota *resource_quota) { + return ((double)(gpr_atm_no_barrier_load( + &resource_quota->memory_usage_estimation))) / + ((double)MEMORY_USAGE_ESTIMATION_MAX); +} + /* Public API */ void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, size_t size) { diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index d1127ce9ea..b9f62cbf83 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -84,6 +84,12 @@ void grpc_resource_quota_unref_internal(grpc_exec_ctx *exec_ctx, grpc_resource_quota *grpc_resource_quota_from_channel_args( const grpc_channel_args *channel_args); +/* Return a number indicating current memory pressure: + 0.0 ==> no memory usage + 1.0 ==> maximum memory usage */ +double grpc_resource_quota_get_memory_pressure( + grpc_resource_quota *resource_quota); + typedef struct grpc_resource_user grpc_resource_user; grpc_resource_user *grpc_resource_user_create( diff --git a/src/core/lib/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c index 44bc2f968b..ffa62cb53c 100644 --- a/src/core/lib/iomgr/sockaddr_utils.c +++ b/src/core/lib/iomgr/sockaddr_utils.c @@ -190,31 +190,37 @@ int grpc_sockaddr_to_string(char **out, } char *grpc_sockaddr_to_uri(const grpc_resolved_address *resolved_addr) { - char *temp; - char *result; grpc_resolved_address addr_normalized; - const struct sockaddr *addr; - if (grpc_sockaddr_is_v4mapped(resolved_addr, &addr_normalized)) { resolved_addr = &addr_normalized; } + const char *scheme = grpc_sockaddr_get_uri_scheme(resolved_addr); + if (scheme == NULL || strcmp("unix", scheme) == 0) { + return grpc_sockaddr_to_uri_unix_if_possible(resolved_addr); + } + char *path = NULL; + char *uri_str = NULL; + if (grpc_sockaddr_to_string(&path, resolved_addr, + false /* suppress errors */) && + scheme != NULL) { + gpr_asprintf(&uri_str, "%s:%s", scheme, path); + } + gpr_free(path); + return uri_str != NULL ? uri_str : NULL; +} - addr = (const struct sockaddr *)resolved_addr->addr; - +const char *grpc_sockaddr_get_uri_scheme( + const grpc_resolved_address *resolved_addr) { + const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr; switch (addr->sa_family) { case AF_INET: - grpc_sockaddr_to_string(&temp, resolved_addr, 0); - gpr_asprintf(&result, "ipv4:%s", temp); - gpr_free(temp); - return result; + return "ipv4"; case AF_INET6: - grpc_sockaddr_to_string(&temp, resolved_addr, 0); - gpr_asprintf(&result, "ipv6:%s", temp); - gpr_free(temp); - return result; - default: - return grpc_sockaddr_to_uri_unix_if_possible(resolved_addr); + return "ipv6"; + case AF_UNIX: + return "unix"; } + return NULL; } int grpc_sockaddr_get_port(const grpc_resolved_address *resolved_addr) { diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h index 5371e360c5..2b22f11b49 100644 --- a/src/core/lib/iomgr/sockaddr_utils.h +++ b/src/core/lib/iomgr/sockaddr_utils.h @@ -84,6 +84,10 @@ int grpc_sockaddr_set_port(const grpc_resolved_address *addr, int port); int grpc_sockaddr_to_string(char **out, const grpc_resolved_address *addr, int normalize); +/* Returns the URI string corresponding to \a addr */ char *grpc_sockaddr_to_uri(const grpc_resolved_address *addr); +/* Returns the URI scheme corresponding to \a addr */ +const char *grpc_sockaddr_get_uri_scheme(const grpc_resolved_address *addr); + #endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H */ diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index 1e84ec3a1e..c8dc9e64bd 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -135,12 +135,10 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { /* Tries to issue one async connection, then schedules both an IOCP notification request for the connection, and one timeout alert. */ -void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, - grpc_endpoint **endpoint, - grpc_pollset_set *interested_parties, - const grpc_channel_args *channel_args, - const grpc_resolved_address *addr, - gpr_timespec deadline) { +static void tcp_client_connect_impl( + grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_endpoint **endpoint, + grpc_pollset_set *interested_parties, const grpc_channel_args *channel_args, + const grpc_resolved_address *addr, gpr_timespec deadline) { SOCKET sock = INVALID_SOCKET; BOOL success; int status; @@ -252,4 +250,21 @@ failure: grpc_closure_sched(exec_ctx, on_done, final_error); } +// overridden by api_fuzzer.c +void (*grpc_tcp_client_connect_impl)( + grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep, + grpc_pollset_set *interested_parties, const grpc_channel_args *channel_args, + const grpc_resolved_address *addr, + gpr_timespec deadline) = tcp_client_connect_impl; + +void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_endpoint **ep, + grpc_pollset_set *interested_parties, + const grpc_channel_args *channel_args, + const grpc_resolved_address *addr, + gpr_timespec deadline) { + grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties, + channel_args, addr, deadline); +} + #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index e9e7511c9c..36f878fdd4 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -44,6 +44,7 @@ #include <errno.h> #include <fcntl.h> +#include <ifaddrs.h> #include <limits.h> #include <netinet/in.h> #include <netinet/tcp.h> @@ -115,6 +116,8 @@ struct grpc_tcp_server { bool shutdown; /* use SO_REUSEPORT */ bool so_reuseport; + /* expand wildcard addresses to a list of all local addresses */ + bool expand_wildcard_addrs; /* linked list of server ports */ grpc_tcp_listener *head; @@ -161,6 +164,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; s->resource_quota = grpc_resource_quota_create(NULL); + s->expand_wildcard_addrs = false; for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) { if (args->args[i].type == GRPC_ARG_INTEGER) { @@ -183,6 +187,15 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); } + } else if (0 == strcmp(GRPC_ARG_EXPAND_WILDCARD_ADDRS, args->args[i].key)) { + if (args->args[i].type == GRPC_ARG_INTEGER) { + s->expand_wildcard_addrs = (args->args[i].value.integer != 0); + } else { + grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); + gpr_free(s); + return GRPC_ERROR_CREATE(GRPC_ARG_EXPAND_WILDCARD_ADDRS + " must be an integer"); + } } } gpr_ref_init(&s->refs, 1); @@ -504,9 +517,224 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, int fd, return err; } -/* Insert count new listeners after listener. Every new listener will have the - same listen address as listener (SO_REUSEPORT must be enabled). Every new - listener is a sibling of listener. */ +/* If successful, add a listener to s for addr, set *dsmode for the socket, and + return the *listener. */ +static grpc_error *add_addr_to_server(grpc_tcp_server *s, + const grpc_resolved_address *addr, + unsigned port_index, unsigned fd_index, + grpc_dualstack_mode *dsmode, + grpc_tcp_listener **listener) { + grpc_resolved_address addr4_copy; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (*dsmode == GRPC_DSMODE_IPV4 && + grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { + addr = &addr4_copy; + } + return add_socket_to_server(s, fd, addr, port_index, fd_index, listener); +} + +/* Bind to "::" to get a port number not used by any address. */ +static grpc_error *get_unused_port(int *port) { + grpc_resolved_address wild; + grpc_sockaddr_make_wildcard6(0, &wild); + grpc_dualstack_mode dsmode; + int fd; + grpc_error *err = + grpc_create_dualstack_socket(&wild, SOCK_STREAM, 0, &dsmode, &fd); + if (err != GRPC_ERROR_NONE) { + return err; + } + if (dsmode == GRPC_DSMODE_IPV4) { + grpc_sockaddr_make_wildcard4(0, &wild); + } + if (bind(fd, (const struct sockaddr *)wild.addr, (socklen_t)wild.len) != 0) { + err = GRPC_OS_ERROR(errno, "bind"); + close(fd); + return err; + } + if (getsockname(fd, (struct sockaddr *)wild.addr, (socklen_t *)&wild.len) != + 0) { + err = GRPC_OS_ERROR(errno, "getsockname"); + close(fd); + return err; + } + close(fd); + *port = grpc_sockaddr_get_port(&wild); + return *port <= 0 ? GRPC_ERROR_CREATE("Bad port") : GRPC_ERROR_NONE; +} + +/* Return the listener in s with address addr or NULL. */ +static grpc_tcp_listener *find_listener_with_addr(grpc_tcp_server *s, + grpc_resolved_address *addr) { + grpc_tcp_listener *l; + gpr_mu_lock(&s->mu); + for (l = s->head; l != NULL; l = l->next) { + if (l->addr.len != addr->len) { + continue; + } + if (memcmp(l->addr.addr, addr->addr, addr->len) == 0) { + break; + } + } + gpr_mu_unlock(&s->mu); + return l; +} + +/* Get all addresses assigned to network interfaces on the machine and create a + listener for each. requested_port is the port to use for every listener, or 0 + to select one random port that will be used for every listener. Set *out_port + to the port selected. Return GRPC_ERROR_NONE only if all listeners were + added. */ +static grpc_error *add_all_local_addrs_to_server(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + struct ifaddrs *ifa = NULL; + struct ifaddrs *ifa_it; + unsigned fd_index = 0; + grpc_tcp_listener *sp = NULL; + grpc_error *err = GRPC_ERROR_NONE; + if (requested_port == 0) { + /* Note: There could be a race where some local addrs can listen on the + selected port and some can't. The sane way to handle this would be to + retry by recreating the whole grpc_tcp_server. Backing out individual + listeners and orphaning the FDs looks like too much trouble. */ + if ((err = get_unused_port(&requested_port)) != GRPC_ERROR_NONE) { + return err; + } else if (requested_port <= 0) { + return GRPC_ERROR_CREATE("Bad get_unused_port()"); + } + gpr_log(GPR_DEBUG, "Picked unused port %d", requested_port); + } + if (getifaddrs(&ifa) != 0 || ifa == NULL) { + return GRPC_OS_ERROR(errno, "getifaddrs"); + } + for (ifa_it = ifa; ifa_it != NULL; ifa_it = ifa_it->ifa_next) { + grpc_resolved_address addr; + char *addr_str = NULL; + grpc_dualstack_mode dsmode; + grpc_tcp_listener *new_sp = NULL; + const char *ifa_name = (ifa_it->ifa_name ? ifa_it->ifa_name : "<unknown>"); + if (ifa_it->ifa_addr == NULL) { + continue; + } else if (ifa_it->ifa_addr->sa_family == AF_INET) { + addr.len = sizeof(struct sockaddr_in); + } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { + addr.len = sizeof(struct sockaddr_in6); + } else { + continue; + } + memcpy(addr.addr, ifa_it->ifa_addr, addr.len); + if (!grpc_sockaddr_set_port(&addr, requested_port)) { + /* Should never happen, because we check sa_family above. */ + err = GRPC_ERROR_CREATE("Failed to set port"); + break; + } + if (grpc_sockaddr_to_string(&addr_str, &addr, 0) < 0) { + addr_str = gpr_strdup("<error>"); + } + gpr_log(GPR_DEBUG, + "Adding local addr from interface %s flags 0x%x to server: %s", + ifa_name, ifa_it->ifa_flags, addr_str); + /* We could have multiple interfaces with the same address (e.g., bonding), + so look for duplicates. */ + if (find_listener_with_addr(s, &addr) != NULL) { + gpr_log(GPR_DEBUG, "Skipping duplicate addr %s on interface %s", addr_str, + ifa_name); + gpr_free(addr_str); + continue; + } + if ((err = add_addr_to_server(s, &addr, port_index, fd_index, &dsmode, + &new_sp)) != GRPC_ERROR_NONE) { + char *err_str = NULL; + grpc_error *root_err; + if (gpr_asprintf(&err_str, "Failed to add listener: %s", addr_str) < 0) { + err_str = gpr_strdup("Failed to add listener"); + } + root_err = GRPC_ERROR_CREATE(err_str); + gpr_free(err_str); + gpr_free(addr_str); + err = grpc_error_add_child(root_err, err); + break; + } else { + GPR_ASSERT(requested_port == new_sp->port); + ++fd_index; + if (sp != NULL) { + new_sp->is_sibling = 1; + sp->sibling = new_sp; + } + sp = new_sp; + } + gpr_free(addr_str); + } + freeifaddrs(ifa); + if (err != GRPC_ERROR_NONE) { + return err; + } else if (sp == NULL) { + return GRPC_ERROR_CREATE("No local addresses"); + } else { + *out_port = sp->port; + return GRPC_ERROR_NONE; + } +} + +/* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ +static grpc_error *add_wildcard_addrs_to_server(grpc_tcp_server *s, + unsigned port_index, + int requested_port, + int *out_port) { + grpc_resolved_address wild4; + grpc_resolved_address wild6; + unsigned fd_index = 0; + grpc_dualstack_mode dsmode; + grpc_tcp_listener *sp = NULL; + grpc_tcp_listener *sp2 = NULL; + grpc_error *v6_err = GRPC_ERROR_NONE; + grpc_error *v4_err = GRPC_ERROR_NONE; + *out_port = -1; + if (s->expand_wildcard_addrs) { + return add_all_local_addrs_to_server(s, port_index, requested_port, + out_port); + } + grpc_sockaddr_make_wildcards(requested_port, &wild4, &wild6); + /* Try listening on IPv6 first. */ + if ((v6_err = add_addr_to_server(s, &wild6, port_index, fd_index, &dsmode, + &sp)) == GRPC_ERROR_NONE) { + ++fd_index; + requested_port = *out_port = sp->port; + if (dsmode == GRPC_DSMODE_DUALSTACK || dsmode == GRPC_DSMODE_IPV4) { + return GRPC_ERROR_NONE; + } + } + /* If we got a v6-only socket or nothing, try adding 0.0.0.0. */ + grpc_sockaddr_set_port(&wild4, requested_port); + if ((v4_err = add_addr_to_server(s, &wild4, port_index, fd_index, &dsmode, + &sp2)) == GRPC_ERROR_NONE) { + *out_port = sp2->port; + if (sp != NULL) { + sp2->is_sibling = 1; + sp->sibling = sp2; + } + } + if (*out_port > 0) { + GRPC_LOG_IF_ERROR("Failed to add :: listener", v6_err); + GRPC_LOG_IF_ERROR("Failed to add 0.0.0.0 listener", v4_err); + return GRPC_ERROR_NONE; + } else { + grpc_error *root_err = + GRPC_ERROR_CREATE("Failed to add any wildcard listeners"); + GPR_ASSERT(v6_err != GRPC_ERROR_NONE && v4_err != GRPC_ERROR_NONE); + root_err = grpc_error_add_child(root_err, v6_err); + root_err = grpc_error_add_child(root_err, v4_err); + return root_err; + } +} + static grpc_error *clone_port(grpc_tcp_listener *listener, unsigned count) { grpc_tcp_listener *sp = NULL; char *addr_str; @@ -559,19 +787,13 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, const grpc_resolved_address *addr, int *out_port) { grpc_tcp_listener *sp; - grpc_tcp_listener *sp2 = NULL; - int fd; - grpc_dualstack_mode dsmode; - grpc_resolved_address addr6_v4mapped; - grpc_resolved_address wild4; - grpc_resolved_address wild6; - grpc_resolved_address addr4_copy; - grpc_resolved_address *allocated_addr = NULL; grpc_resolved_address sockname_temp; - int port; + grpc_resolved_address addr6_v4mapped; + int requested_port = grpc_sockaddr_get_port(addr); unsigned port_index = 0; - unsigned fd_index = 0; - grpc_error *errs[2] = {GRPC_ERROR_NONE, GRPC_ERROR_NONE}; + grpc_dualstack_mode dsmode; + grpc_error *err; + *out_port = -1; if (s->tail != NULL) { port_index = s->tail->port_index + 1; } @@ -579,85 +801,34 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s, /* Check if this is a wildcard port, and if so, try to keep the port the same as some previously created listener. */ - if (grpc_sockaddr_get_port(addr) == 0) { + if (requested_port == 0) { for (sp = s->head; sp; sp = sp->next) { sockname_temp.len = sizeof(struct sockaddr_storage); - if (0 == getsockname(sp->fd, (struct sockaddr *)sockname_temp.addr, + if (0 == getsockname(sp->fd, (struct sockaddr *)&sockname_temp.addr, (socklen_t *)&sockname_temp.len)) { - port = grpc_sockaddr_get_port(&sockname_temp); - if (port > 0) { - allocated_addr = gpr_malloc(sizeof(grpc_resolved_address)); - memcpy(allocated_addr, addr, addr->len); - grpc_sockaddr_set_port(allocated_addr, port); - addr = allocated_addr; + int used_port = grpc_sockaddr_get_port(&sockname_temp); + if (used_port > 0) { + memcpy(&sockname_temp, addr, sizeof(grpc_resolved_address)); + grpc_sockaddr_set_port(&sockname_temp, used_port); + requested_port = used_port; + addr = &sockname_temp; break; } } } } - - sp = NULL; - + if (grpc_sockaddr_is_wildcard(addr, &requested_port)) { + return add_wildcard_addrs_to_server(s, port_index, requested_port, + out_port); + } if (grpc_sockaddr_to_v4mapped(addr, &addr6_v4mapped)) { addr = &addr6_v4mapped; } - - /* Treat :: or 0.0.0.0 as a family-agnostic wildcard. */ - if (grpc_sockaddr_is_wildcard(addr, &port)) { - grpc_sockaddr_make_wildcards(port, &wild4, &wild6); - - /* Try listening on IPv6 first. */ - addr = &wild6; - errs[0] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); - if (errs[0] == GRPC_ERROR_NONE) { - errs[0] = add_socket_to_server(s, fd, addr, port_index, fd_index, &sp); - if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { - goto done; - } - if (sp != NULL) { - ++fd_index; - } - /* If we didn't get a dualstack socket, also listen on 0.0.0.0. */ - if (port == 0 && sp != NULL) { - grpc_sockaddr_set_port(&wild4, sp->port); - } - } - addr = &wild4; - } - - errs[1] = grpc_create_dualstack_socket(addr, SOCK_STREAM, 0, &dsmode, &fd); - if (errs[1] == GRPC_ERROR_NONE) { - if (dsmode == GRPC_DSMODE_IPV4 && - grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { - addr = &addr4_copy; - } - sp2 = sp; - errs[1] = add_socket_to_server(s, fd, addr, port_index, fd_index, &sp); - if (sp2 != NULL && sp != NULL) { - sp2->sibling = sp; - sp->is_sibling = 1; - } - } - -done: - gpr_free(allocated_addr); - if (sp != NULL) { + if ((err = add_addr_to_server(s, addr, port_index, 0, &dsmode, &sp)) == + GRPC_ERROR_NONE) { *out_port = sp->port; - GRPC_ERROR_UNREF(errs[0]); - GRPC_ERROR_UNREF(errs[1]); - return GRPC_ERROR_NONE; - } else { - *out_port = -1; - char *addr_str = grpc_sockaddr_to_uri(addr); - grpc_error *err = grpc_error_set_str( - GRPC_ERROR_CREATE_REFERENCING("Failed to add port to server", errs, - GPR_ARRAY_SIZE(errs)), - GRPC_ERROR_STR_TARGET_ADDRESS, addr_str); - GRPC_ERROR_UNREF(errs[0]); - GRPC_ERROR_UNREF(errs[1]); - gpr_free(addr_str); - return err; } + return err; } /* Return listener at port_index or NULL. Should only be called with s->mu diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 3b23b47d4f..2a1c8d39fa 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -76,8 +76,10 @@ struct grpc_udp_listener { grpc_udp_server *server; grpc_resolved_address addr; grpc_closure read_closure; + grpc_closure write_closure; grpc_closure destroyed_closure; grpc_udp_server_read_cb read_cb; + grpc_udp_server_write_cb write_cb; grpc_udp_server_orphan_cb orphan_cb; struct grpc_udp_listener *next; @@ -176,7 +178,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { /* Call the orphan_cb to signal that the FD is about to be closed and * should no longer be used. */ GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd); grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); @@ -202,7 +204,7 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd); grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE("Server destroyed")); } @@ -304,9 +306,33 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_mu_unlock(&sp->server->mu); } +static void on_write(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_udp_listener *sp = arg; + + gpr_mu_lock(&(sp->server->mu)); + if (error != GRPC_ERROR_NONE) { + if (0 == --sp->server->active_ports) { + gpr_mu_unlock(&sp->server->mu); + deactivated_all_ports(exec_ctx, sp->server); + } else { + gpr_mu_unlock(&sp->server->mu); + } + return; + } + + /* Tell the registered callback that the socket is writeable. */ + GPR_ASSERT(sp->write_cb); + sp->write_cb(exec_ctx, sp->emfd); + + /* Re-arm the notification event so we get another chance to write. */ + grpc_fd_notify_on_write(exec_ctx, sp->emfd, &sp->write_closure); + gpr_mu_unlock(&sp->server->mu); +} + static int add_socket_to_server(grpc_udp_server *s, int fd, const grpc_resolved_address *addr, grpc_udp_server_read_cb read_cb, + grpc_udp_server_write_cb write_cb, grpc_udp_server_orphan_cb orphan_cb) { grpc_udp_listener *sp; int port; @@ -333,6 +359,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, sp->emfd = grpc_fd_create(fd, name); memcpy(&sp->addr, addr, sizeof(grpc_resolved_address)); sp->read_cb = read_cb; + sp->write_cb = write_cb; sp->orphan_cb = orphan_cb; GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); @@ -345,6 +372,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, int grpc_udp_server_add_port(grpc_udp_server *s, const grpc_resolved_address *addr, grpc_udp_server_read_cb read_cb, + grpc_udp_server_write_cb write_cb, grpc_udp_server_orphan_cb orphan_cb) { grpc_udp_listener *sp; int allocated_port1 = -1; @@ -391,7 +419,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s, // TODO(rjshade): Test and propagate the returned grpc_error*: GRPC_ERROR_UNREF(grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode, &fd)); - allocated_port1 = add_socket_to_server(s, fd, addr, read_cb, orphan_cb); + allocated_port1 = + add_socket_to_server(s, fd, addr, read_cb, write_cb, orphan_cb); if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { goto done; } @@ -413,7 +442,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s, grpc_sockaddr_is_v4mapped(addr, &addr4_copy)) { addr = &addr4_copy; } - allocated_port2 = add_socket_to_server(s, fd, addr, read_cb, orphan_cb); + allocated_port2 = + add_socket_to_server(s, fd, addr, read_cb, write_cb, orphan_cb); done: gpr_free(allocated_addr); @@ -451,6 +481,10 @@ void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, grpc_schedule_on_exec_ctx); grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); + grpc_closure_init(&sp->write_closure, on_write, sp, + grpc_schedule_on_exec_ctx); + grpc_fd_notify_on_write(exec_ctx, sp->emfd, &sp->write_closure); + s->active_ports++; sp = sp->next; } diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index f3c466a031..ed63fa7d81 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -49,8 +49,13 @@ typedef struct grpc_udp_server grpc_udp_server; typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, struct grpc_server *server); +/* Called when the socket is writeable. */ +typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, + grpc_fd *emfd); + /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ -typedef void (*grpc_udp_server_orphan_cb)(grpc_fd *emfd); +typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, + grpc_fd *emfd); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); @@ -75,6 +80,7 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index); int grpc_udp_server_add_port(grpc_udp_server *s, const grpc_resolved_address *addr, grpc_udp_server_read_cb read_cb, + grpc_udp_server_write_cb write_cb, grpc_udp_server_orphan_cb orphan_cb); void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *server, diff --git a/src/core/lib/profiling/basic_timers.c b/src/core/lib/profiling/basic_timers.c index bdf9af2339..1f1987fb8e 100644 --- a/src/core/lib/profiling/basic_timers.c +++ b/src/core/lib/profiling/basic_timers.c @@ -43,6 +43,9 @@ #include <grpc/support/thd.h> #include <grpc/support/time.h> #include <stdio.h> +#include <string.h> + +#include "src/core/lib/support/env.h" typedef enum { BEGIN = '{', END = '}', MARK = '.' } marker_type; @@ -74,7 +77,7 @@ typedef struct gpr_timer_log_list { static __thread gpr_timer_log *g_thread_log; static gpr_once g_once_init = GPR_ONCE_INIT; static FILE *output_file; -static const char *output_filename = "latency_trace.txt"; +static const char *output_filename_or_null = NULL; static pthread_mutex_t g_mu; static pthread_cond_t g_cv; static gpr_timer_log_list g_in_progress_logs; @@ -85,6 +88,17 @@ static __thread int g_thread_id; static int g_next_thread_id; static int g_writing_enabled = 1; +static const char *output_filename() { + if (output_filename_or_null == NULL) { + output_filename_or_null = gpr_getenv("LATENCY_TRACE"); + if (output_filename_or_null == NULL || + strlen(output_filename_or_null) == 0) { + output_filename_or_null = "latency_trace.txt"; + } + } + return output_filename_or_null; +} + static int timer_log_push_back(gpr_timer_log_list *list, gpr_timer_log *log) { if (list->head == NULL) { list->head = list->tail = log; @@ -134,7 +148,7 @@ static void timer_log_remove(gpr_timer_log_list *list, gpr_timer_log *log) { static void write_log(gpr_timer_log *log) { size_t i; if (output_file == NULL) { - output_file = fopen(output_filename, "w"); + output_file = fopen(output_filename(), "w"); } for (i = 0; i < log->num_entries; i++) { gpr_timer_entry *entry = &(log->log[i]); @@ -198,7 +212,7 @@ static void finish_writing(void) { } void gpr_timers_set_log_filename(const char *filename) { - output_filename = filename; + output_filename_or_null = filename; } static void init_output() { diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c index 9781a22a86..b24697ce54 100644 --- a/src/core/lib/security/credentials/credentials.c +++ b/src/core/lib/security/credentials/credentials.c @@ -160,6 +160,53 @@ grpc_channel_credentials_duplicate_without_call_credentials( } } +static void credentials_pointer_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { + grpc_channel_credentials_unref(exec_ctx, p); +} + +static void *credentials_pointer_arg_copy(void *p) { + return grpc_channel_credentials_ref(p); +} + +static int credentials_pointer_cmp(void *a, void *b) { return GPR_ICMP(a, b); } + +static const grpc_arg_pointer_vtable credentials_pointer_vtable = { + credentials_pointer_arg_copy, credentials_pointer_arg_destroy, + credentials_pointer_cmp}; + +grpc_arg grpc_channel_credentials_to_arg( + grpc_channel_credentials *credentials) { + grpc_arg result; + result.type = GRPC_ARG_POINTER; + result.key = GRPC_ARG_CHANNEL_CREDENTIALS; + result.value.pointer.vtable = &credentials_pointer_vtable; + result.value.pointer.p = credentials; + return result; +} + +grpc_channel_credentials *grpc_channel_credentials_from_arg( + const grpc_arg *arg) { + if (strcmp(arg->key, GRPC_ARG_CHANNEL_CREDENTIALS)) return NULL; + if (arg->type != GRPC_ARG_POINTER) { + gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, + GRPC_ARG_CHANNEL_CREDENTIALS); + return NULL; + } + return arg->value.pointer.p; +} + +grpc_channel_credentials *grpc_channel_credentials_find_in_args( + const grpc_channel_args *args) { + size_t i; + if (args == NULL) return NULL; + for (i = 0; i < args->num_args; i++) { + grpc_channel_credentials *credentials = + grpc_channel_credentials_from_arg(&args->args[i]); + if (credentials != NULL) return credentials; + } + return NULL; +} + grpc_server_credentials *grpc_server_credentials_ref( grpc_server_credentials *creds) { if (creds == NULL) return NULL; diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 3011df6b8a..510b79552a 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -100,6 +100,8 @@ void grpc_override_well_known_credentials_path_getter( /* --- grpc_channel_credentials. --- */ +#define GRPC_ARG_CHANNEL_CREDENTIALS "grpc.channel_credentials" + typedef struct { void (*destruct)(grpc_exec_ctx *exec_ctx, grpc_channel_credentials *c); @@ -140,6 +142,17 @@ grpc_channel_credentials * grpc_channel_credentials_duplicate_without_call_credentials( grpc_channel_credentials *creds); +/* Util to encapsulate the channel credentials in a channel arg. */ +grpc_arg grpc_channel_credentials_to_arg(grpc_channel_credentials *credentials); + +/* Util to get the channel credentials from a channel arg. */ +grpc_channel_credentials *grpc_channel_credentials_from_arg( + const grpc_arg *arg); + +/* Util to find the channel credentials from channel args. */ +grpc_channel_credentials *grpc_channel_credentials_find_in_args( + const grpc_channel_args *args); + /* --- grpc_credentials_md. --- */ typedef struct { diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c index a8679d097d..a0629f76ce 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.c +++ b/src/core/lib/security/credentials/fake/fake_credentials.c @@ -35,13 +35,13 @@ #include <string.h> -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/iomgr/executor.h" - #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/support/string.h" + /* -- Fake transport security credentials. -- */ static grpc_security_status fake_transport_security_create_security_connector( @@ -49,7 +49,7 @@ static grpc_security_status fake_transport_security_create_security_connector( grpc_call_credentials *call_creds, const char *target, const grpc_channel_args *args, grpc_channel_security_connector **sc, grpc_channel_args **new_args) { - *sc = grpc_fake_channel_security_connector_create(call_creds); + *sc = grpc_fake_channel_security_connector_create(call_creds, target, args); return GRPC_SECURITY_OK; } diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index 9cf38084a3..0fe98417c6 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -38,6 +38,21 @@ /* -- Fake transport security credentials. -- */ +/* Used to verify the target names given to the fake transport security + * connector. + * + * Its syntax by example: + * For LB channels: + * "backend_target_1,backend_target_2,...;lb_target_1,lb_target_2,..." + * For regular channels: + * "backend_taget_1,backend_target_2,..." + * + * That is to say, LB channels have a heading list of LB targets separated from + * the list of backend targets by a semicolon. For non-LB channels, only the + * latter is present. */ +#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ + "grpc.test_only.fake_security.expected_target" + /* Creates a fake transport security credentials object for testing. */ grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index cf056e8008..b9bbe1b304 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -335,7 +335,7 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, grpc_channel_element_args *args) { grpc_security_connector *sc = - grpc_find_security_connector_in_args(args->channel_args); + grpc_security_connector_find_in_args(args->channel_args); grpc_auth_context *auth_context = grpc_find_auth_context_in_args(args->channel_args); diff --git a/src/core/lib/security/transport/lb_targets_info.c b/src/core/lib/security/transport/lb_targets_info.c new file mode 100644 index 0000000000..e73483c039 --- /dev/null +++ b/src/core/lib/security/transport/lb_targets_info.c @@ -0,0 +1,70 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/log.h> + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/security/transport/lb_targets_info.h" + +/* Channel arg key for the mapping of LB server addresses to their names for + * secure naming purposes. */ +#define GRPC_ARG_LB_SECURE_NAMING_MAP "grpc.lb_secure_naming_map" + +static void *targets_info_copy(void *p) { return grpc_slice_hash_table_ref(p); } +static void targets_info_destroy(grpc_exec_ctx *exec_ctx, void *p) { + grpc_slice_hash_table_unref(exec_ctx, p); +} +static int targets_info_cmp(void *a, void *b) { return GPR_ICMP(a, b); } +static const grpc_arg_pointer_vtable server_to_balancer_names_vtable = { + targets_info_copy, targets_info_destroy, targets_info_cmp}; + +grpc_arg grpc_lb_targets_info_create_channel_arg( + grpc_slice_hash_table *targets_info) { + grpc_arg arg; + arg.type = GRPC_ARG_POINTER; + arg.key = GRPC_ARG_LB_SECURE_NAMING_MAP; + arg.value.pointer.p = targets_info; + arg.value.pointer.vtable = &server_to_balancer_names_vtable; + return arg; +} + +grpc_slice_hash_table *grpc_lb_targets_info_find_in_args( + const grpc_channel_args *args) { + const grpc_arg *targets_info_arg = + grpc_channel_args_find(args, GRPC_ARG_LB_SECURE_NAMING_MAP); + if (targets_info_arg != NULL) { + GPR_ASSERT(targets_info_arg->type == GRPC_ARG_POINTER); + return targets_info_arg->value.pointer.p; + } + return NULL; +} diff --git a/src/core/lib/security/transport/lb_targets_info.h b/src/core/lib/security/transport/lb_targets_info.h new file mode 100644 index 0000000000..5e6cacc197 --- /dev/null +++ b/src/core/lib/security/transport/lb_targets_info.h @@ -0,0 +1,47 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H +#define GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H + +#include "src/core/lib/slice/slice_hash_table.h" + +/** Return a channel argument containing \a targets_info. */ +grpc_arg grpc_lb_targets_info_create_channel_arg( + grpc_slice_hash_table *targets_info); + +/** Return the instance of targets info in \a args or NULL */ +grpc_slice_hash_table *grpc_lb_targets_info_find_in_args( + const grpc_channel_args *args); + +#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_LB_TARGETS_INFO_H */ diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index b09127811b..aeb04e33a3 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -43,10 +43,13 @@ #include <grpc/support/string_util.h> #include "src/core/ext/transport/chttp2/alpn/alpn.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/context/security_context.h" #include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/credentials/fake/fake_credentials.h" +#include "src/core/lib/security/transport/lb_targets_info.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/support/env.h" @@ -205,23 +208,23 @@ static const grpc_arg_pointer_vtable connector_pointer_vtable = { grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc) { grpc_arg result; result.type = GRPC_ARG_POINTER; - result.key = GRPC_SECURITY_CONNECTOR_ARG; + result.key = GRPC_ARG_SECURITY_CONNECTOR; result.value.pointer.vtable = &connector_pointer_vtable; result.value.pointer.p = sc; return result; } grpc_security_connector *grpc_security_connector_from_arg(const grpc_arg *arg) { - if (strcmp(arg->key, GRPC_SECURITY_CONNECTOR_ARG)) return NULL; + if (strcmp(arg->key, GRPC_ARG_SECURITY_CONNECTOR)) return NULL; if (arg->type != GRPC_ARG_POINTER) { gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type, - GRPC_SECURITY_CONNECTOR_ARG); + GRPC_ARG_SECURITY_CONNECTOR); return NULL; } return arg->value.pointer.p; } -grpc_security_connector *grpc_find_security_connector_in_args( +grpc_security_connector *grpc_security_connector_find_in_args( const grpc_channel_args *args) { size_t i; if (args == NULL) return NULL; @@ -235,11 +238,21 @@ grpc_security_connector *grpc_find_security_connector_in_args( /* -- Fake implementation. -- */ +typedef struct { + grpc_channel_security_connector base; + char *target; + char *expected_targets; + bool is_lb_channel; +} grpc_fake_channel_security_connector; + static void fake_channel_destroy(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc) { - grpc_channel_security_connector *c = (grpc_channel_security_connector *)sc; - grpc_call_credentials_unref(exec_ctx, c->request_metadata_creds); - gpr_free(sc); + grpc_fake_channel_security_connector *c = + (grpc_fake_channel_security_connector *)sc; + grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); + gpr_free(c->target); + gpr_free(c->expected_targets); + gpr_free(c); } static void fake_server_destroy(grpc_exec_ctx *exec_ctx, @@ -247,6 +260,68 @@ static void fake_server_destroy(grpc_exec_ctx *exec_ctx, gpr_free(sc); } +static bool fake_check_target(const char *target_type, const char *target, + const char *set_str) { + GPR_ASSERT(target_type != NULL); + GPR_ASSERT(target != NULL); + char **set = NULL; + size_t set_size = 0; + gpr_string_split(set_str, ",", &set, &set_size); + bool found = false; + for (size_t i = 0; i < set_size; ++i) { + if (set[i] != NULL && strcmp(target, set[i]) == 0) found = true; + } + for (size_t i = 0; i < set_size; ++i) { + gpr_free(set[i]); + } + gpr_free(set); + return found; +} + +static void fake_secure_name_check(const char *target, + const char *expected_targets, + bool is_lb_channel) { + if (expected_targets == NULL) return; + char **lbs_and_backends = NULL; + size_t lbs_and_backends_size = 0; + bool success = false; + gpr_string_split(expected_targets, ";", &lbs_and_backends, + &lbs_and_backends_size); + if (lbs_and_backends_size > 2 || lbs_and_backends_size == 0) { + gpr_log(GPR_ERROR, "Invalid expected targets arg value: '%s'", + expected_targets); + goto done; + } + if (is_lb_channel) { + if (lbs_and_backends_size != 2) { + gpr_log(GPR_ERROR, + "Invalid expected targets arg value: '%s'. Expectations for LB " + "channels must be of the form 'be1,be2,be3,...;lb1,lb2,...", + expected_targets); + goto done; + } + if (!fake_check_target("LB", target, lbs_and_backends[1])) { + gpr_log(GPR_ERROR, "LB target '%s' not found in expected set '%s'", + target, lbs_and_backends[1]); + goto done; + } + success = true; + } else { + if (!fake_check_target("Backend", target, lbs_and_backends[0])) { + gpr_log(GPR_ERROR, "Backend target '%s' not found in expected set '%s'", + target, lbs_and_backends[0]); + goto done; + } + success = true; + } +done: + for (size_t i = 0; i < lbs_and_backends_size; ++i) { + gpr_free(lbs_and_backends[i]); + } + gpr_free(lbs_and_backends); + if (!success) abort(); +} + static void fake_check_peer(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc, tsi_peer peer, grpc_auth_context **auth_context, @@ -277,12 +352,28 @@ static void fake_check_peer(grpc_exec_ctx *exec_ctx, grpc_auth_context_add_cstring_property( *auth_context, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME, GRPC_FAKE_TRANSPORT_SECURITY_TYPE); - end: grpc_closure_sched(exec_ctx, on_peer_checked, error); tsi_peer_destruct(&peer); } +static void fake_channel_check_peer(grpc_exec_ctx *exec_ctx, + grpc_security_connector *sc, tsi_peer peer, + grpc_auth_context **auth_context, + grpc_closure *on_peer_checked) { + fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked); + grpc_fake_channel_security_connector *c = + (grpc_fake_channel_security_connector *)sc; + fake_secure_name_check(c->target, c->expected_targets, c->is_lb_channel); +} + +static void fake_server_check_peer(grpc_exec_ctx *exec_ctx, + grpc_security_connector *sc, tsi_peer peer, + grpc_auth_context **auth_context, + grpc_closure *on_peer_checked) { + fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked); +} + static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc, const char *host, @@ -313,22 +404,32 @@ static void fake_server_add_handshakers(grpc_exec_ctx *exec_ctx, } static grpc_security_connector_vtable fake_channel_vtable = { - fake_channel_destroy, fake_check_peer}; + fake_channel_destroy, fake_channel_check_peer}; -static grpc_security_connector_vtable fake_server_vtable = {fake_server_destroy, - fake_check_peer}; +static grpc_security_connector_vtable fake_server_vtable = { + fake_server_destroy, fake_server_check_peer}; grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_call_credentials *request_metadata_creds) { - grpc_channel_security_connector *c = gpr_malloc(sizeof(*c)); + grpc_call_credentials *request_metadata_creds, const char *target, + const grpc_channel_args *args) { + grpc_fake_channel_security_connector *c = gpr_malloc(sizeof(*c)); memset(c, 0, sizeof(*c)); - gpr_ref_init(&c->base.refcount, 1); - c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; - c->base.vtable = &fake_channel_vtable; - c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds); - c->check_call_host = fake_channel_check_call_host; - c->add_handshakers = fake_channel_add_handshakers; - return c; + gpr_ref_init(&c->base.base.refcount, 1); + c->base.base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME; + c->base.base.vtable = &fake_channel_vtable; + c->base.request_metadata_creds = + grpc_call_credentials_ref(request_metadata_creds); + c->base.check_call_host = fake_channel_check_call_host; + c->base.add_handshakers = fake_channel_add_handshakers; + c->target = gpr_strdup(target); + const grpc_arg *expected_target_arg = + grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); + if (expected_target_arg != NULL) { + GPR_ASSERT(expected_target_arg->type == GRPC_ARG_STRING); + c->expected_targets = gpr_strdup(expected_target_arg->value.string); + } + c->is_lb_channel = (grpc_lb_targets_info_find_in_args(args) != NULL); + return &c->base; } grpc_server_security_connector *grpc_fake_server_security_connector_create( diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index eba4e6d1d7..3df2fecd39 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -57,7 +57,7 @@ typedef enum { GRPC_SECURITY_OK = 0, GRPC_SECURITY_ERROR } grpc_security_status; typedef struct grpc_security_connector grpc_security_connector; -#define GRPC_SECURITY_CONNECTOR_ARG "grpc.security_connector" +#define GRPC_ARG_SECURITY_CONNECTOR "grpc.security_connector" typedef struct { void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc); @@ -115,7 +115,7 @@ grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc); grpc_security_connector *grpc_security_connector_from_arg(const grpc_arg *arg); /* Util to find the connector from channel args. */ -grpc_security_connector *grpc_find_security_connector_in_args( +grpc_security_connector *grpc_security_connector_find_in_args( const grpc_channel_args *args); /* --- channel_security_connector object. --- @@ -175,7 +175,8 @@ void grpc_server_security_connector_add_handshakers( /* For TESTING ONLY! Creates a fake connector that emulates real channel security. */ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( - grpc_call_credentials *request_metadata_creds); + grpc_call_credentials *request_metadata_creds, const char *target, + const grpc_channel_args *args); /* For TESTING ONLY! Creates a fake connector that emulates real server security. */ diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index bb8a3bf6cd..5d57543ac5 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -451,7 +451,7 @@ static void client_handshaker_factory_add_handshakers( grpc_exec_ctx *exec_ctx, grpc_handshaker_factory *handshaker_factory, const grpc_channel_args *args, grpc_handshake_manager *handshake_mgr) { grpc_channel_security_connector *security_connector = - (grpc_channel_security_connector *)grpc_find_security_connector_in_args( + (grpc_channel_security_connector *)grpc_security_connector_find_in_args( args); grpc_channel_security_connector_add_handshakers(exec_ctx, security_connector, handshake_mgr); @@ -461,7 +461,7 @@ static void server_handshaker_factory_add_handshakers( grpc_exec_ctx *exec_ctx, grpc_handshaker_factory *hf, const grpc_channel_args *args, grpc_handshake_manager *handshake_mgr) { grpc_server_security_connector *security_connector = - (grpc_server_security_connector *)grpc_find_security_connector_in_args( + (grpc_server_security_connector *)grpc_security_connector_find_in_args( args); grpc_server_security_connector_add_handshakers(exec_ctx, security_connector, handshake_mgr); diff --git a/src/core/lib/slice/slice_intern.c b/src/core/lib/slice/slice_intern.c index 7cbd17bffd..32adc4df97 100644 --- a/src/core/lib/slice/slice_intern.c +++ b/src/core/lib/slice/slice_intern.c @@ -215,7 +215,9 @@ bool grpc_slice_is_interned(grpc_slice slice) { } grpc_slice grpc_slice_intern(grpc_slice slice) { + GPR_TIMER_BEGIN("grpc_slice_intern", 0); if (GRPC_IS_STATIC_METADATA_STRING(slice)) { + GPR_TIMER_END("grpc_slice_intern", 0); return slice; } @@ -225,6 +227,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)]; if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT && grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) { + GPR_TIMER_END("grpc_slice_intern", 0); return grpc_static_slice_table[ent.idx]; } } @@ -247,7 +250,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { /* and treat this as if we were never here... sshhh */ } else { gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); + GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } } @@ -275,6 +278,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { gpr_mu_unlock(&shard->mu); + GPR_TIMER_END("grpc_slice_intern", 0); return materialize(s); } diff --git a/src/core/lib/support/log_posix.c b/src/core/lib/support/log_posix.c index f972da0887..79458dd7a3 100644 --- a/src/core/lib/support/log_posix.c +++ b/src/core/lib/support/log_posix.c @@ -37,6 +37,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> #include <grpc/support/time.h> #include <pthread.h> #include <stdarg.h> @@ -93,10 +94,13 @@ void gpr_default_log(gpr_log_func_args *args) { strcpy(time_buffer, "error:strftime"); } - fprintf(stderr, "%s%s.%09d %7tu %s:%d] %s\n", - gpr_log_severity_string(args->severity), time_buffer, - (int)(now.tv_nsec), gettid(), display_file, args->line, - args->message); + char *prefix; + gpr_asprintf(&prefix, "%s%s.%09d %7tu %s:%d]", + gpr_log_severity_string(args->severity), time_buffer, + (int)(now.tv_nsec), gettid(), display_file, args->line); + + fprintf(stderr, "%-70s %s\n", prefix, args->message); + gpr_free(prefix); } #endif /* defined(GPR_POSIX_LOG) */ diff --git a/src/core/lib/support/sync_posix.c b/src/core/lib/support/sync_posix.c index dcb0969a4e..de0f0484b5 100644 --- a/src/core/lib/support/sync_posix.c +++ b/src/core/lib/support/sync_posix.c @@ -42,11 +42,18 @@ #include <time.h> #include "src/core/lib/profiling/timers.h" +#ifdef GPR_MU_COUNTERS +gpr_atm grpc_mu_locks = 0; +#endif + void gpr_mu_init(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_init(mu, NULL) == 0); } void gpr_mu_destroy(gpr_mu* mu) { GPR_ASSERT(pthread_mutex_destroy(mu) == 0); } void gpr_mu_lock(gpr_mu* mu) { +#ifdef GPR_MU_COUNTERS + gpr_atm_no_barrier_fetch_add(&grpc_mu_locks, 1); +#endif GPR_TIMER_BEGIN("gpr_mu_lock", 0); GPR_ASSERT(pthread_mutex_lock(mu) == 0); GPR_TIMER_END("gpr_mu_lock", 0); diff --git a/src/core/lib/support/time_windows.c b/src/core/lib/support/time_windows.c index 6459732879..7b94a5b7bf 100644 --- a/src/core/lib/support/time_windows.c +++ b/src/core/lib/support/time_windows.c @@ -56,7 +56,7 @@ void gpr_time_init(void) { g_time_scale = 1.0 / (double)frequency.QuadPart; } -gpr_timespec gpr_now(gpr_clock_type clock) { +static gpr_timespec now_impl(gpr_clock_type clock) { gpr_timespec now_tv; LONGLONG diff; struct _timeb now_tb; @@ -84,6 +84,12 @@ gpr_timespec gpr_now(gpr_clock_type clock) { return now_tv; } +gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type) = now_impl; + +gpr_timespec gpr_now(gpr_clock_type clock_type) { + return gpr_now_impl(clock_type); +} + void gpr_sleep_until(gpr_timespec until) { gpr_timespec now; gpr_timespec delta; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index d64c2410bf..3352e427cd 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -86,8 +86,11 @@ typedef enum { /* Status came from 'the wire' - or somewhere below the surface layer */ STATUS_FROM_WIRE, - /* Status was created by some internal channel stack operation */ + /* Status was created by some internal channel stack operation: must come via + add_batch_error */ STATUS_FROM_CORE, + /* Status was created by some surface error */ + STATUS_FROM_SURFACE, /* Status came from the server sending status */ STATUS_FROM_SERVER_STATUS, STATUS_SOURCE_COUNT @@ -212,6 +215,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; +int grpc_call_error_trace = 0; + #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) #define CALL_ELEM_FROM_CALL(call, idx) \ @@ -221,11 +226,11 @@ struct grpc_call { static void execute_op(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_transport_stream_op *op); -static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_status_code status, - const char *description); +static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + status_source source, grpc_status_code status, + const char *description); static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_error *error); + status_source source, grpc_error *error); static void destroy_call(grpc_exec_ctx *exec_ctx, void *call_stack, grpc_error *error); static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, @@ -352,7 +357,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, call->start_time, send_deadline, CALL_STACK_FROM_CALL(call))); if (error != GRPC_ERROR_NONE) { - cancel_with_error(exec_ctx, call, GRPC_ERROR_REF(error)); + cancel_with_error(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_ERROR_REF(error)); } if (args->cq != NULL) { GPR_ASSERT( @@ -527,7 +533,6 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, grpc_status_code status, const char *description, void *reserved) { - grpc_call_error r; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GRPC_API_TRACE( "grpc_call_cancel_with_status(" @@ -535,16 +540,16 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, 4, (c, (int)status, description, reserved)); GPR_ASSERT(reserved == NULL); gpr_mu_lock(&c->mu); - r = cancel_with_status(&exec_ctx, c, status, description); + cancel_with_status(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, status, + description); gpr_mu_unlock(&c->mu); grpc_exec_ctx_finish(&exec_ctx); - return r; + return GRPC_CALL_OK; } typedef struct termination_closure { grpc_closure closure; grpc_call *call; - grpc_error *error; grpc_transport_stream_op op; } termination_closure; @@ -559,36 +564,27 @@ static void send_termination(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) { termination_closure *tc = tcp; memset(&tc->op, 0, sizeof(tc->op)); - tc->op.cancel_error = tc->error; + tc->op.cancel_error = GRPC_ERROR_REF(error); /* reuse closure to catch completion */ - grpc_closure_init(&tc->closure, done_termination, tc, - grpc_schedule_on_exec_ctx); - tc->op.on_complete = &tc->closure; + tc->op.on_complete = grpc_closure_init(&tc->closure, done_termination, tc, + grpc_schedule_on_exec_ctx); execute_op(exec_ctx, tc->call, &tc->op); } -static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx, - termination_closure *tc) { - set_status_from_error(exec_ctx, tc->call, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_REF(tc->error)); - grpc_closure_init(&tc->closure, send_termination, tc, - grpc_schedule_on_exec_ctx); - GRPC_CALL_INTERNAL_REF(tc->call, "termination"); - grpc_closure_sched(exec_ctx, &tc->closure, GRPC_ERROR_NONE); - return GRPC_CALL_OK; -} - -static grpc_call_error terminate_with_error(grpc_exec_ctx *exec_ctx, - grpc_call *c, grpc_error *error) { +static void terminate_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, + grpc_error *error) { termination_closure *tc = gpr_malloc(sizeof(*tc)); memset(tc, 0, sizeof(*tc)); tc->call = c; - tc->error = error; - return terminate_with_status(exec_ctx, tc); + GRPC_CALL_INTERNAL_REF(tc->call, "termination"); + grpc_closure_sched(exec_ctx, grpc_closure_init(&tc->closure, send_termination, + tc, grpc_schedule_on_exec_ctx), + error); } static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_error *error) { + status_source source, grpc_error *error) { + set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error)); terminate_with_error(exec_ctx, c, error); } @@ -600,32 +596,35 @@ static grpc_error *error_from_status(grpc_status_code status, GRPC_ERROR_INT_GRPC_STATUS, status); } -static grpc_call_error cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_status_code status, - const char *description) { - return terminate_with_error(exec_ctx, c, - error_from_status(status, description)); +static void cancel_with_status(grpc_exec_ctx *exec_ctx, grpc_call *c, + status_source source, grpc_status_code status, + const char *description) { + cancel_with_error(exec_ctx, c, source, + error_from_status(status, description)); } /******************************************************************************* * FINAL STATUS CODE MANIPULATION */ -static void get_final_status_from(grpc_call *call, status_source from_source, - void (*set_value)(grpc_status_code code, - void *user_data), - void *set_value_user_data, - grpc_slice *details) { +static bool get_final_status_from( + grpc_call *call, status_source from_source, bool allow_ok_status, + void (*set_value)(grpc_status_code code, void *user_data), + void *set_value_user_data, grpc_slice *details) { grpc_status_code code; const char *msg = NULL; grpc_error_get_status(call->status[from_source].error, call->send_deadline, &code, &msg, NULL); + if (code == GRPC_STATUS_OK && !allow_ok_status) { + return false; + } set_value(code, set_value_user_data); if (details != NULL) { *details = msg == NULL ? grpc_empty_slice() : grpc_slice_from_copied_string(msg); } + return true; } static void get_final_status(grpc_call *call, @@ -633,22 +632,37 @@ static void get_final_status(grpc_call *call, void *user_data), void *set_value_user_data, grpc_slice *details) { int i; - /* search for the best status we can present: ideally the error we use has a - clearly defined grpc-status, and we'll prefer that. */ - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - if (call->status[i].is_set && - grpc_error_has_clear_grpc_status(call->status[i].error)) { - get_final_status_from(call, (status_source)i, set_value, - set_value_user_data, details); - return; + if (grpc_call_error_trace) { + gpr_log(GPR_DEBUG, "get_final_status %s", call->is_client ? "CLI" : "SVR"); + for (i = 0; i < STATUS_SOURCE_COUNT; i++) { + if (call->status[i].is_set) { + gpr_log(GPR_DEBUG, " %d: %s", i, + grpc_error_string(call->status[i].error)); + } } } - /* If no clearly defined status exists, search for 'anything' */ - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - if (call->status[i].is_set) { - get_final_status_from(call, (status_source)i, set_value, - set_value_user_data, details); - return; + /* first search through ignoring "OK" statuses: if something went wrong, + * ensure we report it */ + for (int allow_ok_status = 0; allow_ok_status < 2; allow_ok_status++) { + /* search for the best status we can present: ideally the error we use has a + clearly defined grpc-status, and we'll prefer that. */ + for (i = 0; i < STATUS_SOURCE_COUNT; i++) { + if (call->status[i].is_set && + grpc_error_has_clear_grpc_status(call->status[i].error)) { + if (get_final_status_from(call, (status_source)i, allow_ok_status != 0, + set_value, set_value_user_data, details)) { + return; + } + } + } + /* If no clearly defined status exists, search for 'anything' */ + for (i = 0; i < STATUS_SOURCE_COUNT; i++) { + if (call->status[i].is_set) { + if (get_final_status_from(call, (status_source)i, allow_ok_status != 0, + set_value, set_value_user_data, details)) { + return; + } + } } } /* If nothing exists, set some default */ @@ -883,7 +897,7 @@ static void recv_common_filter(grpc_exec_ctx *exec_ctx, grpc_call *call, error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, msg); gpr_free(msg); grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_message); - } else { + } else if (error != GRPC_ERROR_NONE) { error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, ""); } @@ -1029,11 +1043,6 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&call->mu); - if (error != GRPC_ERROR_NONE) { - set_status_from_error(exec_ctx, call, STATUS_FROM_CORE, - GRPC_ERROR_REF(error)); - } - if (bctl->send_initial_metadata) { grpc_metadata_batch_destroy( exec_ctx, @@ -1176,7 +1185,8 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_call *call = bctl->call; gpr_mu_lock(&bctl->call->mu); if (error != GRPC_ERROR_NONE) { - cancel_with_error(exec_ctx, call, GRPC_ERROR_REF(error)); + cancel_with_error(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_ERROR_REF(error)); } if (call->has_initial_md_been_received || error != GRPC_ERROR_NONE || call->receiving_stream == NULL) { @@ -1203,7 +1213,8 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.", algo); gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg); + cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_STATUS_UNIMPLEMENTED, error_msg); } else if (grpc_compression_options_is_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ @@ -1212,7 +1223,8 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(exec_ctx, call, GRPC_STATUS_UNIMPLEMENTED, error_msg); + cancel_with_status(exec_ctx, call, STATUS_FROM_SURFACE, + GRPC_STATUS_UNIMPLEMENTED, error_msg); } else { call->incoming_compression_algorithm = algo; } @@ -1242,7 +1254,10 @@ static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error) { if (error == GRPC_ERROR_NONE) return; int idx = (int)gpr_atm_no_barrier_fetch_add(&bctl->num_errors, 1); - if (idx == 0) cancel_with_error(exec_ctx, bctl->call, GRPC_ERROR_REF(error)); + if (idx == 0) { + cancel_with_error(exec_ctx, bctl->call, STATUS_FROM_CORE, + GRPC_ERROR_REF(error)); + } bctl->errors[idx] = error; } @@ -1468,6 +1483,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } bctl->send_final_op = 1; call->sent_final_op = 1; + GPR_ASSERT(call->send_extra_metadata_count == 0); call->send_extra_metadata_count = 1; call->send_extra_metadata[0].md = grpc_channel_get_reffed_status_elem( exec_ctx, call->channel, op->data.send_status_from_server.status); @@ -1496,6 +1512,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, (int)op->data.send_status_from_server.trailing_metadata_count, op->data.send_status_from_server.trailing_metadata, 1, 1, NULL, 0)) { + for (int n = 0; n < call->send_extra_metadata_count; n++) { + GRPC_MDELEM_UNREF(exec_ctx, call->send_extra_metadata[n].md); + } + call->send_extra_metadata_count = 0; error = GRPC_CALL_ERROR_INVALID_METADATA; goto done_with_error; } diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 8c46a83d42..b70343ddf1 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -125,6 +125,8 @@ uint8_t grpc_call_is_client(grpc_call *call); grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level); +extern int grpc_call_error_trace; + #ifdef __cplusplus } #endif diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index cfa1882775..b338ac4c48 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -63,6 +63,7 @@ #include "src/core/lib/surface/init.h" #include "src/core/lib/surface/lame_client.h" #include "src/core/lib/surface/server.h" +#include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport_impl.h" @@ -192,6 +193,7 @@ void grpc_init(void) { grpc_register_tracer("queue_pluck", &grpc_cq_pluck_trace); grpc_register_tracer("combiner", &grpc_combiner_trace); grpc_register_tracer("server_channel", &grpc_server_channel_trace); + grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace); // Default pluck trace to 1 grpc_cq_pluck_trace = 1; grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace); @@ -199,6 +201,7 @@ void grpc_init(void) { grpc_cq_event_timeout_trace = 1; grpc_register_tracer("op_failure", &grpc_trace_operation_failures); grpc_register_tracer("resource_quota", &grpc_resource_quota_trace); + grpc_register_tracer("call_error", &grpc_call_error_trace); #ifndef NDEBUG grpc_register_tracer("pending_tags", &grpc_trace_pending_tags); #endif diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c index a44407d3bb..46b9a8f922 100644 --- a/src/core/lib/surface/init_secure.c +++ b/src/core/lib/surface/init_secure.c @@ -56,7 +56,7 @@ static bool maybe_prepend_client_auth_filter( grpc_channel_stack_builder_get_channel_arguments(builder); if (args) { for (size_t i = 0; i < args->num_args; i++) { - if (0 == strcmp(GRPC_SECURITY_CONNECTOR_ARG, args->args[i].key)) { + if (0 == strcmp(GRPC_ARG_SECURITY_CONNECTOR, args->args[i].key)) { return grpc_channel_stack_builder_prepend_filter( builder, &grpc_client_auth_filter, NULL, NULL); } diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c new file mode 100644 index 0000000000..e1483677fd --- /dev/null +++ b/src/core/lib/transport/bdp_estimator.c @@ -0,0 +1,104 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/transport/bdp_estimator.h" + +#include <stdlib.h> + +#include <grpc/support/log.h> +#include <grpc/support/useful.h> + +int grpc_bdp_estimator_trace = 0; + +void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { + estimator->estimate = 65536; + estimator->ping_state = GRPC_BDP_PING_UNSCHEDULED; + estimator->name = name; +} + +bool grpc_bdp_estimator_get_estimate(grpc_bdp_estimator *estimator, + int64_t *estimate) { + *estimate = estimator->estimate; + return true; +} + +bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator, + int64_t num_bytes) { + estimator->accumulator += num_bytes; + switch (estimator->ping_state) { + case GRPC_BDP_PING_UNSCHEDULED: + return true; + case GRPC_BDP_PING_SCHEDULED: + return false; + case GRPC_BDP_PING_STARTED: + return false; + } + GPR_UNREACHABLE_CODE(return false); +} + +void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { + if (grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64, + estimator->name, estimator->accumulator, estimator->estimate); + } + GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_UNSCHEDULED); + estimator->ping_state = GRPC_BDP_PING_SCHEDULED; + estimator->accumulator = 0; +} + +void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { + if (grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64, + estimator->name, estimator->accumulator, estimator->estimate); + } + GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_SCHEDULED); + estimator->ping_state = GRPC_BDP_PING_STARTED; + estimator->accumulator = 0; +} + +void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator) { + if (grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64, + estimator->name, estimator->accumulator, estimator->estimate); + } + GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_STARTED); + if (estimator->accumulator > 2 * estimator->estimate / 3) { + estimator->estimate *= 2; + if (grpc_bdp_estimator_trace) { + gpr_log(GPR_DEBUG, "bdp[%s]: estimate increased to %" PRId64, + estimator->name, estimator->estimate); + } + } + estimator->ping_state = GRPC_BDP_PING_UNSCHEDULED; + estimator->accumulator = 0; +} diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h new file mode 100644 index 0000000000..bcaf899910 --- /dev/null +++ b/src/core/lib/transport/bdp_estimator.h @@ -0,0 +1,76 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H +#define GRPC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H + +#include <stdbool.h> +#include <stdint.h> + +#define GRPC_BDP_SAMPLES 16 +#define GRPC_BDP_MIN_SAMPLES_FOR_ESTIMATE 3 + +extern int grpc_bdp_estimator_trace; + +typedef enum { + GRPC_BDP_PING_UNSCHEDULED, + GRPC_BDP_PING_SCHEDULED, + GRPC_BDP_PING_STARTED +} grpc_bdp_estimator_ping_state; + +typedef struct grpc_bdp_estimator { + grpc_bdp_estimator_ping_state ping_state; + int64_t accumulator; + int64_t estimate; + const char *name; +} grpc_bdp_estimator; + +void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name); + +// Returns true if a reasonable estimate could be obtained +bool grpc_bdp_estimator_get_estimate(grpc_bdp_estimator *estimator, + int64_t *estimate); +// Returns true if the user should schedule a ping +bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator, + int64_t num_bytes); +// Schedule a ping: call in response to receiving a true from +// grpc_bdp_estimator_add_incoming_bytes once a ping has been scheduled by a +// transport (but not necessarily started) +void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator); +// Start a ping: call after calling grpc_bdp_estimator_schedule_ping and once +// the ping is on the wire +void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator); +// Completes a previously started ping +void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator); + +#endif diff --git a/src/core/lib/transport/pid_controller.c b/src/core/lib/transport/pid_controller.c index 3cef225d4b..19cb1c0b36 100644 --- a/src/core/lib/transport/pid_controller.c +++ b/src/core/lib/transport/pid_controller.c @@ -32,26 +32,46 @@ */ #include "src/core/lib/transport/pid_controller.h" +#include <grpc/support/useful.h> void grpc_pid_controller_init(grpc_pid_controller *pid_controller, - double gain_p, double gain_i, double gain_d) { - pid_controller->gain_p = gain_p; - pid_controller->gain_i = gain_i; - pid_controller->gain_d = gain_d; + grpc_pid_controller_args args) { + pid_controller->args = args; + pid_controller->last_control_value = args.initial_control_value; grpc_pid_controller_reset(pid_controller); } void grpc_pid_controller_reset(grpc_pid_controller *pid_controller) { pid_controller->last_error = 0.0; + pid_controller->last_dc_dt = 0.0; pid_controller->error_integral = 0.0; } double grpc_pid_controller_update(grpc_pid_controller *pid_controller, double error, double dt) { - pid_controller->error_integral += error * dt; + /* integrate error using the trapezoid rule */ + pid_controller->error_integral += + dt * (pid_controller->last_error + error) * 0.5; + pid_controller->error_integral = GPR_CLAMP( + pid_controller->error_integral, -pid_controller->args.integral_range, + pid_controller->args.integral_range); double diff_error = (error - pid_controller->last_error) / dt; + /* calculate derivative of control value vs time */ + double dc_dt = pid_controller->args.gain_p * error + + pid_controller->args.gain_i * pid_controller->error_integral + + pid_controller->args.gain_d * diff_error; + /* and perform trapezoidal integration */ + double new_control_value = pid_controller->last_control_value + + dt * (pid_controller->last_dc_dt + dc_dt) * 0.5; + new_control_value = + GPR_CLAMP(new_control_value, pid_controller->args.min_control_value, + pid_controller->args.max_control_value); pid_controller->last_error = error; - return dt * (pid_controller->gain_p * error + - pid_controller->gain_i * pid_controller->error_integral + - pid_controller->gain_d * diff_error); + pid_controller->last_dc_dt = dc_dt; + pid_controller->last_control_value = new_control_value; + return new_control_value; +} + +double grpc_pid_controller_last(grpc_pid_controller *pid_controller) { + return pid_controller->last_control_value; } diff --git a/src/core/lib/transport/pid_controller.h b/src/core/lib/transport/pid_controller.h index 83c82d6471..0a86521e90 100644 --- a/src/core/lib/transport/pid_controller.h +++ b/src/core/lib/transport/pid_controller.h @@ -45,20 +45,33 @@ typedef struct { double gain_p; double gain_i; double gain_d; + double initial_control_value; + double min_control_value; + double max_control_value; + double integral_range; +} grpc_pid_controller_args; + +typedef struct { double last_error; double error_integral; + double last_control_value; + double last_dc_dt; + grpc_pid_controller_args args; } grpc_pid_controller; /** Initialize the controller */ void grpc_pid_controller_init(grpc_pid_controller *pid_controller, - double gain_p, double gain_i, double gain_d); + grpc_pid_controller_args args); /** Reset the controller: useful when things have changed significantly */ void grpc_pid_controller_reset(grpc_pid_controller *pid_controller); /** Update the controller: given a current error estimate, and the time since - the last update, returns a delta to the control value */ + the last update, returns a new control value */ double grpc_pid_controller_update(grpc_pid_controller *pid_controller, double error, double dt); +/** Returns the last control value calculated */ +double grpc_pid_controller_last(grpc_pid_controller *pid_controller); + #endif /* GRPC_CORE_LIB_TRANSPORT_PID_CONTROLLER_H */ diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index f6a7f1d755..29898a4209 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -478,6 +478,7 @@ int Server::AddListeningPort(const grpc::string& addr, bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { GPR_ASSERT(!started_); + global_callbacks_->PreServerStart(this); started_ = true; grpc_server_start(server_); diff --git a/src/csharp/Grpc.Core.Testing/.gitignore b/src/csharp/Grpc.Core.Testing/.gitignore new file mode 100644 index 0000000000..1746e3269e --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/.gitignore @@ -0,0 +1,2 @@ +bin +obj diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj new file mode 100644 index 0000000000..9b0b3abf10 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj @@ -0,0 +1,68 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> + <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> + <ProjectGuid>{3AB047CA-6CF9-435D-AA61-2D86C6FA2457}</ProjectGuid> + <OutputType>Library</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>Grpc.Core.Testing</RootNamespace> + <AssemblyName>Grpc.Core.Testing</AssemblyName> + <TargetFrameworkVersion>v4.5</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + <DocumentationFile>bin\$(Configuration)\Grpc.Core.Testing.Xml</DocumentationFile> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> + <DebugSymbols>true</DebugSymbols> + <DebugType>full</DebugType> + <Optimize>false</Optimize> + <OutputPath>bin\Debug\</OutputPath> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> + <DebugType>pdbonly</DebugType> + <Optimize>true</Optimize> + <OutputPath>bin\Release\</OutputPath> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <ItemGroup> + <Reference Include="System" /> + <Reference Include="System.Core" /> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Xml" /> + <Reference Include="System.Interactive.Async"> + <HintPath>..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll</HintPath> + </Reference> + </ItemGroup> + <ItemGroup> + <Compile Include="..\Grpc.Core\Version.cs"> + <Link>Version.cs</Link> + </Compile> + <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="TestCalls.cs" /> + </ItemGroup> + <ItemGroup> + <None Include="Grpc.Core.Testing.project.json" /> + <None Include="packages.config" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\Grpc.Core\Grpc.Core.csproj"> + <Project>{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}</Project> + <Name>Grpc.Core</Name> + </ProjectReference> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> + <!-- To modify your build process, add your task inside one of the targets below and uncomment it. + Other similar extension points exist, see Microsoft.Common.targets. + <Target Name="BeforeBuild"> + </Target> + <Target Name="AfterBuild"> + </Target> + --> +</Project>
\ No newline at end of file diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.project.json b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.project.json new file mode 100644 index 0000000000..c2f5bcb163 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.project.json @@ -0,0 +1,8 @@ +{ + "frameworks": { + "net45": { } + }, + "runtimes": { + "win": { } + } +} diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.xproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.xproj new file mode 100644 index 0000000000..c972387003 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.xproj @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <PropertyGroup> + <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion> + <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath> + </PropertyGroup> + <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" /> + <PropertyGroup Label="Globals"> + <ProjectGuid>2b372155-80ba-4cf9-82d6-4b938e8ec3a0</ProjectGuid> + <RootNamespace>Grpc.Core.Testing</RootNamespace> + <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath> + <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath> + </PropertyGroup> + <PropertyGroup> + <SchemaVersion>2.0</SchemaVersion> + </PropertyGroup> + <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" /> +</Project>
\ No newline at end of file diff --git a/src/csharp/Grpc.Core.Testing/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core.Testing/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..d3ca0c4d0e --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/Properties/AssemblyInfo.cs @@ -0,0 +1,44 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System.Reflection; +using System.Runtime.CompilerServices; + +[assembly: AssemblyTitle("Grpc.Core.Testing")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("")] +[assembly: AssemblyCopyright("Google Inc. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] diff --git a/src/csharp/Grpc.Core.Testing/Settings.StyleCop b/src/csharp/Grpc.Core.Testing/Settings.StyleCop new file mode 100644 index 0000000000..2942add962 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/Settings.StyleCop @@ -0,0 +1,10 @@ +<StyleCopSettings Version="105"> + <SourceFileList> + <SourceFile>Health.cs</SourceFile> + <Settings> + <GlobalSettings> + <BooleanProperty Name="RulesEnabledByDefault">False</BooleanProperty> + </GlobalSettings> + </Settings> + </SourceFileList> +</StyleCopSettings> diff --git a/src/csharp/Grpc.Core.Testing/TestCalls.cs b/src/csharp/Grpc.Core.Testing/TestCalls.cs new file mode 100644 index 0000000000..d8c36f22f3 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/TestCalls.cs @@ -0,0 +1,91 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.Threading.Tasks; +using Grpc.Core; + +namespace Grpc.Core.Testing +{ + /// <summary> + /// Test doubles for client-side call objects. + /// </summary> + public static class TestCalls + { + /// <summary> + /// Creates a test double for <c>AsyncUnaryCall</c>. Only for testing. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public static AsyncUnaryCall<TResponse> AsyncUnaryCall<TResponse> ( + Task<TResponse> responseAsync, Task<Metadata> responseHeadersAsync, Func<Status> getStatusFunc, + Func<Metadata> getTrailersFunc, Action disposeAction) + { + return new AsyncUnaryCall<TResponse>(responseAsync, responseHeadersAsync, getStatusFunc, getTrailersFunc, disposeAction); + } + + /// <summary> + /// Creates a test double for <c>AsyncClientStreamingCall</c>. Only for testing. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public static AsyncClientStreamingCall<TRequest, TResponse> AsyncClientStreamingCall<TRequest, TResponse>( + IClientStreamWriter<TRequest> requestStream, Task<TResponse> responseAsync, + Task<Metadata> responseHeadersAsync, Func<Status> getStatusFunc, + Func<Metadata> getTrailersFunc, Action disposeAction) + { + return new AsyncClientStreamingCall<TRequest, TResponse>(requestStream, responseAsync, responseHeadersAsync, getStatusFunc, getTrailersFunc, disposeAction); + } + + /// <summary> + /// Creates a test double for <c>AsyncServerStreamingCall</c>. Only for testing. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public static AsyncServerStreamingCall<TResponse> AsyncServerStreamingCall<TResponse>( + IAsyncStreamReader<TResponse> responseStream, Task<Metadata> responseHeadersAsync, + Func<Status> getStatusFunc, Func<Metadata> getTrailersFunc, Action disposeAction) + { + return new AsyncServerStreamingCall<TResponse>(responseStream, responseHeadersAsync, getStatusFunc, getTrailersFunc, disposeAction); + } + + /// <summary> + /// Creates a test double for <c>AsyncDuplexStreamingCall</c>. Only for testing. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public static AsyncDuplexStreamingCall<TRequest, TResponse> AsyncDuplexStreamingCall<TResponse, TRequest>( + IClientStreamWriter<TRequest> requestStream, IAsyncStreamReader<TResponse> responseStream, + Task<Metadata> responseHeadersAsync, Func<Status> getStatusFunc, + Func<Metadata> getTrailersFunc, Action disposeAction) + { + return new AsyncDuplexStreamingCall<TRequest, TResponse>(requestStream, responseStream, responseHeadersAsync, getStatusFunc, getTrailersFunc, disposeAction); + } + } +} diff --git a/src/csharp/Grpc.Core.Testing/packages.config b/src/csharp/Grpc.Core.Testing/packages.config new file mode 100644 index 0000000000..53cfad52f0 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/packages.config @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="utf-8"?> +<packages> + <package id="System.Interactive.Async" version="3.1.1" targetFramework="net45" /> +</packages>
\ No newline at end of file diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json new file mode 100644 index 0000000000..02be957812 --- /dev/null +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -0,0 +1,39 @@ +{ + "version": "1.2.0-dev", + "title": "gRPC C# Core Testing", + "authors": [ "Google Inc." ], + "copyright": "Copyright 2017, Google Inc.", + "packOptions": { + "summary": "Testing support for gRPC C#", + "description": "Useful when testing code that uses gRPC.", + "owners": [ "grpc-packages" ], + "licenseUrl": "https://github.com/grpc/grpc/blob/master/LICENSE", + "projectUrl": "https://github.com/grpc/grpc", + "requireLicenseAcceptance": false, + "tags": [ "gRPC test testing" ] + }, + "buildOptions": { + "define": [ "SIGNED" ], + "keyFile": "../keys/Grpc.snk", + "xmlDoc": true, + "compile": { + "includeFiles": [ "../Grpc.Core/Version.cs" ] + } + }, + "dependencies": { + "Grpc.Core": "1.2.0-dev" + }, + "frameworks": { + "net45": { + "frameworkAssemblies": { + "System.Runtime": "", + "System.IO": "" + } + }, + "netstandard1.5": { + "dependencies": { + "NETStandard.Library": "1.6.0" + } + } + } +} diff --git a/src/csharp/Grpc.Core.Tests/project.json b/src/csharp/Grpc.Core.Tests/project.json index 509084a71a..045207a413 100644 --- a/src/csharp/Grpc.Core.Tests/project.json +++ b/src/csharp/Grpc.Core.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Core/NativeDeps.Windows.targets b/src/csharp/Grpc.Core/NativeDeps.Windows.targets index 93db0935bc..623f58b95b 100644 --- a/src/csharp/Grpc.Core/NativeDeps.Windows.targets +++ b/src/csharp/Grpc.Core/NativeDeps.Windows.targets @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8"?> <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <ItemGroup> - <Content Include="..\..\..\vsprojects\$(NativeDependenciesConfiguration)\grpc_csharp_ext.dll"> + <Content Include="..\..\..\cmake\build\Win32\$(NativeDependenciesConfiguration)\grpc_csharp_ext.dll"> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <Link>grpc_csharp_ext.x86.dll</Link> </Content> diff --git a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs index 99158fbc8b..77ac347c7d 100644 --- a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs +++ b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs @@ -49,6 +49,11 @@ using System.Runtime.CompilerServices; "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" + "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" + "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")] +[assembly: InternalsVisibleTo("Grpc.Core.Testing,PublicKey=" + + "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" + + "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" + + "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" + + "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")] [assembly: InternalsVisibleTo("Grpc.IntegrationTesting,PublicKey=" + "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" + "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" + @@ -56,5 +61,6 @@ using System.Runtime.CompilerServices; "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")] #else [assembly: InternalsVisibleTo("Grpc.Core.Tests")] +[assembly: InternalsVisibleTo("Grpc.Core.Testing")] [assembly: InternalsVisibleTo("Grpc.IntegrationTesting")] #endif diff --git a/src/csharp/Grpc.Examples.MathClient/project.json b/src/csharp/Grpc.Examples.MathClient/project.json index 9a8880b5d4..81c17151aa 100644 --- a/src/csharp/Grpc.Examples.MathClient/project.json +++ b/src/csharp/Grpc.Examples.MathClient/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Examples.MathServer/project.json b/src/csharp/Grpc.Examples.MathServer/project.json index 9a8880b5d4..81c17151aa 100644 --- a/src/csharp/Grpc.Examples.MathServer/project.json +++ b/src/csharp/Grpc.Examples.MathServer/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Examples.Tests/project.json b/src/csharp/Grpc.Examples.Tests/project.json index 3e130beeac..e509621a29 100644 --- a/src/csharp/Grpc.Examples.Tests/project.json +++ b/src/csharp/Grpc.Examples.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.HealthCheck.Tests/project.json b/src/csharp/Grpc.HealthCheck.Tests/project.json index addc782afe..654454d1cb 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/project.json +++ b/src/csharp/Grpc.HealthCheck.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.Client/project.json b/src/csharp/Grpc.IntegrationTesting.Client/project.json index ad81cbc48b..f90528151b 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/project.json +++ b/src/csharp/Grpc.IntegrationTesting.Client/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json index 2c7643b74f..161e602abc 100644 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.Server/project.json b/src/csharp/Grpc.IntegrationTesting.Server/project.json index ad81cbc48b..f90528151b 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/project.json +++ b/src/csharp/Grpc.IntegrationTesting.Server/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json index ad81cbc48b..f90528151b 100644 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/project.json +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.IntegrationTesting/project.json b/src/csharp/Grpc.IntegrationTesting/project.json index e47b5953da..eba54318a5 100644 --- a/src/csharp/Grpc.IntegrationTesting/project.json +++ b/src/csharp/Grpc.IntegrationTesting/project.json @@ -16,8 +16,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -37,8 +37,8 @@ "data/ca.pem": "../Grpc.IntegrationTesting/data/ca.pem", "data/server1.key": "../Grpc.IntegrationTesting/data/server1.key", "data/server1.pem": "../Grpc.IntegrationTesting/data/server1.pem", - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.Reflection.Tests/project.json b/src/csharp/Grpc.Reflection.Tests/project.json index 61d3b7e47b..b90834a25e 100644 --- a/src/csharp/Grpc.Reflection.Tests/project.json +++ b/src/csharp/Grpc.Reflection.Tests/project.json @@ -13,8 +13,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Debug/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Debug/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Debug/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/dbg/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/dbg/libgrpc_csharp_ext.dylib" } @@ -31,8 +31,8 @@ }, "copyToOutput": { "mappings": { - "grpc_csharp_ext.x64.dll": "../../../vsprojects/x64/Release/grpc_csharp_ext.dll", - "grpc_csharp_ext.x86.dll": "../../../vsprojects/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x64.dll": "../../../cmake/build/x64/Release/grpc_csharp_ext.dll", + "grpc_csharp_ext.x86.dll": "../../../cmake/build/Win32/Release/grpc_csharp_ext.dll", "libgrpc_csharp_ext.x64.so": "../../../libs/opt/libgrpc_csharp_ext.so", "libgrpc_csharp_ext.x64.dylib": "../../../libs/opt/libgrpc_csharp_ext.dylib" } diff --git a/src/csharp/Grpc.sln b/src/csharp/Grpc.sln index 179e731380..84ba46047f 100644 --- a/src/csharp/Grpc.sln +++ b/src/csharp/Grpc.sln @@ -40,6 +40,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grpc.Reflection", "Grpc.Ref EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grpc.Reflection.Tests", "Grpc.Reflection.Tests\Grpc.Reflection.Tests.csproj", "{B88F91D6-436D-4C78-8B99-47800FA8DE03}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Grpc.Core.Testing", "Grpc.Core.Testing\Grpc.Core.Testing.csproj", "{3AB047CA-6CF9-435D-AA61-2D86C6FA2457}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -110,6 +112,10 @@ Global {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Release|Any CPU.Build.0 = Release|Any CPU
+ {3AB047CA-6CF9-435D-AA61-2D86C6FA2457}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {3AB047CA-6CF9-435D-AA61-2D86C6FA2457}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {3AB047CA-6CF9-435D-AA61-2D86C6FA2457}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {3AB047CA-6CF9-435D-AA61-2D86C6FA2457}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(NestedProjects) = preSolution
EndGlobalSection
diff --git a/src/csharp/README.md b/src/csharp/README.md index 0405ff88a0..8468eb991e 100644 --- a/src/csharp/README.md +++ b/src/csharp/README.md @@ -37,23 +37,15 @@ BUILD FROM SOURCE You only need to go through these steps if you are planning to develop gRPC C#. If you are a user of gRPC C#, go to Usage section above. -**Windows** +**Windows, Linux or Mac OS X** -- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution. Open the - solution `vsprojects/grpc_csharp_ext.sln` in Visual Studio and build it. - -- Open `src\csharp\Grpc.sln` (path is relative to gRPC repository root) - using Visual Studio - -**Linux and Mac OS X** - -- The grpc_csharp_ext native library needs to be built so you can build the gRPC C# solution: - ```sh +- The easiest way to build is using the `run_tests.py` script that will take care of building the `grpc_csharp_ext` native library: + ``` # from the gRPC repository root - $ tools/run_tests/run_tests.py -c dbg -l csharp --build_only + $ python tools/run_tests/run_tests.py -c dbg -l csharp --build_only ``` -- Use MonoDevelop / Xamarin Studio to open the solution Grpc.sln +- Use Visual Studio / MonoDevelop / Xamarin Studio to open the solution Grpc.sln RUNNING TESTS ------------- diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 3b9306b18e..b99fdcbdfd 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -59,6 +59,7 @@ xcopy /Y /I ..\..\architecture=x64,language=protoc,platform=macos\artifacts\* pr %DOTNET% restore . || goto :error %DOTNET% pack --configuration Release Grpc.Core\project.json --output ..\..\artifacts || goto :error +%DOTNET% pack --configuration Release Grpc.Core.Testing\project.json --output ..\..\artifacts || goto :error %DOTNET% pack --configuration Release Grpc.Auth\project.json --output ..\..\artifacts || goto :error %DOTNET% pack --configuration Release Grpc.HealthCheck\project.json --output ..\..\artifacts || goto :error %DOTNET% pack --configuration Release Grpc.Reflection\project.json --output ..\..\artifacts || goto :error diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index dcb4fe3104..442e3acad2 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -61,6 +61,7 @@ cp $EXTERNAL_GIT_ROOT/architecture=x64,language=protoc,platform=macos/artifacts/ dotnet restore . dotnet pack --configuration Release Grpc.Core/project.json --output ../../artifacts +dotnet pack --configuration Release Grpc.Core.Testing/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts diff --git a/src/csharp/buildall.bat b/src/csharp/buildall.bat deleted file mode 100644 index 0beb30c198..0000000000 --- a/src/csharp/buildall.bat +++ /dev/null @@ -1,59 +0,0 @@ -@rem Copyright 2016, Google Inc. -@rem All rights reserved. -@rem -@rem Redistribution and use in source and binary forms, with or without -@rem modification, are permitted provided that the following conditions are -@rem met: -@rem -@rem * Redistributions of source code must retain the above copyright -@rem notice, this list of conditions and the following disclaimer. -@rem * Redistributions in binary form must reproduce the above -@rem copyright notice, this list of conditions and the following disclaimer -@rem in the documentation and/or other materials provided with the -@rem distribution. -@rem * Neither the name of Google Inc. nor the names of its -@rem contributors may be used to endorse or promote products derived from -@rem this software without specific prior written permission. -@rem -@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -@rem Convenience script to build gRPC C# from command line - -setlocal - -@rem enter this directory -cd /d %~dp0 - -@rem Set VS variables (uses Visual Studio 2013) -@call "%VS120COMNTOOLS%\..\..\vc\vcvarsall.bat" x86 - -@rem Fetch all dependencies -nuget restore ..\..\vsprojects\grpc.sln || goto :error -nuget restore ..\..\vsprojects\grpc_csharp_ext.sln || goto :error -nuget restore ..\..\vsprojects\grpc_protoc_plugins.sln || goto :error -nuget restore Grpc.sln || goto :error - -@rem Build the C# native extension -msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Debug /p:PlatformToolset=v120 || goto :error -msbuild ..\..\vsprojects\grpc_csharp_ext.sln /p:Configuration=Release /p:PlatformToolset=v120 || goto :error - -msbuild Grpc.sln /p:Configuration=Debug || goto :error -msbuild Grpc.sln /p:Configuration=Release || goto :error - -endlocal - -goto :EOF - -:error -echo Failed! -exit /b %errorlevel% diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c index e1f4d7cdf3..6a241190b2 100644 --- a/src/csharp/ext/grpc_csharp_ext.c +++ b/src/csharp/ext/grpc_csharp_ext.c @@ -761,7 +761,7 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_send_status_from_server( ops[nops].data.send_message.send_message = ctx->send_message; ops[nops].flags = write_flags; ops[nops].reserved = NULL; - nops ++; + nops++; } if (send_empty_initial_metadata) { ops[nops].op = GRPC_OP_SEND_INITIAL_METADATA; diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index bcc2bb6126..1a3b775c60 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -1,5 +1,11 @@ -# CocoaPods podspec for the gRPC Proto Compiler Plugin +# This file has been automatically generated from a template file. +# Please make modifications to +# `templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template` +# instead. This file can be regenerated from the template by running +# `tools/buildgen/generate_projects.sh`. +# CocoaPods podspec for the gRPC Proto Compiler Plugin +# # Copyright 2016, Google Inc. # All rights reserved. # @@ -36,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.0.2' + v = '1.2.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC @@ -84,10 +90,7 @@ Pod::Spec.new do |s| repo = 'grpc/grpc' file = "grpc_objective_c_plugin-#{v}-macos-x86_64.zip" s.source = { - # TODO(mxyan): Change back to "https://github.com/#{repo}/releases/download/v#{v}/#{file}" for - # next release - # :http => "https://github.com/#{repo}/releases/download/v#{v}/#{file}", - :http => "https://github.com/#{repo}/releases/download/objective-c-v#{v}/#{file}", + :http => "https://github.com/#{repo}/releases/download/v#{v}/#{file}", # TODO(jcanizales): Add sha1 or sha256 # :sha1 => '??', } @@ -98,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.0.2' + s.dependency '!ProtoCompiler', '3.1.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index b55f6c93c6..dc4d8e964e 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.0.2' + v = '3.1.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index 47b5b1a2e7..908bb0b5e5 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - version = '7.0' + version = '8.0' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: @@ -388,42 +388,42 @@ Pod::Spec.new do |s| 0x28340c19, 0x283480ac, 0x283500ea, - 0x2c322910, - 0x2c32a91e, - 0x2c332930, - 0x2c33a942, - 0x2c342956, - 0x2c34a968, - 0x2c352983, - 0x2c35a995, - 0x2c3629a8, + 0x2c3228ca, + 0x2c32a8d8, + 0x2c3328ea, + 0x2c33a8fc, + 0x2c342910, + 0x2c34a922, + 0x2c35293d, + 0x2c35a94f, + 0x2c362962, 0x2c36832d, - 0x2c3729b5, - 0x2c37a9c7, - 0x2c3829da, - 0x2c38a9f1, - 0x2c3929ff, - 0x2c39aa0f, - 0x2c3a2a21, - 0x2c3aaa35, - 0x2c3b2a46, - 0x2c3baa65, - 0x2c3c2a79, - 0x2c3caa8f, - 0x2c3d2aa8, - 0x2c3daac5, - 0x2c3e2ad6, - 0x2c3eaae4, - 0x2c3f2afc, - 0x2c3fab14, - 0x2c402b21, + 0x2c37296f, + 0x2c37a981, + 0x2c382994, + 0x2c38a9ab, + 0x2c3929b9, + 0x2c39a9c9, + 0x2c3a29db, + 0x2c3aa9ef, + 0x2c3b2a00, + 0x2c3baa1f, + 0x2c3c2a33, + 0x2c3caa49, + 0x2c3d2a62, + 0x2c3daa7f, + 0x2c3e2a90, + 0x2c3eaa9e, + 0x2c3f2ab6, + 0x2c3faace, + 0x2c402adb, 0x2c4090e7, - 0x2c412b32, - 0x2c41ab45, + 0x2c412aec, + 0x2c41aaff, 0x2c4210c0, - 0x2c42ab56, + 0x2c42ab10, 0x2c430720, - 0x2c43aa57, + 0x2c43aa11, 0x30320000, 0x30328015, 0x3033001f, @@ -639,74 +639,74 @@ Pod::Spec.new do |s| 0x405b1e9e, 0x405b9eaf, 0x405c1ec2, - 0x405c9ee3, - 0x405d1ef0, - 0x405d9f07, - 0x405e1f27, + 0x405c9ed3, + 0x405d1ee0, + 0x405d9ef7, + 0x405e1f17, 0x405e8a95, - 0x405f1f48, - 0x405f9f55, - 0x40601f63, - 0x40609f85, - 0x40611fad, - 0x40619fc2, - 0x40621fd9, - 0x40629fea, - 0x40631ffb, - 0x4063a010, - 0x40642027, - 0x4064a053, - 0x4065206e, - 0x4065a085, - 0x4066209d, - 0x4066a0c7, - 0x406720f2, - 0x4067a113, - 0x40682126, - 0x4068a147, - 0x40692179, - 0x4069a1a7, - 0x406a21c8, - 0x406aa1e8, - 0x406b2370, - 0x406ba393, - 0x406c23a9, - 0x406ca60b, - 0x406d263a, - 0x406da662, - 0x406e2690, - 0x406ea6a8, - 0x406f26c7, - 0x406fa6dc, - 0x407026ef, - 0x4070a70c, + 0x405f1f38, + 0x405f9f45, + 0x40601f53, + 0x40609f75, + 0x40611f9d, + 0x40619fb2, + 0x40621fc9, + 0x40629fda, + 0x40631feb, + 0x4063a000, + 0x40642017, + 0x4064a043, + 0x4065205e, + 0x4065a075, + 0x4066208d, + 0x4066a0b7, + 0x406720e2, + 0x4067a103, + 0x40682116, + 0x4068a137, + 0x40692169, + 0x4069a197, + 0x406a21b8, + 0x406aa1d8, + 0x406b2360, + 0x406ba383, + 0x406c2399, + 0x406ca5c5, + 0x406d25f4, + 0x406da61c, + 0x406e264a, + 0x406ea662, + 0x406f2681, + 0x406fa696, + 0x407026a9, + 0x4070a6c6, 0x40710800, - 0x4071a71e, - 0x40722731, - 0x4072a74a, - 0x40732762, + 0x4071a6d8, + 0x407226eb, + 0x4072a704, + 0x4073271c, 0x4073936d, - 0x40742776, - 0x4074a790, - 0x407527a1, - 0x4075a7b5, - 0x407627c3, + 0x40742730, + 0x4074a74a, + 0x4075275b, + 0x4075a76f, + 0x4076277d, 0x407691aa, - 0x407727e8, - 0x4077a80a, - 0x40782825, - 0x4078a85e, - 0x40792875, - 0x4079a88b, - 0x407a2897, - 0x407aa8aa, - 0x407b28bf, - 0x407ba8d1, - 0x407c28e6, - 0x407ca8ef, - 0x407d2162, + 0x407727a2, + 0x4077a7c4, + 0x407827df, + 0x4078a818, + 0x4079282f, + 0x4079a845, + 0x407a2851, + 0x407aa864, + 0x407b2879, + 0x407ba88b, + 0x407c28a0, + 0x407ca8a9, + 0x407d2152, 0x407d9c57, - 0x407e283a, + 0x407e27f4, 0x407e9e16, 0x407f1a67, 0x407f9887, @@ -714,45 +714,42 @@ Pod::Spec.new do |s| 0x40809a8f, 0x40811cd9, 0x40819c08, - 0x4082267b, + 0x40822635, 0x4082986d, 0x40831df1, - 0x4083a038, + 0x4083a028, 0x40841aa3, 0x40849e4e, - 0x40851ed3, - 0x41f4229b, - 0x41f9232d, - 0x41fe2220, - 0x41fea3fc, - 0x41ff24ed, - 0x420322b4, - 0x420822d6, - 0x4208a312, - 0x42092204, - 0x4209a34c, - 0x420a225b, - 0x420aa23b, - 0x420b227b, - 0x420ba2f4, - 0x420c2509, - 0x420ca3c9, - 0x420d23e3, - 0x420da41a, - 0x42122434, - 0x421724d0, - 0x4217a476, - 0x421c2498, - 0x421f2453, - 0x42212520, - 0x422624b3, - 0x422b25ef, - 0x422ba59d, - 0x422c25d7, - 0x422ca55c, - 0x422d253b, - 0x422da5bc, - 0x422e2582, + 0x41f4228b, + 0x41f9231d, + 0x41fe2210, + 0x41fea3ec, + 0x41ff24dd, + 0x420322a4, + 0x420822c6, + 0x4208a302, + 0x420921f4, + 0x4209a33c, + 0x420a224b, + 0x420aa22b, + 0x420b226b, + 0x420ba2e4, + 0x420c24f9, + 0x420ca3b9, + 0x420d23d3, + 0x420da40a, + 0x42122424, + 0x421724c0, + 0x4217a466, + 0x421c2488, + 0x421f2443, + 0x42212510, + 0x422624a3, + 0x422b25a9, + 0x422ba572, + 0x422c2591, + 0x422ca54c, + 0x422d252b, 0x4432072b, 0x4432873a, 0x44330746, @@ -795,69 +792,69 @@ Pod::Spec.new do |s| 0x4c3d136d, 0x4c3d937c, 0x4c3e1389, - 0x50322b68, - 0x5032ab77, - 0x50332b82, - 0x5033ab92, - 0x50342bab, - 0x5034abc5, - 0x50352bd3, - 0x5035abe9, - 0x50362bfb, - 0x5036ac11, - 0x50372c2a, - 0x5037ac3d, - 0x50382c55, - 0x5038ac66, - 0x50392c7b, - 0x5039ac8f, - 0x503a2caf, - 0x503aacc5, - 0x503b2cdd, - 0x503bacef, - 0x503c2d0b, - 0x503cad22, - 0x503d2d3b, - 0x503dad51, - 0x503e2d5e, - 0x503ead74, - 0x503f2d86, + 0x50322b22, + 0x5032ab31, + 0x50332b3c, + 0x5033ab4c, + 0x50342b65, + 0x5034ab7f, + 0x50352b8d, + 0x5035aba3, + 0x50362bb5, + 0x5036abcb, + 0x50372be4, + 0x5037abf7, + 0x50382c0f, + 0x5038ac20, + 0x50392c35, + 0x5039ac49, + 0x503a2c69, + 0x503aac7f, + 0x503b2c97, + 0x503baca9, + 0x503c2cc5, + 0x503cacdc, + 0x503d2cf5, + 0x503dad0b, + 0x503e2d18, + 0x503ead2e, + 0x503f2d40, 0x503f8382, - 0x50402d99, - 0x5040ada9, - 0x50412dc3, - 0x5041add2, - 0x50422dec, - 0x5042ae09, - 0x50432e19, - 0x5043ae29, - 0x50442e38, + 0x50402d53, + 0x5040ad63, + 0x50412d7d, + 0x5041ad8c, + 0x50422da6, + 0x5042adc3, + 0x50432dd3, + 0x5043ade3, + 0x50442df2, 0x5044843f, - 0x50452e4c, - 0x5045ae6a, - 0x50462e7d, - 0x5046ae93, - 0x50472ea5, - 0x5047aeba, - 0x50482ee0, - 0x5048aeee, - 0x50492f01, - 0x5049af16, - 0x504a2f2c, - 0x504aaf3c, - 0x504b2f5c, - 0x504baf6f, - 0x504c2f92, - 0x504cafc0, - 0x504d2fd2, - 0x504dafef, - 0x504e300a, - 0x504eb026, - 0x504f3038, - 0x504fb04f, - 0x5050305e, + 0x50452e06, + 0x5045ae24, + 0x50462e37, + 0x5046ae4d, + 0x50472e5f, + 0x5047ae74, + 0x50482e9a, + 0x5048aea8, + 0x50492ebb, + 0x5049aed0, + 0x504a2ee6, + 0x504aaef6, + 0x504b2f16, + 0x504baf29, + 0x504c2f4c, + 0x504caf7a, + 0x504d2f8c, + 0x504dafa9, + 0x504e2fc4, + 0x504eafe0, + 0x504f2ff2, + 0x504fb009, + 0x50503018, 0x505086ef, - 0x50513071, + 0x5051302b, 0x58320ec9, 0x68320e8b, 0x68328c25, @@ -1292,7 +1289,6 @@ Pod::Spec.new do |s| "NO_RENEGOTIATION\\0" "NO_REQUIRED_DIGEST\\0" "NO_SHARED_CIPHER\\0" - "NO_SHARED_GROUP\\0" "NULL_SSL_CTX\\0" "NULL_SSL_METHOD_PASSED\\0" "OLD_SESSION_CIPHER_NOT_RETURNED\\0" @@ -1353,9 +1349,7 @@ Pod::Spec.new do |s| "TLSV1_ALERT_USER_CANCELLED\\0" "TLSV1_BAD_CERTIFICATE_HASH_VALUE\\0" "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE\\0" - "TLSV1_CERTIFICATE_REQUIRED\\0" "TLSV1_CERTIFICATE_UNOBTAINABLE\\0" - "TLSV1_UNKNOWN_PSK_IDENTITY\\0" "TLSV1_UNRECOGNIZED_NAME\\0" "TLSV1_UNSUPPORTED_EXTENSION\\0" "TLS_PEER_DID_NOT_RESPOND_WITH_CERTIFICATE_LIST\\0" diff --git a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m index 40afca2d3d..0e2fa13f2c 100644 --- a/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m +++ b/src/objective-c/GRPCClient/private/GRPCConnectivityMonitor.m @@ -159,11 +159,13 @@ static void PassFlagsToContextInfoBlock(SCNetworkReachabilityRef target, if (strongSelf) { if (lossHandler && !flags.reachable) { lossHandler(); +#if TARGET_OS_IPHONE } else if (wifiStatusChangeHandler && strongSelf->_previousReachabilityFlags && (flags.isWWAN ^ strongSelf->_previousReachabilityFlags.isWWAN)) { wifiStatusChangeHandler(); +#endif } strongSelf->_previousReachabilityFlags = flags; } diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 450bec36e0..246af560cd 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -45,13 +45,10 @@ #import "GRPCCompletionQueue.h" #import "GRPCConnectivityMonitor.h" #import "NSDictionary+GRPC.h" +#import "version.h" NS_ASSUME_NONNULL_BEGIN -// TODO(jcanizales): Generate the version in a standalone header, from templates. Like -// templates/src/core/surface/version.c.template . -#define GRPC_OBJC_VERSION_STRING @"1.0.2" - static NSMutableDictionary *kHostCache; // This connectivity monitor flushes the host cache when connectivity status diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h new file mode 100644 index 0000000000..e569faa25b --- /dev/null +++ b/src/objective-c/GRPCClient/private/version.h @@ -0,0 +1,41 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// This file is autogenerated from a template file. Please make +// modifications to +// `templates/src/objective-c/GRPCClient/private/version.h.template` +// instead. This file can be regenerated from the template by running +// `tools/buildgen/generate_projects.sh`. + + +#define GRPC_OBJC_VERSION_STRING @"1.2.0-dev" diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index 01612a84b2..1e0c8024ca 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -206,7 +206,7 @@ static char *roots_filename; inDomains:NSUserDomainMask] lastObject]; NSLog(@"Documents directory: %@", url); [Cronet start]; - [Cronet startNetLogToFile:@"Documents/cronet_netlog.json" logBytes:YES]; + [Cronet startNetLogToFile:@"cronet_netlog.json" logBytes:YES]; } // The tearDown() function is run after all test cases finish running diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index c3935ce1e0..5584246ad9 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -169,6 +169,8 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +#ifndef GRPC_COMPILE_WITH_CRONET +// TODO (mxyan): Fix this test - (void)testResponsesOverMaxSizeFailWithActionableMessage { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"ResponseOverMaxSize"]; @@ -189,6 +191,7 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +#endif - (void)testResponsesOver4MBAreAcceptedIfOptedIn { XCTAssertNotNil(self.class.host); diff --git a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme index d1d616c4cf..5524a27ffd 100644 --- a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme +++ b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme @@ -49,6 +49,16 @@ </Test> </SkippedTests> </TestableReference> + <TestableReference + skipped = "NO"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "5E8A5DA31D3840B4000F8BC4" + BuildableName = "CoreCronetEnd2EndTests.xctest" + BlueprintName = "CoreCronetEnd2EndTests" + ReferencedContainer = "container:Tests.xcodeproj"> + </BuildableReference> + </TestableReference> </Testables> <MacroExpansion> <BuildableReference @@ -90,15 +100,6 @@ savedToolIdentifier = "" useCustomWorkingDirectory = "NO" debugDocumentVersioning = "YES"> - <MacroExpansion> - <BuildableReference - BuildableIdentifier = "primary" - BlueprintIdentifier = "63423F431B150A5F006CF63C" - BuildableName = "AllTests.xctest" - BlueprintName = "AllTests" - ReferencedContainer = "container:Tests.xcodeproj"> - </BuildableReference> - </MacroExpansion> </ProfileAction> <AnalyzeAction buildConfiguration = "Debug"> diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 81c2885320..677459e082 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -61,3 +61,11 @@ xcodebuild \ test \ | egrep "$XCODEBUILD_FILTER" \ | egrep -v "(GPBDictionary|GPBArray)" - + +xcodebuild \ + -workspace Tests.xcworkspace \ + -scheme InteropTestsRemoteWithCronet \ + -destination name="iPhone 6" \ + test \ + | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "(GPBDictionary|GPBArray)" - diff --git a/src/php/README.md b/src/php/README.md index 320220d3e4..ed91d2fbe5 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -12,24 +12,28 @@ shared C library. * `composer` * `phpunit` (optional) -**Ubuntu/Debian:** +**Install PHP and PECL on Ubuntu/Debian:** ```sh -$ sudo apt-get install php5 php5-dev +$ sudo apt-get install php5 php5-dev php-pear + +OR + +$ sudo apt-get install php7.0 php7.0-dev php-pear ``` -**PEAR/PECL:** +**Install PECL on Mac:** ```sh $ curl -O http://pear.php.net/go-pear.phar $ sudo php -d detect_unicode=0 go-pear.phar ``` -**Composer:** +**Install Composer (Linux or Mac):** ```sh $ curl -sS https://getcomposer.org/installer | php $ sudo mv composer.phar /usr/local/bin/composer ``` -**PHPUnit:** +**Install PHPUnit (Linux or Mac):** ```sh $ wget https://phar.phpunit.de/phpunit-old.phar $ chmod +x phpunit-old.phar @@ -48,6 +52,14 @@ This will compile and install the gRPC PHP extension into the standard PHP extension directory. You should be able to run the [unit tests](#unit-tests), with the PHP extension installed. +**Update php.ini** + +Add this line to your `php.ini` file, e.g. `/etc/php5/cli/php.ini` + +```sh +extension=grpc.so +``` + **Add the gRPC PHP library as a Composer dependency** @@ -55,7 +67,7 @@ You need to add this to your project's `composer.json` file. ``` "require": { - "grpc/grpc": "v1.0.0" + "grpc/grpc": "v1.1.0" } ``` @@ -96,14 +108,6 @@ $ make $ sudo make install ``` -### Update php.ini - -Add this line to your `php.ini` file, e.g. `/etc/php5/cli/php.ini` - -```sh -extension=grpc.so -``` - ## Unit Tests You will need the source code to run tests @@ -138,7 +142,7 @@ $ composer install ### Protobuf compiler Again if you don't have it already, you need to install the protobuf compiler -`protoc`, version 3.1.0+. +`protoc`, version 3.2.0+. If `protoc` hasn't been installed, you can download the `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). @@ -209,6 +213,7 @@ $ sudo apt-get install apache2 ``` Add this line to your `php.ini` file, e.g. `/etc/php5/apache2/php.ini` +or `/etc/php/7.0/apache2/php.ini` ```sh extension=grpc.so @@ -235,7 +240,7 @@ $ cd grpc/src/php $ composer install ``` -Make sure you have generated the client stub `math.php` +Make sure you have generated the client stubs ```sh $ ./bin/generate_proto_php.sh @@ -247,11 +252,10 @@ Copy the `math_client.php` file into your Apache document root, e.g. $ cp tests/generated_code/math_client.php /var/www/html ``` -You may have to fix the first two lines to point the includes to your installation: +You may have to fix the first line to point the includes to your installation: ```php include 'vendor/autoload.php'; -include 'tests/generated_code/math.php'; ``` Connect to `localhost/math_client.php` in your browser, or run this from command line: @@ -266,6 +270,10 @@ Install `nginx` and `php5-fpm`, in addition to `php5` above ```sh $ sudo apt-get install nginx php5-fpm + +OR + +$ sudo apt-get install nginx php7.0-fpm ``` Add this line to your `php.ini` file, e.g. `/etc/php5/fpm/php.ini` @@ -305,7 +313,7 @@ $ cd grpc/src/php $ composer install ``` -Make sure you have generated the client stub `math.php` +Make sure you have generated the client stubs ```sh $ ./bin/generate_proto_php.sh @@ -317,11 +325,10 @@ Copy the `math_client.php` file into your Nginx document root, e.g. $ cp tests/generated_code/math_client.php /var/www/html ``` -You may have to fix the first two lines to point the includes to your installation: +You may have to fix the first line to point the includes to your installation: ```php include 'vendor/autoload.php'; -include 'tests/generated_code/math.php'; ``` Connect to `localhost/math_client.php` in your browser, or run this from command line: diff --git a/src/php/composer.json b/src/php/composer.json index f3529161f8..8528304c81 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -5,7 +5,7 @@ "version": "1.2.0", "require": { "php": ">=5.5.0", - "google/protobuf": "v3.1.0-alpha-1" + "google/protobuf": "v3.1.0" }, "require-dev": { "google/auth": "v0.9" @@ -13,10 +13,8 @@ "autoload": { "psr-4": { "Grpc\\": "lib/Grpc/", - "Grpc\\Testing\\": "tests/interop/Grpc/Testing/", - "GPBMetadata\\Src\\Proto\\Grpc\\Testing\\": "tests/interop/GPBMetadata/Src/Proto/Grpc/Testing/", - "Math\\": "tests/generated_code/Math/", - "GPBMetadata\\": "tests/generated_code/GPBMetadata/" + "": ["tests/interop/", + "tests/generated_code/"] } } } diff --git a/src/php/tests/generated_code/math_client.php b/src/php/tests/generated_code/math_client.php index 6ee92bc465..9ddb1c8f3a 100644 --- a/src/php/tests/generated_code/math_client.php +++ b/src/php/tests/generated_code/math_client.php @@ -32,9 +32,10 @@ * */ -# Fix the following two lines to point to your installation +# Fix the following line to point to your installation +# This assumes that you are using protoc 3.2.0+ and the generated stubs +# were being autoloaded via composer. include 'vendor/autoload.php'; -include 'tests/generated_code/math.php'; function p($line) { @@ -43,7 +44,7 @@ function p($line) $host = 'localhost:50051'; p("Connecting to host: $host"); -$client = new math\MathClient($host, [ +$client = new Math\MathClient($host, [ 'credentials' => Grpc\ChannelCredentials::createInsecure(), ]); p('Client class: '.get_class($client)); @@ -52,7 +53,7 @@ p(''); p('Running unary call test:'); $dividend = 7; $divisor = 4; -$div_arg = new math\DivArgs(); +$div_arg = new Math\DivArgs(); $div_arg->setDividend($dividend); $div_arg->setDivisor($divisor); $call = $client->Div($div_arg); @@ -65,7 +66,7 @@ p(''); p('Running server streaming test:'); $limit = 7; -$fib_arg = new math\FibArgs(); +$fib_arg = new Math\FibArgs(); $fib_arg->setLimit($limit); $call = $client->Fib($fib_arg); $result_array = iterator_to_array($call->responses()); @@ -79,7 +80,7 @@ p(''); p('Running client streaming test:'); $call = $client->Sum(); for ($i = 0; $i <= $limit; ++$i) { - $num = new math\Num(); + $num = new Math\Num(); $num->setNum($i); $call->write($num); } @@ -91,7 +92,7 @@ p(''); p('Running bidi-streaming test:'); $call = $client->DivMany(); for ($i = 0; $i < 7; ++$i) { - $div_arg = new math\DivArgs(); + $div_arg = new Math\DivArgs(); $dividend = 2 * $i + 1; $divisor = 3; $div_arg->setDividend($dividend); diff --git a/src/proto/census/trace_context.proto b/src/proto/census/trace_context.proto index a5d5a9595d..fc0a54d2ce 100644 --- a/src/proto/census/trace_context.proto +++ b/src/proto/census/trace_context.proto @@ -31,18 +31,14 @@ syntax = "proto3"; package google.trace; -// A TraceId uniquely represents a single Trace. It is a 128-bit nonce. -message TraceId { - fixed64 hi = 1; - fixed64 lo = 2; -} - // Tracing information that is propagated with RPC's. message TraceContext { - // Trace identifer. Must be present. - TraceId trace_id = 1; - // ID of parent (client) span. Must be present. - fixed64 span_id = 2; - // true if this trace is sampled. - bool is_sampled = 3; + // A TraceId uniquely represents a single Trace. It is a 128-bit nonce. + // The 128-bit ID is split into 2 64-bit chunks. (REQUIRED) + fixed64 trace_id_hi = 1; + fixed64 trace_id_lo = 2; + // ID of parent (client) span. (REQUIRED) + fixed64 span_id = 3; + // Span option flags. First bit is true if this trace is sampled. (OPTIONAL) + fixed32 span_options = 4; } diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index 529313cfb1..8f0d25c2c9 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -139,7 +139,10 @@ message ServerConfig { int32 async_server_threads = 7; // Specify the number of cores to limit server to, if desired int32 core_limit = 8; - // payload config, used in generic server + // payload config, used in generic server. + // Note this must NOT be used in proto (non-generic) servers. For proto servers, + // 'response sizes' must be configured from the 'response_size' field of the + // 'SimpleRequest' objects in RPC requests. PayloadConfig payload_config = 9; // Specify the cores we should run the server on, if desired diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index d813df5f44..e50ccbe23e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -205,9 +205,7 @@ def check_and_update_cythonization(extensions): base, file_ext = os.path.splitext(source) if file_ext == '.pyx': generated_pyx_source = next((base + gen_ext - for gen_ext in ( - '.c', - '.cpp',) + for gen_ext in ('.c', '.cpp',) if os.path.isfile(base + gen_ext)), None) if generated_pyx_source: diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index fe29971799..b64a708cc7 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1297,47 +1297,22 @@ def server(thread_pool, handlers=None, options=None): ################################### __all__ ################################# -__all__ = ( - 'FutureTimeoutError', - 'FutureCancelledError', - 'Future', - 'ChannelConnectivity', - 'StatusCode', - 'RpcError', - 'RpcContext', - 'Call', - 'ChannelCredentials', - 'CallCredentials', - 'AuthMetadataContext', - 'AuthMetadataPluginCallback', - 'AuthMetadataPlugin', - 'ServerCredentials', - 'UnaryUnaryMultiCallable', - 'UnaryStreamMultiCallable', - 'StreamUnaryMultiCallable', - 'StreamStreamMultiCallable', - 'Channel', - 'ServicerContext', - 'RpcMethodHandler', - 'HandlerCallDetails', - 'GenericRpcHandler', - 'ServiceRpcHandler', - 'Server', - 'unary_unary_rpc_method_handler', - 'unary_stream_rpc_method_handler', - 'stream_unary_rpc_method_handler', - 'stream_stream_rpc_method_handler', - 'method_handlers_generic_handler', - 'ssl_channel_credentials', - 'metadata_call_credentials', - 'access_token_call_credentials', - 'composite_call_credentials', - 'composite_channel_credentials', - 'ssl_server_credentials', - 'channel_ready_future', - 'insecure_channel', - 'secure_channel', - 'server',) +__all__ = ('FutureTimeoutError', 'FutureCancelledError', 'Future', + 'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext', + 'Call', 'ChannelCredentials', 'CallCredentials', + 'AuthMetadataContext', 'AuthMetadataPluginCallback', + 'AuthMetadataPlugin', 'ServerCredentials', 'UnaryUnaryMultiCallable', + 'UnaryStreamMultiCallable', 'StreamUnaryMultiCallable', + 'StreamStreamMultiCallable', 'Channel', 'ServicerContext', + 'RpcMethodHandler', 'HandlerCallDetails', 'GenericRpcHandler', + 'ServiceRpcHandler', 'Server', 'unary_unary_rpc_method_handler', + 'unary_stream_rpc_method_handler', 'stream_unary_rpc_method_handler', + 'stream_stream_rpc_method_handler', + 'method_handlers_generic_handler', 'ssl_channel_credentials', + 'metadata_call_credentials', 'access_token_call_credentials', + 'composite_call_credentials', 'composite_channel_credentials', + 'ssl_server_credentials', 'channel_ready_future', 'insecure_channel', + 'secure_channel', 'server',) ############################### Extension Shims ################################ diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 77412236cc..691e96a322 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -45,28 +45,24 @@ _EMPTY_FLAGS = 0 _INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) _EMPTY_METADATA = cygrpc.Metadata(()) -_UNARY_UNARY_INITIAL_DUE = ( - cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.send_message, - cygrpc.OperationType.send_close_from_client, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_message, - cygrpc.OperationType.receive_status_on_client,) -_UNARY_STREAM_INITIAL_DUE = ( - cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.send_message, - cygrpc.OperationType.send_close_from_client, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_status_on_client,) -_STREAM_UNARY_INITIAL_DUE = ( - cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_message, - cygrpc.OperationType.receive_status_on_client,) -_STREAM_STREAM_INITIAL_DUE = ( - cygrpc.OperationType.send_initial_metadata, - cygrpc.OperationType.receive_initial_metadata, - cygrpc.OperationType.receive_status_on_client,) +_UNARY_UNARY_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.send_message, + cygrpc.OperationType.send_close_from_client, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_message, + cygrpc.OperationType.receive_status_on_client,) +_UNARY_STREAM_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.send_message, + cygrpc.OperationType.send_close_from_client, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_status_on_client,) +_STREAM_UNARY_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_message, + cygrpc.OperationType.receive_status_on_client,) +_STREAM_STREAM_INITIAL_DUE = (cygrpc.OperationType.send_initial_metadata, + cygrpc.OperationType.receive_initial_metadata, + cygrpc.OperationType.receive_status_on_client,) _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = ( 'Exception calling channel subscription callback!') @@ -568,9 +564,9 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): )), event_handler) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), _EMPTY_FLAGS), - cygrpc.operation_send_message(serialized_request, - _EMPTY_FLAGS), + _common.cygrpc_metadata(metadata), + _EMPTY_FLAGS), cygrpc.operation_send_message( + serialized_request, _EMPTY_FLAGS), cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) call_error = call.start_client_batch( @@ -828,10 +824,7 @@ def _deliver(state, initial_connectivity, initial_callbacks): def _spawn_delivery(state, callbacks): delivering_thread = threading.Thread( - target=_deliver, args=( - state, - state.connectivity, - callbacks,)) + target=_deliver, args=(state, state.connectivity, callbacks,)) delivering_thread.start() state.delivering = True diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py index 7ef2571379..a5f1f38ea9 100644 --- a/src/python/grpcio/grpc/_common.py +++ b/src/python/grpcio/grpc/_common.py @@ -40,12 +40,16 @@ from grpc._cython import cygrpc _EMPTY_METADATA = cygrpc.Metadata(()) CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY = { - cygrpc.ConnectivityState.idle: grpc.ChannelConnectivity.IDLE, - cygrpc.ConnectivityState.connecting: grpc.ChannelConnectivity.CONNECTING, - cygrpc.ConnectivityState.ready: grpc.ChannelConnectivity.READY, + cygrpc.ConnectivityState.idle: + grpc.ChannelConnectivity.IDLE, + cygrpc.ConnectivityState.connecting: + grpc.ChannelConnectivity.CONNECTING, + cygrpc.ConnectivityState.ready: + grpc.ChannelConnectivity.READY, cygrpc.ConnectivityState.transient_failure: grpc.ChannelConnectivity.TRANSIENT_FAILURE, - cygrpc.ConnectivityState.shutdown: grpc.ChannelConnectivity.SHUTDOWN, + cygrpc.ConnectivityState.shutdown: + grpc.ChannelConnectivity.SHUTDOWN, } CYGRPC_STATUS_CODE_TO_STATUS_CODE = { diff --git a/src/python/grpcio/grpc/_plugin_wrapping.py b/src/python/grpcio/grpc/_plugin_wrapping.py index bb9a42f3ff..69c46aa546 100644 --- a/src/python/grpcio/grpc/_plugin_wrapping.py +++ b/src/python/grpcio/grpc/_plugin_wrapping.py @@ -37,8 +37,7 @@ from grpc._cython import cygrpc class AuthMetadataContext( collections.namedtuple('AuthMetadataContext', ( - 'service_url', - 'method_name',)), grpc.AuthMetadataContext): + 'service_url', 'method_name',)), grpc.AuthMetadataContext): pass diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 31551e0f1b..bf2743c16b 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -91,8 +91,7 @@ def _details(state): class _HandlerCallDetails( collections.namedtuple('_HandlerCallDetails', ( - 'method', - 'invocation_metadata',)), grpc.HandlerCallDetails): + 'method', 'invocation_metadata',)), grpc.HandlerCallDetails): pass @@ -143,12 +142,11 @@ def _abort(state, call, code, details): effective_code = _abortion_code(state, code) effective_details = details if state.details is None else state.details if state.initial_metadata_allowed: - operations = ( - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( - _common.cygrpc_metadata(state.trailing_metadata), - effective_code, effective_details, _EMPTY_FLAGS),) + operations = (cygrpc.operation_send_initial_metadata( + _EMPTY_METADATA, _EMPTY_FLAGS), + cygrpc.operation_send_status_from_server( + _common.cygrpc_metadata(state.trailing_metadata), + effective_code, effective_details, _EMPTY_FLAGS),) token = _SEND_INITIAL_METADATA_AND_SEND_STATUS_FROM_SERVER_TOKEN else: operations = (cygrpc.operation_send_status_from_server( @@ -417,11 +415,10 @@ def _send_response(rpc_event, state, serialized_response): return False else: if state.initial_metadata_allowed: - operations = ( - cygrpc.operation_send_initial_metadata(_EMPTY_METADATA, - _EMPTY_FLAGS), - cygrpc.operation_send_message(serialized_response, - _EMPTY_FLAGS),) + operations = (cygrpc.operation_send_initial_metadata( + _EMPTY_METADATA, _EMPTY_FLAGS), + cygrpc.operation_send_message(serialized_response, + _EMPTY_FLAGS),) state.initial_metadata_allowed = False token = _SEND_INITIAL_METADATA_AND_SEND_MESSAGE_TOKEN else: @@ -559,10 +556,8 @@ def _handle_unrecognized_method(rpc_event): _EMPTY_METADATA, cygrpc.StatusCode.unimplemented, b'Method not found!', _EMPTY_FLAGS),) rpc_state = _RPCState() - rpc_event.operation_call.start_server_batch(operations, - lambda ignored_event: ( - rpc_state, - (),)) + rpc_event.operation_call.start_server_batch( + operations, lambda ignored_event: (rpc_state, (),)) return rpc_state diff --git a/src/python/grpcio/grpc/_utilities.py b/src/python/grpcio/grpc/_utilities.py index 7c602eb37e..d5a7b2aeef 100644 --- a/src/python/grpcio/grpc/_utilities.py +++ b/src/python/grpcio/grpc/_utilities.py @@ -44,14 +44,9 @@ _DONE_CALLBACK_EXCEPTION_LOG_MESSAGE = ( class RpcMethodHandler( collections.namedtuple('_RpcMethodHandler', ( - 'request_streaming', - 'response_streaming', - 'request_deserializer', - 'response_serializer', - 'unary_unary', - 'unary_stream', - 'stream_unary', - 'stream_stream',)), grpc.RpcMethodHandler): + 'request_streaming', 'response_streaming', 'request_deserializer', + 'response_serializer', 'unary_unary', 'unary_stream', + 'stream_unary', 'stream_stream',)), grpc.RpcMethodHandler): pass diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index e5b28e9408..b53395e2a2 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -454,12 +454,9 @@ class _GenericStub(face.GenericStub): metadata=None, with_call=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _blocking_unary_unary(self._channel, group, method, timeout, with_call, protocol_options, metadata, self._metadata_transformer, request, @@ -472,12 +469,9 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _future_unary_unary(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request, @@ -490,12 +484,9 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _unary_stream(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request, @@ -509,12 +500,9 @@ class _GenericStub(face.GenericStub): metadata=None, with_call=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _blocking_stream_unary( self._channel, group, method, timeout, with_call, protocol_options, metadata, self._metadata_transformer, request_iterator, @@ -527,12 +515,9 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _future_stream_unary( self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request_iterator, request_serializer, @@ -545,12 +530,9 @@ class _GenericStub(face.GenericStub): timeout, metadata=None, protocol_options=None): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _stream_stream(self._channel, group, method, timeout, protocol_options, metadata, self._metadata_transformer, request_iterator, @@ -599,45 +581,33 @@ class _GenericStub(face.GenericStub): raise NotImplementedError() def unary_unary(self, group, method): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _UnaryUnaryMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def unary_stream(self, group, method): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _UnaryStreamMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def stream_unary(self, group, method): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _StreamUnaryMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) def stream_stream(self, group, method): - request_serializer = self._request_serializers.get(( - group, - method,)) - response_deserializer = self._response_deserializers.get(( - group, - method,)) + request_serializer = self._request_serializers.get((group, method,)) + response_deserializer = self._response_deserializers.get((group, + method,)) return _StreamStreamMultiCallable( self._channel, group, method, self._metadata_transformer, request_serializer, response_deserializer) diff --git a/src/python/grpcio/grpc/beta/_connectivity_channel.py b/src/python/grpcio/grpc/beta/_connectivity_channel.py index 39020d2b4e..bfb847f80a 100644 --- a/src/python/grpcio/grpc/beta/_connectivity_channel.py +++ b/src/python/grpcio/grpc/beta/_connectivity_channel.py @@ -85,9 +85,7 @@ class ConnectivityChannel(object): def _spawn_delivery(self, connectivity, callbacks): delivering_thread = threading.Thread( - target=self._deliver, args=( - connectivity, - callbacks,)) + target=self._deliver, args=(connectivity, callbacks,)) delivering_thread.start() self._delivering = True diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py index 206bd7e468..174af2d642 100644 --- a/src/python/grpcio/grpc/beta/_server_adaptations.py +++ b/src/python/grpcio/grpc/beta/_server_adaptations.py @@ -256,14 +256,9 @@ def _adapt_stream_stream_event(stream_stream_event): class _SimpleMethodHandler( collections.namedtuple('_MethodHandler', ( - 'request_streaming', - 'response_streaming', - 'request_deserializer', - 'response_serializer', - 'unary_unary', - 'unary_stream', - 'stream_unary', - 'stream_stream',)), grpc.RpcMethodHandler): + 'request_streaming', 'response_streaming', 'request_deserializer', + 'response_serializer', 'unary_unary', 'unary_stream', + 'stream_unary', 'stream_stream',)), grpc.RpcMethodHandler): pass diff --git a/src/python/grpcio/grpc/framework/interfaces/base/utilities.py b/src/python/grpcio/grpc/framework/interfaces/base/utilities.py index 461706ff9f..80d5d25c12 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/utilities.py +++ b/src/python/grpcio/grpc/framework/interfaces/base/utilities.py @@ -34,19 +34,14 @@ from grpc.framework.interfaces.base import base class _Completion(base.Completion, - collections.namedtuple('_Completion', ( - 'terminal_metadata', - 'code', - 'message',))): + collections.namedtuple('_Completion', ('terminal_metadata', + 'code', 'message',))): """A trivial implementation of base.Completion.""" class _Subscription(base.Subscription, collections.namedtuple('_Subscription', ( - 'kind', - 'termination_callback', - 'allowance', - 'operator', + 'kind', 'termination_callback', 'allowance', 'operator', 'protocol_receiver',))): """A trivial implementation of base.Subscription.""" diff --git a/src/python/grpcio/grpc/framework/interfaces/face/face.py b/src/python/grpcio/grpc/framework/interfaces/face/face.py index 36ddca18c1..6c7e2a3af6 100644 --- a/src/python/grpcio/grpc/framework/interfaces/face/face.py +++ b/src/python/grpcio/grpc/framework/interfaces/face/face.py @@ -63,18 +63,13 @@ class NoSuchMethodError(Exception): self.method = method def __repr__(self): - return 'face.NoSuchMethodError(%s, %s)' % ( - self.group, - self.method,) + return 'face.NoSuchMethodError(%s, %s)' % (self.group, self.method,) class Abortion( - collections.namedtuple('Abortion', ( - 'kind', - 'initial_metadata', - 'terminal_metadata', - 'code', - 'details',))): + collections.namedtuple('Abortion', + ('kind', 'initial_metadata', 'terminal_metadata', + 'code', 'details',))): """A value describing RPC abortion. Attributes: diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 68e40da4cf..a9f20e6d2a 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -180,6 +180,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/server.c', 'src/core/lib/surface/validate_metadata.c', 'src/core/lib/surface/version.c', + 'src/core/lib/transport/bdp_estimator.c', 'src/core/lib/transport/byte_stream.c', 'src/core/lib/transport/connectivity_state.c', 'src/core/lib/transport/error_utils.c', @@ -230,6 +231,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/credentials/plugin/plugin_credentials.c', 'src/core/lib/security/credentials/ssl/ssl_credentials.c', 'src/core/lib/security/transport/client_auth_filter.c', + 'src/core/lib/security/transport/lb_targets_info.c', 'src/core/lib/security/transport/secure_endpoint.c', 'src/core/lib/security/transport/security_connector.c', 'src/core/lib/security/transport/security_handshaker.c', @@ -270,6 +272,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/client/insecure/channel_create.c', 'src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c', 'src/core/ext/lb_policy/grpclb/grpclb.c', + 'src/core/ext/lb_policy/grpclb/grpclb_channel_secure.c', 'src/core/ext/lb_policy/grpclb/load_balancer_api.c', 'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'third_party/nanopb/pb_common.c', diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py index a228ba4a48..edc6def838 100644 --- a/src/python/grpcio/support.py +++ b/src/python/grpcio/support.py @@ -53,7 +53,9 @@ Could not find <Python.h>. This could mean the following: (check your environment variables or try re-installing?) """ -C_CHECKS = {C_PYTHON_DEV: C_PYTHON_DEV_ERROR_MESSAGE,} +C_CHECKS = { + C_PYTHON_DEV: C_PYTHON_DEV_ERROR_MESSAGE, +} def _compile(compiler, source_string): diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 4c3991dcc4..072c3263c6 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -40,14 +40,15 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) import health_commands import grpc_version -PACKAGE_DIRECTORIES = {'': '.',} +PACKAGE_DIRECTORIES = { + '': '.', +} SETUP_REQUIRES = ( 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) -INSTALL_REQUIRES = ( - 'protobuf>=3.0.0', - 'grpcio>={version}'.format(version=grpc_version.VERSION),) +INSTALL_REQUIRES = ('protobuf>=3.0.0', + 'grpcio>={version}'.format(version=grpc_version.VERSION),) COMMAND_CLASS = { # Run preprocess from the repository *before* doing any packaging! diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 2926923029..19aafe443a 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -40,14 +40,15 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) import reflection_commands import grpc_version -PACKAGE_DIRECTORIES = {'': '.',} +PACKAGE_DIRECTORIES = { + '': '.', +} SETUP_REQUIRES = ( 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) -INSTALL_REQUIRES = ( - 'protobuf>=3.0.0', - 'grpcio>={version}'.format(version=grpc_version.VERSION),) +INSTALL_REQUIRES = ('protobuf>=3.0.0', + 'grpcio>={version}'.format(version=grpc_version.VERSION),) COMMAND_CLASS = { # Run preprocess from the repository *before* doing any packaging! diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index af0ffe3475..40f9b83454 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Provides distutils command classes for the gRPC Python setup process.""" -import distutils +from distutils import errors as _errors import glob import os import os.path @@ -186,7 +186,7 @@ class RunInterop(test.test): def finalize_options(self): if self.client and self.server: - raise DistutilsOptionError( + raise _errors.DistutilsOptionError( 'you may only specify one of client or server') def run(self): diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index f0407d1a55..b0c73fc575 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -47,18 +47,16 @@ import grpc_version LICENSE = '3-clause BSD' -PACKAGE_DIRECTORIES = {'': '.',} +PACKAGE_DIRECTORIES = { + '': '.', +} INSTALL_REQUIRES = ( - 'coverage>=4.0', - 'enum34>=1.0.4', - 'futures>=2.2.0', + 'coverage>=4.0', 'enum34>=1.0.4', 'futures>=2.2.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', - 'protobuf>=3.0.0', - 'six>=1.10',) + 'oauth2client>=1.4.7', 'protobuf>=3.0.0', 'six>=1.10',) COMMAND_CLASS = { # Run `preprocess` *before* doing any packaging! @@ -75,7 +73,9 @@ PACKAGE_DATA = { 'credentials/server1.key', 'credentials/server1.pem', ], - 'tests.protoc_plugin.protos.invocation_testing': ['same.proto',], + 'tests.protoc_plugin.protos.invocation_testing': [ + 'same.proto', + ], 'tests.protoc_plugin.protos.invocation_testing.split_messages': [ 'messages.proto', ], diff --git a/src/python/grpcio_tests/tests/_result.py b/src/python/grpcio_tests/tests/_result.py index 794b7540f1..22c667b460 100644 --- a/src/python/grpcio_tests/tests/_result.py +++ b/src/python/grpcio_tests/tests/_result.py @@ -453,7 +453,9 @@ def jenkins_junit_xml(result): }) for case in result.cases.values(): if case.kind is CaseResult.Kind.SUCCESS: - ElementTree.SubElement(suite, 'testcase', {'name': case.name,}) + ElementTree.SubElement(suite, 'testcase', { + 'name': case.name, + }) elif case.kind in (CaseResult.Kind.ERROR, CaseResult.Kind.FAILURE): case_xml = ElementTree.SubElement(suite, 'testcase', { 'name': case.name, diff --git a/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py b/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py index b28406ed3f..5fe929b99e 100644 --- a/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py +++ b/src/python/grpcio_tests/tests/interop/_secure_intraop_test.py @@ -55,9 +55,9 @@ class SecureIntraopTest(_intraop_test_case.IntraopTestCase, unittest.TestCase): self.stub = test_pb2.TestServiceStub( grpc.secure_channel('localhost:{}'.format(port), grpc.ssl_channel_credentials( - resources.test_root_certificates()), (( - 'grpc.ssl_target_name_override', - _SERVER_HOST_OVERRIDE,),))) + resources.test_root_certificates()), ( + ('grpc.ssl_target_name_override', + _SERVER_HOST_OVERRIDE,),))) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index f177896e8e..833818e662 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -116,9 +116,8 @@ def _stub(args): channel_credentials = grpc.composite_channel_credentials( channel_credentials, call_credentials) - channel = grpc.secure_channel(target, channel_credentials, (( - 'grpc.ssl_target_name_override', - args.server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, ( + ('grpc.ssl_target_name_override', args.server_host_override,),)) else: channel = grpc.insecure_channel(target) if args.test_case == "unimplemented_service": diff --git a/src/python/grpcio_tests/tests/interop/methods.py b/src/python/grpcio_tests/tests/interop/methods.py index 1f9b356eb2..662ea9ce57 100644 --- a/src/python/grpcio_tests/tests/interop/methods.py +++ b/src/python/grpcio_tests/tests/interop/methods.py @@ -165,11 +165,7 @@ def _large_unary(stub): def _client_streaming(stub): - payload_body_sizes = ( - 27182, - 8, - 1828, - 45904,) + payload_body_sizes = (27182, 8, 1828, 45904,) payloads = (messages_pb2.Payload(body=b'\x00' * size) for size in payload_body_sizes) requests = (messages_pb2.StreamingInputCallRequest(payload=payload) @@ -181,19 +177,14 @@ def _client_streaming(stub): def _server_streaming(stub): - sizes = ( - 31415, - 9, - 2653, - 58979,) + sizes = (31415, 9, 2653, 58979,) request = messages_pb2.StreamingOutputCallRequest( response_type=messages_pb2.COMPRESSABLE, - response_parameters=( - messages_pb2.ResponseParameters(size=sizes[0]), - messages_pb2.ResponseParameters(size=sizes[1]), - messages_pb2.ResponseParameters(size=sizes[2]), - messages_pb2.ResponseParameters(size=sizes[3]),)) + response_parameters=(messages_pb2.ResponseParameters(size=sizes[0]), + messages_pb2.ResponseParameters(size=sizes[1]), + messages_pb2.ResponseParameters(size=sizes[2]), + messages_pb2.ResponseParameters(size=sizes[3]),)) response_iterator = stub.StreamingOutputCall(request) for index, response in enumerate(response_iterator): _validate_payload_type_and_length(response, messages_pb2.COMPRESSABLE, @@ -240,16 +231,8 @@ class _Pipe(object): def _ping_pong(stub): - request_response_sizes = ( - 31415, - 9, - 2653, - 58979,) - request_payload_sizes = ( - 27182, - 8, - 1828, - 45904,) + request_response_sizes = (31415, 9, 2653, 58979,) + request_payload_sizes = (27182, 8, 1828, 45904,) with _Pipe() as pipe: response_iterator = stub.FullDuplexCall(pipe) @@ -277,16 +260,8 @@ def _cancel_after_begin(stub): def _cancel_after_first_response(stub): - request_response_sizes = ( - 31415, - 9, - 2653, - 58979,) - request_payload_sizes = ( - 27182, - 8, - 1828, - 45904,) + request_response_sizes = (31415, 9, 2653, 58979,) + request_payload_sizes = (27182, 8, 1828, 45904,) with _Pipe() as pipe: response_iterator = stub.FullDuplexCall(pipe) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index ae5da2c3db..a6e3ca0b23 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -134,10 +134,8 @@ class _ServicerMethods(object): class _Service( - collections.namedtuple('_Service', ( - 'servicer_methods', - 'server', - 'stub',))): + collections.namedtuple('_Service', ('servicer_methods', 'server', + 'stub',))): """A live and running service. Attributes: diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index bcc01f3978..db938e6545 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -48,6 +48,8 @@ from grpc_tools import protoc from tests.unit.framework.common import test_constants _MESSAGES_IMPORT = b'import "messages.proto";' +_SPLIT_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing.split;' +_COMMON_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing;' @contextlib.contextmanager @@ -163,7 +165,10 @@ class SameSeparateTest(unittest.TestCase, SeparateTestMixin): os.makedirs(self.grpc_python_out_directory) same_proto_file = os.path.join(self.proto_directory, 'same_separate.proto') - open(same_proto_file, 'wb').write(same_proto_contents) + open(same_proto_file, 'wb').write( + same_proto_contents.replace( + _COMMON_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.same_separate;')) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -199,7 +204,11 @@ class SameCommonTest(unittest.TestCase, CommonTestMixin): os.makedirs(self.python_out_directory) same_proto_file = os.path.join(self.proto_directory, 'same_common.proto') - open(same_proto_file, 'wb').write(same_proto_contents) + open(same_proto_file, 'wb').write( + same_proto_contents.replace( + _COMMON_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.same_common;')) + protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -240,8 +249,14 @@ class SplitCommonTest(unittest.TestCase, CommonTestMixin): 'split_common_messages.proto') open(services_proto_file, 'wb').write( services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_common_messages.proto";')) - open(messages_proto_file, 'wb').write(messages_proto_contents) + _MESSAGES_IMPORT, b'import "split_common_messages.proto";') + .replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_common;')) + open(messages_proto_file, 'wb').write( + messages_proto_contents.replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_common;')) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), @@ -285,8 +300,16 @@ class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): 'split_separate_messages.proto') open(services_proto_file, 'wb').write( services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_separate_messages.proto";')) - open(messages_proto_file, 'wb').write(messages_proto_contents) + _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') + .replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_separate;' + )) + open(messages_proto_file, 'wb').write( + messages_proto_contents.replace( + _SPLIT_NAMESPACE, + b'package grpc_protoc_plugin.invocation_testing.split_separate;' + )) protoc_result = protoc.main([ '', '--proto_path={}'.format(self.proto_directory), diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 76e89ca039..d06ff064e2 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -69,11 +69,10 @@ class ReflectionServicerTest(unittest.TestCase): self._stub = reflection_pb2.ServerReflectionStub(channel) def testFileByName(self): - requests = ( - reflection_pb2.ServerReflectionRequest( - file_by_filename=_EMPTY_PROTO_FILE_NAME), - reflection_pb2.ServerReflectionRequest( - file_by_filename='i-donut-exist'),) + requests = (reflection_pb2.ServerReflectionRequest( + file_by_filename=_EMPTY_PROTO_FILE_NAME), + reflection_pb2.ServerReflectionRequest( + file_by_filename='i-donut-exist'),) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -90,12 +89,10 @@ class ReflectionServicerTest(unittest.TestCase): self.assertSequenceEqual(expected_responses, responses) def testFileBySymbol(self): - requests = ( - reflection_pb2.ServerReflectionRequest( - file_containing_symbol=_EMPTY_PROTO_SYMBOL_NAME), - reflection_pb2.ServerReflectionRequest( - file_containing_symbol='i.donut.exist.co.uk.org.net.me.name.foo' - ),) + requests = (reflection_pb2.ServerReflectionRequest( + file_containing_symbol=_EMPTY_PROTO_SYMBOL_NAME + ), reflection_pb2.ServerReflectionRequest( + file_containing_symbol='i.donut.exist.co.uk.org.net.me.name.foo'),) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -115,15 +112,14 @@ class ReflectionServicerTest(unittest.TestCase): 'TODO(atash): implement file-containing-extension reflection ' '(see https://github.com/google/protobuf/issues/2248)') def testFileContainingExtension(self): - requests = ( - reflection_pb2.ServerReflectionRequest( - file_containing_extension=reflection_pb2.ExtensionRequest( - containing_type='grpc.testing.proto2.Empty', - extension_number=125,),), - reflection_pb2.ServerReflectionRequest( - file_containing_extension=reflection_pb2.ExtensionRequest( - containing_type='i.donut.exist.co.uk.org.net.me.name.foo', - extension_number=55,),),) + requests = (reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type='grpc.testing.proto2.Empty', + extension_number=125,), + ), reflection_pb2.ServerReflectionRequest( + file_containing_extension=reflection_pb2.ExtensionRequest( + containing_type='i.donut.exist.co.uk.org.net.me.name.foo', + extension_number=55,),),) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = ( reflection_pb2.ServerReflectionResponse( @@ -140,7 +136,8 @@ class ReflectionServicerTest(unittest.TestCase): self.assertSequenceEqual(expected_responses, responses) def testListServices(self): - requests = (reflection_pb2.ServerReflectionRequest(list_services='',),) + requests = (reflection_pb2.ServerReflectionRequest( + list_services='',),) responses = tuple(self._stub.ServerReflectionInfo(iter(requests))) expected_responses = (reflection_pb2.ServerReflectionResponse( valid_host='', diff --git a/src/python/grpcio_tests/tests/stress/client.py b/src/python/grpcio_tests/tests/stress/client.py index 61f9e1c6b1..b9dbe61d44 100644 --- a/src/python/grpcio_tests/tests/stress/client.py +++ b/src/python/grpcio_tests/tests/stress/client.py @@ -117,9 +117,8 @@ def _get_channel(target, args): root_certificates = None # will load default roots. channel_credentials = grpc.ssl_channel_credentials( root_certificates=root_certificates) - options = (( - 'grpc.ssl_target_name_override', - args.server_host_override,),) + options = (('grpc.ssl_target_name_override', + args.server_host_override,),) channel = grpc.secure_channel( target, channel_credentials, options=options) else: diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py index 5435c5500c..40c4f0a22a 100644 --- a/src/python/grpcio_tests/tests/unit/_api_test.py +++ b/src/python/grpcio_tests/tests/unit/_api_test.py @@ -41,46 +41,23 @@ class AllTest(unittest.TestCase): def testAll(self): expected_grpc_code_elements = ( - 'FutureTimeoutError', - 'FutureCancelledError', - 'Future', - 'ChannelConnectivity', - 'StatusCode', - 'RpcError', - 'RpcContext', - 'Call', - 'ChannelCredentials', - 'CallCredentials', - 'AuthMetadataContext', - 'AuthMetadataPluginCallback', - 'AuthMetadataPlugin', - 'ServerCredentials', - 'UnaryUnaryMultiCallable', - 'UnaryStreamMultiCallable', - 'StreamUnaryMultiCallable', - 'StreamStreamMultiCallable', - 'Channel', - 'ServicerContext', - 'RpcMethodHandler', - 'HandlerCallDetails', - 'GenericRpcHandler', - 'ServiceRpcHandler', - 'Server', - 'unary_unary_rpc_method_handler', - 'unary_stream_rpc_method_handler', + 'FutureTimeoutError', 'FutureCancelledError', 'Future', + 'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext', + 'Call', 'ChannelCredentials', 'CallCredentials', + 'AuthMetadataContext', 'AuthMetadataPluginCallback', + 'AuthMetadataPlugin', 'ServerCredentials', + 'UnaryUnaryMultiCallable', 'UnaryStreamMultiCallable', + 'StreamUnaryMultiCallable', 'StreamStreamMultiCallable', 'Channel', + 'ServicerContext', 'RpcMethodHandler', 'HandlerCallDetails', + 'GenericRpcHandler', 'ServiceRpcHandler', 'Server', + 'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler', 'stream_unary_rpc_method_handler', 'stream_stream_rpc_method_handler', - 'method_handlers_generic_handler', - 'ssl_channel_credentials', - 'metadata_call_credentials', - 'access_token_call_credentials', - 'composite_call_credentials', - 'composite_channel_credentials', - 'ssl_server_credentials', - 'channel_ready_future', - 'insecure_channel', - 'secure_channel', - 'server',) + 'method_handlers_generic_handler', 'ssl_channel_credentials', + 'metadata_call_credentials', 'access_token_call_credentials', + 'composite_call_credentials', 'composite_channel_credentials', + 'ssl_server_credentials', 'channel_ready_future', + 'insecure_channel', 'secure_channel', 'server',) six.assertCountEqual(self, expected_grpc_code_elements, _from_grpc_import_star.GRPC_ELEMENTS) @@ -89,13 +66,12 @@ class AllTest(unittest.TestCase): class ChannelConnectivityTest(unittest.TestCase): def testChannelConnectivity(self): - self.assertSequenceEqual(( - grpc.ChannelConnectivity.IDLE, - grpc.ChannelConnectivity.CONNECTING, - grpc.ChannelConnectivity.READY, - grpc.ChannelConnectivity.TRANSIENT_FAILURE, - grpc.ChannelConnectivity.SHUTDOWN,), - tuple(grpc.ChannelConnectivity)) + self.assertSequenceEqual( + (grpc.ChannelConnectivity.IDLE, grpc.ChannelConnectivity.CONNECTING, + grpc.ChannelConnectivity.READY, + grpc.ChannelConnectivity.TRANSIENT_FAILURE, + grpc.ChannelConnectivity.SHUTDOWN,), + tuple(grpc.ChannelConnectivity)) class ChannelTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_channel_args_test.py b/src/python/grpcio_tests/tests/unit/_channel_args_test.py index 845db777a4..4ff8f1a186 100644 --- a/src/python/grpcio_tests/tests/unit/_channel_args_test.py +++ b/src/python/grpcio_tests/tests/unit/_channel_args_test.py @@ -39,12 +39,8 @@ class TestPointerWrapper(object): return 123456 -TEST_CHANNEL_ARGS = ( - ('arg1', b'bytes_val'), - ('arg2', 'str_val'), - ('arg3', 1), - (b'arg4', 'str_val'), - ('arg6', TestPointerWrapper()),) +TEST_CHANNEL_ARGS = (('arg1', b'bytes_val'), ('arg2', 'str_val'), ('arg3', 1), + (b'arg4', 'str_val'), ('arg6', TestPointerWrapper()),) class ChannelArgsTest(unittest.TestCase): diff --git a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py index d77f5ecb27..ee7a16ad00 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py @@ -167,11 +167,8 @@ class CancelManyCallsTest(unittest.TestCase): state = _State() - server_thread_args = ( - state, - server, - server_completion_queue, - server_thread_pool,) + server_thread_args = (state, server, server_completion_queue, + server_thread_pool,) server_thread = threading.Thread(target=_serve, args=server_thread_args) server_thread.start() diff --git a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py index 5c7f903015..81307af770 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py @@ -71,9 +71,7 @@ class ChannelTest(unittest.TestCase): def test_single_channel_lonely_connectivity(self): channel, completion_queue = _channel_and_completion_queue() - _in_parallel(_connectivity_loop, ( - channel, - completion_queue,)) + _in_parallel(_connectivity_loop, (channel, completion_queue,)) completion_queue.shutdown() def test_multiple_channels_lonely_connectivity(self): diff --git a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py index 9fbfcbb9c0..2cc8aeef6d 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py @@ -151,11 +151,9 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): server_send_first_message_tag = 'server_send_first_message_tag' server_send_second_message_tag = 'server_send_second_message_tag' server_complete_rpc_tag = 'server_complete_rpc_tag' - server_call_due = set(( - server_send_initial_metadata_tag, - server_send_first_message_tag, - server_send_second_message_tag, - server_complete_rpc_tag,)) + server_call_due = set( + (server_send_initial_metadata_tag, server_send_first_message_tag, + server_send_second_message_tag, server_complete_rpc_tag,)) server_call_completion_queue = cygrpc.CompletionQueue() server_call_driver = _QueueDriver(server_call_condition, server_call_completion_queue, diff --git a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py index efeb237874..f2e3898ed6 100644 --- a/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py +++ b/src/python/grpcio_tests/tests/unit/_invocation_defects_test.py @@ -77,9 +77,7 @@ class _Handler(object): def handle_unary_unary(self, request, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) return request def handle_unary_stream(self, request, servicer_context): @@ -88,9 +86,7 @@ class _Handler(object): yield request self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) def handle_stream_unary(self, request_iterator, servicer_context): if servicer_context is not None: @@ -102,17 +98,13 @@ class _Handler(object): response_elements.append(request) self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) return b''.join(response_elements) def handle_stream_stream(self, request_iterator, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) for request in request_iterator: self._control.control() yield request diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py index af2ce64dce..a2cccfc701 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py @@ -164,7 +164,8 @@ class _Servicer(object): def _generic_handler(servicer): method_handlers = { - _UNARY_UNARY: grpc.unary_unary_rpc_method_handler( + _UNARY_UNARY: + grpc.unary_unary_rpc_method_handler( servicer.unary_unary, request_deserializer=_REQUEST_DESERIALIZER, response_serializer=_RESPONSE_SERIALIZER), @@ -172,7 +173,8 @@ def _generic_handler(servicer): grpc.unary_stream_rpc_method_handler(servicer.unary_stream), _STREAM_UNARY: grpc.stream_unary_rpc_method_handler(servicer.stream_unary), - _STREAM_STREAM: grpc.stream_stream_rpc_method_handler( + _STREAM_STREAM: + grpc.stream_stream_rpc_method_handler( servicer.stream_stream, request_deserializer=_REQUEST_DESERIALIZER, response_serializer=_RESPONSE_SERIALIZER), @@ -192,25 +194,15 @@ class MetadataCodeDetailsTest(unittest.TestCase): channel = grpc.insecure_channel('localhost:{}'.format(port)) self._unary_unary = channel.unary_unary( - '/'.join(( - '', - _SERVICE, - _UNARY_UNARY,)), + '/'.join(('', _SERVICE, _UNARY_UNARY,)), request_serializer=_REQUEST_SERIALIZER, response_deserializer=_RESPONSE_DESERIALIZER,) - self._unary_stream = channel.unary_stream('/'.join(( - '', - _SERVICE, - _UNARY_STREAM,)),) - self._stream_unary = channel.stream_unary('/'.join(( - '', - _SERVICE, - _STREAM_UNARY,)),) + self._unary_stream = channel.unary_stream( + '/'.join(('', _SERVICE, _UNARY_STREAM,)),) + self._stream_unary = channel.stream_unary( + '/'.join(('', _SERVICE, _STREAM_UNARY,)),) self._stream_stream = channel.stream_stream( - '/'.join(( - '', - _SERVICE, - _STREAM_STREAM,)), + '/'.join(('', _SERVICE, _STREAM_STREAM,)), request_serializer=_REQUEST_SERIALIZER, response_deserializer=_RESPONSE_DESERIALIZER,) diff --git a/src/python/grpcio_tests/tests/unit/_rpc_test.py b/src/python/grpcio_tests/tests/unit/_rpc_test.py index 2b1c85a82d..0bfcfc58a1 100644 --- a/src/python/grpcio_tests/tests/unit/_rpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_rpc_test.py @@ -78,9 +78,7 @@ class _Handler(object): def handle_unary_unary(self, request, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) # TODO(https://github.com/grpc/grpc/issues/8483): test the values # returned by these methods rather than only "smoke" testing that # the return after having been called. @@ -94,9 +92,7 @@ class _Handler(object): yield request self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) def handle_stream_unary(self, request_iterator, servicer_context): if servicer_context is not None: @@ -108,17 +104,13 @@ class _Handler(object): response_elements.append(request) self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) return b''.join(response_elements) def handle_stream_stream(self, request_iterator, servicer_context): self._control.control() if servicer_context is not None: - servicer_context.set_trailing_metadata((( - 'testkey', - 'testvalue',),)) + servicer_context.set_trailing_metadata((('testkey', 'testvalue',),)) for request in request_iterator: self._control.control() yield request diff --git a/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py b/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py index b5fdac26c1..7d6df5bb5b 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_beta_features_test.py @@ -177,9 +177,9 @@ class BetaFeaturesTest(unittest.TestCase): thread_pool_size=test_constants.POOL_SIZE) self._server = implementations.server( method_implementations, options=server_options) - server_credentials = implementations.ssl_server_credentials([( - resources.private_key(), - resources.certificate_chain(),),]) + server_credentials = implementations.ssl_server_credentials([ + (resources.private_key(), resources.certificate_chain(),), + ]) port = self._server.add_secure_port('[::]:0', server_credentials) self._server.start() self._channel_credentials = implementations.ssl_channel_credentials( @@ -303,9 +303,9 @@ class ContextManagementAndLifecycleTest(unittest.TestCase): self._server_options = implementations.server_options( thread_pool_size=test_constants.POOL_SIZE) - self._server_credentials = implementations.ssl_server_credentials([( - resources.private_key(), - resources.certificate_chain(),),]) + self._server_credentials = implementations.ssl_server_credentials([ + (resources.private_key(), resources.certificate_chain(),), + ]) self._channel_credentials = implementations.ssl_channel_credentials( resources.test_root_certificates()) self._stub_options = implementations.stub_options( diff --git a/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py b/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py index f421442624..e35a12cf62 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_face_interface_test.py @@ -47,10 +47,8 @@ _SERVER_HOST_OVERRIDE = 'foo.test.google.fr' class _SerializationBehaviors( collections.namedtuple('_SerializationBehaviors', ( - 'request_serializers', - 'request_deserializers', - 'response_serializers', - 'response_deserializers',))): + 'request_serializers', 'request_deserializers', + 'response_serializers', 'response_deserializers',))): pass @@ -89,9 +87,9 @@ class _Implementation(test_interfaces.Implementation): thread_pool_size=test_constants.POOL_SIZE) server = implementations.server( method_implementations, options=server_options) - server_credentials = implementations.ssl_server_credentials([( - resources.private_key(), - resources.certificate_chain(),),]) + server_credentials = implementations.ssl_server_credentials([ + (resources.private_key(), resources.certificate_chain(),), + ]) port = server.add_secure_port('[::]:0', server_credentials) server.start() channel_credentials = implementations.ssl_channel_credentials( diff --git a/src/python/grpcio_tests/tests/unit/beta/test_utilities.py b/src/python/grpcio_tests/tests/unit/beta/test_utilities.py index f542420683..78b4622a87 100644 --- a/src/python/grpcio_tests/tests/unit/beta/test_utilities.py +++ b/src/python/grpcio_tests/tests/unit/beta/test_utilities.py @@ -48,7 +48,6 @@ def not_really_secure_channel(host, port, channel_credentials, conducted. """ target = '%s:%d' % (host, port) - channel = grpc.secure_channel(target, channel_credentials, (( - 'grpc.ssl_target_name_override', - server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, ( + ('grpc.ssl_target_name_override', server_host_override,),)) return implementations.Channel(channel) diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py index 0411da0a66..cc4855cc33 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_digest.py @@ -49,12 +49,9 @@ _IDENTITY = lambda x: x class TestServiceDigest( collections.namedtuple('TestServiceDigest', ( - 'methods', - 'inline_method_implementations', - 'event_method_implementations', - 'multi_method_implementation', - 'unary_unary_messages_sequences', - 'unary_stream_messages_sequences', + 'methods', 'inline_method_implementations', + 'event_method_implementations', 'multi_method_implementation', + 'unary_unary_messages_sequences', 'unary_stream_messages_sequences', 'stream_unary_messages_sequences', 'stream_stream_messages_sequences',))): """A transformation of a service.TestService. diff --git a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py index 4e144a3635..6afbd2b5be 100644 --- a/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py +++ b/src/python/grpcio_tests/tests/unit/framework/interfaces/face/_invocation.py @@ -206,7 +206,5 @@ def invoker_constructors(): Returns: A sequence of InvokerConstructors. """ - return ( - _GenericInvokerConstructor(), - _MultiCallableInvokerConstructor(), - _DynamicInvokerConstructor(),) + return (_GenericInvokerConstructor(), _MultiCallableInvokerConstructor(), + _DynamicInvokerConstructor(),) diff --git a/src/python/grpcio_tests/tests/unit/test_common.py b/src/python/grpcio_tests/tests/unit/test_common.py index 00fbe0567a..4b3ba911b9 100644 --- a/src/python/grpcio_tests/tests/unit/test_common.py +++ b/src/python/grpcio_tests/tests/unit/test_common.py @@ -33,18 +33,9 @@ import collections import grpc import six -INVOCATION_INITIAL_METADATA = ( - ('0', 'abc'), - ('1', 'def'), - ('2', 'ghi'),) -SERVICE_INITIAL_METADATA = ( - ('3', 'jkl'), - ('4', 'mno'), - ('5', 'pqr'),) -SERVICE_TERMINAL_METADATA = ( - ('6', 'stu'), - ('7', 'vwx'), - ('8', 'yza'),) +INVOCATION_INITIAL_METADATA = (('0', 'abc'), ('1', 'def'), ('2', 'ghi'),) +SERVICE_INITIAL_METADATA = (('3', 'jkl'), ('4', 'mno'), ('5', 'pqr'),) +SERVICE_TERMINAL_METADATA = (('6', 'stu'), ('7', 'vwx'), ('8', 'yza'),) DETAILS = 'test details' @@ -103,7 +94,6 @@ def test_secure_channel(target, channel_credentials, server_host_override): An implementations.Channel to the remote host through which RPCs may be conducted. """ - channel = grpc.secure_channel(target, channel_credentials, (( - 'grpc.ssl_target_name_override', - server_host_override,),)) + channel = grpc.secure_channel(target, channel_credentials, ( + ('grpc.ssl_target_name_override', server_host_override,),)) return channel |