diff options
51 files changed, 584 insertions, 582 deletions
diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 16f31f9e94..efbf5314af 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -8,3 +8,4 @@ wheel>=0.29 futures>=2.2.0 google-auth>=1.0.0 oauth2client==4.1.0 +requests>=2.14.2 diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 35c8bb27ee..bb3ea400d1 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -933,11 +933,6 @@ typedef struct client_channel_call_data { grpc_closure pick_closure; grpc_closure pick_cancel_closure; - // state needed to support channelz interception of recv trailing metadata. - grpc_closure recv_trailing_metadata_ready_channelz; - grpc_closure* original_recv_trailing_metadata; - grpc_metadata_batch* recv_trailing_metadata; - grpc_polling_entity* pollent; bool pollent_added_to_interested_parties; @@ -999,8 +994,6 @@ static void start_internal_recv_trailing_metadata(grpc_call_element* elem); static void on_complete(void* arg, grpc_error* error); static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored); static void start_pick_locked(void* arg, grpc_error* ignored); -static void maybe_intercept_recv_trailing_metadata_for_channelz( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch); // // send op data caching @@ -1299,7 +1292,6 @@ static void pending_batches_resume(grpc_call_element* elem) { pending_batch* pending = &calld->pending_batches[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr) { - maybe_intercept_recv_trailing_metadata_for_channelz(elem, batch); batch->handler_private.extra_arg = calld->subchannel_call; GRPC_CLOSURE_INIT(&batch->handler_private.closure, resume_pending_batch_in_call_combiner, batch, @@ -1977,15 +1969,6 @@ static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { batch_data->batch.payload->recv_trailing_metadata.recv_trailing_metadata; get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, &server_pushback_md); - grpc_core::channelz::SubchannelNode* channelz_subchannel = - calld->pick.connected_subchannel->channelz_subchannel(); - if (channelz_subchannel != nullptr) { - if (status == GRPC_STATUS_OK) { - channelz_subchannel->RecordCallSucceeded(); - } else { - channelz_subchannel->RecordCallFailed(); - } - } if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call finished, status=%s", chand, calld, grpc_status_code_to_string(status)); @@ -2590,69 +2573,6 @@ static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored) { } // -// Channelz -// - -static void recv_trailing_metadata_ready_channelz(void* arg, - grpc_error* error) { - grpc_call_element* elem = static_cast<grpc_call_element*>(arg); - channel_data* chand = static_cast<channel_data*>(elem->channel_data); - call_data* calld = static_cast<call_data*>(elem->call_data); - if (grpc_client_channel_trace.enabled()) { - gpr_log(GPR_INFO, - "chand=%p calld=%p: got recv_trailing_metadata_ready_channelz, " - "error=%s", - chand, calld, grpc_error_string(error)); - } - GPR_ASSERT(calld->recv_trailing_metadata != nullptr); - grpc_status_code status = GRPC_STATUS_OK; - grpc_metadata_batch* md_batch = calld->recv_trailing_metadata; - get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, nullptr); - grpc_core::channelz::SubchannelNode* channelz_subchannel = - calld->pick.connected_subchannel->channelz_subchannel(); - GPR_ASSERT(channelz_subchannel != nullptr); - if (status == GRPC_STATUS_OK) { - channelz_subchannel->RecordCallSucceeded(); - } else { - channelz_subchannel->RecordCallFailed(); - } - calld->recv_trailing_metadata = nullptr; - GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata, error); -} - -// If channelz is enabled, intercept recv_trailing so that we may check the -// status and associate it to a subchannel. -// Returns true if callback was intercepted, false otherwise. -static void maybe_intercept_recv_trailing_metadata_for_channelz( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { - call_data* calld = static_cast<call_data*>(elem->call_data); - // only intercept payloads with recv trailing. - if (!batch->recv_trailing_metadata) { - return; - } - // only add interceptor is channelz is enabled. - if (calld->pick.connected_subchannel->channelz_subchannel() == nullptr) { - return; - } - if (grpc_client_channel_trace.enabled()) { - gpr_log(GPR_INFO, - "calld=%p batch=%p: intercepting recv trailing for channelz", calld, - batch); - } - GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready_channelz, - recv_trailing_metadata_ready_channelz, elem, - grpc_schedule_on_exec_ctx); - // save some state needed for the interception callback. - GPR_ASSERT(calld->recv_trailing_metadata == nullptr); - calld->recv_trailing_metadata = - batch->payload->recv_trailing_metadata.recv_trailing_metadata; - calld->original_recv_trailing_metadata = - batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; - batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = - &calld->recv_trailing_metadata_ready_channelz; -} - -// // LB pick // diff --git a/src/core/ext/filters/client_channel/http_connect_handshaker.cc b/src/core/ext/filters/client_channel/http_connect_handshaker.cc index 7ce8da8c00..bfabc68c66 100644 --- a/src/core/ext/filters/client_channel/http_connect_handshaker.cc +++ b/src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -351,6 +351,7 @@ static grpc_handshaker* grpc_http_connect_handshaker_create() { static void handshaker_factory_add_handshakers( grpc_handshaker_factory* factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add(handshake_mgr, grpc_http_connect_handshaker_create()); diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 384e7931b5..5511df7a27 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1491,7 +1491,7 @@ void GrpcLb::OnBalancerChannelConnectivityChangedLocked(void* arg, grpclb_policy->lb_call_backoff_.Reset(); grpclb_policy->StartBalancerCallLocked(); } - // Fall through. + // fallthrough case GRPC_CHANNEL_SHUTDOWN: done: grpclb_policy->watching_lb_channel_ = false; diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 4a668b0fa7..3a1c14c6f1 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -49,6 +49,8 @@ #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/connectivity_state.h" +#include "src/core/lib/transport/error_utils.h" +#include "src/core/lib/transport/status_metadata.h" #define INTERNAL_REF_BITS 16 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1)) @@ -144,6 +146,11 @@ struct grpc_subchannel { struct grpc_subchannel_call { grpc_core::ConnectedSubchannel* connection; grpc_closure* schedule_closure_after_destroy; + // state needed to support channelz interception of recv trailing metadata. + grpc_closure recv_trailing_metadata_ready; + grpc_closure* original_recv_trailing_metadata; + grpc_metadata_batch* recv_trailing_metadata; + grpc_millis deadline; }; #define SUBCHANNEL_CALL_TO_CALL_STACK(call) \ @@ -652,7 +659,7 @@ static bool publish_transport_locked(grpc_subchannel* c) { /* publish */ c->connected_subchannel.reset(grpc_core::New<grpc_core::ConnectedSubchannel>( - stk, c->channelz_subchannel.get(), socket_uuid)); + stk, c->channelz_subchannel, socket_uuid)); gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p", c->connected_subchannel.get(), c); @@ -745,9 +752,68 @@ void grpc_subchannel_call_unref( GRPC_CALL_STACK_UNREF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } +// Sets *status based on md_batch and error. +static void get_call_status(grpc_subchannel_call* call, + grpc_metadata_batch* md_batch, grpc_error* error, + grpc_status_code* status) { + if (error != GRPC_ERROR_NONE) { + grpc_error_get_status(error, call->deadline, status, nullptr, nullptr, + nullptr); + } else { + GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr); + *status = + grpc_get_status_code_from_metadata(md_batch->idx.named.grpc_status->md); + } + GRPC_ERROR_UNREF(error); +} + +static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { + grpc_subchannel_call* call = static_cast<grpc_subchannel_call*>(arg); + GPR_ASSERT(call->recv_trailing_metadata != nullptr); + grpc_status_code status = GRPC_STATUS_OK; + grpc_metadata_batch* md_batch = call->recv_trailing_metadata; + get_call_status(call, md_batch, GRPC_ERROR_REF(error), &status); + grpc_core::channelz::SubchannelNode* channelz_subchannel = + call->connection->channelz_subchannel(); + GPR_ASSERT(channelz_subchannel != nullptr); + if (status == GRPC_STATUS_OK) { + channelz_subchannel->RecordCallSucceeded(); + } else { + channelz_subchannel->RecordCallFailed(); + } + GRPC_CLOSURE_RUN(call->original_recv_trailing_metadata, + GRPC_ERROR_REF(error)); +} + +// If channelz is enabled, intercept recv_trailing so that we may check the +// status and associate it to a subchannel. +static void maybe_intercept_recv_trailing_metadata( + grpc_subchannel_call* call, grpc_transport_stream_op_batch* batch) { + // only intercept payloads with recv trailing. + if (!batch->recv_trailing_metadata) { + return; + } + // only add interceptor is channelz is enabled. + if (call->connection->channelz_subchannel() == nullptr) { + return; + } + GRPC_CLOSURE_INIT(&call->recv_trailing_metadata_ready, + recv_trailing_metadata_ready, call, + grpc_schedule_on_exec_ctx); + // save some state needed for the interception callback. + GPR_ASSERT(call->recv_trailing_metadata == nullptr); + call->recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata; + call->original_recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &call->recv_trailing_metadata_ready; +} + void grpc_subchannel_call_process_op(grpc_subchannel_call* call, grpc_transport_stream_op_batch* batch) { GPR_TIMER_SCOPE("grpc_subchannel_call_process_op", 0); + maybe_intercept_recv_trailing_metadata(call, batch); grpc_call_stack* call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call); grpc_call_element* top_elem = grpc_call_stack_element(call_stack, 0); GRPC_CALL_LOG_OP(GPR_INFO, top_elem, batch); @@ -822,10 +888,12 @@ namespace grpc_core { ConnectedSubchannel::ConnectedSubchannel( grpc_channel_stack* channel_stack, - channelz::SubchannelNode* channelz_subchannel, intptr_t socket_uuid) + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel, + intptr_t socket_uuid) : RefCountedWithTracing<ConnectedSubchannel>(&grpc_trace_stream_refcount), channel_stack_(channel_stack), - channelz_subchannel_(channelz_subchannel), + channelz_subchannel_(std::move(channelz_subchannel)), socket_uuid_(socket_uuid) {} ConnectedSubchannel::~ConnectedSubchannel() { @@ -872,6 +940,7 @@ grpc_error* ConnectedSubchannel::CreateCall(const CallArgs& args, Ref(DEBUG_LOCATION, "subchannel_call"); connection.release(); // Ref is passed to the grpc_subchannel_call object. (*call)->connection = this; + (*call)->deadline = args.deadline; const grpc_call_element_args call_args = { callstk, /* call_stack */ nullptr, /* server_transport_data */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 699f93a8e7..c53b13e37e 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -85,9 +85,11 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { size_t parent_data_size; }; - explicit ConnectedSubchannel(grpc_channel_stack* channel_stack, - channelz::SubchannelNode* channelz_subchannel, - intptr_t socket_uuid); + explicit ConnectedSubchannel( + grpc_channel_stack* channel_stack, + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel, + intptr_t socket_uuid); ~ConnectedSubchannel(); grpc_channel_stack* channel_stack() { return channel_stack_; } @@ -97,15 +99,16 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { void Ping(grpc_closure* on_initiate, grpc_closure* on_ack); grpc_error* CreateCall(const CallArgs& args, grpc_subchannel_call** call); channelz::SubchannelNode* channelz_subchannel() { - return channelz_subchannel_; + return channelz_subchannel_.get(); } intptr_t socket_uuid() { return socket_uuid_; } private: grpc_channel_stack* channel_stack_; - // backpointer to the channelz node in this connected subchannel's + // ref counted pointer to the channelz node in this connected subchannel's // owning subchannel. - channelz::SubchannelNode* channelz_subchannel_; + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel_; // uuid of this subchannel's socket. 0 if this subchannel is not connected. const intptr_t socket_uuid_; }; diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.cc b/src/core/ext/transport/chttp2/client/chttp2_connector.cc index 0ac84032fd..5229304fa4 100644 --- a/src/core/ext/transport/chttp2/client/chttp2_connector.cc +++ b/src/core/ext/transport/chttp2/client/chttp2_connector.cc @@ -160,7 +160,7 @@ static void on_handshake_done(void* arg, grpc_error* error) { static void start_handshake_locked(chttp2_connector* c) { c->handshake_mgr = grpc_handshake_manager_create(); grpc_handshakers_add(HANDSHAKER_CLIENT, c->args.channel_args, - c->handshake_mgr); + c->args.interested_parties, c->handshake_mgr); grpc_endpoint_add_to_pollset_set(c->endpoint, c->args.interested_parties); grpc_handshake_manager_do_handshake( c->handshake_mgr, c->args.interested_parties, c->endpoint, diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index a5fc6062aa..6ed88dfb5e 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -69,6 +69,7 @@ typedef struct { grpc_timer timer; grpc_closure on_timeout; grpc_closure on_receive_settings; + grpc_pollset_set* interested_parties; } server_connection_state; static void server_connection_state_unref( @@ -78,6 +79,9 @@ static void server_connection_state_unref( GRPC_CHTTP2_UNREF_TRANSPORT(connection_state->transport, "receive settings timeout"); } + grpc_pollset_set_del_pollset(connection_state->interested_parties, + connection_state->accepting_pollset); + grpc_pollset_set_destroy(connection_state->interested_parties); gpr_free(connection_state); } } @@ -192,7 +196,11 @@ static void on_accept(void* arg, grpc_endpoint* tcp, connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; connection_state->handshake_mgr = handshake_mgr; + connection_state->interested_parties = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(connection_state->interested_parties, + connection_state->accepting_pollset); grpc_handshakers_add(HANDSHAKER_SERVER, state->args, + connection_state->interested_parties, connection_state->handshake_mgr); const grpc_arg* timeout_arg = grpc_channel_args_find(state->args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS); diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index f3d0c03715..33577d890a 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -43,8 +43,10 @@ namespace grpc_core { namespace channelz { -BaseNode::BaseNode(EntityType type) - : type_(type), uuid_(ChannelzRegistry::Register(this)) {} +BaseNode::BaseNode(EntityType type) : type_(type), uuid_(-1) { + // The registry will set uuid_ under its lock. + ChannelzRegistry::Register(this); +} BaseNode::~BaseNode() { ChannelzRegistry::Unregister(uuid_); } diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index d8113585c2..fddef793fb 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -92,8 +92,10 @@ class BaseNode : public RefCounted<BaseNode> { intptr_t uuid() const { return uuid_; } private: + // to allow the ChannelzRegistry to set uuid_ under its lock. + friend class ChannelzRegistry; const EntityType type_; - const intptr_t uuid_; + intptr_t uuid_; }; // This class is a helper class for channelz entities that deal with Channels, diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index 1b54b19be3..67e56ed791 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -53,10 +53,10 @@ ChannelzRegistry::ChannelzRegistry() { gpr_mu_init(&mu_); } ChannelzRegistry::~ChannelzRegistry() { gpr_mu_destroy(&mu_); } -intptr_t ChannelzRegistry::InternalRegister(BaseNode* node) { +void ChannelzRegistry::InternalRegister(BaseNode* node) { MutexLock lock(&mu_); entities_.push_back(node); - return ++uuid_generator_; + node->uuid_ = ++uuid_generator_; } void ChannelzRegistry::MaybePerformCompactionLocked() { diff --git a/src/core/lib/channel/channelz_registry.h b/src/core/lib/channel/channelz_registry.h index 9c43d960d3..ea6ab6c8e5 100644 --- a/src/core/lib/channel/channelz_registry.h +++ b/src/core/lib/channel/channelz_registry.h @@ -44,7 +44,7 @@ class ChannelzRegistry { // To be called in grpc_shutdown(); static void Shutdown(); - static intptr_t Register(BaseNode* node) { + static void Register(BaseNode* node) { return Default()->InternalRegister(node); } static void Unregister(intptr_t uuid) { Default()->InternalUnregister(uuid); } @@ -74,7 +74,7 @@ class ChannelzRegistry { static ChannelzRegistry* Default(); // globally registers an Entry. Returns its unique uuid - intptr_t InternalRegister(BaseNode* node); + void InternalRegister(BaseNode* node); // globally unregisters the object that is associated to uuid. Also does // sanity check that an object doesn't try to unregister the wrong type. diff --git a/src/core/lib/channel/handshaker_factory.cc b/src/core/lib/channel/handshaker_factory.cc index 4fd43635b6..8ade8fe4e2 100644 --- a/src/core/lib/channel/handshaker_factory.cc +++ b/src/core/lib/channel/handshaker_factory.cc @@ -24,11 +24,12 @@ void grpc_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (handshaker_factory != nullptr) { GPR_ASSERT(handshaker_factory->vtable != nullptr); - handshaker_factory->vtable->add_handshakers(handshaker_factory, args, - handshake_mgr); + handshaker_factory->vtable->add_handshakers( + handshaker_factory, args, interested_parties, handshake_mgr); } } diff --git a/src/core/lib/channel/handshaker_factory.h b/src/core/lib/channel/handshaker_factory.h index 3e45fcf20e..e17a678179 100644 --- a/src/core/lib/channel/handshaker_factory.h +++ b/src/core/lib/channel/handshaker_factory.h @@ -32,6 +32,7 @@ typedef struct grpc_handshaker_factory grpc_handshaker_factory; typedef struct { void (*add_handshakers)(grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); void (*destroy)(grpc_handshaker_factory* handshaker_factory); } grpc_handshaker_factory_vtable; @@ -42,6 +43,7 @@ struct grpc_handshaker_factory { void grpc_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); void grpc_handshaker_factory_destroy( diff --git a/src/core/lib/channel/handshaker_registry.cc b/src/core/lib/channel/handshaker_registry.cc index eec3e1b352..fbafc43e79 100644 --- a/src/core/lib/channel/handshaker_registry.cc +++ b/src/core/lib/channel/handshaker_registry.cc @@ -51,9 +51,11 @@ static void grpc_handshaker_factory_list_register( static void grpc_handshaker_factory_list_add_handshakers( grpc_handshaker_factory_list* list, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { for (size_t i = 0; i < list->num_factories; ++i) { - grpc_handshaker_factory_add_handshakers(list->list[i], args, handshake_mgr); + grpc_handshaker_factory_add_handshakers(list->list[i], args, + interested_parties, handshake_mgr); } } @@ -91,7 +93,9 @@ void grpc_handshaker_factory_register(bool at_start, void grpc_handshakers_add(grpc_handshaker_type handshaker_type, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshaker_factory_list_add_handshakers( - &g_handshaker_factory_lists[handshaker_type], args, handshake_mgr); + &g_handshaker_factory_lists[handshaker_type], args, interested_parties, + handshake_mgr); } diff --git a/src/core/lib/channel/handshaker_registry.h b/src/core/lib/channel/handshaker_registry.h index 82ad9c5b9a..3dd4316de6 100644 --- a/src/core/lib/channel/handshaker_registry.h +++ b/src/core/lib/channel/handshaker_registry.h @@ -43,6 +43,7 @@ void grpc_handshaker_factory_register(bool at_start, void grpc_handshakers_add(grpc_handshaker_type handshaker_type, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_REGISTRY_H */ diff --git a/src/core/lib/http/httpcli_security_connector.cc b/src/core/lib/http/httpcli_security_connector.cc index 50078c37a1..5a5e0b72d5 100644 --- a/src/core/lib/http/httpcli_security_connector.cc +++ b/src/core/lib/http/httpcli_security_connector.cc @@ -29,6 +29,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker_registry.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/ssl_transport_security.h" @@ -51,6 +52,7 @@ static void httpcli_ssl_destroy(grpc_security_connector* sc) { } static void httpcli_ssl_add_handshakers(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_httpcli_ssl_channel_security_connector* c = reinterpret_cast<grpc_httpcli_ssl_channel_security_connector*>(sc); @@ -189,7 +191,8 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host, grpc_arg channel_arg = grpc_security_connector_to_arg(&sc->base); grpc_channel_args args = {1, &channel_arg}; c->handshake_mgr = grpc_handshake_manager_create(); - grpc_handshakers_add(HANDSHAKER_CLIENT, &args, c->handshake_mgr); + grpc_handshakers_add(HANDSHAKER_CLIENT, &args, + nullptr /* interested_parties */, c->handshake_mgr); grpc_handshake_manager_do_handshake( c->handshake_mgr, nullptr /* interested_parties */, tcp, nullptr /* channel_args */, deadline, nullptr /* acceptor */, diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index 54f19a9259..ceba79f678 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -245,7 +245,7 @@ static void timer_main_loop() { gpr_log(GPR_INFO, "timers not checked: expect another thread to"); } next = GRPC_MILLIS_INF_FUTURE; - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: if (!wait_until(next)) { return; diff --git a/src/core/lib/security/security_connector/alts_security_connector.cc b/src/core/lib/security/security_connector/alts_security_connector.cc index 35a787871a..d38c0ff044 100644 --- a/src/core/lib/security/security_connector/alts_security_connector.cc +++ b/src/core/lib/security/security_connector/alts_security_connector.cc @@ -64,29 +64,29 @@ static void alts_server_destroy(grpc_security_connector* sc) { } static void alts_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; auto c = reinterpret_cast<grpc_alts_channel_security_connector*>(sc); grpc_alts_credentials* creds = reinterpret_cast<grpc_alts_credentials*>(c->base.channel_creds); - GPR_ASSERT(alts_tsi_handshaker_create(creds->options, c->target_name, - creds->handshaker_service_url, true, - &handshaker) == TSI_OK); + GPR_ASSERT(alts_tsi_handshaker_create( + creds->options, c->target_name, creds->handshaker_service_url, + true, interested_parties, &handshaker) == TSI_OK); grpc_handshake_manager_add(handshake_manager, grpc_security_handshaker_create( handshaker, &sc->base)); } static void alts_server_add_handshakers( - grpc_server_security_connector* sc, + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; auto c = reinterpret_cast<grpc_alts_server_security_connector*>(sc); grpc_alts_server_credentials* creds = reinterpret_cast<grpc_alts_server_credentials*>(c->base.server_creds); - GPR_ASSERT(alts_tsi_handshaker_create(creds->options, nullptr, - creds->handshaker_service_url, false, - &handshaker) == TSI_OK); + GPR_ASSERT(alts_tsi_handshaker_create( + creds->options, nullptr, creds->handshaker_service_url, false, + interested_parties, &handshaker) == TSI_OK); grpc_handshake_manager_add(handshake_manager, grpc_security_handshaker_create( handshaker, &sc->base)); } diff --git a/src/core/lib/security/security_connector/local_security_connector.cc b/src/core/lib/security/security_connector/local_security_connector.cc index c436a7906b..911013ae58 100644 --- a/src/core/lib/security/security_connector/local_security_connector.cc +++ b/src/core/lib/security/security_connector/local_security_connector.cc @@ -30,6 +30,7 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/security/credentials/local/local_credentials.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/tsi/local_transport_security.h" @@ -68,7 +69,7 @@ static void local_server_destroy(grpc_security_connector* sc) { } static void local_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; GPR_ASSERT(local_tsi_handshaker_create(true /* is_client */, &handshaker) == @@ -78,7 +79,7 @@ static void local_channel_add_handshakers( } static void local_server_add_handshakers( - grpc_server_security_connector* sc, + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; GPR_ASSERT(local_tsi_handshaker_create(false /* is_client */, &handshaker) == diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 6246613e7b..7028ae8d16 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -120,17 +120,19 @@ const tsi_peer_property* tsi_peer_get_property_by_name(const tsi_peer* peer, void grpc_channel_security_connector_add_handshakers( grpc_channel_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (connector != nullptr) { - connector->add_handshakers(connector, handshake_mgr); + connector->add_handshakers(connector, interested_parties, handshake_mgr); } } void grpc_server_security_connector_add_handshakers( grpc_server_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (connector != nullptr) { - connector->add_handshakers(connector, handshake_mgr); + connector->add_handshakers(connector, interested_parties, handshake_mgr); } } @@ -519,7 +521,7 @@ static void fake_channel_cancel_check_call_host( } static void fake_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add( handshake_mgr, @@ -528,6 +530,7 @@ static void fake_channel_add_handshakers( } static void fake_server_add_handshakers(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add( handshake_mgr, @@ -669,6 +672,7 @@ static void ssl_server_destroy(grpc_security_connector* sc) { } static void ssl_channel_add_handshakers(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_ssl_channel_security_connector* c = reinterpret_cast<grpc_ssl_channel_security_connector*>(sc); @@ -779,6 +783,7 @@ static bool try_fetch_ssl_server_credentials( } static void ssl_server_add_handshakers(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_ssl_server_security_connector* c = reinterpret_cast<grpc_ssl_server_security_connector*>(sc); diff --git a/src/core/lib/security/security_connector/security_connector.h b/src/core/lib/security/security_connector/security_connector.h index 67a506b576..d8df3cd72f 100644 --- a/src/core/lib/security/security_connector/security_connector.h +++ b/src/core/lib/security/security_connector/security_connector.h @@ -27,6 +27,7 @@ #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/tcp_server.h" #include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security_interface.h" @@ -125,6 +126,7 @@ struct grpc_channel_security_connector { grpc_closure* on_call_host_checked, grpc_error* error); void (*add_handshakers)(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); }; @@ -151,6 +153,7 @@ void grpc_channel_security_connector_cancel_check_call_host( /* Registers handshakers with \a handshake_mgr. */ void grpc_channel_security_connector_add_handshakers( grpc_channel_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); /* --- server_security_connector object. --- @@ -164,6 +167,7 @@ struct grpc_server_security_connector { grpc_security_connector base; grpc_server_credentials* server_creds; void (*add_handshakers)(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); }; @@ -172,7 +176,8 @@ int grpc_server_security_connector_cmp(grpc_server_security_connector* sc1, grpc_server_security_connector* sc2); void grpc_server_security_connector_add_handshakers( - grpc_server_security_connector* sc, grpc_handshake_manager* handshake_mgr); + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, + grpc_handshake_manager* handshake_mgr); /* --- Creation security connectors. --- */ diff --git a/src/core/lib/security/transport/security_handshaker.cc b/src/core/lib/security/transport/security_handshaker.cc index d76d582638..4d6b133809 100644 --- a/src/core/lib/security/transport/security_handshaker.cc +++ b/src/core/lib/security/transport/security_handshaker.cc @@ -475,22 +475,24 @@ static grpc_handshaker* fail_handshaker_create() { static void client_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_channel_security_connector* security_connector = reinterpret_cast<grpc_channel_security_connector*>( grpc_security_connector_find_in_args(args)); - grpc_channel_security_connector_add_handshakers(security_connector, - handshake_mgr); + grpc_channel_security_connector_add_handshakers( + security_connector, interested_parties, handshake_mgr); } static void server_handshaker_factory_add_handshakers( grpc_handshaker_factory* hf, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_server_security_connector* security_connector = reinterpret_cast<grpc_server_security_connector*>( grpc_security_connector_find_in_args(args)); - grpc_server_security_connector_add_handshakers(security_connector, - handshake_mgr); + grpc_server_security_connector_add_handshakers( + security_connector, interested_parties, handshake_mgr); } static void handshaker_factory_destroy( diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 34608a3de1..dfdd659b87 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -347,7 +347,8 @@ static void init_shared_resources(const char* handshaker_service_url) { tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, - const char* handshaker_service_url, bool is_client, tsi_handshaker** self) { + const char* handshaker_service_url, bool is_client, + grpc_pollset_set* interested_parties, tsi_handshaker** self) { if (handshaker_service_url == nullptr || self == nullptr || options == nullptr || (is_client && target_name == nullptr)) { gpr_log(GPR_ERROR, "Invalid arguments to alts_tsi_handshaker_create()"); diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index 227b30ce53..48ce69b1da 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -23,6 +23,7 @@ #include <grpc/grpc.h> +#include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h" #include "src/core/tsi/alts_transport_security.h" #include "src/core/tsi/transport_security.h" @@ -51,6 +52,7 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; * "host:port". * - is_client: boolean value indicating if the handshaker is used at the client * (is_client = true) or server (is_client = false) side. + * - interested_parties: set of pollsets interested in this connection. * - self: address of ALTS TSI handshaker instance to be returned from the * method. * @@ -58,7 +60,8 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; */ tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, - const char* handshaker_service_url, bool is_client, tsi_handshaker** self); + const char* handshaker_service_url, bool is_client, + grpc_pollset_set* interested_parties, tsi_handshaker** self); /** * This method handles handshaker response returned from ALTS handshaker diff --git a/src/proto/grpc/reflection/v1alpha/BUILD b/src/proto/grpc/reflection/v1alpha/BUILD index 4605418447..4d919d029e 100644 --- a/src/proto/grpc/reflection/v1alpha/BUILD +++ b/src/proto/grpc/reflection/v1alpha/BUILD @@ -22,3 +22,11 @@ grpc_proto_library( name = "reflection_proto", srcs = ["reflection.proto"], ) + +filegroup( + name = "reflection_proto_file", + srcs = [ + "reflection.proto", + ], +) + diff --git a/src/proto/grpc/testing/BUILD b/src/proto/grpc/testing/BUILD index 16721ff2ed..7048911b9a 100644 --- a/src/proto/grpc/testing/BUILD +++ b/src/proto/grpc/testing/BUILD @@ -15,6 +15,8 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package") +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") grpc_package(name = "testing", visibility = "public") @@ -58,12 +60,30 @@ grpc_proto_library( has_services = False, ) +py_proto_library( + name = "py_empty_proto", + protos = ["empty.proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + grpc_proto_library( name = "messages_proto", srcs = ["messages.proto"], has_services = False, ) +py_proto_library( + name = "py_messages_proto", + protos = ["messages.proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + grpc_proto_library( name = "metrics_proto", srcs = ["metrics.proto"], @@ -116,3 +136,17 @@ grpc_proto_library( "messages_proto", ], ) + +py_proto_library( + name = "py_test_proto", + protos = ["test.proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], + proto_deps = [ + ":py_empty_proto", + ":py_messages_proto", + ] +) + diff --git a/src/proto/grpc/testing/proto2/BUILD.bazel b/src/proto/grpc/testing/proto2/BUILD.bazel new file mode 100644 index 0000000000..c4c4f004ef --- /dev/null +++ b/src/proto/grpc/testing/proto2/BUILD.bazel @@ -0,0 +1,30 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + +package(default_visibility = ["//visibility:public"]) + +py_proto_library( + name = "empty2_proto", + protos = [ + "empty2.proto", + ], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + +py_proto_library( + name = "empty2_extensions_proto", + protos = [ + "empty2_extensions.proto", + ], + proto_deps = [ + ":empty2_proto", + ], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel new file mode 100644 index 0000000000..3a2ba26371 --- /dev/null +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/BUILD.bazel @@ -0,0 +1,34 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + +package(default_visibility = ["//visibility:public"]) + +genrule( + name = "mv_reflection_proto", + srcs = [ + "//src/proto/grpc/reflection/v1alpha:reflection_proto_file", + ], + outs = ["reflection.proto",], + cmd = "cp $< $@", +) + +py_proto_library( + name = "py_reflection_proto", + protos = [":mv_reflection_proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + +py_library( + name = "grpc_reflection", + srcs = ["reflection.py",], + deps = [ + ":py_reflection_proto", + "//src/python/grpcio/grpc:grpcio", + requirement('protobuf'), + ], + imports=["../../",], +) + diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel new file mode 100644 index 0000000000..a39f30d32a --- /dev/null +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -0,0 +1,97 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "_intraop_test_case", + srcs = ["_intraop_test_case.py"], + deps = [ + ":methods", + ], + imports=["../../",], +) + +py_library( + name = "client", + srcs = ["client.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ":methods", + ":resources", + "//src/proto/grpc/testing:py_test_proto", + requirement('google-auth'), + ], + imports=["../../",], +) + +py_library( + name = "methods", + srcs = ["methods.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/proto/grpc/testing:py_empty_proto", + "//src/proto/grpc/testing:py_messages_proto", + "//src/proto/grpc/testing:py_test_proto", + requirement('google-auth'), + requirement('requests'), + requirement('enum34'), + ], + imports=["../../",], +) + +py_library( + name = "resources", + srcs = ["resources.py"], + data = [ + "//src/python/grpcio_tests/tests/interop/credentials", + ], +) + +py_library( + name = "server", + srcs = ["server.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ":methods", + ":resources", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/proto/grpc/testing:py_test_proto", + ], + imports=["../../",], +) + +py_test( + name="_insecure_intraop_test", + size="small", + srcs=["_insecure_intraop_test.py",], + main="_insecure_intraop_test.py", + deps=[ + "//src/python/grpcio/grpc:grpcio", + ":_intraop_test_case", + ":methods", + ":server", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/proto/grpc/testing:py_test_proto", + ], + imports=["../../",], + data=[ + "//src/python/grpcio_tests/tests/unit/credentials", + ], +) + +py_test( + name="_secure_intraop_test", + size="small", + srcs=["_secure_intraop_test.py",], + main="_secure_intraop_test.py", + deps=[ + "//src/python/grpcio/grpc:grpcio", + ":_intraop_test_case", + ":methods", + ":server", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/proto/grpc/testing:py_test_proto", + ], + imports=["../../",], +) + diff --git a/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel b/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel new file mode 100644 index 0000000000..bc2b997292 --- /dev/null +++ b/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel @@ -0,0 +1,9 @@ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name="credentials", + srcs=glob([ + "**", + ]), +) + diff --git a/src/python/grpcio_tests/tests/reflection/BUILD.bazel b/src/python/grpcio_tests/tests/reflection/BUILD.bazel new file mode 100644 index 0000000000..c0efb0b7ce --- /dev/null +++ b/src/python/grpcio_tests/tests/reflection/BUILD.bazel @@ -0,0 +1,21 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_test( + name="_reflection_servicer_test", + size="small", + timeout="moderate", + srcs=["_reflection_servicer_test.py",], + main="_reflection_servicer_test.py", + deps=[ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_reflection/grpc_reflection/v1alpha:grpc_reflection", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/proto/grpc/testing:py_empty_proto", + "//src/proto/grpc/testing/proto2:empty2_extensions_proto", + requirement('protobuf'), + ], + imports=["../../",], +) + diff --git a/test/core/handshake/readahead_handshaker_server_ssl.cc b/test/core/handshake/readahead_handshaker_server_ssl.cc index 97e9c20ee4..14d96b5d89 100644 --- a/test/core/handshake/readahead_handshaker_server_ssl.cc +++ b/test/core/handshake/readahead_handshaker_server_ssl.cc @@ -75,6 +75,7 @@ static grpc_handshaker* readahead_handshaker_create() { static void readahead_handshaker_factory_add_handshakers( grpc_handshaker_factory* hf, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add(handshake_mgr, readahead_handshaker_create()); } diff --git a/test/core/iomgr/tcp_client_posix_test.cc b/test/core/iomgr/tcp_client_posix_test.cc index a4c38af86f..90fc5aecfc 100644 --- a/test/core/iomgr/tcp_client_posix_test.cc +++ b/test/core/iomgr/tcp_client_posix_test.cc @@ -170,7 +170,7 @@ void test_fails(void) { break; case GRPC_TIMERS_NOT_CHECKED: polling_deadline = 0; - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_work", diff --git a/test/core/iomgr/tcp_client_uv_test.cc b/test/core/iomgr/tcp_client_uv_test.cc index 0c6250ed7f..ffcc7937c7 100644 --- a/test/core/iomgr/tcp_client_uv_test.cc +++ b/test/core/iomgr/tcp_client_uv_test.cc @@ -165,7 +165,7 @@ void test_fails(void) { break; case GRPC_TIMERS_NOT_CHECKED: polling_deadline = grpc_timespec_to_millis_round_up(now); - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_work", diff --git a/test/core/security/ssl_server_fuzzer.cc b/test/core/security/ssl_server_fuzzer.cc index cb74e3bae1..1e04691ce2 100644 --- a/test/core/security/ssl_server_fuzzer.cc +++ b/test/core/security/ssl_server_fuzzer.cc @@ -91,7 +91,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { struct handshake_state state; state.done_callback_called = false; grpc_handshake_manager* handshake_mgr = grpc_handshake_manager_create(); - grpc_server_security_connector_add_handshakers(sc, handshake_mgr); + grpc_server_security_connector_add_handshakers(sc, nullptr, handshake_mgr); grpc_handshake_manager_do_handshake( handshake_mgr, nullptr /* interested_parties */, mock_endpoint, nullptr /* channel_args */, deadline, nullptr /* acceptor */, diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 85a58114ba..e9eb7e175f 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -421,7 +421,7 @@ static tsi_handshaker* create_test_handshaker(bool used_for_success_test, alts_mock_handshaker_client_create(used_for_success_test); grpc_alts_credentials_options* options = grpc_alts_credentials_client_options_create(); - alts_tsi_handshaker_create(options, "target_name", "lame", is_client, + alts_tsi_handshaker_create(options, "target_name", "lame", is_client, nullptr, &handshaker); alts_tsi_handshaker* alts_handshaker = reinterpret_cast<alts_tsi_handshaker*>(handshaker); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index d97ea071d1..6ecb957801 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -953,6 +953,114 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { EXPECT_TRUE(recv_status.ok()); } +// 1 ping, 2 pongs. +TEST_P(AsyncEnd2endTest, ServerInitialMetadataServerStreaming) { + ResetStub(); + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + ServerContext srv_ctx; + ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx); + + std::pair<::grpc::string, ::grpc::string> meta1("key1", "val1"); + std::pair<::grpc::string, ::grpc::string> meta2("key2", "val2"); + + std::unique_ptr<ClientAsyncReader<EchoResponse>> cli_stream( + stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); + cli_stream->ReadInitialMetadata(tag(11)); + service_->RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, + cq_.get(), cq_.get(), tag(2)); + + Verifier().Expect(1, true).Expect(2, true).Verify(cq_.get()); + + srv_ctx.AddInitialMetadata(meta1.first, meta1.second); + srv_ctx.AddInitialMetadata(meta2.first, meta2.second); + srv_stream.SendInitialMetadata(tag(10)); + Verifier().Expect(10, true).Expect(11, true).Verify(cq_.get()); + auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); + EXPECT_EQ(meta1.second, + ToString(server_initial_metadata.find(meta1.first)->second)); + EXPECT_EQ(meta2.second, + ToString(server_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(static_cast<size_t>(2), server_initial_metadata.size()); + + srv_stream.Write(send_response, tag(3)); + + cli_stream->Read(&recv_response, tag(4)); + Verifier().Expect(3, true).Expect(4, true).Verify(cq_.get()); + + srv_stream.Write(send_response, tag(5)); + cli_stream->Read(&recv_response, tag(6)); + Verifier().Expect(5, true).Expect(6, true).Verify(cq_.get()); + + srv_stream.Finish(Status::OK, tag(7)); + cli_stream->Read(&recv_response, tag(8)); + Verifier().Expect(7, true).Expect(8, false).Verify(cq_.get()); + + cli_stream->Finish(&recv_status, tag(9)); + Verifier().Expect(9, true).Verify(cq_.get()); + + EXPECT_TRUE(recv_status.ok()); +} + +// 1 ping, 2 pongs. +// Test for server initial metadata being sent implicitly +TEST_P(AsyncEnd2endTest, ServerInitialMetadataServerStreamingImplicit) { + ResetStub(); + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + ServerContext srv_ctx; + ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx); + + send_request.set_message(GetParam().message_content); + std::pair<::grpc::string, ::grpc::string> meta1("key1", "val1"); + std::pair<::grpc::string, ::grpc::string> meta2("key2", "val2"); + + std::unique_ptr<ClientAsyncReader<EchoResponse>> cli_stream( + stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); + service_->RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, + cq_.get(), cq_.get(), tag(2)); + + Verifier().Expect(1, true).Expect(2, true).Verify(cq_.get()); + EXPECT_EQ(send_request.message(), recv_request.message()); + + srv_ctx.AddInitialMetadata(meta1.first, meta1.second); + srv_ctx.AddInitialMetadata(meta2.first, meta2.second); + send_response.set_message(recv_request.message()); + srv_stream.Write(send_response, tag(3)); + + cli_stream->Read(&recv_response, tag(4)); + Verifier().Expect(3, true).Expect(4, true).Verify(cq_.get()); + EXPECT_EQ(send_response.message(), recv_response.message()); + + auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); + EXPECT_EQ(meta1.second, + ToString(server_initial_metadata.find(meta1.first)->second)); + EXPECT_EQ(meta2.second, + ToString(server_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(static_cast<size_t>(2), server_initial_metadata.size()); + + srv_stream.Write(send_response, tag(5)); + cli_stream->Read(&recv_response, tag(6)); + Verifier().Expect(5, true).Expect(6, true).Verify(cq_.get()); + + srv_stream.Finish(Status::OK, tag(7)); + cli_stream->Read(&recv_response, tag(8)); + Verifier().Expect(7, true).Expect(8, false).Verify(cq_.get()); + + cli_stream->Finish(&recv_status, tag(9)); + Verifier().Expect(9, true).Verify(cq_.get()); + + EXPECT_TRUE(recv_status.ok()); +} + TEST_P(AsyncEnd2endTest, ServerTrailingMetadataRpc) { ResetStub(); diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index a4b1a85f85..1d7fa73aa8 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -137,7 +137,8 @@ int main(int argc, char** argv) { &grpc::testing::InteropClient::DoTimeoutOnSleepingServer, &client); actions["empty_stream"] = std::bind(&grpc::testing::InteropClient::DoEmptyStream, &client); - if (FLAGS_use_tls) { + if (FLAGS_use_tls || + FLAGS_custom_credentials_type == "google_default_credentials") { actions["compute_engine_creds"] = std::bind(&grpc::testing::InteropClient::DoComputeEngineCreds, &client, FLAGS_default_service_account, FLAGS_oauth_scope); diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh deleted file mode 100755 index 205c0bf8c5..0000000000 --- a/tools/gce/create_interop_worker.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates an interop worker on GCE. -# IMPORTANT: After this script finishes, there are still some manual -# steps needed there are hard to automatize. -# See go/grpc-jenkins-setup for followup instructions. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-east1-a # canary gateway is reachable from this zone - -INSTANCE_NAME="${1:-grpc-canary-interop2}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image ubuntu-15-10 \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_kokoro_performance_worker.sh b/tools/gce/create_linux_kokoro_performance_worker.sh index d08a1aa2c6..63d54a48bf 100755 --- a/tools/gce/create_linux_kokoro_performance_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker.sh @@ -15,6 +15,13 @@ # Creates a performance worker on GCE to be used on Kokoro. +# IMPORTANT: Instructions for updating +# If the VM configuration / installed software is updated, +# - all existing performance worker VMs need to be updated to reflect the changes +# - a new GCE image named "grpc-performance-kokoro-v1" needs to be created, +# incrementing the version number. +# - kokoro jobs need to be reconfigured to use the new image version + set -ex cd "$(dirname "$0")" @@ -30,7 +37,7 @@ gcloud compute instances create "$INSTANCE_NAME" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ + --image-family ubuntu-1804-lts \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh deleted file mode 100755 index e9033ec443..0000000000 --- a/tools/gce/create_linux_performance_worker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a performance worker on GCE. -# IMPORTANT: After creating the worker, one needs to manually add the pubkey -# of jenkins@the-machine-where-jenkins-starts-perf-tests -# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. -# See tools/run_tests/run_performance_tests.py for details. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-b # this zone allows 32core machines - -INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-32 - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type $MACHINE_TYPE \ - --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ - --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh deleted file mode 100755 index a93d8c5e83..0000000000 --- a/tools/gce/create_linux_worker.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a standard jenkins worker on GCE. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-a - -INSTANCE_NAME="${1:-grpc-jenkins-worker1}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image=ubuntu-1510 \ - --image-project=grpc-testing \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub deleted file mode 100644 index e9853224e1..0000000000 --- a/tools/gce/jenkins_master.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/kokoro_performance.pub b/tools/gce/kokoro_performance.pub index 1154debe78..4eca916b8a 100644 --- a/tools/gce/kokoro_performance.pub +++ b/tools/gce/kokoro_performance.pub @@ -1 +1,4 @@ +# Enable Kokoro CI to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com +# Enable kokoro multi-machine benchmark driver VM to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQ5UEX4AFefec9BKICupFS7x9Hoq4ZyLKy+QX0J31I49ew9mG2AJlr3sp8ql15eX+A2Ml9MKJkmgZGHpJtw+SfvmI94SmomSyiCLAK92sQ85NMzaRdo4b9e30E9nhXnAvAaemvIEQbgCMYFvzk0C8AtXj6+htCrN4jFaLqTCPISJhX3ETc4TgX1qaHQHyl31tdaXHYlITvBDsfokcGcZQnhmCUDtD8wyaSC8GFk9gZbXshkfaYCuuLPPA0vwWGBw+YPbonHsFCsOog1IYSzYPCkIjq8dt6evsusK6Kaoyw/Z+l2kYty2FKTj+wU3l06QMoxwcfNT4WxdhcnVbY71r kbuilder@kokoro-performance-driver diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 4a1e3e608b..b78695d802 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -47,7 +47,6 @@ sudo apt-get install -y \ libtool \ make \ strace \ - pypy \ python-dev \ python-pip \ python-setuptools \ @@ -68,30 +67,34 @@ sudo apt-get install -y google-perftools libgoogle-perftools-dev # netperf sudo apt-get install -y netperf +# required to run kokoro_log_reader.py +sudo apt-get install -y python-psutil python3-psutil + +# gcloud tools, including gsutil +sudo apt-get install -y google-cloud-sdk + # C++ dependencies sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies sudo pip install --upgrade pip==10.0.1 sudo pip install tabulate -sudo pip install google-api-python-client +sudo pip install google-api-python-client oauth2client sudo pip install virtualenv -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - +# pypy is used instead of python for postprocessing benchmark outputs +# because some reports are huge and pypy is much faster. +# TODO(jtattermusch): get rid of pypy once possible, it's hard to +# keep track of all the installed variants of python. +sudo apt-get install -y pypy pypy-dev curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py sudo pypy -m pip install tabulate -sudo pip install google-api-python-client +sudo pypy -m pip install google-api-python-client oauth2client +# TODO(jtattermusch): for some reason, we need psutil installed +# in pypy for kokoro_log_reader.py (strange, because the comand is +# "python kokoro_log_reader.py" and pypy is not the system default) +sudo pypy -m pip install psutil # Node dependencies (nvm has to be installed under user kbuilder) touch .profile @@ -104,31 +107,31 @@ nvm install 4 && npm config set cache /tmp/npm-cache nvm install 5 && npm config set cache /tmp/npm-cache nvm alias default 4 +# C# dependencies +sudo apt-get install -y cmake + # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list sudo apt-get update -sudo apt-get install -y mono-devel nuget +sudo apt-get install -y mono-devel -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 +# C# .NET Core dependencies (https://www.microsoft.com/net/download) +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb + +sudo apt-get install -y apt-transport-https sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# C# 1.0.4 SDK -curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -sudo mkdir -p /opt/dotnet -sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet -sudo ln -s /opt/dotnet/dotnet /usr/local/bin - -# C# .NET dependencies -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb -sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb -sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb -sudo apt-get update && sudo apt-get install -y libicu55 +sudo apt-get install -y dotnet-sdk-2.1 + +# Install .NET Core 1.0.5 Runtime (required to run netcoreapp1.0) +wget -q https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz +mkdir -p dotnet105_download +tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C dotnet105_download +sudo cp -r dotnet105_download/shared/Microsoft.NETCore.App/1.0.5/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ +# To prevent "Failed to initialize CoreCLR, HRESULT: 0x80131500" with .NET Core 1.0.5 runtime +wget -q http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.4_amd64.deb +sudo dpkg -i libicu55_55.1-7ubuntu0.4_amd64.deb # Ruby dependencies gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 @@ -163,7 +166,7 @@ sudo mv composer.phar /usr/local/bin/composer # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 +GO_VERSION=1.10 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz @@ -190,11 +193,22 @@ git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts sudo apt-get install -y python-scipy python-numpy +# Install docker +curl -sSL https://get.docker.com/ | sh +# Enable kbuilder to use docker without sudo: +sudo usermod -aG docker kbuilder + # Add pubkey of Kokoro driver VM to allow SSH # silence false-positive shellcheck warning ("< redirect does not affect sudo") # shellcheck disable=SC2024 sudo tee --append ~kbuilder/.ssh/authorized_keys < kokoro_performance.pub +# Kokoro requires /tmpfs/READY file to exist the directory and file itself should +# be owned by kbuilder. +sudo mkdir /tmpfs +sudo chown kbuilder /tmpfs +touch /tmpfs/READY + # Restart for VM to pick up kernel update echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' sleep 10 diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh deleted file mode 100755 index 7222cef9a2..0000000000 --- a/tools/gce/linux_performance_worker_init.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux performance worker. -# You shouldn't run this script on your own, -# use create_linux_performance_worker.sh instead. - -set -ex - -sudo apt-get update - -# Install Java 8 JDK (to build gRPC Java) -sudo apt-get install -y openjdk-8-jdk -sudo apt-get install -y unzip lsof - -sudo apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libcurl4-openssl-dev \ - libgtest-dev \ - libreadline-dev \ - libssl-dev \ - libtool \ - make \ - strace \ - pypy \ - python-dev \ - python-pip \ - python-setuptools \ - python-yaml \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-yaml \ - telnet \ - unzip \ - wget \ - zip \ - zlib1g-dev - -# perftools -sudo apt-get install -y google-perftools libgoogle-perftools-dev - -# netperf -sudo apt-get install -y netperf - -# C++ dependencies -sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang - -# Python dependencies -sudo pip install --upgrade pip==10.0.1 -sudo pip install tabulate -sudo pip install google-api-python-client -sudo pip install virtualenv - -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - -curl -O https://bootstrap.pypa.io/get-pip.py -sudo pypy get-pip.py -sudo pypy -m pip install tabulate -sudo pip install google-api-python-client - -# Node dependencies (nvm has to be installed under user jenkins) -touch .profile -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -# silence shellcheck warning as it cannot follow the `source` path statically: -# shellcheck disable=SC1090 -source ~/.nvm/nvm.sh -nvm install 0.12 && npm config set cache /tmp/npm-cache -nvm install 4 && npm config set cache /tmp/npm-cache -nvm install 5 && npm config set cache /tmp/npm-cache -nvm alias default 4 - -# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list -sudo apt-get update -sudo apt-get install -y mono-devel nuget - -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 -sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# Ruby dependencies -git clone https://github.com/rbenv/rbenv.git ~/.rbenv -export PATH="$HOME/.rbenv/bin:$PATH" -eval "$(rbenv init -)" - -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" - -rbenv install 2.4.0 -rbenv global 2.4.0 -ruby -v - -# Install bundler (prerequisite for gRPC Ruby) -gem install bundler - -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - -# Java dependencies - nothing as we already have Java JDK 8 - -# Go dependencies -# Currently, the golang package available via apt-get doesn't have the latest go. -# Significant performance improvements with grpc-go have been observed after -# upgrading from go 1.5 to a later version, so a later go version is preferred. -# Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 -OS=linux -ARCH=amd64 -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -# Put go on the PATH, keep the usual installation dir -sudo ln -s /usr/local/go/bin/go /usr/bin/go -rm go$GO_VERSION.$OS-$ARCH.tar.gz - -# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) -sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" -# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar -echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid -# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol -echo 0 | sudo tee /proc/sys/kernel/kptr_restrict - -# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in -# order to not lose perf events or time out -echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb - -# Fetch scripts to generate flame graphs from perf data collected -# on benchmarks -git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph - -# Install scipy and numpy for benchmarking scripts -sudo apt-get install -y python-scipy python-numpy - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. -# silence false-positive shellcheck warning ("< redirect does not affect sudo") -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for VM to pick up kernel update -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 -sudo reboot diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh deleted file mode 100755 index 05855354ff..0000000000 --- a/tools/gce/linux_worker_init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux worker. -# You shouldn't run this script on your own, use create_linux_worker.sh -# instead. - -set -ex - -# Create some swap space -sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 -sudo chmod 600 /swap -sudo mkswap /swap -sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab -sudo swapon -a - -# Typical apt-get maintenance -sudo apt-get update - -# Install JRE -sudo apt-get install -y openjdk-8-jre -sudo apt-get install -y unzip lsof - -# Install Docker -curl -sSL https://get.docker.com/ | sh - -# Setup jenkins user (or the user will already exist bcuz magic) -sudo adduser jenkins --disabled-password || true - -# Enable jenkins to use docker without sudo: -sudo usermod -aG docker jenkins - -# Use "overlay" storage driver for docker -# see https://github.com/grpc/grpc/issues/4988 -printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json - -# Install pip and Google API library to enable using GCP services -sudo apt-get install -y python-pip -sudo pip install google-api-python-client - -# Install RVM -# TODO(jtattermusch): why is RVM needed? -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby - -# Upgrade Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. - -# disable superfluous warning by shellcheck: -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for docker to pick up the config changes. -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 - -sudo reboot diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29801a5b86..d93e8979fc 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -19,10 +19,20 @@ cd "$(dirname "$0")/../../.." mkdir -p artifacts/ -# All the python packages have been built in the artifact phase already -# and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true +strip_binary_wheel() { + TEMP_WHEEL_DIR=$(mktemp -d) + unzip "$1" -d "$TEMP_WHEEL_DIR" + find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" + find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" + (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1" +} + +for wheel in artifacts/*.whl; do + strip_binary_wheel "$wheel" +done + # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index f235582579..9e6e72d97b 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -61,6 +61,10 @@ do "node"|"node_purejs") tools/run_tests/performance/build_performance_node.sh ;; + "python") + # python workers are only run with python2.7 and building with multiple python versions is costly + python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --compiler python2.7 --build_only -j 8 + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e8d6b59687..9752e741fb 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1376,7 +1376,8 @@ try: transport_security='tls') jobs.append(tls_test_job) if str(language) in [ - 'go' + 'c++', + 'go', ]: # Add more languages to the list to turn on tests. google_default_creds_test_job = cloud_to_prod_jobspec( language, |