diff options
48 files changed, 417 insertions, 665 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index b3fad22141..69a9d1c5e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5249,6 +5249,7 @@ add_library(qps ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/worker_service.grpc.pb.h test/cpp/qps/benchmark_config.cc test/cpp/qps/client_async.cc + test/cpp/qps/client_callback.cc test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc test/cpp/qps/parse_json.cc @@ -7510,6 +7510,7 @@ LIBQPS_SRC = \ $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc \ test/cpp/qps/benchmark_config.cc \ test/cpp/qps/client_async.cc \ + test/cpp/qps/client_callback.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ test/cpp/qps/parse_json.cc \ @@ -7566,6 +7567,7 @@ endif endif $(OBJDIR)/$(CONFIG)/test/cpp/qps/benchmark_config.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_callback.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/parse_json.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc @@ -24917,6 +24919,7 @@ test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) test/cpp/qps/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) +test/cpp/qps/client_callback.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) test/cpp/qps/parse_json.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 7ce3214511..f994801408 100644 --- a/build.yaml +++ b/build.yaml @@ -1965,6 +1965,7 @@ libs: - src/proto/grpc/testing/worker_service.proto - test/cpp/qps/benchmark_config.cc - test/cpp/qps/client_async.cc + - test/cpp/qps/client_callback.cc - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc - test/cpp/qps/parse_json.cc @@ -1717,6 +1717,7 @@ 'src/proto/grpc/testing/worker_service.proto', 'test/cpp/qps/benchmark_config.cc', 'test/cpp/qps/client_async.cc', + 'test/cpp/qps/client_callback.cc', 'test/cpp/qps/client_sync.cc', 'test/cpp/qps/driver.cc', 'test/cpp/qps/parse_json.cc', diff --git a/include/grpcpp/opencensus.h b/include/grpcpp/opencensus.h index 07a1333986..29b221f767 100644 --- a/include/grpcpp/opencensus.h +++ b/include/grpcpp/opencensus.h @@ -19,10 +19,6 @@ #ifndef GRPCPP_OPENCENSUS_H #define GRPCPP_OPENCENSUS_H -#ifndef GRPC_BAZEL_BUILD -#error OpenCensus for gRPC is only supported when building with bazel. -#endif - #include "opencensus/trace/span.h" namespace grpc { diff --git a/requirements.bazel.txt b/requirements.bazel.txt index efbf5314af..16f31f9e94 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -8,4 +8,3 @@ wheel>=0.29 futures>=2.2.0 google-auth>=1.0.0 oauth2client==4.1.0 -requests>=2.14.2 diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index bb3ea400d1..d63e2c66c2 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -2194,9 +2194,9 @@ static void add_retriable_send_initial_metadata_op( .grpc_previous_rpc_attempts); } if (GPR_UNLIKELY(calld->num_attempts_completed > 0)) { - grpc_mdelem retry_md = grpc_mdelem_from_slices( + grpc_mdelem retry_md = grpc_mdelem_create( GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS, - *retry_count_strings[calld->num_attempts_completed - 1]); + *retry_count_strings[calld->num_attempts_completed - 1], nullptr); grpc_error* error = grpc_metadata_batch_add_tail( &retry_state->send_initial_metadata, &retry_state->send_initial_metadata_storage[calld->send_initial_metadata diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc index 1ca20ebb26..6383f12594 100644 --- a/src/core/ext/filters/http/client_authority_filter.cc +++ b/src/core/ext/filters/http/client_authority_filter.cc @@ -59,9 +59,8 @@ void authority_start_transport_stream_op_batch( initial_metadata->idx.named.authority == nullptr) { grpc_error* error = grpc_metadata_batch_add_head( initial_metadata, &calld->authority_storage, - grpc_mdelem_from_slices( - GRPC_MDSTR_AUTHORITY, - grpc_slice_ref_internal(chand->default_authority))); + grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, chand->default_authority, + nullptr)); if (error != GRPC_ERROR_NONE) { grpc_transport_stream_op_batch_finish_with_failure(batch, error, calld->call_combiner); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 2d4b4da4c6..8a481bb7d5 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -2128,8 +2128,7 @@ void grpc_chttp2_fake_status(grpc_chttp2_transport* t, grpc_chttp2_stream* s, "add_status_message", grpc_chttp2_incoming_metadata_buffer_replace_or_add( &s->metadata_buffer[1], - grpc_mdelem_from_slices(GRPC_MDSTR_GRPC_MESSAGE, - grpc_slice_ref_internal(slice)))); + grpc_mdelem_create(GRPC_MDSTR_GRPC_MESSAGE, slice, nullptr))); } s->published_metadata[1] = GRPC_METADATA_SYNTHESIZED_FROM_FAKE; grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s); diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index f3d0c03715..33577d890a 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -43,8 +43,10 @@ namespace grpc_core { namespace channelz { -BaseNode::BaseNode(EntityType type) - : type_(type), uuid_(ChannelzRegistry::Register(this)) {} +BaseNode::BaseNode(EntityType type) : type_(type), uuid_(-1) { + // The registry will set uuid_ under its lock. + ChannelzRegistry::Register(this); +} BaseNode::~BaseNode() { ChannelzRegistry::Unregister(uuid_); } diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index d8113585c2..88551befc8 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -42,13 +42,13 @@ /** This is the default value for whether or not to enable channelz. If * GRPC_ARG_ENABLE_CHANNELZ is set, it will override this default value. */ -#define GRPC_ENABLE_CHANNELZ_DEFAULT false +#define GRPC_ENABLE_CHANNELZ_DEFAULT true /** This is the default value for the maximum amount of memory used by trace * events per channel trace node. If * GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE is set, it will override * this default value. */ -#define GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT 0 +#define GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT 1024 * 4 namespace grpc_core { @@ -92,8 +92,10 @@ class BaseNode : public RefCounted<BaseNode> { intptr_t uuid() const { return uuid_; } private: + // to allow the ChannelzRegistry to set uuid_ under its lock. + friend class ChannelzRegistry; const EntityType type_; - const intptr_t uuid_; + intptr_t uuid_; }; // This class is a helper class for channelz entities that deal with Channels, diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index 1b54b19be3..67e56ed791 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -53,10 +53,10 @@ ChannelzRegistry::ChannelzRegistry() { gpr_mu_init(&mu_); } ChannelzRegistry::~ChannelzRegistry() { gpr_mu_destroy(&mu_); } -intptr_t ChannelzRegistry::InternalRegister(BaseNode* node) { +void ChannelzRegistry::InternalRegister(BaseNode* node) { MutexLock lock(&mu_); entities_.push_back(node); - return ++uuid_generator_; + node->uuid_ = ++uuid_generator_; } void ChannelzRegistry::MaybePerformCompactionLocked() { diff --git a/src/core/lib/channel/channelz_registry.h b/src/core/lib/channel/channelz_registry.h index 9c43d960d3..ea6ab6c8e5 100644 --- a/src/core/lib/channel/channelz_registry.h +++ b/src/core/lib/channel/channelz_registry.h @@ -44,7 +44,7 @@ class ChannelzRegistry { // To be called in grpc_shutdown(); static void Shutdown(); - static intptr_t Register(BaseNode* node) { + static void Register(BaseNode* node) { return Default()->InternalRegister(node); } static void Unregister(intptr_t uuid) { Default()->InternalUnregister(uuid); } @@ -74,7 +74,7 @@ class ChannelzRegistry { static ChannelzRegistry* Default(); // globally registers an Entry. Returns its unique uuid - intptr_t InternalRegister(BaseNode* node); + void InternalRegister(BaseNode* node); // globally unregisters the object that is associated to uuid. Also does // sanity check that an object doesn't try to unregister the wrong type. diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index e40bf81c90..aa2704ce26 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -468,7 +468,9 @@ static void tcp_do_read(grpc_tcp* tcp) { GRPC_STATS_INC_TCP_READ_SIZE(read_bytes); add_to_estimate(tcp, static_cast<size_t>(read_bytes)); GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); - if (static_cast<size_t>(read_bytes) < tcp->incoming_buffer->length) { + if (static_cast<size_t>(read_bytes) == tcp->incoming_buffer->length) { + finish_estimate(tcp); + } else if (static_cast<size_t>(read_bytes) < tcp->incoming_buffer->length) { grpc_slice_buffer_trim_end( tcp->incoming_buffer, tcp->incoming_buffer->length - static_cast<size_t>(read_bytes), @@ -498,7 +500,7 @@ static void tcp_read_allocation_done(void* tcpp, grpc_error* error) { static void tcp_continue_read(grpc_tcp* tcp) { size_t target_read_size = get_target_read_size(tcp); - if (tcp->incoming_buffer->length < target_read_size && + if (tcp->incoming_buffer->length < target_read_size / 2 && tcp->incoming_buffer->count < MAX_READ_IOVEC) { if (grpc_tcp_trace.enabled()) { gpr_log(GPR_INFO, "TCP:%p alloc_slices", tcp); diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.cc b/src/core/lib/security/credentials/plugin/plugin_credentials.cc index 73946ce039..4015124298 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.cc +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.cc @@ -102,8 +102,7 @@ static grpc_error* process_plugin_result( } else { for (size_t i = 0; i < num_md; ++i) { grpc_mdelem mdelem = - grpc_mdelem_from_slices(grpc_slice_ref_internal(md[i].key), - grpc_slice_ref_internal(md[i].value)); + grpc_mdelem_create(md[i].key, md[i].value, nullptr); grpc_credentials_mdelem_array_add(r->md_array, mdelem); GRPC_MDELEM_UNREF(mdelem); } diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 4294434504..d7095c24d4 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -336,9 +336,8 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel, grpc_core::ExecCtx exec_ctx; grpc_call* call = grpc_channel_create_call_internal( channel, parent_call, propagation_mask, cq, nullptr, - grpc_mdelem_from_slices(GRPC_MDSTR_PATH, grpc_slice_ref_internal(method)), - host != nullptr ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY, - grpc_slice_ref_internal(*host)) + grpc_mdelem_create(GRPC_MDSTR_PATH, method, nullptr), + host != nullptr ? grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, *host, nullptr) : GRPC_MDNULL, grpc_timespec_to_millis_round_up(deadline)); @@ -347,14 +346,13 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel, grpc_call* grpc_channel_create_pollset_set_call( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, - grpc_pollset_set* pollset_set, grpc_slice method, const grpc_slice* host, - grpc_millis deadline, void* reserved) { + grpc_pollset_set* pollset_set, const grpc_slice& method, + const grpc_slice* host, grpc_millis deadline, void* reserved) { GPR_ASSERT(!reserved); return grpc_channel_create_call_internal( channel, parent_call, propagation_mask, nullptr, pollset_set, - grpc_mdelem_from_slices(GRPC_MDSTR_PATH, grpc_slice_ref_internal(method)), - host != nullptr ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY, - grpc_slice_ref_internal(*host)) + grpc_mdelem_create(GRPC_MDSTR_PATH, method, nullptr), + host != nullptr ? grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, *host, nullptr) : GRPC_MDNULL, deadline); } diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h index e5ff2c3596..4ac76b8a29 100644 --- a/src/core/lib/surface/channel.h +++ b/src/core/lib/surface/channel.h @@ -45,8 +45,8 @@ grpc_channel* grpc_channel_create_with_builder( value of \a propagation_mask (see propagation_bits.h for possible values) */ grpc_call* grpc_channel_create_pollset_set_call( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, - grpc_pollset_set* pollset_set, grpc_slice method, const grpc_slice* host, - grpc_millis deadline, void* reserved); + grpc_pollset_set* pollset_set, const grpc_slice& method, + const grpc_slice* host, grpc_millis deadline, void* reserved); /** Get a (borrowed) pointer to this channels underlying channel stack */ grpc_channel_stack* grpc_channel_get_channel_stack(grpc_channel* channel); diff --git a/src/core/lib/transport/metadata.cc b/src/core/lib/transport/metadata.cc index d164502280..60af22393e 100644 --- a/src/core/lib/transport/metadata.cc +++ b/src/core/lib/transport/metadata.cc @@ -237,7 +237,7 @@ static void rehash_mdtab(mdtab_shard* shard) { } grpc_mdelem grpc_mdelem_create( - grpc_slice key, grpc_slice value, + const grpc_slice& key, const grpc_slice& value, grpc_mdelem_data* compatible_external_backing_store) { if (!grpc_slice_is_interned(key) || !grpc_slice_is_interned(value)) { if (compatible_external_backing_store != nullptr) { @@ -324,7 +324,8 @@ grpc_mdelem grpc_mdelem_create( return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED); } -grpc_mdelem grpc_mdelem_from_slices(grpc_slice key, grpc_slice value) { +grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key, + const grpc_slice& value) { grpc_mdelem out = grpc_mdelem_create(key, value, nullptr); grpc_slice_unref_internal(key); grpc_slice_unref_internal(value); diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 338082276c..989c7544c1 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -109,7 +109,8 @@ struct grpc_mdelem { (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT)) /* Unrefs the slices. */ -grpc_mdelem grpc_mdelem_from_slices(grpc_slice key, grpc_slice value); +grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key, + const grpc_slice& value); /* Cheaply convert a grpc_metadata to a grpc_mdelem; may use the grpc_metadata object as backing storage (so lifetimes should align) */ @@ -120,7 +121,7 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata); compatible_external_backing_store if it is non-NULL (in which case it's the users responsibility to ensure that it outlives usage) */ grpc_mdelem grpc_mdelem_create( - grpc_slice key, grpc_slice value, + const grpc_slice& key, const grpc_slice& value, grpc_mdelem_data* compatible_external_backing_store); bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b); diff --git a/src/proto/grpc/testing/BUILD b/src/proto/grpc/testing/BUILD index 7048911b9a..16721ff2ed 100644 --- a/src/proto/grpc/testing/BUILD +++ b/src/proto/grpc/testing/BUILD @@ -15,8 +15,6 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package") -load("@grpc_python_dependencies//:requirements.bzl", "requirement") -load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") grpc_package(name = "testing", visibility = "public") @@ -60,30 +58,12 @@ grpc_proto_library( has_services = False, ) -py_proto_library( - name = "py_empty_proto", - protos = ["empty.proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], -) - grpc_proto_library( name = "messages_proto", srcs = ["messages.proto"], has_services = False, ) -py_proto_library( - name = "py_messages_proto", - protos = ["messages.proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], -) - grpc_proto_library( name = "metrics_proto", srcs = ["metrics.proto"], @@ -136,17 +116,3 @@ grpc_proto_library( "messages_proto", ], ) - -py_proto_library( - name = "py_test_proto", - protos = ["test.proto",], - with_grpc = True, - deps = [ - requirement('protobuf'), - ], - proto_deps = [ - ":py_empty_proto", - ":py_messages_proto", - ] -) - diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index a4a9c8fe57..4cfdc2cafb 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -25,6 +25,7 @@ enum ClientType { SYNC_CLIENT = 0; ASYNC_CLIENT = 1; OTHER_CLIENT = 2; // used for some language-specific variants + CALLBACK_CLIENT = 3; } enum ServerType { diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 3494c9b15a..eeeb4ddb33 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -981,4 +981,7 @@ class Channel(grpc.Channel): # then deletion of this grpc._channel.Channel instance can be made to # effect closure of the underlying cygrpc.Channel instance. cygrpc.fork_unregister_channel(self) - _moot(self._connectivity_state) + # This prevent the failed-at-initializing object removal from failing. + # Though the __init__ failed, the removal will still trigger __del__. + if hasattr(self, "_connectivity_state"): + _moot(self._connectivity_state) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi index 6cb1bc0c05..e0e068e452 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi @@ -32,7 +32,7 @@ cdef class _ArgumentProcessor: cdef grpc_arg c_argument - cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references) + cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references) except * cdef class _ArgumentsProcessor: @@ -42,5 +42,5 @@ cdef class _ArgumentsProcessor: cdef readonly list _references cdef grpc_channel_args _c_arguments - cdef grpc_channel_args *c(self, grpc_arg_pointer_vtable *vtable) + cdef grpc_channel_args *c(self, grpc_arg_pointer_vtable *vtable) except * cdef un_c(self) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi index 2239e26b32..b7a4277ff6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi @@ -52,7 +52,7 @@ cdef grpc_arg _unwrap_grpc_arg(tuple wrapped_arg): cdef class _ArgumentProcessor: - cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references): + cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references) except *: key, value = argument cdef bytes encoded_key = _encode(key) if encoded_key is not key: @@ -89,7 +89,7 @@ cdef class _ArgumentsProcessor: self._argument_processors = [] self._references = [] - cdef grpc_channel_args *c(self, grpc_arg_pointer_vtable *vtable): + cdef grpc_channel_args *c(self, grpc_arg_pointer_vtable *vtable) except *: self._c_arguments.arguments_length = len(self._arguments) if self._c_arguments.arguments_length == 0: return NULL diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 0dfbf3180b..6931d93ef0 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -116,6 +116,8 @@ class TestGevent(setuptools.Command): # eventually succeed, but need to dig into performance issues. 'unit._cython._no_messages_server_completion_queue_per_call_test.Test.test_rpcs', 'unit._cython._no_messages_single_server_completion_queue_test.Test.test_rpcs', + # TODO(https://github.com/grpc/grpc/issues/16890) enable this test + 'unit._cython._channel_test.ChannelTest.test_multiple_channels_lonely_connectivity', # I have no idea why this doesn't work in gevent, but it shouldn't even be # using the c-core 'testing._client_test.ClientTest.test_infinite_request_stream_real_time', diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel deleted file mode 100644 index a39f30d32a..0000000000 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ /dev/null @@ -1,97 +0,0 @@ -load("@grpc_python_dependencies//:requirements.bzl", "requirement") - -package(default_visibility = ["//visibility:public"]) - -py_library( - name = "_intraop_test_case", - srcs = ["_intraop_test_case.py"], - deps = [ - ":methods", - ], - imports=["../../",], -) - -py_library( - name = "client", - srcs = ["client.py"], - deps = [ - "//src/python/grpcio/grpc:grpcio", - ":methods", - ":resources", - "//src/proto/grpc/testing:py_test_proto", - requirement('google-auth'), - ], - imports=["../../",], -) - -py_library( - name = "methods", - srcs = ["methods.py"], - deps = [ - "//src/python/grpcio/grpc:grpcio", - "//src/proto/grpc/testing:py_empty_proto", - "//src/proto/grpc/testing:py_messages_proto", - "//src/proto/grpc/testing:py_test_proto", - requirement('google-auth'), - requirement('requests'), - requirement('enum34'), - ], - imports=["../../",], -) - -py_library( - name = "resources", - srcs = ["resources.py"], - data = [ - "//src/python/grpcio_tests/tests/interop/credentials", - ], -) - -py_library( - name = "server", - srcs = ["server.py"], - deps = [ - "//src/python/grpcio/grpc:grpcio", - ":methods", - ":resources", - "//src/python/grpcio_tests/tests/unit:test_common", - "//src/proto/grpc/testing:py_test_proto", - ], - imports=["../../",], -) - -py_test( - name="_insecure_intraop_test", - size="small", - srcs=["_insecure_intraop_test.py",], - main="_insecure_intraop_test.py", - deps=[ - "//src/python/grpcio/grpc:grpcio", - ":_intraop_test_case", - ":methods", - ":server", - "//src/python/grpcio_tests/tests/unit:test_common", - "//src/proto/grpc/testing:py_test_proto", - ], - imports=["../../",], - data=[ - "//src/python/grpcio_tests/tests/unit/credentials", - ], -) - -py_test( - name="_secure_intraop_test", - size="small", - srcs=["_secure_intraop_test.py",], - main="_secure_intraop_test.py", - deps=[ - "//src/python/grpcio/grpc:grpcio", - ":_intraop_test_case", - ":methods", - ":server", - "//src/python/grpcio_tests/tests/unit:test_common", - "//src/proto/grpc/testing:py_test_proto", - ], - imports=["../../",], -) - diff --git a/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel b/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel deleted file mode 100644 index bc2b997292..0000000000 --- a/src/python/grpcio_tests/tests/interop/credentials/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -filegroup( - name="credentials", - srcs=glob([ - "**", - ]), -) - diff --git a/src/python/grpcio_tests/tests/unit/_channel_args_test.py b/src/python/grpcio_tests/tests/unit/_channel_args_test.py index 869c2f4d2f..dd1d2969a2 100644 --- a/src/python/grpcio_tests/tests/unit/_channel_args_test.py +++ b/src/python/grpcio_tests/tests/unit/_channel_args_test.py @@ -33,6 +33,14 @@ TEST_CHANNEL_ARGS = ( ('arg6', TestPointerWrapper()), ) +INVALID_TEST_CHANNEL_ARGS = [ + { + 'foo': 'bar' + }, + (('key',),), + 'str', +] + class ChannelArgsTest(unittest.TestCase): @@ -44,6 +52,14 @@ class ChannelArgsTest(unittest.TestCase): futures.ThreadPoolExecutor(max_workers=1), options=TEST_CHANNEL_ARGS) + def test_invalid_client_args(self): + for invalid_arg in INVALID_TEST_CHANNEL_ARGS: + self.assertRaises( + ValueError, + grpc.insecure_channel, + 'localhost:8080', + options=invalid_arg) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/test/core/channel/channelz_test.cc b/test/core/channel/channelz_test.cc index aed1fba47c..057d53b4ab 100644 --- a/test/core/channel/channelz_test.cc +++ b/test/core/channel/channelz_test.cc @@ -126,12 +126,12 @@ void ValidateGetServers(size_t expected_servers) { class ChannelFixture { public: ChannelFixture(int max_tracer_event_memory = 0) { - grpc_arg client_a[2]; - client_a[0] = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), - max_tracer_event_memory); - client_a[1] = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true); + grpc_arg client_a[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + max_tracer_event_memory), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true)}; grpc_channel_args client_args = {GPR_ARRAY_SIZE(client_a), client_a}; channel_ = grpc_insecure_channel_create("fake_target", &client_args, nullptr); @@ -238,8 +238,16 @@ TEST_P(ChannelzChannelTest, BasicChannel) { TEST(ChannelzChannelTest, ChannelzDisabled) { grpc_core::ExecCtx exec_ctx; + // explicitly disable channelz + grpc_arg arg[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + 0), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), false)}; + grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg}; grpc_channel* channel = - grpc_insecure_channel_create("fake_target", nullptr, nullptr); + grpc_insecure_channel_create("fake_target", &args, nullptr); ChannelNode* channelz_channel = grpc_channel_get_channelz_node(channel); ASSERT_EQ(channelz_channel, nullptr); grpc_channel_destroy(channel); diff --git a/test/core/end2end/tests/channelz.cc b/test/core/end2end/tests/channelz.cc index 41549190e3..a812994435 100644 --- a/test/core/end2end/tests/channelz.cc +++ b/test/core/end2end/tests/channelz.cc @@ -199,9 +199,13 @@ static void run_one_request(grpc_end2end_test_config config, static void test_channelz(grpc_end2end_test_config config) { grpc_end2end_test_fixture f; - grpc_arg arg = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true); - grpc_channel_args args = {1, &arg}; + grpc_arg arg[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + 0), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true)}; + grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg}; f = begin_test(config, "test_channelz", &args, &args); grpc_core::channelz::ChannelNode* channelz_channel = @@ -267,12 +271,12 @@ static void test_channelz(grpc_end2end_test_config config) { static void test_channelz_with_channel_trace(grpc_end2end_test_config config) { grpc_end2end_test_fixture f; - grpc_arg arg[2]; - arg[0] = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), - 1024 * 1024); - arg[1] = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true); + grpc_arg arg[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + 1024 * 1024), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true)}; grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg}; f = begin_test(config, "test_channelz_with_channel_trace", &args, &args); @@ -307,7 +311,15 @@ static void test_channelz_with_channel_trace(grpc_end2end_test_config config) { static void test_channelz_disabled(grpc_end2end_test_config config) { grpc_end2end_test_fixture f; - f = begin_test(config, "test_channelz_disabled", nullptr, nullptr); + grpc_arg arg[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE), + 0), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), false)}; + grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg}; + + f = begin_test(config, "test_channelz_disabled", &args, &args); grpc_core::channelz::ChannelNode* channelz_channel = grpc_channel_get_channelz_node(f.client); GPR_ASSERT(channelz_channel == nullptr); diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc index 941d9ae319..9c018962f8 100644 --- a/test/core/end2end/tests/simple_request.cc +++ b/test/core/end2end/tests/simple_request.cc @@ -108,7 +108,9 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_stats_data* after = static_cast<grpc_stats_data*>(gpr_malloc(sizeof(grpc_stats_data))); +#if defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) grpc_stats_collect(before); +#endif /* defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) */ gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call(f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq, @@ -225,17 +227,18 @@ static void simple_request_body(grpc_end2end_test_config config, cq_verifier_destroy(cqv); + int expected_calls = 1; + if (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) { + expected_calls *= 2; + } +#if defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) + grpc_stats_collect(after); char* stats = grpc_stats_data_as_json(after); gpr_log(GPR_DEBUG, "%s", stats); gpr_free(stats); - int expected_calls = 1; - if (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) { - expected_calls *= 2; - } -#if defined(GRPC_COLLECT_STATS) || !defined(NDEBUG) GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] - before->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] == expected_calls); diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index 1d7fa73aa8..a4b1a85f85 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -137,8 +137,7 @@ int main(int argc, char** argv) { &grpc::testing::InteropClient::DoTimeoutOnSleepingServer, &client); actions["empty_stream"] = std::bind(&grpc::testing::InteropClient::DoEmptyStream, &client); - if (FLAGS_use_tls || - FLAGS_custom_credentials_type == "google_default_credentials") { + if (FLAGS_use_tls) { actions["compute_engine_creds"] = std::bind(&grpc::testing::InteropClient::DoComputeEngineCreds, &client, FLAGS_default_service_account, FLAGS_oauth_scope); diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index 483b29b1b3..2ef7441371 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -31,6 +31,7 @@ grpc_cc_library( name = "qps_worker_impl", srcs = [ "client_async.cc", + "client_callback.cc", "client_sync.cc", "qps_server_builder.cc", "qps_worker.cc", diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 0b4b2ff0a9..4ed34e0405 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -533,6 +533,7 @@ class ClientImpl : public Client { std::unique_ptr<Client> CreateSynchronousClient(const ClientConfig& args); std::unique_ptr<Client> CreateAsyncClient(const ClientConfig& args); +std::unique_ptr<Client> CreateCallbackClient(const ClientConfig& args); std::unique_ptr<Client> CreateGenericAsyncStreamingClient( const ClientConfig& args); diff --git a/test/cpp/qps/client_callback.cc b/test/cpp/qps/client_callback.cc new file mode 100644 index 0000000000..87889e36dc --- /dev/null +++ b/test/cpp/qps/client_callback.cc @@ -0,0 +1,219 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <list> +#include <memory> +#include <mutex> +#include <sstream> +#include <string> +#include <thread> +#include <utility> +#include <vector> + +#include <grpc/grpc.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpcpp/alarm.h> +#include <grpcpp/channel.h> +#include <grpcpp/client_context.h> + +#include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" +#include "test/cpp/qps/client.h" +#include "test/cpp/qps/usage_timer.h" + +namespace grpc { +namespace testing { + +/** + * Maintains context info per RPC + */ +struct CallbackClientRpcContext { + CallbackClientRpcContext(BenchmarkService::Stub* stub) : stub_(stub) {} + + ~CallbackClientRpcContext() {} + + SimpleResponse response_; + ClientContext context_; + Alarm alarm_; + BenchmarkService::Stub* stub_; +}; + +static std::unique_ptr<BenchmarkService::Stub> BenchmarkStubCreator( + const std::shared_ptr<Channel>& ch) { + return BenchmarkService::NewStub(ch); +} + +class CallbackClient + : public ClientImpl<BenchmarkService::Stub, SimpleRequest> { + public: + CallbackClient(const ClientConfig& config) + : ClientImpl<BenchmarkService::Stub, SimpleRequest>( + config, BenchmarkStubCreator) { + num_threads_ = NumThreads(config); + rpcs_done_ = 0; + SetupLoadTest(config, num_threads_); + total_outstanding_rpcs_ = + config.client_channels() * config.outstanding_rpcs_per_channel(); + } + + virtual ~CallbackClient() {} + + protected: + size_t num_threads_; + size_t total_outstanding_rpcs_; + // The below mutex and condition variable is used by main benchmark thread to + // wait on completion of all RPCs before shutdown + std::mutex shutdown_mu_; + std::condition_variable shutdown_cv_; + // Number of rpcs done after thread completion + size_t rpcs_done_; + // Vector of Context data pointers for running a RPC + std::vector<std::unique_ptr<CallbackClientRpcContext>> ctx_; + + virtual void InitThreadFuncImpl(size_t thread_idx) = 0; + virtual bool ThreadFuncImpl(Thread* t, size_t thread_idx) = 0; + + void ThreadFunc(size_t thread_idx, Thread* t) override { + InitThreadFuncImpl(thread_idx); + ThreadFuncImpl(t, thread_idx); + } + + virtual void ScheduleRpc(Thread* t, size_t thread_idx, + size_t ctx_vector_idx) = 0; + + /** + * The main thread of the benchmark will be waiting on DestroyMultithreading. + * Increment the rpcs_done_ variable to signify that the Callback RPC + * after thread completion is done. When the last outstanding rpc increments + * the counter it should also signal the main thread's conditional variable. + */ + void NotifyMainThreadOfThreadCompletion() { + std::lock_guard<std::mutex> l(shutdown_mu_); + rpcs_done_++; + if (rpcs_done_ == total_outstanding_rpcs_) { + shutdown_cv_.notify_one(); + } + } + + private: + int NumThreads(const ClientConfig& config) { + int num_threads = config.async_client_threads(); + if (num_threads <= 0) { // Use dynamic sizing + num_threads = cores_; + gpr_log(GPR_INFO, "Sizing callback client to %d threads", num_threads); + } + return num_threads; + } + + /** + * Wait until all outstanding Callback RPCs are done + */ + void DestroyMultithreading() final { + std::unique_lock<std::mutex> l(shutdown_mu_); + while (rpcs_done_ != total_outstanding_rpcs_) { + shutdown_cv_.wait(l); + } + EndThreads(); + } +}; + +class CallbackUnaryClient final : public CallbackClient { + public: + CallbackUnaryClient(const ClientConfig& config) : CallbackClient(config) { + for (int ch = 0; ch < config.client_channels(); ch++) { + for (int i = 0; i < config.outstanding_rpcs_per_channel(); i++) { + ctx_.emplace_back( + new CallbackClientRpcContext(channels_[ch].get_stub())); + } + } + StartThreads(num_threads_); + } + ~CallbackUnaryClient() {} + + protected: + bool ThreadFuncImpl(Thread* t, size_t thread_idx) override { + for (size_t vector_idx = thread_idx; vector_idx < total_outstanding_rpcs_; + vector_idx += num_threads_) { + ScheduleRpc(t, thread_idx, vector_idx); + } + return true; + } + + void InitThreadFuncImpl(size_t thread_idx) override { return; } + + private: + void ScheduleRpc(Thread* t, size_t thread_idx, size_t vector_idx) override { + if (!closed_loop_) { + gpr_timespec next_issue_time = NextIssueTime(thread_idx); + // Start an alarm callback to run the internal callback after + // next_issue_time + ctx_[vector_idx]->alarm_.experimental().Set( + next_issue_time, [this, t, thread_idx, vector_idx](bool ok) { + IssueUnaryCallbackRpc(t, thread_idx, vector_idx); + }); + } else { + IssueUnaryCallbackRpc(t, thread_idx, vector_idx); + } + } + + void IssueUnaryCallbackRpc(Thread* t, size_t thread_idx, size_t vector_idx) { + GPR_TIMER_SCOPE("CallbackUnaryClient::ThreadFunc", 0); + double start = UsageTimer::Now(); + ctx_[vector_idx]->stub_->experimental_async()->UnaryCall( + (&ctx_[vector_idx]->context_), &request_, &ctx_[vector_idx]->response_, + [this, t, thread_idx, start, vector_idx](grpc::Status s) { + // Update Histogram with data from the callback run + HistogramEntry entry; + if (s.ok()) { + entry.set_value((UsageTimer::Now() - start) * 1e9); + } + entry.set_status(s.error_code()); + t->UpdateHistogram(&entry); + + if (ThreadCompleted() || !s.ok()) { + // Notify thread of completion + NotifyMainThreadOfThreadCompletion(); + } else { + // Reallocate ctx for next RPC + ctx_[vector_idx].reset( + new CallbackClientRpcContext(ctx_[vector_idx]->stub_)); + // Schedule a new RPC + ScheduleRpc(t, thread_idx, vector_idx); + } + }); + } +}; + +std::unique_ptr<Client> CreateCallbackClient(const ClientConfig& config) { + switch (config.rpc_type()) { + case UNARY: + return std::unique_ptr<Client>(new CallbackUnaryClient(config)); + case STREAMING: + case STREAMING_FROM_CLIENT: + case STREAMING_FROM_SERVER: + case STREAMING_BOTH_WAYS: + assert(false); + return nullptr; + default: + assert(false); + return nullptr; + } +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index 7ddf3c1cf3..d97d95d8f3 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -60,6 +60,8 @@ static std::unique_ptr<Client> CreateClient(const ClientConfig& config) { return config.payload_config().has_bytebuf_params() ? CreateGenericAsyncStreamingClient(config) : CreateAsyncClient(config); + case ClientType::CALLBACK_CLIENT: + return CreateCallbackClient(config); default: abort(); } diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh deleted file mode 100755 index 205c0bf8c5..0000000000 --- a/tools/gce/create_interop_worker.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates an interop worker on GCE. -# IMPORTANT: After this script finishes, there are still some manual -# steps needed there are hard to automatize. -# See go/grpc-jenkins-setup for followup instructions. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-east1-a # canary gateway is reachable from this zone - -INSTANCE_NAME="${1:-grpc-canary-interop2}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image ubuntu-15-10 \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_kokoro_performance_worker.sh b/tools/gce/create_linux_kokoro_performance_worker.sh index d08a1aa2c6..63d54a48bf 100755 --- a/tools/gce/create_linux_kokoro_performance_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker.sh @@ -15,6 +15,13 @@ # Creates a performance worker on GCE to be used on Kokoro. +# IMPORTANT: Instructions for updating +# If the VM configuration / installed software is updated, +# - all existing performance worker VMs need to be updated to reflect the changes +# - a new GCE image named "grpc-performance-kokoro-v1" needs to be created, +# incrementing the version number. +# - kokoro jobs need to be reconfigured to use the new image version + set -ex cd "$(dirname "$0")" @@ -30,7 +37,7 @@ gcloud compute instances create "$INSTANCE_NAME" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ + --image-family ubuntu-1804-lts \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh deleted file mode 100755 index e9033ec443..0000000000 --- a/tools/gce/create_linux_performance_worker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a performance worker on GCE. -# IMPORTANT: After creating the worker, one needs to manually add the pubkey -# of jenkins@the-machine-where-jenkins-starts-perf-tests -# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. -# See tools/run_tests/run_performance_tests.py for details. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-b # this zone allows 32core machines - -INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-32 - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type $MACHINE_TYPE \ - --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ - --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh deleted file mode 100755 index a93d8c5e83..0000000000 --- a/tools/gce/create_linux_worker.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a standard jenkins worker on GCE. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-a - -INSTANCE_NAME="${1:-grpc-jenkins-worker1}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image=ubuntu-1510 \ - --image-project=grpc-testing \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub deleted file mode 100644 index e9853224e1..0000000000 --- a/tools/gce/jenkins_master.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/kokoro_performance.pub b/tools/gce/kokoro_performance.pub index 1154debe78..4eca916b8a 100644 --- a/tools/gce/kokoro_performance.pub +++ b/tools/gce/kokoro_performance.pub @@ -1 +1,4 @@ +# Enable Kokoro CI to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com +# Enable kokoro multi-machine benchmark driver VM to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQ5UEX4AFefec9BKICupFS7x9Hoq4ZyLKy+QX0J31I49ew9mG2AJlr3sp8ql15eX+A2Ml9MKJkmgZGHpJtw+SfvmI94SmomSyiCLAK92sQ85NMzaRdo4b9e30E9nhXnAvAaemvIEQbgCMYFvzk0C8AtXj6+htCrN4jFaLqTCPISJhX3ETc4TgX1qaHQHyl31tdaXHYlITvBDsfokcGcZQnhmCUDtD8wyaSC8GFk9gZbXshkfaYCuuLPPA0vwWGBw+YPbonHsFCsOog1IYSzYPCkIjq8dt6evsusK6Kaoyw/Z+l2kYty2FKTj+wU3l06QMoxwcfNT4WxdhcnVbY71r kbuilder@kokoro-performance-driver diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 4a1e3e608b..b78695d802 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -47,7 +47,6 @@ sudo apt-get install -y \ libtool \ make \ strace \ - pypy \ python-dev \ python-pip \ python-setuptools \ @@ -68,30 +67,34 @@ sudo apt-get install -y google-perftools libgoogle-perftools-dev # netperf sudo apt-get install -y netperf +# required to run kokoro_log_reader.py +sudo apt-get install -y python-psutil python3-psutil + +# gcloud tools, including gsutil +sudo apt-get install -y google-cloud-sdk + # C++ dependencies sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies sudo pip install --upgrade pip==10.0.1 sudo pip install tabulate -sudo pip install google-api-python-client +sudo pip install google-api-python-client oauth2client sudo pip install virtualenv -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - +# pypy is used instead of python for postprocessing benchmark outputs +# because some reports are huge and pypy is much faster. +# TODO(jtattermusch): get rid of pypy once possible, it's hard to +# keep track of all the installed variants of python. +sudo apt-get install -y pypy pypy-dev curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py sudo pypy -m pip install tabulate -sudo pip install google-api-python-client +sudo pypy -m pip install google-api-python-client oauth2client +# TODO(jtattermusch): for some reason, we need psutil installed +# in pypy for kokoro_log_reader.py (strange, because the comand is +# "python kokoro_log_reader.py" and pypy is not the system default) +sudo pypy -m pip install psutil # Node dependencies (nvm has to be installed under user kbuilder) touch .profile @@ -104,31 +107,31 @@ nvm install 4 && npm config set cache /tmp/npm-cache nvm install 5 && npm config set cache /tmp/npm-cache nvm alias default 4 +# C# dependencies +sudo apt-get install -y cmake + # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list sudo apt-get update -sudo apt-get install -y mono-devel nuget +sudo apt-get install -y mono-devel -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 +# C# .NET Core dependencies (https://www.microsoft.com/net/download) +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb + +sudo apt-get install -y apt-transport-https sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# C# 1.0.4 SDK -curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -sudo mkdir -p /opt/dotnet -sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet -sudo ln -s /opt/dotnet/dotnet /usr/local/bin - -# C# .NET dependencies -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb -sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb -sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb -sudo apt-get update && sudo apt-get install -y libicu55 +sudo apt-get install -y dotnet-sdk-2.1 + +# Install .NET Core 1.0.5 Runtime (required to run netcoreapp1.0) +wget -q https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz +mkdir -p dotnet105_download +tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C dotnet105_download +sudo cp -r dotnet105_download/shared/Microsoft.NETCore.App/1.0.5/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ +# To prevent "Failed to initialize CoreCLR, HRESULT: 0x80131500" with .NET Core 1.0.5 runtime +wget -q http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.4_amd64.deb +sudo dpkg -i libicu55_55.1-7ubuntu0.4_amd64.deb # Ruby dependencies gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 @@ -163,7 +166,7 @@ sudo mv composer.phar /usr/local/bin/composer # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 +GO_VERSION=1.10 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz @@ -190,11 +193,22 @@ git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts sudo apt-get install -y python-scipy python-numpy +# Install docker +curl -sSL https://get.docker.com/ | sh +# Enable kbuilder to use docker without sudo: +sudo usermod -aG docker kbuilder + # Add pubkey of Kokoro driver VM to allow SSH # silence false-positive shellcheck warning ("< redirect does not affect sudo") # shellcheck disable=SC2024 sudo tee --append ~kbuilder/.ssh/authorized_keys < kokoro_performance.pub +# Kokoro requires /tmpfs/READY file to exist the directory and file itself should +# be owned by kbuilder. +sudo mkdir /tmpfs +sudo chown kbuilder /tmpfs +touch /tmpfs/READY + # Restart for VM to pick up kernel update echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' sleep 10 diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh deleted file mode 100755 index 7222cef9a2..0000000000 --- a/tools/gce/linux_performance_worker_init.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux performance worker. -# You shouldn't run this script on your own, -# use create_linux_performance_worker.sh instead. - -set -ex - -sudo apt-get update - -# Install Java 8 JDK (to build gRPC Java) -sudo apt-get install -y openjdk-8-jdk -sudo apt-get install -y unzip lsof - -sudo apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libcurl4-openssl-dev \ - libgtest-dev \ - libreadline-dev \ - libssl-dev \ - libtool \ - make \ - strace \ - pypy \ - python-dev \ - python-pip \ - python-setuptools \ - python-yaml \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-yaml \ - telnet \ - unzip \ - wget \ - zip \ - zlib1g-dev - -# perftools -sudo apt-get install -y google-perftools libgoogle-perftools-dev - -# netperf -sudo apt-get install -y netperf - -# C++ dependencies -sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang - -# Python dependencies -sudo pip install --upgrade pip==10.0.1 -sudo pip install tabulate -sudo pip install google-api-python-client -sudo pip install virtualenv - -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - -curl -O https://bootstrap.pypa.io/get-pip.py -sudo pypy get-pip.py -sudo pypy -m pip install tabulate -sudo pip install google-api-python-client - -# Node dependencies (nvm has to be installed under user jenkins) -touch .profile -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -# silence shellcheck warning as it cannot follow the `source` path statically: -# shellcheck disable=SC1090 -source ~/.nvm/nvm.sh -nvm install 0.12 && npm config set cache /tmp/npm-cache -nvm install 4 && npm config set cache /tmp/npm-cache -nvm install 5 && npm config set cache /tmp/npm-cache -nvm alias default 4 - -# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list -sudo apt-get update -sudo apt-get install -y mono-devel nuget - -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 -sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# Ruby dependencies -git clone https://github.com/rbenv/rbenv.git ~/.rbenv -export PATH="$HOME/.rbenv/bin:$PATH" -eval "$(rbenv init -)" - -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" - -rbenv install 2.4.0 -rbenv global 2.4.0 -ruby -v - -# Install bundler (prerequisite for gRPC Ruby) -gem install bundler - -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - -# Java dependencies - nothing as we already have Java JDK 8 - -# Go dependencies -# Currently, the golang package available via apt-get doesn't have the latest go. -# Significant performance improvements with grpc-go have been observed after -# upgrading from go 1.5 to a later version, so a later go version is preferred. -# Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 -OS=linux -ARCH=amd64 -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -# Put go on the PATH, keep the usual installation dir -sudo ln -s /usr/local/go/bin/go /usr/bin/go -rm go$GO_VERSION.$OS-$ARCH.tar.gz - -# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) -sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" -# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar -echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid -# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol -echo 0 | sudo tee /proc/sys/kernel/kptr_restrict - -# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in -# order to not lose perf events or time out -echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb - -# Fetch scripts to generate flame graphs from perf data collected -# on benchmarks -git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph - -# Install scipy and numpy for benchmarking scripts -sudo apt-get install -y python-scipy python-numpy - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. -# silence false-positive shellcheck warning ("< redirect does not affect sudo") -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for VM to pick up kernel update -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 -sudo reboot diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh deleted file mode 100755 index 05855354ff..0000000000 --- a/tools/gce/linux_worker_init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux worker. -# You shouldn't run this script on your own, use create_linux_worker.sh -# instead. - -set -ex - -# Create some swap space -sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 -sudo chmod 600 /swap -sudo mkswap /swap -sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab -sudo swapon -a - -# Typical apt-get maintenance -sudo apt-get update - -# Install JRE -sudo apt-get install -y openjdk-8-jre -sudo apt-get install -y unzip lsof - -# Install Docker -curl -sSL https://get.docker.com/ | sh - -# Setup jenkins user (or the user will already exist bcuz magic) -sudo adduser jenkins --disabled-password || true - -# Enable jenkins to use docker without sudo: -sudo usermod -aG docker jenkins - -# Use "overlay" storage driver for docker -# see https://github.com/grpc/grpc/issues/4988 -printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json - -# Install pip and Google API library to enable using GCP services -sudo apt-get install -y python-pip -sudo pip install google-api-python-client - -# Install RVM -# TODO(jtattermusch): why is RVM needed? -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby - -# Upgrade Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. - -# disable superfluous warning by shellcheck: -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for docker to pick up the config changes. -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 - -sudo reboot diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8749593ed5..b27f5b5037 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7836,6 +7836,7 @@ "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/client_async.cc", + "test/cpp/qps/client_callback.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", "test/cpp/qps/driver.h", diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 9752e741fb..1d639edb82 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -778,7 +778,7 @@ def cloud_to_prod_jobspec(language, if transport_security == 'tls': transport_security_options = ['--use_tls=true'] elif transport_security == 'google_default_credentials' and str( - language) in ['c++', 'go']: + language) in ['c++', 'go', 'java', 'javaokhttp']: transport_security_options = [ '--custom_credentials_type=google_default_credentials' ] @@ -1323,7 +1323,9 @@ try: service_account_key_file, transport_security='tls') jobs.append(tls_test_job) - if str(language) in ['c++', 'go']: + if str(language) in [ + 'c++', 'go', 'java', 'javaokhttp' + ]: google_default_creds_test_job = cloud_to_prod_jobspec( language, test_case, @@ -1376,8 +1378,7 @@ try: transport_security='tls') jobs.append(tls_test_job) if str(language) in [ - 'c++', - 'go', + 'go' ]: # Add more languages to the list to turn on tests. google_default_creds_test_job = cloud_to_prod_jobspec( language, diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c9b4c8b28b..48c40ad724 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -825,6 +825,12 @@ class PythonLanguage(object): minor='6', bits=bits, config_vars=config_vars) + python37_config = _python_config_generator( + name='py37', + major='3', + minor='7', + bits=bits, + config_vars=config_vars) pypy27_config = _pypy_config_generator( name='pypy', major='2', config_vars=config_vars) pypy32_config = _pypy_config_generator( @@ -846,6 +852,8 @@ class PythonLanguage(object): return (python35_config,) elif args.compiler == 'python3.6': return (python36_config,) + elif args.compiler == 'python3.7': + return (python37_config,) elif args.compiler == 'pypy': return (pypy27_config,) elif args.compiler == 'pypy3': @@ -858,6 +866,7 @@ class PythonLanguage(object): python34_config, python35_config, python36_config, + python37_config, ) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -1360,9 +1369,9 @@ argp.add_argument( choices=[ 'default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc7.2', 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'clang7.0', - 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', - 'python_alpine', 'all_the_cpythons', 'electron1.3', 'electron1.6', - 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'python3.7', 'pypy', + 'pypy3', 'python_alpine', 'all_the_cpythons', 'electron1.3', + 'electron1.6', 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' ], default='default', help= |