diff options
166 files changed, 1566 insertions, 834 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c28eb974f8..0a7141c1be 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,8 +2,8 @@ # Uses OWNERS files in different modules throughout the # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @a11r -/bazel/** @nicolasnoble @dgquintas @a11r @vjpai -/cmake/** @jtattermusch @nicolasnoble @mehrdada -/src/core/ext/filters/client_channel/** @markdroth @dgquintas @AspirinSJL -/tools/dockerfile/** @jtattermusch @mehrdada @nicolasnoble +/bazel/** @nicolasnoble @jtattermusch @a11r @vjpai +/cmake/** @jtattermusch @nicolasnoble @apolcyn +/src/core/ext/filters/client_channel/** @markdroth @apolcyn @AspirinSJL +/tools/dockerfile/** @jtattermusch @apolcyn @nicolasnoble /tools/run_tests/performance/** @ncteisen @apolcyn @jtattermusch @@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "gao" +g_stands_for = "gizmo" core_version = "6.0.0-dev" -version = "1.16.0-dev" +version = "1.17.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", @@ -847,8 +847,8 @@ grpc_cc_library( "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", - "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/block_annotate.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", @@ -1550,9 +1550,9 @@ grpc_cc_library( deps = [ "alts_util", "grpc_base", + "grpc_shadow_boringssl", "grpc_transport_chttp2_alpn", "tsi", - "grpc_shadow_boringssl", ], ) @@ -1812,8 +1812,8 @@ grpc_cc_library( deps = [ "gpr", "grpc_base", - "tsi_interface", "grpc_shadow_boringssl", + "tsi_interface", ], ) @@ -1908,9 +1908,9 @@ grpc_cc_library( "alts_util", "gpr", "grpc_base", + "grpc_shadow_boringssl", "grpc_transport_chttp2_client_insecure", "tsi_interface", - "grpc_shadow_boringssl", ], ) @@ -2137,10 +2137,10 @@ grpc_cc_library( grpc_cc_library( name = "grpc_opencensus_plugin", srcs = [ + "src/core/ext/filters/census/grpc_context.cc", "src/cpp/ext/filters/census/channel_filter.cc", "src/cpp/ext/filters/census/client_filter.cc", "src/cpp/ext/filters/census/context.cc", - "src/core/ext/filters/census/grpc_context.cc", "src/cpp/ext/filters/census/grpc_plugin.cc", "src/cpp/ext/filters/census/measures.cc", "src/cpp/ext/filters/census/rpc_encoding.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 66799c456f..69a9d1c5e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.16.0-dev") +set(PACKAGE_VERSION "1.17.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") @@ -5249,6 +5249,7 @@ add_library(qps ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/worker_service.grpc.pb.h test/cpp/qps/benchmark_config.cc test/cpp/qps/client_async.cc + test/cpp/qps/client_callback.cc test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc test/cpp/qps/parse_json.cc @@ -437,9 +437,9 @@ E = @echo Q = @ endif -CORE_VERSION = 6.0.0-dev -CPP_VERSION = 1.16.0-dev -CSHARP_VERSION = 1.16.0-dev +CORE_VERSION = 7.0.0-dev +CPP_VERSION = 1.17.0-dev +CSHARP_VERSION = 1.17.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) @@ -487,7 +487,7 @@ SHARED_EXT_CORE = dll SHARED_EXT_CPP = dll SHARED_EXT_CSHARP = dll SHARED_PREFIX = -SHARED_VERSION_CORE = -6 +SHARED_VERSION_CORE = -7 SHARED_VERSION_CPP = -1 SHARED_VERSION_CSHARP = -1 else ifeq ($(SYSTEM),Darwin) @@ -792,7 +792,7 @@ PC_DESCRIPTION = high performance general RPC framework without SSL PC_CFLAGS = PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) -PC_LIB = -lgrpc +PC_LIB = -lgrpc_unsecure GRPC_UNSECURE_PC_FILE := $(CORE_PC_TEMPLATE) PROTOBUF_PKG_CONFIG = false @@ -863,7 +863,7 @@ PC_DESCRIPTION = C++ wrapper for gRPC without SSL PC_CFLAGS = PC_REQUIRES_PRIVATE = grpc_unsecure $(PC_REQUIRES_GRPCXX) PC_LIBS_PRIVATE = $(PC_LIBS_GRPCXX) -PC_LIB = -lgrpc++ +PC_LIB = -lgrpc++_unsecure GRPCXX_UNSECURE_PC_FILE := $(CPP_PC_TEMPLATE) ifeq ($(MAKECMDGOALS),clean) @@ -3003,7 +3003,7 @@ install-shared_c: shared_c strip-shared_c install-pkg-config_c ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libaddress_sorting.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so.6 + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so.7 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libaddress_sorting.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3012,7 +3012,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgpr.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.6 + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so.7 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgpr.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3021,7 +3021,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3030,7 +3030,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_cronet.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_cronet.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE)" @@ -3039,7 +3039,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE)-dll.a $(prefix)/lib/libgrpc_unsecure.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(prefix)/lib/libgrpc_unsecure.so endif ifneq ($(SYSTEM),MINGW32) @@ -3056,7 +3056,7 @@ install-shared_cxx: shared_cxx strip-shared_cxx install-shared_c install-pkg-con ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" @@ -3065,7 +3065,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_cronet$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_cronet.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_cronet.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" @@ -3074,7 +3074,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_error_details$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_error_details.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc++_error_details$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_error_details.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" @@ -3083,7 +3083,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_reflection$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_reflection.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_reflection.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" @@ -3092,7 +3092,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpc++_unsecure.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc++_unsecure$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpc++_unsecure.so endif $(E) "[INSTALL] Installing $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP)" @@ -3101,7 +3101,7 @@ endif ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz$(SHARED_VERSION_CPP)-dll.a $(prefix)/lib/libgrpcpp_channelz.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpcpp_channelz.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpcpp_channelz.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpcpp_channelz$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP) $(prefix)/lib/libgrpcpp_channelz.so endif ifneq ($(SYSTEM),MINGW32) @@ -3118,7 +3118,7 @@ install-shared_csharp: shared_csharp strip-shared_csharp ifeq ($(SYSTEM),MINGW32) $(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext$(SHARED_VERSION_CSHARP)-dll.a $(prefix)/lib/libgrpc_csharp_ext.a else ifneq ($(SYSTEM),Darwin) - $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so.6 + $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SHARED_EXT_CSHARP) $(prefix)/lib/libgrpc_csharp_ext.so endif ifneq ($(SYSTEM),MINGW32) @@ -3209,8 +3209,8 @@ $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): ifeq ($(SYSTEM),Darwin) $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.6 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.6 + $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libaddress_sorting.so.7 -o $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBADDRESS_SORTING_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so.7 $(Q) ln -sf $(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libaddress_sorting$(SHARED_VERSION_CORE).so endif endif @@ -3404,8 +3404,8 @@ $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGPR_OB ifeq ($(SYSTEM),Darwin) $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.6 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.6 + $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgpr.so.7 -o $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGPR_OBJS) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so.7 $(Q) ln -sf $(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgpr$(SHARED_VERSION_CORE).so endif endif @@ -3831,8 +3831,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(LIBGRPC_ ifeq ($(SYSTEM),Darwin) $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.6 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.6 + $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc.so.7 -o $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc$(SHARED_VERSION_CORE).so endif endif @@ -4196,8 +4196,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $(L ifeq ($(SYSTEM),Darwin) $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_cronet.so.6 -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so.6 + $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_cronet.so.7 -o $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_CRONET_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(OPENSSL_MERGE_LIBS) $(LDLIBS_SECURE) $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_cronet$(SHARED_VERSION_CORE).so endif endif @@ -5110,8 +5110,8 @@ $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE): $ ifeq ($(SYSTEM),Darwin) $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -install_name $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) -dynamiclib -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) else - $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.6 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) - $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.6 + $(Q) $(LD) $(LDFLAGS) -L$(LIBDIR)/$(CONFIG) -shared -Wl,-soname,libgrpc_unsecure.so.7 -o $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBGRPC_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgpr.a $(ZLIB_MERGE_LIBS) $(CARES_MERGE_LIBS) $(ADDRESS_SORTING_MERGE_LIBS) $(LDLIBS) + $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so.7 $(Q) ln -sf $(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure$(SHARED_VERSION_CORE).so endif endif @@ -7510,6 +7510,7 @@ LIBQPS_SRC = \ $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc \ test/cpp/qps/benchmark_config.cc \ test/cpp/qps/client_async.cc \ + test/cpp/qps/client_callback.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ test/cpp/qps/parse_json.cc \ @@ -7566,6 +7567,7 @@ endif endif $(OBJDIR)/$(CONFIG)/test/cpp/qps/benchmark_config.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_callback.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/parse_json.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.pb.cc $(GENDIR)/src/proto/grpc/testing/benchmark_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.pb.cc $(GENDIR)/src/proto/grpc/testing/report_qps_scenario_service.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.pb.cc $(GENDIR)/src/proto/grpc/testing/worker_service.grpc.pb.cc @@ -24917,6 +24919,7 @@ test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) test/cpp/qps/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) +test/cpp/qps/client_callback.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) test/cpp/qps/parse_json.cc: $(OPENSSL_DEP) @@ -34,10 +34,11 @@ pip_import( load("@grpc_python_dependencies//:requirements.bzl", "pip_install") pip_install() +# NOTE(https://github.com/pubref/rules_protobuf/pull/196): Switch to upstream repo after this gets merged. git_repository( name="org_pubref_rules_protobuf", - remote="https://github.com/pubref/rules_protobuf", - tag="v0.8.2", + remote="https://github.com/ghostwriternr/rules_protobuf", + tag="v0.8.2.1-alpha", ) load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") diff --git a/bazel/OWNERS b/bazel/OWNERS index 613ba36be8..30813d1e7a 100644 --- a/bazel/OWNERS +++ b/bazel/OWNERS @@ -1,6 +1,6 @@ set noparent @nicolasnoble -@dgquintas +@jtattermusch @a11r @vjpai diff --git a/build.yaml b/build.yaml index 6f81bf7afc..f994801408 100644 --- a/build.yaml +++ b/build.yaml @@ -12,9 +12,9 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 6.0.0-dev - g_stands_for: gao - version: 1.16.0-dev + core_version: 7.0.0-dev + g_stands_for: gizmo + version: 1.17.0-dev filegroups: - name: alts_proto headers: @@ -1965,6 +1965,7 @@ libs: - src/proto/grpc/testing/worker_service.proto - test/cpp/qps/benchmark_config.cc - test/cpp/qps/client_async.cc + - test/cpp/qps/client_callback.cc - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc - test/cpp/qps/parse_json.cc diff --git a/build_config.rb b/build_config.rb index 644c80fdb6..ab06a13747 100644 --- a/build_config.rb +++ b/build_config.rb @@ -13,5 +13,5 @@ # limitations under the License. module GrpcBuildConfig - CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-6.dll' + CORE_WINDOWS_DLL = '/tmp/libs/opt/grpc-7.dll' end diff --git a/cmake/OWNERS b/cmake/OWNERS index 21981a7c55..a6181f6209 100644 --- a/cmake/OWNERS +++ b/cmake/OWNERS @@ -1,4 +1,4 @@ set noparent @jtattermusch @nicolasnoble -@mehrdada +@apolcyn diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index c89a6cb572..a5a8efb21c 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -15,4 +15,5 @@ - 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x) - 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/v1.14.x) - 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/v1.15.x) -- 1.16 'g' stands for ['gao'](https://github.com/grpc/grpc/tree/master) +- 1.16 'g' stands for ['gao'](https://github.com/grpc/grpc/tree/v1.16.x) +- 1.17 'g' stands for ['gizmo'](https://github.com/grpc/grpc/tree/master) diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md index 3de1ecac00..20fe3cef08 100644 --- a/doc/ssl-performance.md +++ b/doc/ssl-performance.md @@ -28,7 +28,7 @@ Language | From source | Platform | Uses assembly optimizations C# | n/a | Linux, 64bit | :heavy_check_mark: C# | n/a | Linux, 32bit | :x: C# | n/a | MacOS | :heavy_check_mark: -C# | n/a | Windows | :x: +C# | n/a | Windows | :heavy_check_mark: Node.JS | n/a | Linux | :heavy_check_mark: Node.JS | n/a | MacOS | :heavy_check_mark: Node.JS | n/a | Windows | :x: diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index f2d9a814c3..3cc19d5908 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.16.0-dev' + # version = '1.17.0-dev' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.16.0-dev' + grpc_version = '1.17.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d575ca666a..8bb2311414 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.16.0-dev' + version = '1.17.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 75fd592e75..693b873d14 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.16.0-dev' + version = '1.17.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index ecd786e7b3..fd590023e1 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.16.0-dev' + version = '1.17.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 9164b2169a..5e513cb127 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.16.0-dev' + version = '1.17.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' @@ -75,12 +75,12 @@ EXPORTS grpc_resource_quota_arg_vtable grpc_channelz_get_top_channels grpc_channelz_get_servers + grpc_channelz_get_server_sockets grpc_channelz_get_channel grpc_channelz_get_subchannel grpc_channelz_get_socket grpc_insecure_channel_create_from_fd grpc_server_add_insecure_channel_from_fd - grpc_use_signal grpc_auth_property_iterator_next grpc_auth_context_property_iterator grpc_auth_context_peer_identity @@ -1717,6 +1717,7 @@ 'src/proto/grpc/testing/worker_service.proto', 'test/cpp/qps/benchmark_config.cc', 'test/cpp/qps/client_async.cc', + 'test/cpp/qps/client_callback.cc', 'test/cpp/qps/client_sync.cc', 'test/cpp/qps/driver.cc', 'test/cpp/qps/parse_json.cc', diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index a9beee1c9e..02ab6e8ba4 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -503,6 +503,10 @@ GRPCAPI char* grpc_channelz_get_top_channels(intptr_t start_channel_id); /* Gets all servers that exist in the process. */ GRPCAPI char* grpc_channelz_get_servers(intptr_t start_server_id); +/* Gets all server sockets that exist in the server. */ +GRPCAPI char* grpc_channelz_get_server_sockets(intptr_t server_id, + intptr_t start_socket_id); + /* Returns a single Channel, or else a NOT_FOUND code. The returned string is allocated and must be freed by the application. */ GRPCAPI char* grpc_channelz_get_channel(intptr_t channel_id); diff --git a/include/grpc/grpc_posix.h b/include/grpc/grpc_posix.h index 5f1ada5aaf..fbce5e13c1 100644 --- a/include/grpc/grpc_posix.h +++ b/include/grpc/grpc_posix.h @@ -52,14 +52,6 @@ GRPCAPI grpc_channel* grpc_insecure_channel_create_from_fd( GRPCAPI void grpc_server_add_insecure_channel_from_fd(grpc_server* server, void* reserved, int fd); -/** GRPC Core POSIX library may internally use signals to optimize some work. - The library uses (SIGRTMIN + 6) signal by default. Use this API to instruct - the library to use a different signal i.e 'signum' instead. - Note: - - To prevent GRPC library from using any signals, pass a 'signum' of -1 - - This API is optional but if called, it MUST be called before grpc_init() */ -GRPCAPI void grpc_use_signal(int signum); - #ifdef __cplusplus } #endif diff --git a/package.xml b/package.xml index 586bab004f..be3fd0d4c2 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ <date>2018-01-19</date> <time>16:06:07</time> <version> - <release>1.16.0dev</release> - <api>1.16.0dev</api> + <release>1.17.0dev</release> + <api>1.17.0dev</api> </version> <stability> <release>beta</release> diff --git a/src/core/ext/filters/client_channel/OWNERS b/src/core/ext/filters/client_channel/OWNERS index c8760d947b..d38970e0fa 100644 --- a/src/core/ext/filters/client_channel/OWNERS +++ b/src/core/ext/filters/client_channel/OWNERS @@ -1,4 +1,4 @@ set noparent @markdroth -@dgquintas +@apolcyn @AspirinSJL diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 388736b60a..bb3ea400d1 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -933,11 +933,6 @@ typedef struct client_channel_call_data { grpc_closure pick_closure; grpc_closure pick_cancel_closure; - // state needed to support channelz interception of recv trailing metadata. - grpc_closure recv_trailing_metadata_ready_channelz; - grpc_closure* original_recv_trailing_metadata; - grpc_metadata_batch* recv_trailing_metadata; - grpc_polling_entity* pollent; bool pollent_added_to_interested_parties; @@ -999,8 +994,6 @@ static void start_internal_recv_trailing_metadata(grpc_call_element* elem); static void on_complete(void* arg, grpc_error* error); static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored); static void start_pick_locked(void* arg, grpc_error* ignored); -static void maybe_intercept_recv_trailing_metadata_for_channelz( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch); // // send op data caching @@ -1299,7 +1292,6 @@ static void pending_batches_resume(grpc_call_element* elem) { pending_batch* pending = &calld->pending_batches[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr) { - maybe_intercept_recv_trailing_metadata_for_channelz(elem, batch); batch->handler_private.extra_arg = calld->subchannel_call; GRPC_CLOSURE_INIT(&batch->handler_private.closure, resume_pending_batch_in_call_combiner, batch, @@ -1977,15 +1969,6 @@ static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { batch_data->batch.payload->recv_trailing_metadata.recv_trailing_metadata; get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, &server_pushback_md); - grpc_core::channelz::SubchannelNode* channelz_subchannel = - calld->pick.connected_subchannel->channelz_subchannel(); - if (channelz_subchannel != nullptr) { - if (status == GRPC_STATUS_OK) { - channelz_subchannel->RecordCallSucceeded(); - } else { - channelz_subchannel->RecordCallFailed(); - } - } if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call finished, status=%s", chand, calld, grpc_status_code_to_string(status)); @@ -2590,69 +2573,6 @@ static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored) { } // -// Channelz -// - -static void recv_trailing_metadata_ready_channelz(void* arg, - grpc_error* error) { - grpc_call_element* elem = static_cast<grpc_call_element*>(arg); - channel_data* chand = static_cast<channel_data*>(elem->channel_data); - call_data* calld = static_cast<call_data*>(elem->call_data); - if (grpc_client_channel_trace.enabled()) { - gpr_log(GPR_INFO, - "chand=%p calld=%p: got recv_trailing_metadata_ready_channelz, " - "error=%s", - chand, calld, grpc_error_string(error)); - } - GPR_ASSERT(calld->recv_trailing_metadata != nullptr); - grpc_status_code status = GRPC_STATUS_OK; - grpc_metadata_batch* md_batch = calld->recv_trailing_metadata; - get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, nullptr); - grpc_core::channelz::SubchannelNode* channelz_subchannel = - calld->pick.connected_subchannel->channelz_subchannel(); - GPR_ASSERT(channelz_subchannel != nullptr); - if (status == GRPC_STATUS_OK) { - channelz_subchannel->RecordCallSucceeded(); - } else { - channelz_subchannel->RecordCallFailed(); - } - calld->recv_trailing_metadata = nullptr; - GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata, error); -} - -// If channelz is enabled, intercept recv_trailing so that we may check the -// status and associate it to a subchannel. -// Returns true if callback was intercepted, false otherwise. -static void maybe_intercept_recv_trailing_metadata_for_channelz( - grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { - call_data* calld = static_cast<call_data*>(elem->call_data); - // only intercept payloads with recv trailing. - if (!batch->recv_trailing_metadata) { - return; - } - // only add interceptor is channelz is enabled. - if (calld->pick.connected_subchannel->channelz_subchannel() == nullptr) { - return; - } - if (grpc_client_channel_trace.enabled()) { - gpr_log(GPR_INFO, - "calld=%p batch=%p: intercepting recv trailing for channelz", calld, - batch); - } - GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready_channelz, - recv_trailing_metadata_ready_channelz, elem, - grpc_schedule_on_exec_ctx); - // save some state needed for the interception callback. - GPR_ASSERT(calld->recv_trailing_metadata == nullptr); - calld->recv_trailing_metadata = - batch->payload->recv_trailing_metadata.recv_trailing_metadata; - calld->original_recv_trailing_metadata = - batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; - batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = - &calld->recv_trailing_metadata_ready_channelz; -} - -// // LB pick // @@ -3289,8 +3209,9 @@ static void try_to_connect_locked(void* arg, grpc_error* error_ignored) { } void grpc_client_channel_populate_child_refs( - grpc_channel_element* elem, grpc_core::ChildRefsList* child_subchannels, - grpc_core::ChildRefsList* child_channels) { + grpc_channel_element* elem, + grpc_core::channelz::ChildRefsList* child_subchannels, + grpc_core::channelz::ChildRefsList* child_channels) { channel_data* chand = static_cast<channel_data*>(elem->channel_data); if (chand->lb_policy != nullptr) { chand->lb_policy->FillChildRefsForChannelz(child_subchannels, diff --git a/src/core/ext/filters/client_channel/client_channel.h b/src/core/ext/filters/client_channel/client_channel.h index 0b44a17562..d64faaabd2 100644 --- a/src/core/ext/filters/client_channel/client_channel.h +++ b/src/core/ext/filters/client_channel/client_channel.h @@ -41,8 +41,9 @@ extern grpc_core::TraceFlag grpc_client_channel_trace; extern const grpc_channel_filter grpc_client_channel_filter; void grpc_client_channel_populate_child_refs( - grpc_channel_element* elem, grpc_core::ChildRefsList* child_subchannels, - grpc_core::ChildRefsList* child_channels); + grpc_channel_element* elem, + grpc_core::channelz::ChildRefsList* child_subchannels, + grpc_core::channelz::ChildRefsList* child_channels); grpc_connectivity_state grpc_client_channel_check_connectivity_state( grpc_channel_element* elem, int try_to_connect); diff --git a/src/core/ext/filters/client_channel/client_channel_channelz.h b/src/core/ext/filters/client_channel/client_channel_channelz.h index 8ce331e529..8a5c3e7e5e 100644 --- a/src/core/ext/filters/client_channel/client_channel_channelz.h +++ b/src/core/ext/filters/client_channel/client_channel_channelz.h @@ -25,17 +25,10 @@ #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channel_trace.h" #include "src/core/lib/channel/channelz.h" -#include "src/core/lib/gprpp/inlined_vector.h" typedef struct grpc_subchannel grpc_subchannel; namespace grpc_core { - -// TODO(ncteisen), this only contains the uuids of the children for now, -// since that is all that is strictly needed. In a future enhancement we will -// add human readable names as in the channelz.proto -typedef InlinedVector<intptr_t, 10> ChildRefsList; - namespace channelz { // Subtype of ChannelNode that overrides and provides client_channel specific diff --git a/src/core/ext/filters/client_channel/http_connect_handshaker.cc b/src/core/ext/filters/client_channel/http_connect_handshaker.cc index 7ce8da8c00..bfabc68c66 100644 --- a/src/core/ext/filters/client_channel/http_connect_handshaker.cc +++ b/src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -351,6 +351,7 @@ static grpc_handshaker* grpc_http_connect_handshaker_create() { static void handshaker_factory_add_handshakers( grpc_handshaker_factory* factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add(handshake_mgr, grpc_http_connect_handshaker_create()); diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 3c0a9c1118..21f80b7b94 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -151,9 +151,9 @@ class LoadBalancingPolicy /// LB policy's referenced children. This is not invoked from the /// client_channel's combiner. The implementation is responsible for /// providing its own synchronization. - virtual void FillChildRefsForChannelz(ChildRefsList* child_subchannels, - ChildRefsList* child_channels) - GRPC_ABSTRACT; + virtual void FillChildRefsForChannelz( + channelz::ChildRefsList* child_subchannels, + channelz::ChildRefsList* child_channels) GRPC_ABSTRACT; void Orphan() override { // Invoke ShutdownAndUnrefLocked() inside of the combiner. @@ -212,8 +212,8 @@ class LoadBalancingPolicy // Dummy classes needed for alignment issues. // See https://github.com/grpc/grpc/issues/16032 for context. // TODO(ncteisen): remove this as soon as the issue is resolved. - ChildRefsList dummy_list_foo; - ChildRefsList dummy_list_bar; + channelz::ChildRefsList dummy_list_foo; + channelz::ChildRefsList dummy_list_bar; }; } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 0061190016..5511df7a27 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -136,8 +136,9 @@ class GrpcLb : public LoadBalancingPolicy { void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; void ResetBackoffLocked() override; - void FillChildRefsForChannelz(ChildRefsList* child_subchannels, - ChildRefsList* child_channels) override; + void FillChildRefsForChannelz( + channelz::ChildRefsList* child_subchannels, + channelz::ChildRefsList* child_channels) override; private: /// Linked list of pending pick requests. It stores all information needed to @@ -1258,8 +1259,9 @@ bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { return pick_done; } -void GrpcLb::FillChildRefsForChannelz(ChildRefsList* child_subchannels, - ChildRefsList* child_channels) { +void GrpcLb::FillChildRefsForChannelz( + channelz::ChildRefsList* child_subchannels, + channelz::ChildRefsList* child_channels) { // delegate to the RoundRobin to fill the children subchannels. rr_policy_->FillChildRefsForChannelz(child_subchannels, child_channels); MutexLock lock(&lb_channel_mu_); @@ -1489,7 +1491,7 @@ void GrpcLb::OnBalancerChannelConnectivityChangedLocked(void* arg, grpclb_policy->lb_call_backoff_.Reset(); grpclb_policy->StartBalancerCallLocked(); } - // Fall through. + // fallthrough case GRPC_CHANNEL_SHUTDOWN: done: grpclb_policy->watching_lb_channel_ = false; diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index ed8cc60ea1..f4dca146f7 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -59,8 +59,8 @@ class PickFirst : public LoadBalancingPolicy { void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; void ResetBackoffLocked() override; - void FillChildRefsForChannelz(ChildRefsList* child_subchannels, - ChildRefsList* ignored) override; + void FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels, + channelz::ChildRefsList* ignored) override; private: ~PickFirst(); @@ -147,8 +147,8 @@ class PickFirst : public LoadBalancingPolicy { /// Lock and data used to capture snapshots of this channels child /// channels and subchannels. This data is consumed by channelz. gpr_mu child_refs_mu_; - ChildRefsList child_subchannels_; - ChildRefsList child_channels_; + channelz::ChildRefsList child_subchannels_; + channelz::ChildRefsList child_channels_; }; PickFirst::PickFirst(const Args& args) : LoadBalancingPolicy(args) { @@ -300,7 +300,8 @@ void PickFirst::NotifyOnStateChangeLocked(grpc_connectivity_state* current, } void PickFirst::FillChildRefsForChannelz( - ChildRefsList* child_subchannels_to_fill, ChildRefsList* ignored) { + channelz::ChildRefsList* child_subchannels_to_fill, + channelz::ChildRefsList* ignored) { MutexLock lock(&child_refs_mu_); for (size_t i = 0; i < child_subchannels_.size(); ++i) { // TODO(ncteisen): implement a de dup loop that is not O(n^2). Might @@ -320,7 +321,7 @@ void PickFirst::FillChildRefsForChannelz( } void PickFirst::UpdateChildRefsLocked() { - ChildRefsList cs; + channelz::ChildRefsList cs; if (subchannel_list_ != nullptr) { subchannel_list_->PopulateChildRefsList(&cs); } diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 8dd5820bae..e9ed85cf66 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -70,8 +70,8 @@ class RoundRobin : public LoadBalancingPolicy { void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; void ResetBackoffLocked() override; - void FillChildRefsForChannelz(ChildRefsList* child_subchannels, - ChildRefsList* ignored) override; + void FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels, + channelz::ChildRefsList* ignored) override; private: ~RoundRobin(); @@ -223,8 +223,8 @@ class RoundRobin : public LoadBalancingPolicy { /// Lock and data used to capture snapshots of this channel's child /// channels and subchannels. This data is consumed by channelz. gpr_mu child_refs_mu_; - ChildRefsList child_subchannels_; - ChildRefsList child_channels_; + channelz::ChildRefsList child_subchannels_; + channelz::ChildRefsList child_channels_; }; RoundRobin::RoundRobin(const Args& args) : LoadBalancingPolicy(args) { @@ -402,7 +402,8 @@ bool RoundRobin::PickLocked(PickState* pick, grpc_error** error) { } void RoundRobin::FillChildRefsForChannelz( - ChildRefsList* child_subchannels_to_fill, ChildRefsList* ignored) { + channelz::ChildRefsList* child_subchannels_to_fill, + channelz::ChildRefsList* ignored) { MutexLock lock(&child_refs_mu_); for (size_t i = 0; i < child_subchannels_.size(); ++i) { // TODO(ncteisen): implement a de dup loop that is not O(n^2). Might @@ -422,7 +423,7 @@ void RoundRobin::FillChildRefsForChannelz( } void RoundRobin::UpdateChildRefsLocked() { - ChildRefsList cs; + channelz::ChildRefsList cs; if (subchannel_list_ != nullptr) { subchannel_list_->PopulateChildRefsList(&cs); } diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 5e8682e056..e0e0e1e638 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -201,7 +201,7 @@ class SubchannelList bool shutting_down() const { return shutting_down_; } // Populates refs_list with the uuids of this SubchannelLists's subchannels. - void PopulateChildRefsList(ChildRefsList* refs_list) { + void PopulateChildRefsList(channelz::ChildRefsList* refs_list) { for (size_t i = 0; i < subchannels_.size(); ++i) { if (subchannels_[i].subchannel() != nullptr) { grpc_core::channelz::SubchannelNode* subchannel_node = diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 4a668b0fa7..3a1c14c6f1 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -49,6 +49,8 @@ #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/connectivity_state.h" +#include "src/core/lib/transport/error_utils.h" +#include "src/core/lib/transport/status_metadata.h" #define INTERNAL_REF_BITS 16 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1)) @@ -144,6 +146,11 @@ struct grpc_subchannel { struct grpc_subchannel_call { grpc_core::ConnectedSubchannel* connection; grpc_closure* schedule_closure_after_destroy; + // state needed to support channelz interception of recv trailing metadata. + grpc_closure recv_trailing_metadata_ready; + grpc_closure* original_recv_trailing_metadata; + grpc_metadata_batch* recv_trailing_metadata; + grpc_millis deadline; }; #define SUBCHANNEL_CALL_TO_CALL_STACK(call) \ @@ -652,7 +659,7 @@ static bool publish_transport_locked(grpc_subchannel* c) { /* publish */ c->connected_subchannel.reset(grpc_core::New<grpc_core::ConnectedSubchannel>( - stk, c->channelz_subchannel.get(), socket_uuid)); + stk, c->channelz_subchannel, socket_uuid)); gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p", c->connected_subchannel.get(), c); @@ -745,9 +752,68 @@ void grpc_subchannel_call_unref( GRPC_CALL_STACK_UNREF(SUBCHANNEL_CALL_TO_CALL_STACK(c), REF_REASON); } +// Sets *status based on md_batch and error. +static void get_call_status(grpc_subchannel_call* call, + grpc_metadata_batch* md_batch, grpc_error* error, + grpc_status_code* status) { + if (error != GRPC_ERROR_NONE) { + grpc_error_get_status(error, call->deadline, status, nullptr, nullptr, + nullptr); + } else { + GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr); + *status = + grpc_get_status_code_from_metadata(md_batch->idx.named.grpc_status->md); + } + GRPC_ERROR_UNREF(error); +} + +static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { + grpc_subchannel_call* call = static_cast<grpc_subchannel_call*>(arg); + GPR_ASSERT(call->recv_trailing_metadata != nullptr); + grpc_status_code status = GRPC_STATUS_OK; + grpc_metadata_batch* md_batch = call->recv_trailing_metadata; + get_call_status(call, md_batch, GRPC_ERROR_REF(error), &status); + grpc_core::channelz::SubchannelNode* channelz_subchannel = + call->connection->channelz_subchannel(); + GPR_ASSERT(channelz_subchannel != nullptr); + if (status == GRPC_STATUS_OK) { + channelz_subchannel->RecordCallSucceeded(); + } else { + channelz_subchannel->RecordCallFailed(); + } + GRPC_CLOSURE_RUN(call->original_recv_trailing_metadata, + GRPC_ERROR_REF(error)); +} + +// If channelz is enabled, intercept recv_trailing so that we may check the +// status and associate it to a subchannel. +static void maybe_intercept_recv_trailing_metadata( + grpc_subchannel_call* call, grpc_transport_stream_op_batch* batch) { + // only intercept payloads with recv trailing. + if (!batch->recv_trailing_metadata) { + return; + } + // only add interceptor is channelz is enabled. + if (call->connection->channelz_subchannel() == nullptr) { + return; + } + GRPC_CLOSURE_INIT(&call->recv_trailing_metadata_ready, + recv_trailing_metadata_ready, call, + grpc_schedule_on_exec_ctx); + // save some state needed for the interception callback. + GPR_ASSERT(call->recv_trailing_metadata == nullptr); + call->recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata; + call->original_recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &call->recv_trailing_metadata_ready; +} + void grpc_subchannel_call_process_op(grpc_subchannel_call* call, grpc_transport_stream_op_batch* batch) { GPR_TIMER_SCOPE("grpc_subchannel_call_process_op", 0); + maybe_intercept_recv_trailing_metadata(call, batch); grpc_call_stack* call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call); grpc_call_element* top_elem = grpc_call_stack_element(call_stack, 0); GRPC_CALL_LOG_OP(GPR_INFO, top_elem, batch); @@ -822,10 +888,12 @@ namespace grpc_core { ConnectedSubchannel::ConnectedSubchannel( grpc_channel_stack* channel_stack, - channelz::SubchannelNode* channelz_subchannel, intptr_t socket_uuid) + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel, + intptr_t socket_uuid) : RefCountedWithTracing<ConnectedSubchannel>(&grpc_trace_stream_refcount), channel_stack_(channel_stack), - channelz_subchannel_(channelz_subchannel), + channelz_subchannel_(std::move(channelz_subchannel)), socket_uuid_(socket_uuid) {} ConnectedSubchannel::~ConnectedSubchannel() { @@ -872,6 +940,7 @@ grpc_error* ConnectedSubchannel::CreateCall(const CallArgs& args, Ref(DEBUG_LOCATION, "subchannel_call"); connection.release(); // Ref is passed to the grpc_subchannel_call object. (*call)->connection = this; + (*call)->deadline = args.deadline; const grpc_call_element_args call_args = { callstk, /* call_stack */ nullptr, /* server_transport_data */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 699f93a8e7..c53b13e37e 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -85,9 +85,11 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { size_t parent_data_size; }; - explicit ConnectedSubchannel(grpc_channel_stack* channel_stack, - channelz::SubchannelNode* channelz_subchannel, - intptr_t socket_uuid); + explicit ConnectedSubchannel( + grpc_channel_stack* channel_stack, + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel, + intptr_t socket_uuid); ~ConnectedSubchannel(); grpc_channel_stack* channel_stack() { return channel_stack_; } @@ -97,15 +99,16 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { void Ping(grpc_closure* on_initiate, grpc_closure* on_ack); grpc_error* CreateCall(const CallArgs& args, grpc_subchannel_call** call); channelz::SubchannelNode* channelz_subchannel() { - return channelz_subchannel_; + return channelz_subchannel_.get(); } intptr_t socket_uuid() { return socket_uuid_; } private: grpc_channel_stack* channel_stack_; - // backpointer to the channelz node in this connected subchannel's + // ref counted pointer to the channelz node in this connected subchannel's // owning subchannel. - channelz::SubchannelNode* channelz_subchannel_; + grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> + channelz_subchannel_; // uuid of this subchannel's socket. 0 if this subchannel is not connected. const intptr_t socket_uuid_; }; diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.cc b/src/core/ext/transport/chttp2/client/chttp2_connector.cc index 0ac84032fd..5229304fa4 100644 --- a/src/core/ext/transport/chttp2/client/chttp2_connector.cc +++ b/src/core/ext/transport/chttp2/client/chttp2_connector.cc @@ -160,7 +160,7 @@ static void on_handshake_done(void* arg, grpc_error* error) { static void start_handshake_locked(chttp2_connector* c) { c->handshake_mgr = grpc_handshake_manager_create(); grpc_handshakers_add(HANDSHAKER_CLIENT, c->args.channel_args, - c->handshake_mgr); + c->args.interested_parties, c->handshake_mgr); grpc_endpoint_add_to_pollset_set(c->endpoint, c->args.interested_parties); grpc_handshake_manager_do_handshake( c->handshake_mgr, c->args.interested_parties, c->endpoint, diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.cc b/src/core/ext/transport/chttp2/server/chttp2_server.cc index 3f8a26ae32..6ed88dfb5e 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.cc +++ b/src/core/ext/transport/chttp2/server/chttp2_server.cc @@ -53,6 +53,8 @@ typedef struct { grpc_closure tcp_server_shutdown_complete; grpc_closure* server_destroy_listener_done; grpc_handshake_manager* pending_handshake_mgrs; + grpc_core::RefCountedPtr<grpc_core::channelz::ListenSocketNode> + channelz_listen_socket; } server_state; typedef struct { @@ -67,6 +69,7 @@ typedef struct { grpc_timer timer; grpc_closure on_timeout; grpc_closure on_receive_settings; + grpc_pollset_set* interested_parties; } server_connection_state; static void server_connection_state_unref( @@ -76,6 +79,9 @@ static void server_connection_state_unref( GRPC_CHTTP2_UNREF_TRANSPORT(connection_state->transport, "receive settings timeout"); } + grpc_pollset_set_del_pollset(connection_state->interested_parties, + connection_state->accepting_pollset); + grpc_pollset_set_destroy(connection_state->interested_parties); gpr_free(connection_state); } } @@ -133,7 +139,8 @@ static void on_handshake_done(void* arg, grpc_error* error) { grpc_create_chttp2_transport(args->args, args->endpoint, false); grpc_server_setup_transport( connection_state->svr_state->server, transport, - connection_state->accepting_pollset, args->args); + connection_state->accepting_pollset, args->args, + grpc_chttp2_transport_get_socket_uuid(transport)); // Use notify_on_receive_settings callback to enforce the // handshake deadline. connection_state->transport = @@ -189,7 +196,11 @@ static void on_accept(void* arg, grpc_endpoint* tcp, connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; connection_state->handshake_mgr = handshake_mgr; + connection_state->interested_parties = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(connection_state->interested_parties, + connection_state->accepting_pollset); grpc_handshakers_add(HANDSHAKER_SERVER, state->args, + connection_state->interested_parties, connection_state->handshake_mgr); const grpc_arg* timeout_arg = grpc_channel_args_find(state->args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS); @@ -223,6 +234,7 @@ static void tcp_server_shutdown_complete(void* arg, grpc_error* error) { GPR_ASSERT(state->shutdown); grpc_handshake_manager_pending_list_shutdown_all( state->pending_handshake_mgrs, GRPC_ERROR_REF(error)); + state->channelz_listen_socket.reset(); gpr_mu_unlock(&state->mu); // Flush queued work before destroying handshaker factory, since that // may do a synchronous unref. @@ -262,6 +274,8 @@ grpc_error* grpc_chttp2_server_add_port(grpc_server* server, const char* addr, server_state* state = nullptr; grpc_error** errors = nullptr; size_t naddrs = 0; + const grpc_arg* arg = nullptr; + intptr_t socket_uuid = 0; *port_num = -1; @@ -323,9 +337,16 @@ grpc_error* grpc_chttp2_server_add_port(grpc_server* server, const char* addr, } grpc_resolved_addresses_destroy(resolved); + arg = grpc_channel_args_find(args, GRPC_ARG_ENABLE_CHANNELZ); + if (grpc_channel_arg_get_bool(arg, false)) { + state->channelz_listen_socket = + grpc_core::MakeRefCounted<grpc_core::channelz::ListenSocketNode>(); + socket_uuid = state->channelz_listen_socket->uuid(); + } + /* Register with the server only upon success */ grpc_server_add_listener(server, state, server_start_listener, - server_destroy_listener); + server_destroy_listener, socket_uuid); goto done; /* Error path: cleanup and return */ diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc index e4bd91d07b..b9024a87e2 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc @@ -61,7 +61,7 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server* server, grpc_endpoint_add_to_pollset(server_endpoint, pollsets[i]); } - grpc_server_setup_transport(server, transport, nullptr, server_args); + grpc_server_setup_transport(server, transport, nullptr, server_args, 0); grpc_chttp2_transport_start_reading(transport, nullptr, nullptr); } diff --git a/src/core/ext/transport/inproc/inproc_transport.cc b/src/core/ext/transport/inproc/inproc_transport.cc index b0ca7f8207..9dbd095843 100644 --- a/src/core/ext/transport/inproc/inproc_transport.cc +++ b/src/core/ext/transport/inproc/inproc_transport.cc @@ -1256,7 +1256,9 @@ grpc_channel* grpc_inproc_channel_create(grpc_server* server, inproc_transports_create(&server_transport, server_args, &client_transport, client_args); - grpc_server_setup_transport(server, server_transport, nullptr, server_args); + // TODO(ncteisen): design and support channelz GetSocket for inproc. + grpc_server_setup_transport(server, server_transport, nullptr, server_args, + 0); grpc_channel* channel = grpc_channel_create( "inproc", client_args, GRPC_CLIENT_DIRECT_CHANNEL, client_transport); diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index 573292fba1..33577d890a 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -37,13 +37,16 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/server.h" #include "src/core/lib/transport/error_utils.h" namespace grpc_core { namespace channelz { -BaseNode::BaseNode(EntityType type) - : type_(type), uuid_(ChannelzRegistry::Register(this)) {} +BaseNode::BaseNode(EntityType type) : type_(type), uuid_(-1) { + // The registry will set uuid_ under its lock. + ChannelzRegistry::Register(this); +} BaseNode::~BaseNode() { ChannelzRegistry::Unregister(uuid_); } @@ -190,11 +193,45 @@ RefCountedPtr<ChannelNode> ChannelNode::MakeChannelNode( channel, channel_tracer_max_nodes, is_top_level_channel); } -ServerNode::ServerNode(size_t channel_tracer_max_nodes) - : BaseNode(EntityType::kServer), trace_(channel_tracer_max_nodes) {} +ServerNode::ServerNode(grpc_server* server, size_t channel_tracer_max_nodes) + : BaseNode(EntityType::kServer), + server_(server), + trace_(channel_tracer_max_nodes) {} ServerNode::~ServerNode() {} +char* ServerNode::RenderServerSockets(intptr_t start_socket_id) { + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + ChildRefsList socket_refs; + // uuids index into entities one-off (idx 0 is really uuid 1, since 0 is + // reserved). However, we want to support requests coming in with + // start_server_id=0, which signifies "give me everything." + size_t start_idx = start_socket_id == 0 ? 0 : start_socket_id - 1; + grpc_server_populate_server_sockets(server_, &socket_refs, start_idx); + if (!socket_refs.empty()) { + // create list of socket refs + grpc_json* array_parent = grpc_json_create_child( + nullptr, json, "socketRef", nullptr, GRPC_JSON_ARRAY, false); + for (size_t i = 0; i < socket_refs.size(); ++i) { + json_iterator = + grpc_json_create_child(json_iterator, array_parent, nullptr, nullptr, + GRPC_JSON_OBJECT, false); + grpc_json_add_number_string_child(json_iterator, nullptr, "socketId", + socket_refs[i]); + } + } + // For now we do not have any pagination rules. In the future we could + // pick a constant for max_channels_sent for a GetServers request. + // Tracking: https://github.com/grpc/grpc/issues/16019. + json_iterator = grpc_json_create_child(nullptr, json, "end", nullptr, + GRPC_JSON_TRUE, false); + char* json_str = grpc_json_dump_to_string(top_level_json, 0); + grpc_json_destroy(top_level_json); + return json_str; +} + grpc_json* ServerNode::RenderJson() { // We need to track these three json objects to build our object grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); @@ -223,6 +260,20 @@ grpc_json* ServerNode::RenderJson() { } // ask CallCountingHelper to populate trace and call count data. call_counter_.PopulateCallCounts(json); + json = top_level_json; + ChildRefsList listen_sockets; + grpc_server_populate_listen_sockets(server_, &listen_sockets); + if (!listen_sockets.empty()) { + grpc_json* array_parent = grpc_json_create_child( + nullptr, json, "listenSocket", nullptr, GRPC_JSON_ARRAY, false); + for (size_t i = 0; i < listen_sockets.size(); ++i) { + json_iterator = + grpc_json_create_child(json_iterator, array_parent, nullptr, nullptr, + GRPC_JSON_OBJECT, false); + grpc_json_add_number_string_child(json_iterator, nullptr, "socketId", + listen_sockets[i]); + } + } return top_level_json; } @@ -323,5 +374,22 @@ grpc_json* SocketNode::RenderJson() { return top_level_json; } +ListenSocketNode::ListenSocketNode() : BaseNode(EntityType::kSocket) {} + +grpc_json* ListenSocketNode::RenderJson() { + // We need to track these three json objects to build our object + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + // create and fill the ref child + json_iterator = grpc_json_create_child(json_iterator, json, "ref", nullptr, + GRPC_JSON_OBJECT, false); + json = json_iterator; + json_iterator = nullptr; + json_iterator = grpc_json_add_number_string_child(json, json_iterator, + "socketId", uuid()); + return top_level_json; +} + } // namespace channelz } // namespace grpc_core diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index 3f728cda6d..fddef793fb 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -24,6 +24,7 @@ #include <grpc/grpc.h> #include "src/core/lib/channel/channel_trace.h" +#include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -50,8 +51,14 @@ #define GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT 0 namespace grpc_core { + namespace channelz { +// TODO(ncteisen), this only contains the uuids of the children for now, +// since that is all that is strictly needed. In a future enhancement we will +// add human readable names as in the channelz.proto +typedef InlinedVector<intptr_t, 10> ChildRefsList; + namespace testing { class CallCountingHelperPeer; class ChannelNodePeer; @@ -85,8 +92,10 @@ class BaseNode : public RefCounted<BaseNode> { intptr_t uuid() const { return uuid_; } private: + // to allow the ChannelzRegistry to set uuid_ under its lock. + friend class ChannelzRegistry; const EntityType type_; - const intptr_t uuid_; + intptr_t uuid_; }; // This class is a helper class for channelz entities that deal with Channels, @@ -193,11 +202,13 @@ class ChannelNode : public BaseNode { // Handles channelz bookkeeping for servers class ServerNode : public BaseNode { public: - explicit ServerNode(size_t channel_tracer_max_nodes); + ServerNode(grpc_server* server, size_t channel_tracer_max_nodes); ~ServerNode() override; grpc_json* RenderJson() override; + char* RenderServerSockets(intptr_t start_socket_id); + // proxy methods to composed classes. void AddTraceEvent(ChannelTrace::Severity severity, grpc_slice data) { trace_.AddTraceEvent(severity, data); @@ -213,6 +224,7 @@ class ServerNode : public BaseNode { void RecordCallSucceeded() { call_counter_.RecordCallSucceeded(); } private: + grpc_server* server_; CallCountingHelper call_counter_; ChannelTrace trace_; }; @@ -253,6 +265,15 @@ class SocketNode : public BaseNode { UniquePtr<char> peer_string_; }; +// Handles channelz bookkeeping for listen sockets +class ListenSocketNode : public BaseNode { + public: + ListenSocketNode(); + ~ListenSocketNode() override {} + + grpc_json* RenderJson() override; +}; + // Creation functions typedef RefCountedPtr<ChannelNode> (*ChannelNodeCreationFunc)(grpc_channel*, diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index 841f1c6104..67e56ed791 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -53,26 +53,74 @@ ChannelzRegistry::ChannelzRegistry() { gpr_mu_init(&mu_); } ChannelzRegistry::~ChannelzRegistry() { gpr_mu_destroy(&mu_); } -intptr_t ChannelzRegistry::InternalRegister(BaseNode* node) { +void ChannelzRegistry::InternalRegister(BaseNode* node) { MutexLock lock(&mu_); entities_.push_back(node); - intptr_t uuid = entities_.size(); - return uuid; + node->uuid_ = ++uuid_generator_; +} + +void ChannelzRegistry::MaybePerformCompactionLocked() { + constexpr double kEmptinessTheshold = 1 / 3; + double emptiness_ratio = + double(num_empty_slots_) / double(entities_.capacity()); + if (emptiness_ratio > kEmptinessTheshold) { + int front = 0; + for (size_t i = 0; i < entities_.size(); ++i) { + if (entities_[i] != nullptr) { + entities_[front++] = entities_[i]; + } + } + for (int i = 0; i < num_empty_slots_; ++i) { + entities_.pop_back(); + } + num_empty_slots_ = 0; + } +} + +int ChannelzRegistry::FindByUuidLocked(intptr_t target_uuid) { + size_t left = 0; + size_t right = entities_.size() - 1; + while (left <= right) { + size_t true_middle = left + (right - left) / 2; + size_t first_non_null = true_middle; + while (first_non_null < right && entities_[first_non_null] == nullptr) { + first_non_null++; + } + if (entities_[first_non_null] == nullptr) { + right = true_middle - 1; + continue; + } + intptr_t uuid = entities_[first_non_null]->uuid(); + if (uuid == target_uuid) { + return int(first_non_null); + } + if (uuid < target_uuid) { + left = first_non_null + 1; + } else { + right = true_middle - 1; + } + } + return -1; } void ChannelzRegistry::InternalUnregister(intptr_t uuid) { GPR_ASSERT(uuid >= 1); MutexLock lock(&mu_); - GPR_ASSERT(static_cast<size_t>(uuid) <= entities_.size()); - entities_[uuid - 1] = nullptr; + GPR_ASSERT(uuid <= uuid_generator_); + int idx = FindByUuidLocked(uuid); + GPR_ASSERT(idx >= 0); + entities_[idx] = nullptr; + num_empty_slots_++; + MaybePerformCompactionLocked(); } BaseNode* ChannelzRegistry::InternalGet(intptr_t uuid) { MutexLock lock(&mu_); - if (uuid < 1 || uuid > static_cast<intptr_t>(entities_.size())) { + if (uuid < 1 || uuid > uuid_generator_) { return nullptr; } - return entities_[uuid - 1]; + int idx = FindByUuidLocked(uuid); + return idx < 0 ? nullptr : entities_[idx]; } char* ChannelzRegistry::InternalGetTopChannels(intptr_t start_channel_id) { @@ -160,6 +208,21 @@ char* grpc_channelz_get_servers(intptr_t start_server_id) { return grpc_core::channelz::ChannelzRegistry::GetServers(start_server_id); } +char* grpc_channelz_get_server_sockets(intptr_t server_id, + intptr_t start_socket_id) { + grpc_core::channelz::BaseNode* base_node = + grpc_core::channelz::ChannelzRegistry::Get(server_id); + if (base_node == nullptr || + base_node->type() != grpc_core::channelz::BaseNode::EntityType::kServer) { + return nullptr; + } + // This cast is ok since we have just checked to make sure base_node is + // actually a server node + grpc_core::channelz::ServerNode* server_node = + static_cast<grpc_core::channelz::ServerNode*>(base_node); + return server_node->RenderServerSockets(start_socket_id); +} + char* grpc_channelz_get_channel(intptr_t channel_id) { grpc_core::channelz::BaseNode* channel_node = grpc_core::channelz::ChannelzRegistry::Get(channel_id); diff --git a/src/core/lib/channel/channelz_registry.h b/src/core/lib/channel/channelz_registry.h index d0d660600d..ea6ab6c8e5 100644 --- a/src/core/lib/channel/channelz_registry.h +++ b/src/core/lib/channel/channelz_registry.h @@ -30,6 +30,10 @@ namespace grpc_core { namespace channelz { +namespace testing { +class ChannelzRegistryPeer; +} + // singleton registry object to track all objects that are needed to support // channelz bookkeeping. All objects share globally distributed uuids. class ChannelzRegistry { @@ -40,7 +44,7 @@ class ChannelzRegistry { // To be called in grpc_shutdown(); static void Shutdown(); - static intptr_t Register(BaseNode* node) { + static void Register(BaseNode* node) { return Default()->InternalRegister(node); } static void Unregister(intptr_t uuid) { Default()->InternalUnregister(uuid); } @@ -61,6 +65,7 @@ class ChannelzRegistry { private: GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE + friend class testing::ChannelzRegistryPeer; ChannelzRegistry(); ~ChannelzRegistry(); @@ -69,7 +74,7 @@ class ChannelzRegistry { static ChannelzRegistry* Default(); // globally registers an Entry. Returns its unique uuid - intptr_t InternalRegister(BaseNode* node); + void InternalRegister(BaseNode* node); // globally unregisters the object that is associated to uuid. Also does // sanity check that an object doesn't try to unregister the wrong type. @@ -82,9 +87,18 @@ class ChannelzRegistry { char* InternalGetTopChannels(intptr_t start_channel_id); char* InternalGetServers(intptr_t start_server_id); - // protects entities_ and uuid_ + // If entities_ has over a certain threshold of empty slots, it will + // compact the vector and move all used slots to the front. + void MaybePerformCompactionLocked(); + + // Performs binary search on entities_ to find the index with that uuid. + int FindByUuidLocked(intptr_t uuid); + + // protects members gpr_mu mu_; InlinedVector<BaseNode*, 20> entities_; + intptr_t uuid_generator_ = 0; + int num_empty_slots_ = 0; }; } // namespace channelz diff --git a/src/core/lib/channel/handshaker_factory.cc b/src/core/lib/channel/handshaker_factory.cc index 4fd43635b6..8ade8fe4e2 100644 --- a/src/core/lib/channel/handshaker_factory.cc +++ b/src/core/lib/channel/handshaker_factory.cc @@ -24,11 +24,12 @@ void grpc_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (handshaker_factory != nullptr) { GPR_ASSERT(handshaker_factory->vtable != nullptr); - handshaker_factory->vtable->add_handshakers(handshaker_factory, args, - handshake_mgr); + handshaker_factory->vtable->add_handshakers( + handshaker_factory, args, interested_parties, handshake_mgr); } } diff --git a/src/core/lib/channel/handshaker_factory.h b/src/core/lib/channel/handshaker_factory.h index 3e45fcf20e..e17a678179 100644 --- a/src/core/lib/channel/handshaker_factory.h +++ b/src/core/lib/channel/handshaker_factory.h @@ -32,6 +32,7 @@ typedef struct grpc_handshaker_factory grpc_handshaker_factory; typedef struct { void (*add_handshakers)(grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); void (*destroy)(grpc_handshaker_factory* handshaker_factory); } grpc_handshaker_factory_vtable; @@ -42,6 +43,7 @@ struct grpc_handshaker_factory { void grpc_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); void grpc_handshaker_factory_destroy( diff --git a/src/core/lib/channel/handshaker_registry.cc b/src/core/lib/channel/handshaker_registry.cc index eec3e1b352..fbafc43e79 100644 --- a/src/core/lib/channel/handshaker_registry.cc +++ b/src/core/lib/channel/handshaker_registry.cc @@ -51,9 +51,11 @@ static void grpc_handshaker_factory_list_register( static void grpc_handshaker_factory_list_add_handshakers( grpc_handshaker_factory_list* list, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { for (size_t i = 0; i < list->num_factories; ++i) { - grpc_handshaker_factory_add_handshakers(list->list[i], args, handshake_mgr); + grpc_handshaker_factory_add_handshakers(list->list[i], args, + interested_parties, handshake_mgr); } } @@ -91,7 +93,9 @@ void grpc_handshaker_factory_register(bool at_start, void grpc_handshakers_add(grpc_handshaker_type handshaker_type, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshaker_factory_list_add_handshakers( - &g_handshaker_factory_lists[handshaker_type], args, handshake_mgr); + &g_handshaker_factory_lists[handshaker_type], args, interested_parties, + handshake_mgr); } diff --git a/src/core/lib/channel/handshaker_registry.h b/src/core/lib/channel/handshaker_registry.h index 82ad9c5b9a..3dd4316de6 100644 --- a/src/core/lib/channel/handshaker_registry.h +++ b/src/core/lib/channel/handshaker_registry.h @@ -43,6 +43,7 @@ void grpc_handshaker_factory_register(bool at_start, void grpc_handshakers_add(grpc_handshaker_type handshaker_type, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_REGISTRY_H */ diff --git a/src/core/lib/gprpp/inlined_vector.h b/src/core/lib/gprpp/inlined_vector.h index 76e2f0a785..65c2b9634f 100644 --- a/src/core/lib/gprpp/inlined_vector.h +++ b/src/core/lib/gprpp/inlined_vector.h @@ -123,6 +123,14 @@ class InlinedVector { void push_back(T&& value) { emplace_back(std::move(value)); } + void pop_back() { + assert(!empty()); + size_t s = size(); + T& value = data()[s - 1]; + value.~T(); + size_--; + } + void copy_from(const InlinedVector& v) { // if v is allocated, copy over the buffer. if (v.dynamic_ != nullptr) { diff --git a/src/core/lib/http/httpcli_security_connector.cc b/src/core/lib/http/httpcli_security_connector.cc index 50078c37a1..5a5e0b72d5 100644 --- a/src/core/lib/http/httpcli_security_connector.cc +++ b/src/core/lib/http/httpcli_security_connector.cc @@ -29,6 +29,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker_registry.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/ssl_transport_security.h" @@ -51,6 +52,7 @@ static void httpcli_ssl_destroy(grpc_security_connector* sc) { } static void httpcli_ssl_add_handshakers(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_httpcli_ssl_channel_security_connector* c = reinterpret_cast<grpc_httpcli_ssl_channel_security_connector*>(sc); @@ -189,7 +191,8 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host, grpc_arg channel_arg = grpc_security_connector_to_arg(&sc->base); grpc_channel_args args = {1, &channel_arg}; c->handshake_mgr = grpc_handshake_manager_create(); - grpc_handshakers_add(HANDSHAKER_CLIENT, &args, c->handshake_mgr); + grpc_handshakers_add(HANDSHAKER_CLIENT, &args, + nullptr /* interested_parties */, c->handshake_mgr); grpc_handshake_manager_do_handshake( c->handshake_mgr, nullptr /* interested_parties */, tcp, nullptr /* channel_args */, deadline, nullptr /* acceptor */, diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 764fb08142..8a7dc7b004 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -395,6 +395,4 @@ void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { g_event_engine->pollset_set_del_fd(pollset_set, fd); } -void grpc_use_signal(int signum) {} - #endif // GRPC_POSIX_SOCKET_EV diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index c8046b21dc..bf56a7298d 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -62,7 +62,8 @@ #define GRPC_HAVE_UNIX_SOCKET 1 #ifdef LINUX_VERSION_CODE #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) -#define GRPC_LINUX_ERRQUEUE 1 +/* TODO(yashykt): Re-enable once Fathom changes are commited. +#define GRPC_LINUX_ERRQUEUE 1 */ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) */ #endif /* LINUX_VERSION_CODE */ #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1 diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index e40bf81c90..aa2704ce26 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -468,7 +468,9 @@ static void tcp_do_read(grpc_tcp* tcp) { GRPC_STATS_INC_TCP_READ_SIZE(read_bytes); add_to_estimate(tcp, static_cast<size_t>(read_bytes)); GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); - if (static_cast<size_t>(read_bytes) < tcp->incoming_buffer->length) { + if (static_cast<size_t>(read_bytes) == tcp->incoming_buffer->length) { + finish_estimate(tcp); + } else if (static_cast<size_t>(read_bytes) < tcp->incoming_buffer->length) { grpc_slice_buffer_trim_end( tcp->incoming_buffer, tcp->incoming_buffer->length - static_cast<size_t>(read_bytes), @@ -498,7 +500,7 @@ static void tcp_read_allocation_done(void* tcpp, grpc_error* error) { static void tcp_continue_read(grpc_tcp* tcp) { size_t target_read_size = get_target_read_size(tcp); - if (tcp->incoming_buffer->length < target_read_size && + if (tcp->incoming_buffer->length < target_read_size / 2 && tcp->incoming_buffer->count < MAX_READ_IOVEC) { if (grpc_tcp_trace.enabled()) { gpr_log(GPR_INFO, "TCP:%p alloc_slices", tcp); diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc index aba5539199..6a925add80 100644 --- a/src/core/lib/iomgr/timer_generic.cc +++ b/src/core/lib/iomgr/timer_generic.cc @@ -48,22 +48,22 @@ grpc_core::TraceFlag grpc_timer_trace(false, "timer"); grpc_core::TraceFlag grpc_timer_check_trace(false, "timer_check"); /* A "timer shard". Contains a 'heap' and a 'list' of timers. All timers with - * deadlines earlier than 'queue_deadline" cap are maintained in the heap and + * deadlines earlier than 'queue_deadline_cap' are maintained in the heap and * others are maintained in the list (unordered). This helps to keep the number * of elements in the heap low. * * The 'queue_deadline_cap' gets recomputed periodically based on the timer * stats maintained in 'stats' and the relevant timers are then moved from the - * 'list' to 'heap' + * 'list' to 'heap'. */ typedef struct { gpr_mu mu; grpc_time_averaged_stats stats; - /* All and only timers with deadlines <= this will be in the heap. */ + /* All and only timers with deadlines < this will be in the heap. */ grpc_millis queue_deadline_cap; - /* The deadline of the next timer due in this shard */ + /* The deadline of the next timer due in this shard. */ grpc_millis min_deadline; - /* Index of this timer_shard in the g_shard_queue */ + /* Index of this timer_shard in the g_shard_queue. */ uint32_t shard_queue_index; /* This holds all timers with deadlines < queue_deadline_cap. Timers in this list have the top bit of their deadline set to 0. */ @@ -85,7 +85,7 @@ static timer_shard** g_shard_queue; #ifndef NDEBUG -/* == Hash table for duplicate timer detection == */ +/* == DEBUG ONLY: hash table for duplicate timer detection == */ #define NUM_HASH_BUCKETS 1009 /* Prime number close to 1000 */ @@ -177,7 +177,7 @@ static void remove_from_ht(grpc_timer* t) { t->hash_table_next = nullptr; } -/* If a timer is added to a timer shard (either heap or a list), it cannot +/* If a timer is added to a timer shard (either heap or a list), it must * be pending. A timer is added to hash table only-if it is added to the * timer shard. * Therefore, if timer->pending is false, it cannot be in hash table */ @@ -489,7 +489,7 @@ static void timer_cancel(grpc_timer* timer) { 'queue_deadline_cap') into into shard->heap. Returns 'true' if shard->heap has atleast ONE element REQUIRES: shard->mu locked */ -static int refill_heap(timer_shard* shard, grpc_millis now) { +static bool refill_heap(timer_shard* shard, grpc_millis now) { /* Compute the new queue window width and bound by the limits: */ double computed_deadline_delta = grpc_time_averaged_stats_update_average(&shard->stats) * diff --git a/src/core/lib/iomgr/timer_heap.cc b/src/core/lib/iomgr/timer_heap.cc index 0c17d607eb..2c6a599149 100644 --- a/src/core/lib/iomgr/timer_heap.cc +++ b/src/core/lib/iomgr/timer_heap.cc @@ -95,7 +95,7 @@ void grpc_timer_heap_init(grpc_timer_heap* heap) { void grpc_timer_heap_destroy(grpc_timer_heap* heap) { gpr_free(heap->timers); } -int grpc_timer_heap_add(grpc_timer_heap* heap, grpc_timer* timer) { +bool grpc_timer_heap_add(grpc_timer_heap* heap, grpc_timer* timer) { if (heap->timer_count == heap->timer_capacity) { heap->timer_capacity = GPR_MAX(heap->timer_capacity + 1, heap->timer_capacity * 3 / 2); @@ -122,7 +122,7 @@ void grpc_timer_heap_remove(grpc_timer_heap* heap, grpc_timer* timer) { note_changed_priority(heap, heap->timers[i]); } -int grpc_timer_heap_is_empty(grpc_timer_heap* heap) { +bool grpc_timer_heap_is_empty(grpc_timer_heap* heap) { return heap->timer_count == 0; } diff --git a/src/core/lib/iomgr/timer_heap.h b/src/core/lib/iomgr/timer_heap.h index 503365d4cd..7b983e7133 100644 --- a/src/core/lib/iomgr/timer_heap.h +++ b/src/core/lib/iomgr/timer_heap.h @@ -29,8 +29,8 @@ typedef struct { uint32_t timer_capacity; } grpc_timer_heap; -/* return 1 if the new timer is the first timer in the heap */ -int grpc_timer_heap_add(grpc_timer_heap* heap, grpc_timer* timer); +/* return true if the new timer is the first timer in the heap */ +bool grpc_timer_heap_add(grpc_timer_heap* heap, grpc_timer* timer); void grpc_timer_heap_init(grpc_timer_heap* heap); void grpc_timer_heap_destroy(grpc_timer_heap* heap); @@ -39,6 +39,6 @@ void grpc_timer_heap_remove(grpc_timer_heap* heap, grpc_timer* timer); grpc_timer* grpc_timer_heap_top(grpc_timer_heap* heap); void grpc_timer_heap_pop(grpc_timer_heap* heap); -int grpc_timer_heap_is_empty(grpc_timer_heap* heap); +bool grpc_timer_heap_is_empty(grpc_timer_heap* heap); #endif /* GRPC_CORE_LIB_IOMGR_TIMER_HEAP_H */ diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index e272d228f8..ceba79f678 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -100,8 +100,7 @@ static void start_timer_thread_and_unlock(void) { void grpc_timer_manager_tick() { grpc_core::ExecCtx exec_ctx; - grpc_millis next = GRPC_MILLIS_INF_FUTURE; - grpc_timer_check(&next); + grpc_timer_check(nullptr); } static void run_some_timers() { @@ -110,9 +109,12 @@ static void run_some_timers() { // remove a waiter from the pool, and start another thread if necessary --g_waiter_count; if (g_waiter_count == 0 && g_threaded) { + // The number of timer threads is always increasing until all the threads + // are stopped. In rare cases, if a large number of timers fire + // simultaneously, we may end up using a large number of threads. start_timer_thread_and_unlock(); } else { - // if there's no thread waiting with a timeout, kick an existing + // if there's no thread waiting with a timeout, kick an existing untimed // waiter so that the next deadline is not missed if (!g_has_timed_waiter) { if (grpc_timer_check_trace.enabled()) { @@ -243,7 +245,7 @@ static void timer_main_loop() { gpr_log(GPR_INFO, "timers not checked: expect another thread to"); } next = GRPC_MILLIS_INF_FUTURE; - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: if (!wait_until(next)) { return; diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h index 3c4cdda2c8..00dcdc461b 100644 --- a/src/core/lib/iomgr/timer_manager.h +++ b/src/core/lib/iomgr/timer_manager.h @@ -23,8 +23,8 @@ #include <stdbool.h> -/* Timer Manager tries to keep one thread waiting for the next timeout at all - times */ +/* Timer Manager tries to keep only one thread waiting for the next timeout at + all times, and thus effectively preventing the thundering herd problem. */ void grpc_timer_manager_init(void); void grpc_timer_manager_shutdown(void); diff --git a/src/core/lib/security/security_connector/alts_security_connector.cc b/src/core/lib/security/security_connector/alts_security_connector.cc index 35a787871a..d38c0ff044 100644 --- a/src/core/lib/security/security_connector/alts_security_connector.cc +++ b/src/core/lib/security/security_connector/alts_security_connector.cc @@ -64,29 +64,29 @@ static void alts_server_destroy(grpc_security_connector* sc) { } static void alts_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; auto c = reinterpret_cast<grpc_alts_channel_security_connector*>(sc); grpc_alts_credentials* creds = reinterpret_cast<grpc_alts_credentials*>(c->base.channel_creds); - GPR_ASSERT(alts_tsi_handshaker_create(creds->options, c->target_name, - creds->handshaker_service_url, true, - &handshaker) == TSI_OK); + GPR_ASSERT(alts_tsi_handshaker_create( + creds->options, c->target_name, creds->handshaker_service_url, + true, interested_parties, &handshaker) == TSI_OK); grpc_handshake_manager_add(handshake_manager, grpc_security_handshaker_create( handshaker, &sc->base)); } static void alts_server_add_handshakers( - grpc_server_security_connector* sc, + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; auto c = reinterpret_cast<grpc_alts_server_security_connector*>(sc); grpc_alts_server_credentials* creds = reinterpret_cast<grpc_alts_server_credentials*>(c->base.server_creds); - GPR_ASSERT(alts_tsi_handshaker_create(creds->options, nullptr, - creds->handshaker_service_url, false, - &handshaker) == TSI_OK); + GPR_ASSERT(alts_tsi_handshaker_create( + creds->options, nullptr, creds->handshaker_service_url, false, + interested_parties, &handshaker) == TSI_OK); grpc_handshake_manager_add(handshake_manager, grpc_security_handshaker_create( handshaker, &sc->base)); } diff --git a/src/core/lib/security/security_connector/local_security_connector.cc b/src/core/lib/security/security_connector/local_security_connector.cc index c436a7906b..911013ae58 100644 --- a/src/core/lib/security/security_connector/local_security_connector.cc +++ b/src/core/lib/security/security_connector/local_security_connector.cc @@ -30,6 +30,7 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/security/credentials/local/local_credentials.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/tsi/local_transport_security.h" @@ -68,7 +69,7 @@ static void local_server_destroy(grpc_security_connector* sc) { } static void local_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; GPR_ASSERT(local_tsi_handshaker_create(true /* is_client */, &handshaker) == @@ -78,7 +79,7 @@ static void local_channel_add_handshakers( } static void local_server_add_handshakers( - grpc_server_security_connector* sc, + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_manager) { tsi_handshaker* handshaker = nullptr; GPR_ASSERT(local_tsi_handshaker_create(false /* is_client */, &handshaker) == diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 6246613e7b..7028ae8d16 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -120,17 +120,19 @@ const tsi_peer_property* tsi_peer_get_property_by_name(const tsi_peer* peer, void grpc_channel_security_connector_add_handshakers( grpc_channel_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (connector != nullptr) { - connector->add_handshakers(connector, handshake_mgr); + connector->add_handshakers(connector, interested_parties, handshake_mgr); } } void grpc_server_security_connector_add_handshakers( grpc_server_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { if (connector != nullptr) { - connector->add_handshakers(connector, handshake_mgr); + connector->add_handshakers(connector, interested_parties, handshake_mgr); } } @@ -519,7 +521,7 @@ static void fake_channel_cancel_check_call_host( } static void fake_channel_add_handshakers( - grpc_channel_security_connector* sc, + grpc_channel_security_connector* sc, grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add( handshake_mgr, @@ -528,6 +530,7 @@ static void fake_channel_add_handshakers( } static void fake_server_add_handshakers(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add( handshake_mgr, @@ -669,6 +672,7 @@ static void ssl_server_destroy(grpc_security_connector* sc) { } static void ssl_channel_add_handshakers(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_ssl_channel_security_connector* c = reinterpret_cast<grpc_ssl_channel_security_connector*>(sc); @@ -779,6 +783,7 @@ static bool try_fetch_ssl_server_credentials( } static void ssl_server_add_handshakers(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_ssl_server_security_connector* c = reinterpret_cast<grpc_ssl_server_security_connector*>(sc); diff --git a/src/core/lib/security/security_connector/security_connector.h b/src/core/lib/security/security_connector/security_connector.h index 67a506b576..d8df3cd72f 100644 --- a/src/core/lib/security/security_connector/security_connector.h +++ b/src/core/lib/security/security_connector/security_connector.h @@ -27,6 +27,7 @@ #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/tcp_server.h" #include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security_interface.h" @@ -125,6 +126,7 @@ struct grpc_channel_security_connector { grpc_closure* on_call_host_checked, grpc_error* error); void (*add_handshakers)(grpc_channel_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); }; @@ -151,6 +153,7 @@ void grpc_channel_security_connector_cancel_check_call_host( /* Registers handshakers with \a handshake_mgr. */ void grpc_channel_security_connector_add_handshakers( grpc_channel_security_connector* connector, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); /* --- server_security_connector object. --- @@ -164,6 +167,7 @@ struct grpc_server_security_connector { grpc_security_connector base; grpc_server_credentials* server_creds; void (*add_handshakers)(grpc_server_security_connector* sc, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr); }; @@ -172,7 +176,8 @@ int grpc_server_security_connector_cmp(grpc_server_security_connector* sc1, grpc_server_security_connector* sc2); void grpc_server_security_connector_add_handshakers( - grpc_server_security_connector* sc, grpc_handshake_manager* handshake_mgr); + grpc_server_security_connector* sc, grpc_pollset_set* interested_parties, + grpc_handshake_manager* handshake_mgr); /* --- Creation security connectors. --- */ diff --git a/src/core/lib/security/transport/security_handshaker.cc b/src/core/lib/security/transport/security_handshaker.cc index d76d582638..4d6b133809 100644 --- a/src/core/lib/security/transport/security_handshaker.cc +++ b/src/core/lib/security/transport/security_handshaker.cc @@ -475,22 +475,24 @@ static grpc_handshaker* fail_handshaker_create() { static void client_handshaker_factory_add_handshakers( grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_channel_security_connector* security_connector = reinterpret_cast<grpc_channel_security_connector*>( grpc_security_connector_find_in_args(args)); - grpc_channel_security_connector_add_handshakers(security_connector, - handshake_mgr); + grpc_channel_security_connector_add_handshakers( + security_connector, interested_parties, handshake_mgr); } static void server_handshaker_factory_add_handshakers( grpc_handshaker_factory* hf, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_server_security_connector* security_connector = reinterpret_cast<grpc_server_security_connector*>( grpc_security_connector_find_in_args(args)); - grpc_server_security_connector_add_handshakers(security_connector, - handshake_mgr); + grpc_server_security_connector_add_handshakers( + security_connector, interested_parties, handshake_mgr); } static void handshaker_factory_destroy( diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 449aef768e..35ab2c3bce 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -54,6 +54,7 @@ struct listener { size_t pollset_count); void (*destroy)(grpc_server* server, void* arg, grpc_closure* closure); struct listener* next; + intptr_t socket_uuid; grpc_closure destroy_done; }; @@ -104,6 +105,7 @@ struct channel_data { uint32_t registered_method_max_probes; grpc_closure finish_destroy_channel_closure; grpc_closure channel_connectivity_changed; + intptr_t socket_uuid; }; typedef struct shutdown_tag { @@ -1016,7 +1018,7 @@ grpc_server* grpc_server_create(const grpc_channel_args* args, void* reserved) { {GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT, 0, INT_MAX}); server->channelz_server = grpc_core::MakeRefCounted<grpc_core::channelz::ServerNode>( - channel_tracer_max_memory); + server, channel_tracer_max_memory); server->channelz_server->AddTraceEvent( grpc_core::channelz::ChannelTrace::Severity::Info, grpc_slice_from_static_string("Server created")); @@ -1119,7 +1121,8 @@ void grpc_server_get_pollsets(grpc_server* server, grpc_pollset*** pollsets, void grpc_server_setup_transport(grpc_server* s, grpc_transport* transport, grpc_pollset* accepting_pollset, - const grpc_channel_args* args) { + const grpc_channel_args* args, + intptr_t socket_uuid) { size_t num_registered_methods; size_t alloc; registered_method* rm; @@ -1139,6 +1142,7 @@ void grpc_server_setup_transport(grpc_server* s, grpc_transport* transport, chand->server = s; server_ref(s); chand->channel = channel; + chand->socket_uuid = socket_uuid; size_t cq_idx; for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) { @@ -1213,6 +1217,29 @@ void grpc_server_setup_transport(grpc_server* s, grpc_transport* transport, grpc_transport_perform_op(transport, op); } +void grpc_server_populate_server_sockets( + grpc_server* s, grpc_core::channelz::ChildRefsList* server_sockets, + intptr_t start_idx) { + gpr_mu_lock(&s->mu_global); + channel_data* c = nullptr; + for (c = s->root_channel_data.next; c != &s->root_channel_data; c = c->next) { + intptr_t socket_uuid = c->socket_uuid; + if (socket_uuid >= start_idx) { + server_sockets->push_back(socket_uuid); + } + } + gpr_mu_unlock(&s->mu_global); +} + +void grpc_server_populate_listen_sockets( + grpc_server* server, grpc_core::channelz::ChildRefsList* listen_sockets) { + gpr_mu_lock(&server->mu_global); + for (listener* l = server->listeners; l != nullptr; l = l->next) { + listen_sockets->push_back(l->socket_uuid); + } + gpr_mu_unlock(&server->mu_global); +} + void done_published_shutdown(void* done_arg, grpc_cq_completion* storage) { (void)done_arg; gpr_free(storage); @@ -1346,11 +1373,13 @@ void grpc_server_add_listener(grpc_server* server, void* arg, grpc_pollset** pollsets, size_t pollset_count), void (*destroy)(grpc_server* server, void* arg, - grpc_closure* on_done)) { + grpc_closure* on_done), + intptr_t socket_uuid) { listener* l = static_cast<listener*>(gpr_malloc(sizeof(listener))); l->arg = arg; l->start = start; l->destroy = destroy; + l->socket_uuid = socket_uuid; l->next = server->listeners; server->listeners = l; } diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 0196743ff9..33c205417e 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -39,13 +39,24 @@ void grpc_server_add_listener(grpc_server* server, void* listener, grpc_pollset** pollsets, size_t npollsets), void (*destroy)(grpc_server* server, void* arg, - grpc_closure* on_done)); + grpc_closure* on_done), + intptr_t socket_uuid); /* Setup a transport - creates a channel stack, binds the transport to the server */ void grpc_server_setup_transport(grpc_server* server, grpc_transport* transport, grpc_pollset* accepting_pollset, - const grpc_channel_args* args); + const grpc_channel_args* args, + intptr_t socket_uuid); + +/* fills in the uuids of all sockets used for connections on this server */ +void grpc_server_populate_server_sockets( + grpc_server* server, grpc_core::channelz::ChildRefsList* server_sockets, + intptr_t start_idx); + +/* fills in the uuids of all listen sockets on this server */ +void grpc_server_populate_listen_sockets( + grpc_server* server, grpc_core::channelz::ChildRefsList* listen_sockets); grpc_core::channelz::ServerNode* grpc_server_get_channelz_node( grpc_server* server); diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index a44f9acdc3..66890ce65a 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include <grpc/grpc.h> -const char* grpc_version_string(void) { return "6.0.0-dev"; } +const char* grpc_version_string(void) { return "7.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "gao"; } +const char* grpc_g_stands_for(void) { return "gizmo"; } diff --git a/src/core/lib/transport/metadata_batch.cc b/src/core/lib/transport/metadata_batch.cc index 761aad19c4..928ed73cda 100644 --- a/src/core/lib/transport/metadata_batch.cc +++ b/src/core/lib/transport/metadata_batch.cc @@ -139,6 +139,7 @@ static void link_head(grpc_mdelem_list* list, grpc_linked_mdelem* storage) { GPR_ASSERT(!GRPC_MDISNULL(storage->md)); storage->prev = nullptr; storage->next = list->head; + storage->reserved = nullptr; if (list->head != nullptr) { list->head->prev = storage; } else { diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 34608a3de1..dfdd659b87 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -347,7 +347,8 @@ static void init_shared_resources(const char* handshaker_service_url) { tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, - const char* handshaker_service_url, bool is_client, tsi_handshaker** self) { + const char* handshaker_service_url, bool is_client, + grpc_pollset_set* interested_parties, tsi_handshaker** self) { if (handshaker_service_url == nullptr || self == nullptr || options == nullptr || (is_client && target_name == nullptr)) { gpr_log(GPR_ERROR, "Invalid arguments to alts_tsi_handshaker_create()"); diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index 227b30ce53..48ce69b1da 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -23,6 +23,7 @@ #include <grpc/grpc.h> +#include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h" #include "src/core/tsi/alts_transport_security.h" #include "src/core/tsi/transport_security.h" @@ -51,6 +52,7 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; * "host:port". * - is_client: boolean value indicating if the handshaker is used at the client * (is_client = true) or server (is_client = false) side. + * - interested_parties: set of pollsets interested in this connection. * - self: address of ALTS TSI handshaker instance to be returned from the * method. * @@ -58,7 +60,8 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; */ tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, - const char* handshaker_service_url, bool is_client, tsi_handshaker** self); + const char* handshaker_service_url, bool is_client, + grpc_pollset_set* interested_parties, tsi_handshaker** self); /** * This method handles handshaker response returned from ALTS handshaker diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index cc797f1546..8abd45efb7 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include <grpcpp/grpcpp.h> namespace grpc { -grpc::string Version() { return "1.16.0-dev"; } +grpc::string Version() { return "1.17.0-dev"; } } // namespace grpc diff --git a/src/cpp/server/channelz/channelz_service.cc b/src/cpp/server/channelz/channelz_service.cc index 4e3fe8c1c9..79ed9102e5 100644 --- a/src/cpp/server/channelz/channelz_service.cc +++ b/src/cpp/server/channelz/channelz_service.cc @@ -60,6 +60,23 @@ Status ChannelzService::GetServers( return Status::OK; } +Status ChannelzService::GetServerSockets( + ServerContext* unused, const channelz::v1::GetServerSocketsRequest* request, + channelz::v1::GetServerSocketsResponse* response) { + char* json_str = grpc_channelz_get_server_sockets(request->server_id(), + request->start_socket_id()); + if (json_str == nullptr) { + return Status(INTERNAL, "grpc_channelz_get_server_sockets returned null"); + } + google::protobuf::util::Status s = + google::protobuf::util::JsonStringToMessage(json_str, response); + gpr_free(json_str); + if (s != google::protobuf::util::Status::OK) { + return Status(INTERNAL, s.ToString()); + } + return Status::OK; +} + Status ChannelzService::GetChannel( ServerContext* unused, const channelz::v1::GetChannelRequest* request, channelz::v1::GetChannelResponse* response) { @@ -96,7 +113,6 @@ Status ChannelzService::GetSocket(ServerContext* unused, const channelz::v1::GetSocketRequest* request, channelz::v1::GetSocketResponse* response) { char* json_str = grpc_channelz_get_socket(request->socket_id()); - gpr_log(GPR_ERROR, "%s", json_str); if (json_str == nullptr) { return Status(NOT_FOUND, "No object found for that SocketId"); } diff --git a/src/cpp/server/channelz/channelz_service.h b/src/cpp/server/channelz/channelz_service.h index 1be4e01c73..590b5d492e 100644 --- a/src/cpp/server/channelz/channelz_service.h +++ b/src/cpp/server/channelz/channelz_service.h @@ -36,6 +36,11 @@ class ChannelzService final : public channelz::v1::Channelz::Service { Status GetServers(ServerContext* unused, const channelz::v1::GetServersRequest* request, channelz::v1::GetServersResponse* response) override; + // implementation of GetServerSockets rpc + Status GetServerSockets( + ServerContext* unused, + const channelz::v1::GetServerSocketsRequest* request, + channelz::v1::GetServerSocketsResponse* response) override; // implementation of GetChannel rpc Status GetChannel(ServerContext* unused, const channelz::v1::GetChannelRequest* request, diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 18515ea1e8..ed0d884365 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ <!-- This file is generated --> <Project> <PropertyGroup> - <GrpcCsharpVersion>1.16.0-dev</GrpcCsharpVersion> + <GrpcCsharpVersion>1.17.0-dev</GrpcCsharpVersion> <GoogleProtobufVersion>3.6.1</GoogleProtobufVersion> </PropertyGroup> </Project> diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 55d09dda7a..14714c8c4a 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// <summary> /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies /// </summary> - public const string CurrentAssemblyFileVersion = "1.16.0.0"; + public const string CurrentAssemblyFileVersion = "1.17.0.0"; /// <summary> /// Current version of gRPC C# /// </summary> - public const string CurrentVersion = "1.16.0-dev"; + public const string CurrentVersion = "1.17.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index da6d038d23..27688360e9 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.16.0-dev +set VERSION=1.17.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 4e7ac4e414..dd74de0491 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.16.0-dev +set VERSION=1.17.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 200cee5b7b..a95a120d21 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.16.0-dev' + v = '1.17.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 38862e102a..d5463c0b4c 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.16.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.17.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 82833102ad..ca27c03b3c 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.16.0-dev" -#define GRPC_C_VERSION_STRING @"6.0.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.17.0-dev" +#define GRPC_C_VERSION_STRING @"7.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index f31423f8c0..d54db91b5f 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.16.0", + "version": "1.17.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 469a48e782..70f8bbbf40 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.16.0dev" +#define PHP_GRPC_VERSION "1.17.0dev" #endif /* VERSION_H */ diff --git a/src/proto/grpc/health/v1/BUILD b/src/proto/grpc/health/v1/BUILD index d234842883..97642985c9 100644 --- a/src/proto/grpc/health/v1/BUILD +++ b/src/proto/grpc/health/v1/BUILD @@ -22,3 +22,11 @@ grpc_proto_library( name = "health_proto", srcs = ["health.proto"], ) + +filegroup( + name = "health_proto_file", + srcs = [ + "health.proto", + ], +) + diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index a4a9c8fe57..4cfdc2cafb 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -25,6 +25,7 @@ enum ClientType { SYNC_CLIENT = 0; ASYNC_CLIENT = 1; OTHER_CLIENT = 2; // used for some language-specific variants + CALLBACK_CLIENT = 3; } enum ServerType { diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 24e1557578..42b3a1ad49 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.16.0.dev0""" +__version__ = """1.17.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 6ffe1eb827..71113e68d9 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel new file mode 100644 index 0000000000..ce3121fa90 --- /dev/null +++ b/src/python/grpcio_health_checking/grpc_health/v1/BUILD.bazel @@ -0,0 +1,33 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + +package(default_visibility = ["//visibility:public"]) + +genrule( + name = "mv_health_proto", + srcs = [ + "//src/proto/grpc/health/v1:health_proto_file", + ], + outs = ["health.proto",], + cmd = "cp $< $@", +) + +py_proto_library( + name = "py_health_proto", + protos = [":mv_health_proto",], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + +py_library( + name = "grpc_health", + srcs = ["health.py",], + deps = [ + ":py_health_proto", + "//src/python/grpcio/grpc:grpcio", + ], + imports=["../../",], +) + diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index e080bf2cbc..a30aac2e0b 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 4b3b95fee9..aafea9fe76 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index c12aa153a4..876acd3142 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index f4b8a34a46..cc9b41587c 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/src/python/grpcio_tests/tests/health_check/BUILD.bazel b/src/python/grpcio_tests/tests/health_check/BUILD.bazel new file mode 100644 index 0000000000..19e1e1b2e1 --- /dev/null +++ b/src/python/grpcio_tests/tests/health_check/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +py_test( + name = "health_servicer_test", + srcs = ["_health_servicer_test.py"], + main = "_health_servicer_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_health_checking/grpc_health/v1:grpc_health", + "//src/python/grpcio_tests/tests/unit:test_common", + ], + imports = ["../../",], +) + diff --git a/src/python/grpcio_tests/tests/testing/BUILD.bazel b/src/python/grpcio_tests/tests/testing/BUILD.bazel new file mode 100644 index 0000000000..9bdd616c56 --- /dev/null +++ b/src/python/grpcio_tests/tests/testing/BUILD.bazel @@ -0,0 +1,30 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "testing", + srcs = ["__init__.py",], + deps = [ + # ":_application_common", + ":_server_application", + ], +) + +# TODO(ghostwriternr): To be added later. +# py_library( +# name = "_application_common", +# srcs = ["_application_common.py",], +# deps = [ +# "//src/python/grpcio_tests/tests/testing/proto:requests", +# "//src/python/grpcio_tests/tests/testing/proto:services", +# ], +# imports = ["../../",], +# ) + +py_library( + name = "_server_application", + srcs = ["_server_application.py",], + imports = ["../../",], +) + + + diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel new file mode 100644 index 0000000000..dcd6d9fbb2 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -0,0 +1,83 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +GRPCIO_TESTS_UNIT = [ + "_api_test.py", + "_auth_context_test.py", + "_auth_test.py", + "_channel_args_test.py", + "_channel_close_test.py", + "_channel_connectivity_test.py", + "_channel_ready_future_test.py", + "_compression_test.py", + "_credentials_test.py", + "_empty_message_test.py", + "_exit_test.py", + "_interceptor_test.py", + "_invalid_metadata_test.py", + "_invocation_defects_test.py", + "_metadata_code_details_test.py", + "_metadata_test.py", + # TODO: Issue 16336 + # "_reconnect_test.py", + "_resource_exhausted_test.py", + "_rpc_test.py", + # TODO(ghostwriternr): To be added later. + # "_server_ssl_cert_config_test.py", + "_server_test.py", + "_session_cache_test.py", +] + +py_library( + name = "resources", + srcs = ["resources.py"], + data=[ + "//src/python/grpcio_tests/tests/unit/credentials", + ], +) + +py_library( + name = "test_common", + srcs = ["test_common.py"], +) + +py_library( + name = "_exit_scenarios", + srcs = ["_exit_scenarios.py"], +) + +py_library( + name = "_thread_pool", + srcs = ["_thread_pool.py"], +) + +py_library( + name = "_from_grpc_import_star", + srcs = ["_from_grpc_import_star.py"], +) + +[ + py_test( + name=test_file_name[:-3], + size="small", + srcs=[test_file_name], + main=test_file_name, + deps=[ + "//src/python/grpcio/grpc:grpcio", + ":resources", + ":test_common", + ":_exit_scenarios", + ":_thread_pool", + ":_from_grpc_import_star", + "//src/python/grpcio_tests/tests/unit/framework/common", + "//src/python/grpcio_tests/tests/testing", + requirement('six'), + ], + imports=["../../",], + data=[ + "//src/python/grpcio_tests/tests/unit/credentials", + ], + ) for test_file_name in GRPCIO_TESTS_UNIT +] + diff --git a/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel new file mode 100644 index 0000000000..458a6b1fb8 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/BUILD.bazel @@ -0,0 +1,46 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +GRPCIO_TESTS_UNIT_CYTHON = [ + "_cancel_many_calls_test.py", + "_channel_test.py", + "_no_messages_server_completion_queue_per_call_test.py", + "_no_messages_single_server_completion_queue_test.py", + "_read_some_but_not_all_responses_test.py", + "_server_test.py", + "cygrpc_test.py", +] + +py_library( + name = "common", + srcs = ["_common.py"], +) + +py_library( + name = "test_utilities", + srcs = ["test_utilities.py"], +) + +[ + py_test( + name=test_file_name[:-3], + size="small", + srcs=[test_file_name], + main=test_file_name, + deps=[ + "//src/python/grpcio/grpc:grpcio", + ":common", + ":test_utilities", + "//src/python/grpcio_tests/tests/unit/framework/common", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/python/grpcio_tests/tests/unit:resources", + ], + imports=["../../../",], + data=[ + "//src/python/grpcio_tests/tests/unit/credentials", + ], + ) for test_file_name in GRPCIO_TESTS_UNIT_CYTHON +] + + diff --git a/src/python/grpcio_tests/tests/unit/beta/BUILD.bazel b/src/python/grpcio_tests/tests/unit/beta/BUILD.bazel new file mode 100644 index 0000000000..d3e0fe20eb --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/beta/BUILD.bazel @@ -0,0 +1,75 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "test_utilities", + srcs = ["test_utilities.py"], + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + +py_test( + name = "_beta_features_test", + srcs = ["_beta_features_test.py"], + main = "_beta_features_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_tests/tests/unit:resources", + "//src/python/grpcio_tests/tests/unit/framework/common", + ":test_utilities", + ], + imports=["../../../",], +) + +py_test( + name = "_connectivity_channel_test", + srcs = ["_connectivity_channel_test.py"], + main = "_connectivity_channel_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + +# TODO(ghostwriternr): To be added later. +#py_test( +# name = "_implementations_test", +# srcs = ["_implementations_test.py"], +# main = "_implementations_test.py", +# size = "small", +# deps = [ +# "//src/python/grpcio/grpc:grpcio", +# "//src/python/grpcio_tests/tests/unit:resources", +# requirement('oauth2client'), +# ], +# imports=["../../../",], +#) + +py_test( + name = "_not_found_test", + srcs = ["_not_found_test.py"], + main = "_not_found_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_tests/tests/unit/framework/common", + ], + imports=["../../../",], +) + +py_test( + name = "_utilities_test", + srcs = ["_utilities_test.py"], + main = "_utilities_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_tests/tests/unit/framework/common", + ], + imports=["../../../",], +) + + diff --git a/src/python/grpcio_tests/tests/unit/credentials/BUILD.bazel b/src/python/grpcio_tests/tests/unit/credentials/BUILD.bazel new file mode 100644 index 0000000000..358216db58 --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/credentials/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name="credentials", + srcs=glob([ + "**", + ]), +) + + diff --git a/src/python/grpcio_tests/tests/unit/framework/common/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/common/BUILD.bazel new file mode 100644 index 0000000000..c206a04fad --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/framework/common/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "common", + srcs = [ + "test_constants.py", + "test_control.py", + "test_coverage.py", + ], +) + diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 1e7d7f687f..0e192b6201 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -98,12 +98,12 @@ grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_imp grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import; grpc_channelz_get_servers_type grpc_channelz_get_servers_import; +grpc_channelz_get_server_sockets_type grpc_channelz_get_server_sockets_import; grpc_channelz_get_channel_type grpc_channelz_get_channel_import; grpc_channelz_get_subchannel_type grpc_channelz_get_subchannel_import; grpc_channelz_get_socket_type grpc_channelz_get_socket_import; grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import; grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import; -grpc_use_signal_type grpc_use_signal_import; grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; grpc_auth_context_property_iterator_type grpc_auth_context_property_iterator_import; grpc_auth_context_peer_identity_type grpc_auth_context_peer_identity_import; @@ -355,12 +355,12 @@ void grpc_rb_load_imports(HMODULE library) { grpc_resource_quota_arg_vtable_import = (grpc_resource_quota_arg_vtable_type) GetProcAddress(library, "grpc_resource_quota_arg_vtable"); grpc_channelz_get_top_channels_import = (grpc_channelz_get_top_channels_type) GetProcAddress(library, "grpc_channelz_get_top_channels"); grpc_channelz_get_servers_import = (grpc_channelz_get_servers_type) GetProcAddress(library, "grpc_channelz_get_servers"); + grpc_channelz_get_server_sockets_import = (grpc_channelz_get_server_sockets_type) GetProcAddress(library, "grpc_channelz_get_server_sockets"); grpc_channelz_get_channel_import = (grpc_channelz_get_channel_type) GetProcAddress(library, "grpc_channelz_get_channel"); grpc_channelz_get_subchannel_import = (grpc_channelz_get_subchannel_type) GetProcAddress(library, "grpc_channelz_get_subchannel"); grpc_channelz_get_socket_import = (grpc_channelz_get_socket_type) GetProcAddress(library, "grpc_channelz_get_socket"); grpc_insecure_channel_create_from_fd_import = (grpc_insecure_channel_create_from_fd_type) GetProcAddress(library, "grpc_insecure_channel_create_from_fd"); grpc_server_add_insecure_channel_from_fd_import = (grpc_server_add_insecure_channel_from_fd_type) GetProcAddress(library, "grpc_server_add_insecure_channel_from_fd"); - grpc_use_signal_import = (grpc_use_signal_type) GetProcAddress(library, "grpc_use_signal"); grpc_auth_property_iterator_next_import = (grpc_auth_property_iterator_next_type) GetProcAddress(library, "grpc_auth_property_iterator_next"); grpc_auth_context_property_iterator_import = (grpc_auth_context_property_iterator_type) GetProcAddress(library, "grpc_auth_context_property_iterator"); grpc_auth_context_peer_identity_import = (grpc_auth_context_peer_identity_type) GetProcAddress(library, "grpc_auth_context_peer_identity"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index ed4b6264b0..e075db89e8 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -269,6 +269,9 @@ extern grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import typedef char*(*grpc_channelz_get_servers_type)(intptr_t start_server_id); extern grpc_channelz_get_servers_type grpc_channelz_get_servers_import; #define grpc_channelz_get_servers grpc_channelz_get_servers_import +typedef char*(*grpc_channelz_get_server_sockets_type)(intptr_t server_id, intptr_t start_socket_id); +extern grpc_channelz_get_server_sockets_type grpc_channelz_get_server_sockets_import; +#define grpc_channelz_get_server_sockets grpc_channelz_get_server_sockets_import typedef char*(*grpc_channelz_get_channel_type)(intptr_t channel_id); extern grpc_channelz_get_channel_type grpc_channelz_get_channel_import; #define grpc_channelz_get_channel grpc_channelz_get_channel_import @@ -284,9 +287,6 @@ extern grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_fr typedef void(*grpc_server_add_insecure_channel_from_fd_type)(grpc_server* server, void* reserved, int fd); extern grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import; #define grpc_server_add_insecure_channel_from_fd grpc_server_add_insecure_channel_from_fd_import -typedef void(*grpc_use_signal_type)(int signum); -extern grpc_use_signal_type grpc_use_signal_import; -#define grpc_use_signal grpc_use_signal_import typedef const grpc_auth_property*(*grpc_auth_property_iterator_next_type)(grpc_auth_property_iterator* it); extern grpc_auth_property_iterator_next_type grpc_auth_property_iterator_next_import; #define grpc_auth_property_iterator_next grpc_auth_property_iterator_next_import diff --git a/src/ruby/lib/grpc/errors.rb b/src/ruby/lib/grpc/errors.rb index 8f6aea30ad..0c15c2f3ba 100644 --- a/src/ruby/lib/grpc/errors.rb +++ b/src/ruby/lib/grpc/errors.rb @@ -68,7 +68,6 @@ module GRPC codes[OUT_OF_RANGE] = OutOfRange codes[UNIMPLEMENTED] = Unimplemented codes[INTERNAL] = Internal - codes[UNIMPLEMENTED] = Unimplemented codes[UNAVAILABLE] = Unavailable codes[DATA_LOSS] = DataLoss diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 0c3e1ef734..243d566645 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.16.0.dev' + VERSION = '1.17.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 03d977c064..92e85eb882 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.16.0.dev' + VERSION = '1.17.0.dev' end end diff --git a/templates/Makefile.template b/templates/Makefile.template index 2e3d75d819..0b67416d3e 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -692,7 +692,7 @@ PC_CFLAGS = PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) - PC_LIB = -lgrpc + PC_LIB = -lgrpc_unsecure GRPC_UNSECURE_PC_FILE := $(CORE_PC_TEMPLATE) PROTOBUF_PKG_CONFIG = false @@ -769,7 +769,7 @@ PC_CFLAGS = PC_REQUIRES_PRIVATE = grpc_unsecure $(PC_REQUIRES_GRPCXX) PC_LIBS_PRIVATE = $(PC_LIBS_GRPCXX) - PC_LIB = -lgrpc++ + PC_LIB = -lgrpc++_unsecure GRPCXX_UNSECURE_PC_FILE := $(CPP_PC_TEMPLATE) ifeq ($(MAKECMDGOALS),clean) diff --git a/templates/tools/dockerfile/gcp_api_libraries.include b/templates/tools/dockerfile/gcp_api_libraries.include index adecb92c15..5a263649d2 100644 --- a/templates/tools/dockerfile/gcp_api_libraries.include +++ b/templates/tools/dockerfile/gcp_api_libraries.include @@ -1,3 +1,3 @@ # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client diff --git a/test/core/bad_client/bad_client.cc b/test/core/bad_client/bad_client.cc index ade23133c5..4f5d2a2862 100644 --- a/test/core/bad_client/bad_client.cc +++ b/test/core/bad_client/bad_client.cc @@ -66,7 +66,7 @@ static void server_setup_transport(void* ts, grpc_transport* transport) { thd_args* a = static_cast<thd_args*>(ts); grpc_core::ExecCtx exec_ctx; grpc_server_setup_transport(a->server, transport, nullptr, - grpc_server_get_channel_args(a->server)); + grpc_server_get_channel_args(a->server), 0); } /* Sets the read_done event */ diff --git a/test/core/channel/channelz_registry_test.cc b/test/core/channel/channelz_registry_test.cc index c02d525c81..fdfc8eec94 100644 --- a/test/core/channel/channelz_registry_test.cc +++ b/test/core/channel/channelz_registry_test.cc @@ -43,56 +43,151 @@ namespace grpc_core { namespace channelz { namespace testing { -TEST(ChannelzRegistryTest, UuidStartsAboveZeroTest) { - BaseNode* channelz_channel = nullptr; - intptr_t uuid = ChannelzRegistry::Register(channelz_channel); +class ChannelzRegistryPeer { + public: + const InlinedVector<BaseNode*, 20>* entities() { + return &ChannelzRegistry::Default()->entities_; + } + int num_empty_slots() { + return ChannelzRegistry::Default()->num_empty_slots_; + } +}; + +class ChannelzRegistryTest : public ::testing::Test { + protected: + // ensure we always have a fresh registry for tests. + void SetUp() override { ChannelzRegistry::Init(); } + + void TearDown() override { ChannelzRegistry::Shutdown(); } +}; + +TEST_F(ChannelzRegistryTest, UuidStartsAboveZeroTest) { + UniquePtr<BaseNode> channelz_channel = + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel); + intptr_t uuid = channelz_channel->uuid(); EXPECT_GT(uuid, 0) << "First uuid chose must be greater than zero. Zero if " "reserved according to " "https://github.com/grpc/proposal/blob/master/" "A14-channelz.md"; - ChannelzRegistry::Unregister(uuid); } -TEST(ChannelzRegistryTest, UuidsAreIncreasing) { - BaseNode* channelz_channel = nullptr; - std::vector<intptr_t> uuids; - uuids.reserve(10); +TEST_F(ChannelzRegistryTest, UuidsAreIncreasing) { + std::vector<UniquePtr<BaseNode>> channelz_channels; + channelz_channels.reserve(10); for (int i = 0; i < 10; ++i) { - // reregister the same object. It's ok since we are just testing uuids - uuids.push_back(ChannelzRegistry::Register(channelz_channel)); + channelz_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); } - for (size_t i = 1; i < uuids.size(); ++i) { - EXPECT_LT(uuids[i - 1], uuids[i]) << "Uuids must always be increasing"; + for (size_t i = 1; i < channelz_channels.size(); ++i) { + EXPECT_LT(channelz_channels[i - 1]->uuid(), channelz_channels[i]->uuid()) + << "Uuids must always be increasing"; } } -TEST(ChannelzRegistryTest, RegisterGetTest) { - // we hackily jam an intptr_t into this pointer to check for equality later - BaseNode* channelz_channel = (BaseNode*)42; - intptr_t uuid = ChannelzRegistry::Register(channelz_channel); - BaseNode* retrieved = ChannelzRegistry::Get(uuid); - EXPECT_EQ(channelz_channel, retrieved); +TEST_F(ChannelzRegistryTest, RegisterGetTest) { + UniquePtr<BaseNode> channelz_channel = + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel); + BaseNode* retrieved = ChannelzRegistry::Get(channelz_channel->uuid()); + EXPECT_EQ(channelz_channel.get(), retrieved); } -TEST(ChannelzRegistryTest, RegisterManyItems) { - // we hackily jam an intptr_t into this pointer to check for equality later - BaseNode* channelz_channel = (BaseNode*)42; +TEST_F(ChannelzRegistryTest, RegisterManyItems) { + std::vector<UniquePtr<BaseNode>> channelz_channels; for (int i = 0; i < 100; i++) { - intptr_t uuid = ChannelzRegistry::Register(channelz_channel); - BaseNode* retrieved = ChannelzRegistry::Get(uuid); - EXPECT_EQ(channelz_channel, retrieved); + channelz_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + BaseNode* retrieved = ChannelzRegistry::Get(channelz_channels[i]->uuid()); + EXPECT_EQ(channelz_channels[i].get(), retrieved); } } -TEST(ChannelzRegistryTest, NullIfNotPresentTest) { - // we hackily jam an intptr_t into this pointer to check for equality later - BaseNode* channelz_channel = (BaseNode*)42; - intptr_t uuid = ChannelzRegistry::Register(channelz_channel); +TEST_F(ChannelzRegistryTest, NullIfNotPresentTest) { + UniquePtr<BaseNode> channelz_channel = + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel); // try to pull out a uuid that does not exist. - BaseNode* nonexistant = ChannelzRegistry::Get(uuid + 1); + BaseNode* nonexistant = ChannelzRegistry::Get(channelz_channel->uuid() + 1); EXPECT_EQ(nonexistant, nullptr); - BaseNode* retrieved = ChannelzRegistry::Get(uuid); - EXPECT_EQ(channelz_channel, retrieved); + BaseNode* retrieved = ChannelzRegistry::Get(channelz_channel->uuid()); + EXPECT_EQ(channelz_channel.get(), retrieved); +} + +TEST_F(ChannelzRegistryTest, TestCompaction) { + const int kLoopIterations = 100; + // These channels that will stay in the registry for the duration of the test. + std::vector<UniquePtr<BaseNode>> even_channels; + even_channels.reserve(kLoopIterations); + { + // The channels will unregister themselves at the end of the for block. + std::vector<UniquePtr<BaseNode>> odd_channels; + odd_channels.reserve(kLoopIterations); + for (int i = 0; i < kLoopIterations; i++) { + even_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + odd_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + } + } + // without compaction, there would be exactly kLoopIterations empty slots at + // this point. However, one of the unregisters should have triggered + // compaction. + ChannelzRegistryPeer peer; + EXPECT_LT(peer.num_empty_slots(), kLoopIterations); +} + +TEST_F(ChannelzRegistryTest, TestGetAfterCompaction) { + const int kLoopIterations = 100; + // These channels that will stay in the registry for the duration of the test. + std::vector<UniquePtr<BaseNode>> even_channels; + even_channels.reserve(kLoopIterations); + std::vector<intptr_t> odd_uuids; + odd_uuids.reserve(kLoopIterations); + { + // The channels will unregister themselves at the end of the for block. + std::vector<UniquePtr<BaseNode>> odd_channels; + odd_channels.reserve(kLoopIterations); + for (int i = 0; i < kLoopIterations; i++) { + even_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + odd_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + odd_uuids.push_back(odd_channels[i]->uuid()); + } + } + for (int i = 0; i < kLoopIterations; i++) { + BaseNode* retrieved = ChannelzRegistry::Get(even_channels[i]->uuid()); + EXPECT_EQ(even_channels[i].get(), retrieved); + retrieved = ChannelzRegistry::Get(odd_uuids[i]); + EXPECT_EQ(retrieved, nullptr); + } +} + +TEST_F(ChannelzRegistryTest, TestAddAfterCompaction) { + const int kLoopIterations = 100; + // These channels that will stay in the registry for the duration of the test. + std::vector<UniquePtr<BaseNode>> even_channels; + even_channels.reserve(kLoopIterations); + std::vector<intptr_t> odd_uuids; + odd_uuids.reserve(kLoopIterations); + { + // The channels will unregister themselves at the end of the for block. + std::vector<UniquePtr<BaseNode>> odd_channels; + odd_channels.reserve(kLoopIterations); + for (int i = 0; i < kLoopIterations; i++) { + even_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + odd_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + odd_uuids.push_back(odd_channels[i]->uuid()); + } + } + std::vector<UniquePtr<BaseNode>> more_channels; + more_channels.reserve(kLoopIterations); + for (int i = 0; i < kLoopIterations; i++) { + more_channels.push_back( + MakeUnique<BaseNode>(BaseNode::EntityType::kTopLevelChannel)); + BaseNode* retrieved = ChannelzRegistry::Get(more_channels[i]->uuid()); + EXPECT_EQ(more_channels[i].get(), retrieved); + } } } // namespace testing @@ -101,9 +196,7 @@ TEST(ChannelzRegistryTest, NullIfNotPresentTest) { int main(int argc, char** argv) { grpc_test_init(argc, argv); - grpc_init(); ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); - grpc_shutdown(); return ret; } diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.cc b/test/core/end2end/fixtures/h2_sockpair+trace.cc index cdefcf4546..eb71e24c77 100644 --- a/test/core/end2end/fixtures/h2_sockpair+trace.cc +++ b/test/core/end2end/fixtures/h2_sockpair+trace.cc @@ -53,7 +53,7 @@ static void server_setup_transport(void* ts, grpc_transport* transport) { grpc_endpoint_pair* sfd = static_cast<grpc_endpoint_pair*>(f->fixture_data); grpc_endpoint_add_to_pollset(sfd->server, grpc_cq_pollset(f->cq)); grpc_server_setup_transport(f->server, transport, nullptr, - grpc_server_get_channel_args(f->server)); + grpc_server_get_channel_args(f->server), 0); } typedef struct { diff --git a/test/core/end2end/fixtures/h2_sockpair.cc b/test/core/end2end/fixtures/h2_sockpair.cc index 8966cb38d4..904bda5458 100644 --- a/test/core/end2end/fixtures/h2_sockpair.cc +++ b/test/core/end2end/fixtures/h2_sockpair.cc @@ -47,7 +47,7 @@ static void server_setup_transport(void* ts, grpc_transport* transport) { grpc_endpoint_pair* sfd = static_cast<grpc_endpoint_pair*>(f->fixture_data); grpc_endpoint_add_to_pollset(sfd->server, grpc_cq_pollset(f->cq)); grpc_server_setup_transport(f->server, transport, nullptr, - grpc_server_get_channel_args(f->server)); + grpc_server_get_channel_args(f->server), 0); } typedef struct { diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.cc b/test/core/end2end/fixtures/h2_sockpair_1byte.cc index ebf4162217..45f7f254ac 100644 --- a/test/core/end2end/fixtures/h2_sockpair_1byte.cc +++ b/test/core/end2end/fixtures/h2_sockpair_1byte.cc @@ -47,7 +47,7 @@ static void server_setup_transport(void* ts, grpc_transport* transport) { grpc_endpoint_pair* sfd = static_cast<grpc_endpoint_pair*>(f->fixture_data); grpc_endpoint_add_to_pollset(sfd->server, grpc_cq_pollset(f->cq)); grpc_server_setup_transport(f->server, transport, nullptr, - grpc_server_get_channel_args(f->server)); + grpc_server_get_channel_args(f->server), 0); } typedef struct { diff --git a/test/core/end2end/fuzzers/api_fuzzer.cc b/test/core/end2end/fuzzers/api_fuzzer.cc index eacfd4a8c3..9544adb912 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.cc +++ b/test/core/end2end/fuzzers/api_fuzzer.cc @@ -416,7 +416,7 @@ static void do_connect(void* arg, grpc_error* error) { grpc_transport* transport = grpc_create_chttp2_transport(nullptr, server, false); - grpc_server_setup_transport(g_server, transport, nullptr, nullptr); + grpc_server_setup_transport(g_server, transport, nullptr, nullptr, 0); grpc_chttp2_transport_start_reading(transport, nullptr, nullptr); GRPC_CLOSURE_SCHED(fc->closure, GRPC_ERROR_NONE); diff --git a/test/core/end2end/fuzzers/server_fuzzer.cc b/test/core/end2end/fuzzers/server_fuzzer.cc index 248c34cbc1..bd686215dd 100644 --- a/test/core/end2end/fuzzers/server_fuzzer.cc +++ b/test/core/end2end/fuzzers/server_fuzzer.cc @@ -62,7 +62,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { grpc_server_start(server); grpc_transport* transport = grpc_create_chttp2_transport(nullptr, mock_endpoint, false); - grpc_server_setup_transport(server, transport, nullptr, nullptr); + grpc_server_setup_transport(server, transport, nullptr, nullptr, 0); grpc_chttp2_transport_start_reading(transport, nullptr, nullptr); grpc_call* call1 = nullptr; diff --git a/test/core/end2end/tests/channelz.cc b/test/core/end2end/tests/channelz.cc index 0e3ad6a476..41549190e3 100644 --- a/test/core/end2end/tests/channelz.cc +++ b/test/core/end2end/tests/channelz.cc @@ -256,6 +256,10 @@ static void test_channelz(grpc_end2end_test_config config) { GPR_ASSERT(nullptr == strstr(json, "\"severity\":\"CT_INFO\"")); gpr_free(json); + json = channelz_server->RenderServerSockets(0); + GPR_ASSERT(nullptr != strstr(json, "\"socketRef\":")); + gpr_free(json); + end_test(&f); config.tear_down_data(&f); } diff --git a/test/core/gprpp/inlined_vector_test.cc b/test/core/gprpp/inlined_vector_test.cc index e9d1eb2c76..73e0773b31 100644 --- a/test/core/gprpp/inlined_vector_test.cc +++ b/test/core/gprpp/inlined_vector_test.cc @@ -264,6 +264,32 @@ TEST(InlinedVectorTest, MoveAssignmentAllocatedAllocated) { EXPECT_EQ(move_assigned.data(), old_data); } +TEST(InlinedVectorTest, PopBackInlined) { + InlinedVector<UniquePtr<int>, 2> v; + // Add two elements, pop one out + v.push_back(MakeUnique<int>(3)); + EXPECT_EQ(1UL, v.size()); + EXPECT_EQ(3, *v[0]); + v.push_back(MakeUnique<int>(5)); + EXPECT_EQ(2UL, v.size()); + EXPECT_EQ(5, *v[1]); + v.pop_back(); + EXPECT_EQ(1UL, v.size()); +} + +TEST(InlinedVectorTest, PopBackAllocated) { + const int kInlinedSize = 2; + InlinedVector<UniquePtr<int>, kInlinedSize> v; + // Add elements to ensure allocated backing. + for (size_t i = 0; i < kInlinedSize + 1; ++i) { + v.push_back(MakeUnique<int>(3)); + EXPECT_EQ(i + 1, v.size()); + } + size_t sz = v.size(); + v.pop_back(); + EXPECT_EQ(sz - 1, v.size()); +} + } // namespace testing } // namespace grpc_core diff --git a/test/core/handshake/readahead_handshaker_server_ssl.cc b/test/core/handshake/readahead_handshaker_server_ssl.cc index 97e9c20ee4..14d96b5d89 100644 --- a/test/core/handshake/readahead_handshaker_server_ssl.cc +++ b/test/core/handshake/readahead_handshaker_server_ssl.cc @@ -75,6 +75,7 @@ static grpc_handshaker* readahead_handshaker_create() { static void readahead_handshaker_factory_add_handshakers( grpc_handshaker_factory* hf, const grpc_channel_args* args, + grpc_pollset_set* interested_parties, grpc_handshake_manager* handshake_mgr) { grpc_handshake_manager_add(handshake_mgr, readahead_handshaker_create()); } diff --git a/test/core/iomgr/tcp_client_posix_test.cc b/test/core/iomgr/tcp_client_posix_test.cc index a4c38af86f..90fc5aecfc 100644 --- a/test/core/iomgr/tcp_client_posix_test.cc +++ b/test/core/iomgr/tcp_client_posix_test.cc @@ -170,7 +170,7 @@ void test_fails(void) { break; case GRPC_TIMERS_NOT_CHECKED: polling_deadline = 0; - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_work", diff --git a/test/core/iomgr/tcp_client_uv_test.cc b/test/core/iomgr/tcp_client_uv_test.cc index 0c6250ed7f..ffcc7937c7 100644 --- a/test/core/iomgr/tcp_client_uv_test.cc +++ b/test/core/iomgr/tcp_client_uv_test.cc @@ -165,7 +165,7 @@ void test_fails(void) { break; case GRPC_TIMERS_NOT_CHECKED: polling_deadline = grpc_timespec_to_millis_round_up(now); - /* fall through */ + // fallthrough case GRPC_TIMERS_CHECKED_AND_EMPTY: GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_work", diff --git a/test/core/security/ssl_server_fuzzer.cc b/test/core/security/ssl_server_fuzzer.cc index cb74e3bae1..1e04691ce2 100644 --- a/test/core/security/ssl_server_fuzzer.cc +++ b/test/core/security/ssl_server_fuzzer.cc @@ -91,7 +91,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { struct handshake_state state; state.done_callback_called = false; grpc_handshake_manager* handshake_mgr = grpc_handshake_manager_create(); - grpc_server_security_connector_add_handshakers(sc, handshake_mgr); + grpc_server_security_connector_add_handshakers(sc, nullptr, handshake_mgr); grpc_handshake_manager_do_handshake( handshake_mgr, nullptr /* interested_parties */, mock_endpoint, nullptr /* channel_args */, deadline, nullptr /* acceptor */, diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 3ebdb88a08..4640b8e4ca 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -137,6 +137,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_resource_quota_arg_vtable); printf("%lx", (unsigned long) grpc_channelz_get_top_channels); printf("%lx", (unsigned long) grpc_channelz_get_servers); + printf("%lx", (unsigned long) grpc_channelz_get_server_sockets); printf("%lx", (unsigned long) grpc_channelz_get_channel); printf("%lx", (unsigned long) grpc_channelz_get_subchannel); printf("%lx", (unsigned long) grpc_channelz_get_socket); diff --git a/test/core/transport/chttp2/settings_timeout_test.cc b/test/core/transport/chttp2/settings_timeout_test.cc index 39ae587bae..2d6f0a9a62 100644 --- a/test/core/transport/chttp2/settings_timeout_test.cc +++ b/test/core/transport/chttp2/settings_timeout_test.cc @@ -196,6 +196,8 @@ class Client { "grpc_pollset_work", grpc_pollset_work(pollset_, &worker, grpc_core::ExecCtx::Get()->Now() + 1000)); + // Flushes any work scheduled before or during polling. + grpc_core::ExecCtx::Get()->Flush(); gpr_mu_unlock(mu_); if (state != nullptr && state->done()) return true; if (grpc_core::ExecCtx::Get()->Now() >= deadline) return false; diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 85a58114ba..e9eb7e175f 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -421,7 +421,7 @@ static tsi_handshaker* create_test_handshaker(bool used_for_success_test, alts_mock_handshaker_client_create(used_for_success_test); grpc_alts_credentials_options* options = grpc_alts_credentials_client_options_create(); - alts_tsi_handshaker_create(options, "target_name", "lame", is_client, + alts_tsi_handshaker_create(options, "target_name", "lame", is_client, nullptr, &handshaker); alts_tsi_handshaker* alts_handshaker = reinterpret_cast<alts_tsi_handshaker*>(handshaker); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index d97ea071d1..6ecb957801 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -953,6 +953,114 @@ TEST_P(AsyncEnd2endTest, ServerInitialMetadataRpc) { EXPECT_TRUE(recv_status.ok()); } +// 1 ping, 2 pongs. +TEST_P(AsyncEnd2endTest, ServerInitialMetadataServerStreaming) { + ResetStub(); + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + ServerContext srv_ctx; + ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx); + + std::pair<::grpc::string, ::grpc::string> meta1("key1", "val1"); + std::pair<::grpc::string, ::grpc::string> meta2("key2", "val2"); + + std::unique_ptr<ClientAsyncReader<EchoResponse>> cli_stream( + stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); + cli_stream->ReadInitialMetadata(tag(11)); + service_->RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, + cq_.get(), cq_.get(), tag(2)); + + Verifier().Expect(1, true).Expect(2, true).Verify(cq_.get()); + + srv_ctx.AddInitialMetadata(meta1.first, meta1.second); + srv_ctx.AddInitialMetadata(meta2.first, meta2.second); + srv_stream.SendInitialMetadata(tag(10)); + Verifier().Expect(10, true).Expect(11, true).Verify(cq_.get()); + auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); + EXPECT_EQ(meta1.second, + ToString(server_initial_metadata.find(meta1.first)->second)); + EXPECT_EQ(meta2.second, + ToString(server_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(static_cast<size_t>(2), server_initial_metadata.size()); + + srv_stream.Write(send_response, tag(3)); + + cli_stream->Read(&recv_response, tag(4)); + Verifier().Expect(3, true).Expect(4, true).Verify(cq_.get()); + + srv_stream.Write(send_response, tag(5)); + cli_stream->Read(&recv_response, tag(6)); + Verifier().Expect(5, true).Expect(6, true).Verify(cq_.get()); + + srv_stream.Finish(Status::OK, tag(7)); + cli_stream->Read(&recv_response, tag(8)); + Verifier().Expect(7, true).Expect(8, false).Verify(cq_.get()); + + cli_stream->Finish(&recv_status, tag(9)); + Verifier().Expect(9, true).Verify(cq_.get()); + + EXPECT_TRUE(recv_status.ok()); +} + +// 1 ping, 2 pongs. +// Test for server initial metadata being sent implicitly +TEST_P(AsyncEnd2endTest, ServerInitialMetadataServerStreamingImplicit) { + ResetStub(); + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + ClientContext cli_ctx; + ServerContext srv_ctx; + ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx); + + send_request.set_message(GetParam().message_content); + std::pair<::grpc::string, ::grpc::string> meta1("key1", "val1"); + std::pair<::grpc::string, ::grpc::string> meta2("key2", "val2"); + + std::unique_ptr<ClientAsyncReader<EchoResponse>> cli_stream( + stub_->AsyncResponseStream(&cli_ctx, send_request, cq_.get(), tag(1))); + service_->RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, + cq_.get(), cq_.get(), tag(2)); + + Verifier().Expect(1, true).Expect(2, true).Verify(cq_.get()); + EXPECT_EQ(send_request.message(), recv_request.message()); + + srv_ctx.AddInitialMetadata(meta1.first, meta1.second); + srv_ctx.AddInitialMetadata(meta2.first, meta2.second); + send_response.set_message(recv_request.message()); + srv_stream.Write(send_response, tag(3)); + + cli_stream->Read(&recv_response, tag(4)); + Verifier().Expect(3, true).Expect(4, true).Verify(cq_.get()); + EXPECT_EQ(send_response.message(), recv_response.message()); + + auto server_initial_metadata = cli_ctx.GetServerInitialMetadata(); + EXPECT_EQ(meta1.second, + ToString(server_initial_metadata.find(meta1.first)->second)); + EXPECT_EQ(meta2.second, + ToString(server_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(static_cast<size_t>(2), server_initial_metadata.size()); + + srv_stream.Write(send_response, tag(5)); + cli_stream->Read(&recv_response, tag(6)); + Verifier().Expect(5, true).Expect(6, true).Verify(cq_.get()); + + srv_stream.Finish(Status::OK, tag(7)); + cli_stream->Read(&recv_response, tag(8)); + Verifier().Expect(7, true).Expect(8, false).Verify(cq_.get()); + + cli_stream->Finish(&recv_status, tag(9)); + Verifier().Expect(9, true).Verify(cq_.get()); + + EXPECT_TRUE(recv_status.ok()); +} + TEST_P(AsyncEnd2endTest, ServerTrailingMetadataRpc) { ResetStub(); diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index df9e23d50c..7a7a716e79 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -41,6 +41,8 @@ using grpc::channelz::v1::GetChannelRequest; using grpc::channelz::v1::GetChannelResponse; +using grpc::channelz::v1::GetServerSocketsRequest; +using grpc::channelz::v1::GetServerSocketsResponse; using grpc::channelz::v1::GetServersRequest; using grpc::channelz::v1::GetServersResponse; using grpc::channelz::v1::GetSocketRequest; @@ -600,6 +602,30 @@ TEST_F(ChannelzServerTest, StreamingRPC) { kNumMessages); } +TEST_F(ChannelzServerTest, GetServerSocketsTest) { + ResetStubs(); + ConfigureProxy(1); + GetServersRequest get_server_request; + GetServersResponse get_server_response; + get_server_request.set_start_server_id(0); + ClientContext get_server_context; + Status s = channelz_stub_->GetServers(&get_server_context, get_server_request, + &get_server_response); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); + EXPECT_EQ(get_server_response.server_size(), 1); + GetServerSocketsRequest get_server_sockets_request; + GetServerSocketsResponse get_server_sockets_response; + get_server_sockets_request.set_server_id( + get_server_response.server(0).ref().server_id()); + get_server_sockets_request.set_start_socket_id(0); + ClientContext get_server_sockets_context; + s = channelz_stub_->GetServerSockets(&get_server_sockets_context, + get_server_sockets_request, + &get_server_sockets_response); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); + EXPECT_EQ(get_server_sockets_response.socket_ref_size(), 1); +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index a4b1a85f85..1d7fa73aa8 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -137,7 +137,8 @@ int main(int argc, char** argv) { &grpc::testing::InteropClient::DoTimeoutOnSleepingServer, &client); actions["empty_stream"] = std::bind(&grpc::testing::InteropClient::DoEmptyStream, &client); - if (FLAGS_use_tls) { + if (FLAGS_use_tls || + FLAGS_custom_credentials_type == "google_default_credentials") { actions["compute_engine_creds"] = std::bind(&grpc::testing::InteropClient::DoComputeEngineCreds, &client, FLAGS_default_service_account, FLAGS_oauth_scope); diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index 7188a2a28a..e57eb6ddd1 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -200,7 +200,7 @@ class EndpointPairFixture : public BaseFixture { } grpc_server_setup_transport(server_->c_server(), server_transport_, - nullptr, server_args); + nullptr, server_args, 0); grpc_chttp2_transport_start_reading(server_transport_, nullptr, nullptr); } diff --git a/test/cpp/performance/writes_per_rpc_test.cc b/test/cpp/performance/writes_per_rpc_test.cc index 7b51260e5b..32eab1fc44 100644 --- a/test/cpp/performance/writes_per_rpc_test.cc +++ b/test/cpp/performance/writes_per_rpc_test.cc @@ -100,7 +100,7 @@ class EndpointPairFixture { } grpc_server_setup_transport(server_->c_server(), transport, nullptr, - server_args); + server_args, 0); grpc_chttp2_transport_start_reading(transport, nullptr, nullptr); } diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index 483b29b1b3..2ef7441371 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -31,6 +31,7 @@ grpc_cc_library( name = "qps_worker_impl", srcs = [ "client_async.cc", + "client_callback.cc", "client_sync.cc", "qps_server_builder.cc", "qps_worker.cc", diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 0b4b2ff0a9..4ed34e0405 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -533,6 +533,7 @@ class ClientImpl : public Client { std::unique_ptr<Client> CreateSynchronousClient(const ClientConfig& args); std::unique_ptr<Client> CreateAsyncClient(const ClientConfig& args); +std::unique_ptr<Client> CreateCallbackClient(const ClientConfig& args); std::unique_ptr<Client> CreateGenericAsyncStreamingClient( const ClientConfig& args); diff --git a/test/cpp/qps/client_callback.cc b/test/cpp/qps/client_callback.cc new file mode 100644 index 0000000000..87889e36dc --- /dev/null +++ b/test/cpp/qps/client_callback.cc @@ -0,0 +1,219 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <list> +#include <memory> +#include <mutex> +#include <sstream> +#include <string> +#include <thread> +#include <utility> +#include <vector> + +#include <grpc/grpc.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpcpp/alarm.h> +#include <grpcpp/channel.h> +#include <grpcpp/client_context.h> + +#include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" +#include "test/cpp/qps/client.h" +#include "test/cpp/qps/usage_timer.h" + +namespace grpc { +namespace testing { + +/** + * Maintains context info per RPC + */ +struct CallbackClientRpcContext { + CallbackClientRpcContext(BenchmarkService::Stub* stub) : stub_(stub) {} + + ~CallbackClientRpcContext() {} + + SimpleResponse response_; + ClientContext context_; + Alarm alarm_; + BenchmarkService::Stub* stub_; +}; + +static std::unique_ptr<BenchmarkService::Stub> BenchmarkStubCreator( + const std::shared_ptr<Channel>& ch) { + return BenchmarkService::NewStub(ch); +} + +class CallbackClient + : public ClientImpl<BenchmarkService::Stub, SimpleRequest> { + public: + CallbackClient(const ClientConfig& config) + : ClientImpl<BenchmarkService::Stub, SimpleRequest>( + config, BenchmarkStubCreator) { + num_threads_ = NumThreads(config); + rpcs_done_ = 0; + SetupLoadTest(config, num_threads_); + total_outstanding_rpcs_ = + config.client_channels() * config.outstanding_rpcs_per_channel(); + } + + virtual ~CallbackClient() {} + + protected: + size_t num_threads_; + size_t total_outstanding_rpcs_; + // The below mutex and condition variable is used by main benchmark thread to + // wait on completion of all RPCs before shutdown + std::mutex shutdown_mu_; + std::condition_variable shutdown_cv_; + // Number of rpcs done after thread completion + size_t rpcs_done_; + // Vector of Context data pointers for running a RPC + std::vector<std::unique_ptr<CallbackClientRpcContext>> ctx_; + + virtual void InitThreadFuncImpl(size_t thread_idx) = 0; + virtual bool ThreadFuncImpl(Thread* t, size_t thread_idx) = 0; + + void ThreadFunc(size_t thread_idx, Thread* t) override { + InitThreadFuncImpl(thread_idx); + ThreadFuncImpl(t, thread_idx); + } + + virtual void ScheduleRpc(Thread* t, size_t thread_idx, + size_t ctx_vector_idx) = 0; + + /** + * The main thread of the benchmark will be waiting on DestroyMultithreading. + * Increment the rpcs_done_ variable to signify that the Callback RPC + * after thread completion is done. When the last outstanding rpc increments + * the counter it should also signal the main thread's conditional variable. + */ + void NotifyMainThreadOfThreadCompletion() { + std::lock_guard<std::mutex> l(shutdown_mu_); + rpcs_done_++; + if (rpcs_done_ == total_outstanding_rpcs_) { + shutdown_cv_.notify_one(); + } + } + + private: + int NumThreads(const ClientConfig& config) { + int num_threads = config.async_client_threads(); + if (num_threads <= 0) { // Use dynamic sizing + num_threads = cores_; + gpr_log(GPR_INFO, "Sizing callback client to %d threads", num_threads); + } + return num_threads; + } + + /** + * Wait until all outstanding Callback RPCs are done + */ + void DestroyMultithreading() final { + std::unique_lock<std::mutex> l(shutdown_mu_); + while (rpcs_done_ != total_outstanding_rpcs_) { + shutdown_cv_.wait(l); + } + EndThreads(); + } +}; + +class CallbackUnaryClient final : public CallbackClient { + public: + CallbackUnaryClient(const ClientConfig& config) : CallbackClient(config) { + for (int ch = 0; ch < config.client_channels(); ch++) { + for (int i = 0; i < config.outstanding_rpcs_per_channel(); i++) { + ctx_.emplace_back( + new CallbackClientRpcContext(channels_[ch].get_stub())); + } + } + StartThreads(num_threads_); + } + ~CallbackUnaryClient() {} + + protected: + bool ThreadFuncImpl(Thread* t, size_t thread_idx) override { + for (size_t vector_idx = thread_idx; vector_idx < total_outstanding_rpcs_; + vector_idx += num_threads_) { + ScheduleRpc(t, thread_idx, vector_idx); + } + return true; + } + + void InitThreadFuncImpl(size_t thread_idx) override { return; } + + private: + void ScheduleRpc(Thread* t, size_t thread_idx, size_t vector_idx) override { + if (!closed_loop_) { + gpr_timespec next_issue_time = NextIssueTime(thread_idx); + // Start an alarm callback to run the internal callback after + // next_issue_time + ctx_[vector_idx]->alarm_.experimental().Set( + next_issue_time, [this, t, thread_idx, vector_idx](bool ok) { + IssueUnaryCallbackRpc(t, thread_idx, vector_idx); + }); + } else { + IssueUnaryCallbackRpc(t, thread_idx, vector_idx); + } + } + + void IssueUnaryCallbackRpc(Thread* t, size_t thread_idx, size_t vector_idx) { + GPR_TIMER_SCOPE("CallbackUnaryClient::ThreadFunc", 0); + double start = UsageTimer::Now(); + ctx_[vector_idx]->stub_->experimental_async()->UnaryCall( + (&ctx_[vector_idx]->context_), &request_, &ctx_[vector_idx]->response_, + [this, t, thread_idx, start, vector_idx](grpc::Status s) { + // Update Histogram with data from the callback run + HistogramEntry entry; + if (s.ok()) { + entry.set_value((UsageTimer::Now() - start) * 1e9); + } + entry.set_status(s.error_code()); + t->UpdateHistogram(&entry); + + if (ThreadCompleted() || !s.ok()) { + // Notify thread of completion + NotifyMainThreadOfThreadCompletion(); + } else { + // Reallocate ctx for next RPC + ctx_[vector_idx].reset( + new CallbackClientRpcContext(ctx_[vector_idx]->stub_)); + // Schedule a new RPC + ScheduleRpc(t, thread_idx, vector_idx); + } + }); + } +}; + +std::unique_ptr<Client> CreateCallbackClient(const ClientConfig& config) { + switch (config.rpc_type()) { + case UNARY: + return std::unique_ptr<Client>(new CallbackUnaryClient(config)); + case STREAMING: + case STREAMING_FROM_CLIENT: + case STREAMING_FROM_SERVER: + case STREAMING_BOTH_WAYS: + assert(false); + return nullptr; + default: + assert(false); + return nullptr; + } +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc index 7ddf3c1cf3..d97d95d8f3 100644 --- a/test/cpp/qps/qps_worker.cc +++ b/test/cpp/qps/qps_worker.cc @@ -60,6 +60,8 @@ static std::unique_ptr<Client> CreateClient(const ClientConfig& config) { return config.payload_config().has_bytebuf_params() ? CreateGenericAsyncStreamingClient(config) : CreateAsyncClient(config); + case ClientType::CALLBACK_CLIENT: + return CreateCallbackClient(config); default: abort(); } diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 581dab3b4e..4b775e667e 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index 5f0ad58d61..8ad09b7e83 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@mehrdada +@apolcyn @nicolasnoble diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 578bf427cd..030d301a40 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index c4f959fc1d..f9dc8f20d7 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index d07ea9a9b0..76015c8c42 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile index f60f67665d..77aac3cdec 100644 --- a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile @@ -52,7 +52,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index b0d9261af2..09479ba2f7 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index 439baadc2c..3a87639794 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile index b80249d2d8..d4117f83c8 100644 --- a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 986cfcbada..7f871f2f62 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index ad719f330f..3c95554b02 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #================ # C# dependencies diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 1fe44b6dd7..7c36fb3734 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client # Install Electron apt dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index 53ef7b3443..0dff839904 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -62,7 +62,7 @@ RUN cd /var/local/git/php-src \ # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index e884572640..ed59e56995 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index c2b4c1845b..a4c3a9f91e 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index c23e67c904..d94ccc8c74 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d6f7459685..321b501de2 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index cb153012ea..e6bdb4ee03 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 97599be443..40abd726c4 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.16.0-dev +PROJECT_NUMBER = 1.17.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index c852b29fce..8fed272159 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.16.0-dev +PROJECT_NUMBER = 1.17.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 0c2569d0d8..973975ae28 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-dev +PROJECT_NUMBER = 7.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index d9ab712301..d5986fde90 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-dev +PROJECT_NUMBER = 7.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh deleted file mode 100755 index 205c0bf8c5..0000000000 --- a/tools/gce/create_interop_worker.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates an interop worker on GCE. -# IMPORTANT: After this script finishes, there are still some manual -# steps needed there are hard to automatize. -# See go/grpc-jenkins-setup for followup instructions. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-east1-a # canary gateway is reachable from this zone - -INSTANCE_NAME="${1:-grpc-canary-interop2}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image ubuntu-15-10 \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_kokoro_performance_worker.sh b/tools/gce/create_linux_kokoro_performance_worker.sh index d08a1aa2c6..63d54a48bf 100755 --- a/tools/gce/create_linux_kokoro_performance_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker.sh @@ -15,6 +15,13 @@ # Creates a performance worker on GCE to be used on Kokoro. +# IMPORTANT: Instructions for updating +# If the VM configuration / installed software is updated, +# - all existing performance worker VMs need to be updated to reflect the changes +# - a new GCE image named "grpc-performance-kokoro-v1" needs to be created, +# incrementing the version number. +# - kokoro jobs need to be reconfigured to use the new image version + set -ex cd "$(dirname "$0")" @@ -30,7 +37,7 @@ gcloud compute instances create "$INSTANCE_NAME" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ + --image-family ubuntu-1804-lts \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh deleted file mode 100755 index e9033ec443..0000000000 --- a/tools/gce/create_linux_performance_worker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a performance worker on GCE. -# IMPORTANT: After creating the worker, one needs to manually add the pubkey -# of jenkins@the-machine-where-jenkins-starts-perf-tests -# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. -# See tools/run_tests/run_performance_tests.py for details. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-b # this zone allows 32core machines - -INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-32 - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type $MACHINE_TYPE \ - --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ - --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh deleted file mode 100755 index a93d8c5e83..0000000000 --- a/tools/gce/create_linux_worker.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a standard jenkins worker on GCE. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-a - -INSTANCE_NAME="${1:-grpc-jenkins-worker1}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image=ubuntu-1510 \ - --image-project=grpc-testing \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub deleted file mode 100644 index e9853224e1..0000000000 --- a/tools/gce/jenkins_master.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/kokoro_performance.pub b/tools/gce/kokoro_performance.pub index 1154debe78..4eca916b8a 100644 --- a/tools/gce/kokoro_performance.pub +++ b/tools/gce/kokoro_performance.pub @@ -1 +1,4 @@ +# Enable Kokoro CI to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com +# Enable kokoro multi-machine benchmark driver VM to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQ5UEX4AFefec9BKICupFS7x9Hoq4ZyLKy+QX0J31I49ew9mG2AJlr3sp8ql15eX+A2Ml9MKJkmgZGHpJtw+SfvmI94SmomSyiCLAK92sQ85NMzaRdo4b9e30E9nhXnAvAaemvIEQbgCMYFvzk0C8AtXj6+htCrN4jFaLqTCPISJhX3ETc4TgX1qaHQHyl31tdaXHYlITvBDsfokcGcZQnhmCUDtD8wyaSC8GFk9gZbXshkfaYCuuLPPA0vwWGBw+YPbonHsFCsOog1IYSzYPCkIjq8dt6evsusK6Kaoyw/Z+l2kYty2FKTj+wU3l06QMoxwcfNT4WxdhcnVbY71r kbuilder@kokoro-performance-driver diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 4a1e3e608b..b78695d802 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -47,7 +47,6 @@ sudo apt-get install -y \ libtool \ make \ strace \ - pypy \ python-dev \ python-pip \ python-setuptools \ @@ -68,30 +67,34 @@ sudo apt-get install -y google-perftools libgoogle-perftools-dev # netperf sudo apt-get install -y netperf +# required to run kokoro_log_reader.py +sudo apt-get install -y python-psutil python3-psutil + +# gcloud tools, including gsutil +sudo apt-get install -y google-cloud-sdk + # C++ dependencies sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies sudo pip install --upgrade pip==10.0.1 sudo pip install tabulate -sudo pip install google-api-python-client +sudo pip install google-api-python-client oauth2client sudo pip install virtualenv -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - +# pypy is used instead of python for postprocessing benchmark outputs +# because some reports are huge and pypy is much faster. +# TODO(jtattermusch): get rid of pypy once possible, it's hard to +# keep track of all the installed variants of python. +sudo apt-get install -y pypy pypy-dev curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py sudo pypy -m pip install tabulate -sudo pip install google-api-python-client +sudo pypy -m pip install google-api-python-client oauth2client +# TODO(jtattermusch): for some reason, we need psutil installed +# in pypy for kokoro_log_reader.py (strange, because the comand is +# "python kokoro_log_reader.py" and pypy is not the system default) +sudo pypy -m pip install psutil # Node dependencies (nvm has to be installed under user kbuilder) touch .profile @@ -104,31 +107,31 @@ nvm install 4 && npm config set cache /tmp/npm-cache nvm install 5 && npm config set cache /tmp/npm-cache nvm alias default 4 +# C# dependencies +sudo apt-get install -y cmake + # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list sudo apt-get update -sudo apt-get install -y mono-devel nuget +sudo apt-get install -y mono-devel -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 +# C# .NET Core dependencies (https://www.microsoft.com/net/download) +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb + +sudo apt-get install -y apt-transport-https sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# C# 1.0.4 SDK -curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -sudo mkdir -p /opt/dotnet -sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet -sudo ln -s /opt/dotnet/dotnet /usr/local/bin - -# C# .NET dependencies -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb -sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb -sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb -sudo apt-get update && sudo apt-get install -y libicu55 +sudo apt-get install -y dotnet-sdk-2.1 + +# Install .NET Core 1.0.5 Runtime (required to run netcoreapp1.0) +wget -q https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz +mkdir -p dotnet105_download +tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C dotnet105_download +sudo cp -r dotnet105_download/shared/Microsoft.NETCore.App/1.0.5/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ +# To prevent "Failed to initialize CoreCLR, HRESULT: 0x80131500" with .NET Core 1.0.5 runtime +wget -q http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.4_amd64.deb +sudo dpkg -i libicu55_55.1-7ubuntu0.4_amd64.deb # Ruby dependencies gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 @@ -163,7 +166,7 @@ sudo mv composer.phar /usr/local/bin/composer # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 +GO_VERSION=1.10 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz @@ -190,11 +193,22 @@ git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts sudo apt-get install -y python-scipy python-numpy +# Install docker +curl -sSL https://get.docker.com/ | sh +# Enable kbuilder to use docker without sudo: +sudo usermod -aG docker kbuilder + # Add pubkey of Kokoro driver VM to allow SSH # silence false-positive shellcheck warning ("< redirect does not affect sudo") # shellcheck disable=SC2024 sudo tee --append ~kbuilder/.ssh/authorized_keys < kokoro_performance.pub +# Kokoro requires /tmpfs/READY file to exist the directory and file itself should +# be owned by kbuilder. +sudo mkdir /tmpfs +sudo chown kbuilder /tmpfs +touch /tmpfs/READY + # Restart for VM to pick up kernel update echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' sleep 10 diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh deleted file mode 100755 index 7222cef9a2..0000000000 --- a/tools/gce/linux_performance_worker_init.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux performance worker. -# You shouldn't run this script on your own, -# use create_linux_performance_worker.sh instead. - -set -ex - -sudo apt-get update - -# Install Java 8 JDK (to build gRPC Java) -sudo apt-get install -y openjdk-8-jdk -sudo apt-get install -y unzip lsof - -sudo apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libcurl4-openssl-dev \ - libgtest-dev \ - libreadline-dev \ - libssl-dev \ - libtool \ - make \ - strace \ - pypy \ - python-dev \ - python-pip \ - python-setuptools \ - python-yaml \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-yaml \ - telnet \ - unzip \ - wget \ - zip \ - zlib1g-dev - -# perftools -sudo apt-get install -y google-perftools libgoogle-perftools-dev - -# netperf -sudo apt-get install -y netperf - -# C++ dependencies -sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang - -# Python dependencies -sudo pip install --upgrade pip==10.0.1 -sudo pip install tabulate -sudo pip install google-api-python-client -sudo pip install virtualenv - -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - -curl -O https://bootstrap.pypa.io/get-pip.py -sudo pypy get-pip.py -sudo pypy -m pip install tabulate -sudo pip install google-api-python-client - -# Node dependencies (nvm has to be installed under user jenkins) -touch .profile -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -# silence shellcheck warning as it cannot follow the `source` path statically: -# shellcheck disable=SC1090 -source ~/.nvm/nvm.sh -nvm install 0.12 && npm config set cache /tmp/npm-cache -nvm install 4 && npm config set cache /tmp/npm-cache -nvm install 5 && npm config set cache /tmp/npm-cache -nvm alias default 4 - -# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list -sudo apt-get update -sudo apt-get install -y mono-devel nuget - -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 -sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# Ruby dependencies -git clone https://github.com/rbenv/rbenv.git ~/.rbenv -export PATH="$HOME/.rbenv/bin:$PATH" -eval "$(rbenv init -)" - -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" - -rbenv install 2.4.0 -rbenv global 2.4.0 -ruby -v - -# Install bundler (prerequisite for gRPC Ruby) -gem install bundler - -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - -# Java dependencies - nothing as we already have Java JDK 8 - -# Go dependencies -# Currently, the golang package available via apt-get doesn't have the latest go. -# Significant performance improvements with grpc-go have been observed after -# upgrading from go 1.5 to a later version, so a later go version is preferred. -# Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 -OS=linux -ARCH=amd64 -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -# Put go on the PATH, keep the usual installation dir -sudo ln -s /usr/local/go/bin/go /usr/bin/go -rm go$GO_VERSION.$OS-$ARCH.tar.gz - -# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) -sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" -# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar -echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid -# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol -echo 0 | sudo tee /proc/sys/kernel/kptr_restrict - -# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in -# order to not lose perf events or time out -echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb - -# Fetch scripts to generate flame graphs from perf data collected -# on benchmarks -git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph - -# Install scipy and numpy for benchmarking scripts -sudo apt-get install -y python-scipy python-numpy - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. -# silence false-positive shellcheck warning ("< redirect does not affect sudo") -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for VM to pick up kernel update -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 -sudo reboot diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh deleted file mode 100755 index 05855354ff..0000000000 --- a/tools/gce/linux_worker_init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux worker. -# You shouldn't run this script on your own, use create_linux_worker.sh -# instead. - -set -ex - -# Create some swap space -sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 -sudo chmod 600 /swap -sudo mkswap /swap -sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab -sudo swapon -a - -# Typical apt-get maintenance -sudo apt-get update - -# Install JRE -sudo apt-get install -y openjdk-8-jre -sudo apt-get install -y unzip lsof - -# Install Docker -curl -sSL https://get.docker.com/ | sh - -# Setup jenkins user (or the user will already exist bcuz magic) -sudo adduser jenkins --disabled-password || true - -# Enable jenkins to use docker without sudo: -sudo usermod -aG docker jenkins - -# Use "overlay" storage driver for docker -# see https://github.com/grpc/grpc/issues/4988 -printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json - -# Install pip and Google API library to enable using GCP services -sudo apt-get install -y python-pip -sudo pip install google-api-python-client - -# Install RVM -# TODO(jtattermusch): why is RVM needed? -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby - -# Upgrade Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. - -# disable superfluous warning by shellcheck: -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for docker to pick up the config changes. -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 - -sudo reboot diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 054561aac9..d18ea2aca1 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -240,11 +240,10 @@ class CSharpExtArtifact: ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], use_workspace=True) elif self.platform == 'windows': - cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( self.name, [ 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', - cmake_arch_option + self.arch ], use_workspace=True) else: diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index ac2c92b716..713e480f72 100644 --- a/tools/run_tests/artifacts/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat @@ -15,16 +15,33 @@ @rem Builds C# artifacts on Windows set ARCHITECTURE=%1 -set GRPC_SKIP_DOTNET_RESTORE=true -@call tools\run_tests\helper_scripts\pre_build_csharp.bat %ARCHITECTURE% || goto :error -cd cmake\build\%ARCHITECTURE% -cmake --build . --target grpc_csharp_ext --config RelWithDebInfo +@rem enter repo root +cd /d %~dp0\..\..\.. + +mkdir cmake +cd cmake +mkdir build +cd build +mkdir %ARCHITECTURE% +cd %ARCHITECTURE% + +@rem TODO(jtattermusch): is there a better way to force using MSVC? +@rem select the MSVC compiler explicitly to avoid using gcc from mingw or cygwin +@rem (both are on path) +set "MSVC_COMPILER=C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/cl.exe" +if "%ARCHITECTURE%" == "x64" ( + set "MSVC_COMPILER=C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64/cl.exe" +) + +call "%VS140COMNTOOLS%..\..\VC\vcvarsall.bat" %ARCHITECTURE% +cmake -G Ninja -DCMAKE_C_COMPILER="%MSVC_COMPILER%" -DCMAKE_CXX_COMPILER="%MSVC_COMPILER%" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON ../../.. || goto :error +cmake --build . --target grpc_csharp_ext cd ..\..\.. mkdir -p %ARTIFACTS_OUT% -copy /Y cmake\build\Win32\RelWithDebInfo\grpc_csharp_ext.dll %ARTIFACTS_OUT% || copy /Y cmake\build\x64\RelWithDebInfo\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error -copy /Y cmake\build\Win32\RelWithDebInfo\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || copy /Y cmake\build\x64\RelWithDebInfo\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || goto :error +copy /Y cmake\build\%ARCHITECTURE%\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error +copy /Y cmake\build\%ARCHITECTURE%\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || goto :error goto :EOF diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29801a5b86..d93e8979fc 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -19,10 +19,20 @@ cd "$(dirname "$0")/../../.." mkdir -p artifacts/ -# All the python packages have been built in the artifact phase already -# and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true +strip_binary_wheel() { + TEMP_WHEEL_DIR=$(mktemp -d) + unzip "$1" -d "$TEMP_WHEEL_DIR" + find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" + find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" + (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1" +} + +for wheel in artifacts/*.whl; do + strip_binary_wheel "$wheel" +done + # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8749593ed5..b27f5b5037 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7836,6 +7836,7 @@ "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/client_async.cc", + "test/cpp/qps/client_callback.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", "test/cpp/qps/driver.h", diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index 2ae870ebb1..05c6cf0f61 100644 --- a/tools/run_tests/helper_scripts/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -32,9 +32,7 @@ cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC cd ..\..\..\src\csharp -if NOT DEFINED GRPC_SKIP_DOTNET_RESTORE ( - dotnet restore Grpc.sln || goto :error -) +dotnet restore Grpc.sln || goto :error endlocal diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index f235582579..9e6e72d97b 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -61,6 +61,10 @@ do "node"|"node_purejs") tools/run_tests/performance/build_performance_node.sh ;; + "python") + # python workers are only run with python2.7 and building with multiple python versions is costly + python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --compiler python2.7 --build_only -j 8 + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh index d69c85b40e..685c4357c9 100755 --- a/tools/run_tests/performance/remote_host_prepare.sh +++ b/tools/run_tests/performance/remote_host_prepare.sh @@ -38,3 +38,6 @@ ssh "${USER_AT_HOST}" "tar -xf ~/performance_workspace/grpc.tar -C ~/performance # For consistency with local run, invoke the kill_workers script remotely. # shellcheck disable=SC2088 ssh "${USER_AT_HOST}" "~/performance_workspace/grpc/tools/run_tests/performance/kill_workers.sh" + +# make sure the port server is running (required by C++ qps_worker) +ssh "${USER_AT_HOST}" "cd ~/performance_workspace/grpc/ && python tools/run_tests/start_port_server.py" diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e8d6b59687..7ec02b707e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -778,7 +778,7 @@ def cloud_to_prod_jobspec(language, if transport_security == 'tls': transport_security_options = ['--use_tls=true'] elif transport_security == 'google_default_credentials' and str( - language) in ['c++', 'go']: + language) in ['c++', 'go', 'java', 'javaokhttp']: transport_security_options = [ '--custom_credentials_type=google_default_credentials' ] @@ -1323,7 +1323,9 @@ try: service_account_key_file, transport_security='tls') jobs.append(tls_test_job) - if str(language) in ['c++', 'go']: + if str(language) in [ + 'c++', 'go', 'java', 'javaokhttp' + ]: google_default_creds_test_job = cloud_to_prod_jobspec( language, test_case, @@ -1376,7 +1378,8 @@ try: transport_security='tls') jobs.append(tls_test_job) if str(language) in [ - 'go' + 'c++', + 'go', ]: # Add more languages to the list to turn on tests. google_default_creds_test_job = cloud_to_prod_jobspec( language, diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 04e706fa5b..a746d531a1 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -92,8 +92,7 @@ def create_qpsworker_job(language, cmdline = ['timeout', '%s' % (worker_timeout + 30)] + cmdline ssh_cmd.extend([ str(user_at_host), - 'cd ~/performance_workspace/grpc/ && python tools/run_tests/start_port_server.py && %s' - % ' '.join(cmdline) + 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline) ]) cmdline = ssh_cmd @@ -270,6 +269,12 @@ def build_on_remote_hosts(hosts, 'CONFIG': 'opt'}, timeout_seconds=build_timeout)) if build_local: + # start port server locally + build_jobs.append( + jobset.JobSpec( + cmdline=['python', 'tools/run_tests/start_port_server.py'], + shortname='local_start_port_server', + timeout_seconds=2 * 60)) # Build locally as well build_jobs.append( jobset.JobSpec( |