diff options
author | 2017-09-19 10:12:25 -0700 | |
---|---|---|
committer | 2017-09-19 10:12:25 -0700 | |
commit | 3f583b78290fef31202f4a05fac589d88d586111 (patch) | |
tree | 7e13f1d54ec9cec5abce904151258e6cd2670db7 | |
parent | 3218e2b975a71cc32aedad71d00267e07244c109 (diff) | |
parent | aafa875cb389140cf721d8470a3a05cf930f74b2 (diff) |
Merge remote-tracking branch 'upstream/master' into fix-stream-compression-transport-duplicate
221 files changed, 5225 insertions, 1802 deletions
@@ -576,6 +576,8 @@ grpc_cc_library( "src/core/lib/compression/stream_compression.c", "src/core/lib/compression/stream_compression_gzip.c", "src/core/lib/compression/stream_compression_identity.c", + "src/core/lib/debug/stats.c", + "src/core/lib/debug/stats_data.c", "src/core/lib/http/format_request.c", "src/core/lib/http/httpcli.c", "src/core/lib/http/parser.c", @@ -692,8 +694,6 @@ grpc_cc_library( "src/core/lib/transport/timeout_encoding.c", "src/core/lib/transport/transport.c", "src/core/lib/transport/transport_op_string.c", - "src/core/lib/debug/stats.c", - "src/core/lib/debug/stats_data.c", ], hdrs = [ "src/core/lib/channel/channel_args.h", @@ -709,6 +709,8 @@ grpc_cc_library( "src/core/lib/compression/stream_compression.h", "src/core/lib/compression/stream_compression_gzip.h", "src/core/lib/compression/stream_compression_identity.h", + "src/core/lib/debug/stats.h", + "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", @@ -811,8 +813,6 @@ grpc_cc_library( "src/core/lib/transport/timeout_encoding.h", "src/core/lib/transport/transport.h", "src/core/lib/transport/transport_impl.h", - "src/core/lib/debug/stats.h", - "src/core/lib/debug/stats_data.h", ], external_deps = [ "zlib", diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fcfcf77aa..bb159fa6fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -761,6 +761,18 @@ add_dependencies(buildtests_cxx thread_stress_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) +endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -14128,6 +14140,178 @@ target_link_libraries(inproc_nosec_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test_unsecure + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util_unsecure + grpc_test_util_unsecure + gpr_test_util + grpc++_unsecure + grpc_unsecure + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker_unsecure + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) add_executable(api_fuzzer_one_entry test/core/end2end/fuzzers/api_fuzzer.c @@ -1266,6 +1266,10 @@ h2_sockpair+trace_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair+trace_nosec_test h2_sockpair_1byte_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_nosec_test h2_uds_nosec_test: $(BINDIR)/$(CONFIG)/h2_uds_nosec_test inproc_nosec_test: $(BINDIR)/$(CONFIG)/inproc_nosec_test +resolver_component_test_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure +resolver_component_test: $(BINDIR)/$(CONFIG)/resolver_component_test +resolver_component_tests_runner_invoker_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure +resolver_component_tests_runner_invoker: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker api_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry client_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry hpack_parser_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry @@ -1652,6 +1656,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/boringssl_x509_test \ $(BINDIR)/$(CONFIG)/boringssl_tab_test \ $(BINDIR)/$(CONFIG)/boringssl_v3name_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ else buildtests_cxx: privatelibs_cxx \ @@ -1730,6 +1738,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ endif @@ -2141,6 +2153,10 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 ) $(E) "[RUN] Testing writes_per_rpc_test" $(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker_unsecure" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure || ( echo test resolver_component_tests_runner_invoker_unsecure failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker || ( echo test resolver_component_tests_runner_invoker failed ; exit 1 ) flaky_test_cxx: buildtests_cxx @@ -19492,6 +19508,178 @@ ifneq ($(NO_DEPS),true) endif +RESOLVER_COMPONENT_TEST_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TEST_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker_unsecure: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) +endif +endif + + API_FUZZER_ONE_ENTRY_SRC = \ test/core/end2end/fuzzers/api_fuzzer.c \ test/core/util/one_corpus_entry_fuzzer.c \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index ba7ffcc9be..6cfed6b0e7 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -106,6 +106,22 @@ def grpc_sh_test(name, srcs, args = [], data = []): args = args, data = data) +def grpc_sh_binary(name, srcs, data = []): + native.sh_test( + name = name, + srcs = srcs, + data = data) + +def grpc_py_binary(name, srcs, data = [], deps = []): + if name == "test_dns_server": + # TODO: allow running test_dns_server in oss bazel test suite + deps = [] + native.py_binary( + name = name, + srcs = srcs, + data = data, + deps = deps) + def grpc_package(name, visibility = "private", features = []): if visibility == "tests": visibility = ["//test:__subpackages__"] @@ -12,7 +12,7 @@ if test "$PHP_GRPC" != "no"; then LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index b79cd97363..f90f1d5b10 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -50,6 +50,7 @@ some configuration as environment variables that can be set. - channel_stack_builder - traces information about channel stacks being built - executor - traces grpc's internal thread pool ('the executor') - http - traces state in the http2 transport engine + - http2_stream_state - traces all http2 stream state mutations. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - flowctl - traces http2 flow control diff --git a/doc/service_config.md b/doc/service_config.md index 99d985f3bf..0abbd7f324 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -24,10 +24,7 @@ The service config is a JSON string of the following form: // opposed to backend addresses), gRPC will use grpclb (see // https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), // regardless of what LB policy is requested either here or via the - // client API. However, if the resolver returns at least one backend - // address in addition to the balancer address(es), the client may fall - // back to the requested policy if it is unable to reach any of the - // grpclb load balancers. + // client API. 'loadBalancingPolicy': string, // Per-method configuration. Optional. diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index 64da044ea6..ddf6c1aaf3 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -60,11 +60,15 @@ class GreeterClient { // Storage for the status of the RPC upon completion. Status status; - // stub_->AsyncSayHello() performs the RPC call, returning an instance we - // store in "rpc". Because we are using the asynchronous API, we need to - // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. std::unique_ptr<ClientAsyncResponseReader<HelloReply> > rpc( - stub_->AsyncSayHello(&context, request, &cq)); + stub_->PrepareAsyncSayHello(&context, request, &cq)); + + // StartCall initiates the RPC call + rpc->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/examples/cpp/helloworld/greeter_async_client2.cc b/examples/cpp/helloworld/greeter_async_client2.cc index 4192546bef..3154e84d85 100644 --- a/examples/cpp/helloworld/greeter_async_client2.cc +++ b/examples/cpp/helloworld/greeter_async_client2.cc @@ -49,11 +49,15 @@ class GreeterClient { // Call object to store rpc data AsyncClientCall* call = new AsyncClientCall; - // stub_->AsyncSayHello() performs the RPC call, returning an instance to - // store in "call". Because we are using the asynchronous API, we need to - // hold on to the "call" instance in order to get updates on the ongoing RPC. - call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); - + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. + call->response_reader = + stub_->PrepareAsyncSayHello(&call->context, request, &cq_); + + // StartCall initiates the RPC call + call->response_reader->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/grpc.gemspec b/grpc.gemspec index 7cc23f3424..c31abb4b4c 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |s| s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' diff --git a/include/grpc++/generic/generic_stub.h b/include/grpc++/generic/generic_stub.h index 7b269d0887..2b3ff59ea2 100644 --- a/include/grpc++/generic/generic_stub.h +++ b/include/grpc++/generic/generic_stub.h @@ -44,6 +44,13 @@ class GenericStub final { ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag); + /// Setup a call to a named method \a method using \a context, but don't + /// start it. Let it be started explicitly with StartCall and a tag. + /// The return value only indicates whether or not registration of the call + /// succeeded (i.e. the call won't proceed if the return value is nullptr). + std::unique_ptr<GenericClientAsyncReaderWriter> PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq); + private: std::shared_ptr<ChannelInterface> channel_; }; diff --git a/include/grpc++/impl/codegen/async_stream.h b/include/grpc++/impl/codegen/async_stream.h index 9cf7ac30dd..e60572fc93 100644 --- a/include/grpc++/impl/codegen/async_stream.h +++ b/include/grpc++/impl/codegen/async_stream.h @@ -35,6 +35,11 @@ class ClientAsyncStreamingInterface { public: virtual ~ClientAsyncStreamingInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall(void* tag) = 0; + /// Request notification of the reading of the initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -156,20 +161,22 @@ class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface, template <class R> class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Write the first request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started and - /// \a request has been written out. + /// \a request has been written out. If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template <class W> static ClientAsyncReader* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, const W& request, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReader))) - ClientAsyncReader(call, context, request, tag); + ClientAsyncReader(call, context, request, start, tag); } // always allocated against a call arena, no memory free required @@ -177,6 +184,12 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { assert(size == sizeof(ClientAsyncReader)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata /// method for semantics. /// @@ -186,6 +199,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { /// calling code can access the received metadata through the /// \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -194,6 +208,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -208,6 +223,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata received from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -219,19 +235,28 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { private: template <class W> ClientAsyncReader(Call call, ClientContext* context, const W& request, - void* tag) - : context_(context), call_(call) { - init_ops_.set_output_tag(tag); - init_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + bool start, void* tag) + : context_(context), call_(call), started_(start) { // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok()); init_ops_.ClientSendClose(); + if (start) { + StartCallInternal(tag); + } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + init_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + init_ops_.set_output_tag(tag); call_.PerformOps(&init_ops_); } ClientContext* context_; Call call_; + bool started_; CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> init_ops_; CallOpSet<CallOpRecvInitialMetadata> meta_ops_; @@ -257,9 +282,12 @@ class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface, template <class W> class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC if \a start is set /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, \a tag must be nullptr and the actual call + /// must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. /// \a response will be filled in with the single expected response @@ -269,11 +297,11 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { static ClientAsyncWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, R* response, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncWriter))) - ClientAsyncWriter(call, context, response, tag); + ClientAsyncWriter(call, context, response, start, tag); } // always allocated against a call arena, no memory free required @@ -281,6 +309,12 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { assert(size == sizeof(ClientAsyncWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method for /// semantics. /// @@ -289,6 +323,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { /// associated with this call is updated, and the calling code can access /// the received metadata through the \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -297,6 +332,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -304,6 +340,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -315,6 +352,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -328,6 +366,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { /// - attempts to fill in the \a response parameter passed to this class's /// constructor with the server's response message. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -338,25 +377,32 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { private: template <class R> - ClientAsyncWriter(Call call, ClientContext* context, R* response, void* tag) - : context_(context), call_(call) { + ClientAsyncWriter(Call call, ClientContext* context, R* response, bool start, + void* tag) + : context_(context), call_(call), started_(start) { finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - if (context_->initial_metadata_corked_) { - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet<CallOpRecvInitialMetadata> meta_ops_; CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> write_ops_; @@ -388,20 +434,23 @@ template <class W, class R> class ClientAsyncReaderWriter final : public ClientAsyncReaderWriterInterface<W, R> { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC request if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. - /// intitial metadata sent). + /// intitial metadata sent). If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. static ClientAsyncReaderWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, - ClientContext* context, void* tag) { + ClientContext* context, bool start, + void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReaderWriter))) - ClientAsyncReaderWriter(call, context, tag); + ClientAsyncReaderWriter(call, context, start, tag); } // always allocated against a call arena, no memory free required @@ -409,6 +458,12 @@ class ClientAsyncReaderWriter final assert(size == sizeof(ClientAsyncReaderWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method /// for semantics of this method. /// @@ -417,6 +472,7 @@ class ClientAsyncReaderWriter final /// is updated with it, and then the receiving initial metadata can /// be accessed through this \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -425,6 +481,7 @@ class ClientAsyncReaderWriter final } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -434,6 +491,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -441,6 +499,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -452,6 +511,7 @@ class ClientAsyncReaderWriter final } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -462,6 +522,7 @@ class ClientAsyncReaderWriter final /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -471,23 +532,30 @@ class ClientAsyncReaderWriter final } private: - ClientAsyncReaderWriter(Call call, ClientContext* context, void* tag) - : context_(context), call_(call) { - if (context_->initial_metadata_corked_) { - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncReaderWriter(Call call, ClientContext* context, bool start, + void* tag) + : context_(context), call_(call), started_(start) { + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet<CallOpRecvInitialMetadata> meta_ops_; CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_; CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> diff --git a/include/grpc++/impl/codegen/async_unary_call.h b/include/grpc++/impl/codegen/async_unary_call.h index f0f909686b..e472f04f56 100644 --- a/include/grpc++/impl/codegen/async_unary_call.h +++ b/include/grpc++/impl/codegen/async_unary_call.h @@ -32,13 +32,18 @@ namespace grpc { class CompletionQueue; extern CoreCodegenInterface* g_core_codegen_interface; -/// An interface relevant for async client side unary RPCS (which send +/// An interface relevant for async client side unary RPCs (which send /// one request message to a server and receive one response message). template <class R> class ClientAsyncResponseReaderInterface { public: virtual ~ClientAsyncResponseReaderInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall() = 0; + /// Request notification of the reading of initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -70,9 +75,10 @@ template <class R> class ClientAsyncResponseReader final : public ClientAsyncResponseReaderInterface<R> { public: - /// Start a call and write the request out. + /// Start a call and write the request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template <class W> @@ -80,11 +86,11 @@ class ClientAsyncResponseReader final CompletionQueue* cq, const RpcMethod& method, ClientContext* context, - const W& request) { + const W& request, bool start) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncResponseReader))) - ClientAsyncResponseReader(call, context, request); + ClientAsyncResponseReader(call, context, request, start); } // always allocated against a call arena, no memory free required @@ -92,13 +98,20 @@ class ClientAsyncResponseReader final assert(size == sizeof(ClientAsyncResponseReader)); } + void StartCall() override { + assert(!started_); + started_ = true; + StartCallInternal(); + } + /// See \a ClientAsyncResponseReaderInterface::ReadInitialMetadata for /// semantics. /// /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void ReadInitialMetadata(void* tag) { + void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_buf.set_output_tag(tag); @@ -111,7 +124,8 @@ class ClientAsyncResponseReader final /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void Finish(R* msg, Status* status, void* tag) { + void Finish(R* msg, Status* status, void* tag) override { + assert(started_); finish_buf.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_buf.RecvInitialMetadata(context_); @@ -125,15 +139,22 @@ class ClientAsyncResponseReader final private: ClientContext* const context_; Call call_; + bool started_; template <class W> - ClientAsyncResponseReader(Call call, ClientContext* context, const W& request) - : context_(context), call_(call) { - init_buf.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncResponseReader(Call call, ClientContext* context, const W& request, + bool start) + : context_(context), call_(call), started_(start) { + // Bind the metadata at time of StartCallInternal but set up the rest here // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_buf.SendMessage(request).ok()); init_buf.ClientSendClose(); + if (start) StartCallInternal(); + } + + void StartCallInternal() { + init_buf.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); call_.PerformOps(&init_buf); } diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 9dc505f008..7b6befeaf1 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,12 +64,6 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); - /// Set the grpclb fallback timeout (in ms) for the channel. If this amount - /// of time has passed but we have not gotten any non-empty \a serverlist from - /// the balancer, we will fall back to use the backend address(es) returned by - /// the resolver. - void SetGrpclbFallbackTimeout(int fallback_timeout); - /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/compression.h b/include/grpc/compression.h index 3a8de4b7b8..13a8dd66ad 100644 --- a/include/grpc/compression.h +++ b/include/grpc/compression.h @@ -44,13 +44,13 @@ int grpc_stream_compression_algorithm_parse( * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_compression_algorithm_name( - grpc_compression_algorithm algorithm, char **name); + grpc_compression_algorithm algorithm, const char **name); /** Updates \a name with the encoding name corresponding to a valid \a * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name); + grpc_stream_compression_algorithm algorithm, const char **name); /** Returns the compression algorithm corresponding to \a level for the * compression algorithms encoded in the \a accepted_encodings bitset. diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index beb6080c94..748dc717a3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -287,11 +287,7 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" -/* Timeout in milliseconds to wait for the serverlist from the grpclb load - balancer before using fallback backend addresses from the resolver. - If 0, fallback will never be used. */ -#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/package.xml b/package.xml index 8135e8db73..3ccd29ac6e 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ <email>grpc-packages@google.com</email> <active>yes</active> </lead> - <date>2017-05-22</date> + <date>2017-08-24</date> <time>16:06:07</time> <version> <release>1.7.0dev</release> @@ -25,6 +25,9 @@ - Channel are now by default persistent #11878 - Some bug fixes from 1.4 branch #12109, #12123 - Fixed hang bug when fork() was used #11814 +- License changed to Apache 2.0 +- Added support for php_namespace option in codegen plugin #11886 +- Updated gRPC C Core library version 1.6 </notes> <contents> <dir baseinstalldir="/" name="/"> @@ -70,7 +70,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] # Environment variable to determine whether or not the Cython extension should # *use* Cython or use the generated C files. Note that this requires the C files diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index b09bf99677..c2db8eff71 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -165,25 +165,37 @@ void PrintHeaderClientMethodInterfaces( (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "virtual ::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) = 0;\n"); - printer->Print(*vars, - "std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -197,19 +209,26 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, $Response$* " - "response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncWriterInterface< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* " + "response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncWriterInterface< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -223,19 +242,25 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "std::unique_ptr< ::grpc::ClientReaderWriterInterface< " @@ -249,61 +274,83 @@ void PrintHeaderClientMethodInterfaces( "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) = 0;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientWriterInterface< $Request$>*" " $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" - " Async$Method$Raw(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" + " $AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientReaderInterface< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request) = 0;\n"); - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " - "Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "virtual ::grpc::ClientReaderWriterInterface< $Request$, " "$Response$>* " "$Method$Raw(::grpc::ClientContext* context) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncReaderWriterInterface< " - "$Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderWriterInterface< " + "$Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } } } @@ -315,25 +362,35 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) override;\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReader< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -346,18 +403,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -371,19 +434,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -396,53 +464,80 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " - "$Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " + "$Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print(*vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) override;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) " "override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncWriter< $Request$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientReader< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request)" " override;\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReader< $Response$>* Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "::grpc::ClientReaderWriter< $Request$, $Response$>* " "$Method$Raw(::grpc::ClientContext* context) override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } } } @@ -1077,6 +1172,13 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string start; // bool literal expressed as string + grpc::string method_params; // extra arguments to method + grpc::string create_args; // extra arguments to creator + } async_prefixes[] = {{"Async", "true", ", void* tag", ", tag"}, + {"PrepareAsync", "false", "", ", nullptr"}}; if (method->NoStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Stub::$Method$(" @@ -1087,19 +1189,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request, response);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Print(*vars, - " return " - "::grpc::ClientAsyncResponseReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + printer->Print(*vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Print(*vars, + " return " + "::grpc::ClientAsyncResponseReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$);\n" + "}\n\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* " @@ -1111,17 +1217,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, response);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncWriter< $Request$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, response, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncWriter< $Request$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncWriter< $Request$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, response, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -1134,17 +1246,24 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -1157,19 +1276,25 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print( - *vars, - " return " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print( + *vars, + " return " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } } @@ -1460,50 +1585,79 @@ void PrintMockClientMethods(grpc_generator::Printer *printer, (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + int extra_method_param_count; + } async_prefixes[] = {{"Async", ", void* tag", 1}, {"PrepareAsync", "", 0}}; + if (method->NoStreaming()) { printer->Print( *vars, "MOCK_METHOD3($Method$, ::grpc::Status(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "MOCK_METHOD3($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq));\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientWriterInterface< $Request$>*" "(::grpc::ClientContext* context, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncWriterInterface< $Request$>*" - "(::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print(*vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncWriterInterface< $Request$>*" + "(::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientReaderInterface< $Response$>*" "(::grpc::ClientContext* context, const $Request$& request));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, "MOCK_METHOD1($Method$Raw, " "::grpc::ClientReaderWriterInterface< $Request$, $Response$>*" "(::grpc::ClientContext* context));\n"); - printer->Print( - *vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" - "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, " - "void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(2 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" + "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq" + "$AsyncMethodParams$));\n"); + } } } diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 31b177c28e..a60b528e3b 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -769,7 +769,7 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file, PrivateGenerator generator(config_, &pbfile); if (parameter == "grpc_2_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true); - } else if (parameter == "") { + } else if (parameter == "grpc_1_0" || parameter == "") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true) && GenerateGrpc(context, generator, pb2_file_name, false); } else { diff --git a/src/core/ext/census/base_resources.c b/src/core/ext/census/base_resources.c index 2114bf04cd..1f2bb39fe0 100644 --- a/src/core/ext/census/base_resources.c +++ b/src/core/ext/census/base_resources.c @@ -37,20 +37,20 @@ void define_base_resources() { google_census_Resource_BasicUnit numerator = google_census_Resource_BasicUnit_SECS; - resource r = {"client_rpc_latency", // name - "Client RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + resource r = {(char *)"client_rpc_latency", // name + (char *)"Client RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); - r = (resource){"server_rpc_latency", // name - "Server RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + r = (resource){(char *)"server_rpc_latency", // name + (char *)"Server RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); } diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 129d0f368b..016199b1f4 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -375,7 +375,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, } // Extract the following fields from the resolver result, if non-NULL. bool lb_policy_updated = false; - char *lb_policy_name = NULL; + char *lb_policy_name_dup = NULL; bool lb_policy_name_changed = false; grpc_lb_policy *new_lb_policy = NULL; char *service_config_json = NULL; @@ -383,6 +383,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, grpc_slice_hash_table *method_params_table = NULL; if (chand->resolver_result != NULL) { // Find LB policy name. + const char *lb_policy_name = NULL; const grpc_arg *channel_arg = grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_POLICY_NAME); if (channel_arg != NULL) { @@ -473,7 +474,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // Before we clean up, save a copy of lb_policy_name, since it might // be pointing to data inside chand->resolver_result. // The copy will be saved in chand->lb_policy_name below. - lb_policy_name = gpr_strdup(lb_policy_name); + lb_policy_name_dup = gpr_strdup(lb_policy_name); grpc_channel_args_destroy(exec_ctx, chand->resolver_result); chand->resolver_result = NULL; } @@ -481,8 +482,8 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, gpr_log(GPR_DEBUG, "chand=%p: resolver result: lb_policy_name=\"%s\"%s, " "service_config=\"%s\"", - chand, lb_policy_name, lb_policy_name_changed ? " (changed)" : "", - service_config_json); + chand, lb_policy_name_dup, + lb_policy_name_changed ? " (changed)" : "", service_config_json); } // Now swap out fields in chand. Note that the new values may still // be NULL if (e.g.) the resolver failed to return results or the @@ -490,9 +491,9 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // // First, swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); - if (lb_policy_name != NULL) { + if (lb_policy_name_dup != NULL) { gpr_free(chand->info_lb_policy_name); - chand->info_lb_policy_name = lb_policy_name; + chand->info_lb_policy_name = lb_policy_name_dup; } if (service_config_json != NULL) { gpr_free(chand->info_service_config_json); diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index e8aa4cda29..57eac8f875 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -63,6 +63,6 @@ static const grpc_arg_pointer_vtable factory_arg_vtable = { grpc_arg grpc_client_channel_factory_create_channel_arg( grpc_client_channel_factory* factory) { - return grpc_channel_arg_pointer_create(GRPC_ARG_CLIENT_CHANNEL_FACTORY, + return grpc_channel_arg_pointer_create((char*)GRPC_ARG_CLIENT_CHANNEL_FACTORY, factory, &factory_arg_vtable); } diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.c b/src/core/ext/filters/client_channel/client_channel_plugin.c index c32e83d012..1f71c5a7f9 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.c +++ b/src/core/ext/filters/client_channel/client_channel_plugin.c @@ -54,8 +54,8 @@ static bool set_default_host_if_unset(grpc_exec_ctx *exec_ctx, char *default_authority = grpc_get_default_authority( exec_ctx, grpc_channel_stack_builder_get_target(builder)); if (default_authority != NULL) { - grpc_arg arg = grpc_channel_arg_string_create(GRPC_ARG_DEFAULT_AUTHORITY, - default_authority); + grpc_arg arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_DEFAULT_AUTHORITY, default_authority); grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1); grpc_channel_stack_builder_set_channel_arguments(exec_ctx, builder, new_args); diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index ef3512ed83..0c3b009e44 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -157,7 +157,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, } grpc_arg args_to_add[2]; args_to_add[0] = grpc_channel_arg_string_create( - GRPC_ARG_HTTP_CONNECT_SERVER, + (char*)GRPC_ARG_HTTP_CONNECT_SERVER, uri->path[0] == '/' ? uri->path + 1 : uri->path); if (user_cred != NULL) { /* Use base64 encoding for user credentials as stated in RFC 7617 */ @@ -166,8 +166,8 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, char* header; gpr_asprintf(&header, "Proxy-Authorization:Basic %s", encoded_user_cred); gpr_free(encoded_user_cred); - args_to_add[1] = - grpc_channel_arg_string_create(GRPC_ARG_HTTP_CONNECT_HEADERS, header); + args_to_add[1] = grpc_channel_arg_string_create( + (char*)GRPC_ARG_HTTP_CONNECT_HEADERS, header); *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 2); gpr_free(header); } else { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index d3a51c6e6f..85ef7894ea 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -123,7 +123,6 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -300,10 +299,6 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; - /** timeout in milliseconds for before using fallback backend addresses. - * 0 means not using fallback. */ - int lb_fallback_timeout_ms; - /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -330,9 +325,6 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; - /** stores the backend addresses from the resolver */ - grpc_lb_addresses *fallback_backend_addresses; - /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -353,9 +345,6 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; - /** is \a lb_fallback_timer active? */ - bool fallback_timer_active; - /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -378,9 +367,6 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; - /* LB fallback timer callback. */ - grpc_closure lb_on_fallback; - grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -404,9 +390,6 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; - /** LB fallback timer */ - grpc_timer lb_fallback_timer; - bool initial_request_sent; bool seen_initial_response; @@ -553,32 +536,6 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } -/* Returns the backend addresses extracted from the given addresses */ -static grpc_lb_addresses *extract_backend_addresses_locked( - grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { - /* first pass: count the number of backend addresses */ - size_t num_backends = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (!addresses->addresses[i].is_balancer) { - ++num_backends; - } - } - /* second pass: actually populate the addresses and (empty) LB tokens */ - grpc_lb_addresses *backend_addresses = - grpc_lb_addresses_create(num_backends, &lb_token_vtable); - size_t num_copied = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (addresses->addresses[i].is_balancer) continue; - const grpc_resolved_address *addr = &addresses->addresses[i].address; - grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, - addr->len, false /* is_balancer */, - NULL /* balancer_name */, - (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); - ++num_copied; - } - return backend_addresses; -} - static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -646,38 +603,35 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Check for drops if we are not using fallback backend addresses. - if (glb_policy->serverlist != NULL) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. + } + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked( - server->load_balance_token, wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); - gpr_free(wc_arg->free_when_done); - return false; - } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, + wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); gpr_free(wc_arg->free_when_done); - return true; + return false; } + gpr_free(wc_arg->free_when_done); + return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -715,18 +669,8 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses; - if (glb_policy->serverlist != NULL) { - GPR_ASSERT(glb_policy->serverlist->num_servers > 0); - addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); - } else { - // If rr_handover_locked() is invoked when we haven't received any - // serverlist from the balancer, we use the fallback backends returned by - // the resolver. Note that the fallback backend list may be empty, in which - // case the new round_robin policy will keep the requested picks pending. - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); - } + grpc_lb_addresses *addresses = + process_serverlist_locked(exec_ctx, glb_policy->serverlist); GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -832,6 +776,8 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + GPR_ASSERT(glb_policy->serverlist != NULL && + glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -980,9 +926,6 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } - if (glb_policy->fallback_backend_addresses != NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); grpc_subchannel_index_unref(); if (glb_policy->pending_update_args != NULL) { @@ -1124,28 +1067,10 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - /* start a timer to fall back */ - if (glb_policy->lb_fallback_timeout_ms > 0 && - glb_policy->serverlist == NULL) { - gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec deadline = gpr_time_add( - now, - gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); - GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, - glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->fallback_timer_active = true; - grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, - &glb_policy->lb_on_fallback, now); - } - glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1600,15 +1525,6 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); - } else { - /* or dispose of the fallback */ - grpc_lb_addresses_destroy(exec_ctx, - glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = NULL; - if (glb_policy->fallback_timer_active) { - grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); - glb_policy->fallback_timer_active = false; - } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1619,7 +1535,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Received empty server list, ignoring."); + gpr_log(GPR_INFO, + "Received empty server list. Picks will stay pending until " + "a response with > 0 servers is received"); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1642,6 +1560,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_response_received); /* loop */ GPR_ASSERT(GRPC_CALL_OK == call_error); + } else { + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "lb_on_response_received_locked_shutdown"); } } else { /* empty payload: call cancelled. */ /* dispose of the "lb_on_response_received_locked" weak ref taken in @@ -1666,26 +1587,6 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - glb_lb_policy *glb_policy = (glb_lb_policy *)arg; - /* If we receive a serverlist after the timer fires but before this callback - * actually runs, don't do anything. */ - if (glb_policy->serverlist != NULL) return; - glb_policy->fallback_timer_active = false; - if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Falling back to use backends from resolver (grpclb %p)", - (void *)glb_policy); - } - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - rr_handover_locked(exec_ctx, glb_policy); - } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, - "grpclb_fallback_timer"); -} - static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1806,17 +1707,6 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } - - // Propagate update to fallback_backend_addresses if a non-empty serverlist - // hasn't been received from the balancer. - if (glb_policy->serverlist == NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - if (glb_policy->rr_policy != NULL) { - rr_handover_locked(exec_ctx, glb_policy); - } - } } // Invoked as part of the update process. It continues watching the LB channel @@ -1899,7 +1789,13 @@ static const grpc_lb_policy_vtable glb_lb_policy_vtable = { static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. */ + /* Count the number of gRPC-LB addresses. There must be at least one. + * TODO(roth): For now, we ignore non-balancer addresses, but in the + * future, we may change the behavior such that we fall back to using + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1935,24 +1831,14 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); - glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( - arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, - INT_MAX}); - // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. - grpc_arg new_arg = - grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); + grpc_arg new_arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)"grpclb"); static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); - /* Extract the backend addresses (may be empty) from the resolver for - * fallback. */ - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 8ac1a46abd..a3a62e9f3c 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -589,7 +589,7 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, // Dispose of outdated subchannel lists. if (sd->subchannel_list != p->subchannel_list && sd->subchannel_list != p->latest_pending_subchannel_list) { - char *reason = NULL; + const char *reason = NULL; if (sd->subchannel_list->shutting_down) { reason = "sl_outdated_straggler"; rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, reason); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index a9a00f5b2b..4d1405454c 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - const void* address, size_t address_len, + void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); @@ -141,7 +141,7 @@ static const grpc_arg_pointer_vtable lb_addresses_arg_vtable = { grpc_arg grpc_lb_addresses_create_channel_arg( const grpc_lb_addresses* addresses) { return grpc_channel_arg_pointer_create( - GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); + (char*)GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); } grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index cf0f8cb615..9d9fb143df 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - const void *address, size_t address_len, + void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b7082..9bb229ad95 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } @@ -249,7 +249,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, service_config_string); args_to_remove[num_args_to_remove++] = GRPC_ARG_SERVICE_CONFIG; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_SERVICE_CONFIG, service_config_string); + (char *)GRPC_ARG_SERVICE_CONFIG, service_config_string); service_config = grpc_service_config_create(service_config_string); if (service_config != NULL) { const char *lb_policy_name = @@ -257,7 +257,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, if (lb_policy_name != NULL) { args_to_remove[num_args_to_remove++] = GRPC_ARG_LB_POLICY_NAME; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); } } } diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c index c4676e0299..69ea440ae6 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c @@ -210,7 +210,7 @@ grpc_arg grpc_fake_resolver_response_generator_arg( grpc_fake_resolver_response_generator* generator) { grpc_arg arg; arg.type = GRPC_ARG_POINTER; - arg.key = GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; + arg.key = (char*)GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; arg.value.pointer.p = generator; arg.value.pointer.vtable = &response_generator_arg_vtable; return arg; diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index bc9c3cc782..40a51c72d6 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -811,6 +811,6 @@ const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args) { grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr) { return grpc_channel_arg_string_create( - GRPC_ARG_SUBCHANNEL_ADDRESS, + (char *)GRPC_ARG_SUBCHANNEL_ADDRESS, addr->len > 0 ? grpc_sockaddr_to_uri(addr) : gpr_strdup("")); } diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 0145dffab0..f785e1355d 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -244,7 +244,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, &calld->slices, &tmp); if (did_compress) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; const size_t before_size = calld->slices.length; const size_t after_size = tmp.length; const float savings_ratio = 1.0f - (float)after_size / (float)before_size; @@ -258,7 +258,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); gpr_log(GPR_DEBUG, diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c index f56afd27d2..2486ead427 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c +++ b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c @@ -55,7 +55,8 @@ static bool maybe_add_server_load_reporting_filter( } grpc_arg grpc_load_reporting_enable_arg() { - return grpc_channel_arg_integer_create(GRPC_ARG_ENABLE_LOAD_REPORTING, 1); + return grpc_channel_arg_integer_create((char *)GRPC_ARG_ENABLE_LOAD_REPORTING, + 1); } /* Plugin registration */ diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index cccb347bf1..6410a6043d 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -55,7 +55,7 @@ static grpc_channel *client_channel_factory_create_channel( } // Add channel arg containing the server URI. grpc_arg arg = grpc_channel_arg_string_create( - GRPC_ARG_SERVER_URI, + (char *)GRPC_ARG_SERVER_URI, grpc_resolver_factory_add_default_prefix_if_needed(exec_ctx, target)); const char *to_remove[] = {GRPC_ARG_SERVER_URI}; grpc_channel_args *new_args = diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index 0346d50b6c..dd88136f7b 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -42,7 +42,7 @@ grpc_channel *grpc_insecure_channel_create_from_fd( (target, fd, args)); grpc_arg default_authority_arg = grpc_channel_arg_string_create( - GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + (char *)GRPC_ARG_DEFAULT_AUTHORITY, (char *)"test.authority"); grpc_channel_args *final_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c index 78551df9c3..6d09953830 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c @@ -23,6 +23,7 @@ void grpc_chttp2_plugin_init(void) { grpc_register_tracer(&grpc_http_trace); grpc_register_tracer(&grpc_flowctl_trace); + grpc_register_tracer(&grpc_trace_http2_stream_state); #ifndef NDEBUG grpc_register_tracer(&grpc_trace_chttp2_refcount); #endif diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9b89bebc30..ed08aa79c5 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -144,10 +144,11 @@ static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error); -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, - grpc_closure *on_complete); +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_complete, + grpc_chttp2_initiate_write_reason initiate_write_reason); static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); @@ -346,7 +347,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (is_client) { grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( GRPC_CHTTP2_CLIENT_CONNECT_STRING)); - grpc_chttp2_initiate_write(exec_ctx, t, "initial_write"); } /* configure http2 the way we like it */ @@ -578,7 +578,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED; } - grpc_chttp2_initiate_write(exec_ctx, t, "init"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE); post_benign_reclaimer(exec_ctx, t); } @@ -843,13 +844,91 @@ static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } +static void inc_initiate_write_reason( + grpc_exec_ctx *exec_ctx, grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx); + break; + } +} + void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason) { + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason) { GPR_TIMER_BEGIN("grpc_chttp2_initiate_write", 0); switch (t->write_state) { case GRPC_CHTTP2_WRITE_STATE_IDLE: - set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, reason); + inc_initiate_write_reason(exec_ctx, reason); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + grpc_chttp2_initiate_write_reason_string(reason)); t->is_first_write_in_batch = true; GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); GRPC_CLOSURE_SCHED( @@ -861,7 +940,7 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, break; case GRPC_CHTTP2_WRITE_STATE_WRITING: set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE, - reason); + grpc_chttp2_initiate_write_reason_string(reason)); break; case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: break; @@ -869,16 +948,12 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("grpc_chttp2_initiate_write", 0); } -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason) { +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:become"); } - if (also_initiate_write) { - grpc_chttp2_initiate_write(exec_ctx, t, reason); - } } static grpc_closure_scheduler *write_scheduler(grpc_chttp2_transport *t, @@ -1102,7 +1177,9 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); post_destructive_reclaimer(exec_ctx, t); - grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM); } /* cancel out streams that will never be started */ while (t->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -1199,7 +1276,9 @@ static void maybe_become_writable_due_to_send_msg(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { if (s->id != 0 && (!s->write_buffering || s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); } } @@ -1403,14 +1482,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } } else { GPR_ASSERT(s->id != 0); - bool initiate_write = true; - if (op->send_message && - (op->payload->send_message.send_message->flags & - GRPC_WRITE_BUFFER_HINT)) { - initiate_write = false; + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + if (!(op->send_message && + (op->payload->send_message.send_message->flags & + GRPC_WRITE_BUFFER_HINT))) { + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); } - grpc_chttp2_become_writable(exec_ctx, t, s, initiate_write, - "op.send_initial_metadata"); } } else { s->send_initial_metadata = NULL; @@ -1518,8 +1596,9 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } else if (s->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "op.send_trailing_metadata"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA); } } } @@ -1631,15 +1710,17 @@ static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, grpc_closure *on_ack) { +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_ack, + grpc_chttp2_initiate_write_reason initiate_write_reason) { grpc_chttp2_ping_queue *pq = &t->ping_queues[ping_type]; grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_INITIATE], on_initiate, GRPC_ERROR_NONE); if (grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_NEXT], on_ack, GRPC_ERROR_NONE)) { - grpc_chttp2_initiate_write(exec_ctx, t, "send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, initiate_write_reason); } } @@ -1647,7 +1728,8 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)tp; t->ping_state.is_delayed_ping_timer_set = false; - grpc_chttp2_initiate_write(exec_ctx, t, "retry_send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -1662,7 +1744,8 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]); if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) { - grpc_chttp2_initiate_write(exec_ctx, t, "continue_pings"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS); } } @@ -1675,7 +1758,8 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &slice, &http_error); grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)http_error, grpc_slice_ref_internal(slice), &t->qbuf); - grpc_chttp2_initiate_write(exec_ctx, t, "goaway_sent"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT); GRPC_ERROR_UNREF(error); } @@ -1722,7 +1806,8 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, if (op->send_ping) { send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, NULL, - op->send_ping); + op->send_ping, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING); } if (op->on_connectivity_state_change != NULL) { @@ -1971,7 +2056,8 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM); } } if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { @@ -2292,7 +2378,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &s->stats.outgoing)); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); - grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API); } typedef struct { @@ -2327,19 +2414,20 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "immediate stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL); break; case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE: - grpc_chttp2_become_writable(exec_ctx, t, s, false, - "queue stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); break; } switch (action.send_transport_update) { case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_initiate_write(exec_ctx, t, "immediate transport flowctl"); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL); break; // this is the same as no action b/c every time the transport enters the // writing path it will maybe do an update @@ -2357,7 +2445,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, (uint32_t)action.max_frame_size); } if (action.send_setting_update == GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY) { - grpc_chttp2_initiate_write(exec_ctx, t, "immediate setting update"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS); } } if (action.need_ping) { @@ -2365,7 +2454,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, grpc_bdp_estimator_schedule_ping(&t->flow_control.bdp_estimator); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, - &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); + &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING); } } @@ -2444,7 +2534,10 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, if (t->flow_control.initial_window_update > 0) { grpc_chttp2_stream *s; while (grpc_chttp2_list_pop_stalled_by_stream(t, &s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "unstalled"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING); } } t->flow_control.initial_window_update = 0; @@ -2559,7 +2652,8 @@ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive ping end"); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, &t->start_keepalive_ping_locked, - &t->finish_keepalive_ping_locked); + &t->finish_keepalive_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING); } else { GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); grpc_timer_init( @@ -3019,6 +3113,56 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, /******************************************************************************* * MONITORING */ + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + return "INITIAL_WRITE"; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + return "START_NEW_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + return "SEND_MESSAGE"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + return "SEND_INITIAL_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + return "SEND_TRAILING_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + return "RETRY_SEND_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + return "CONTINUE_PINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + return "GOAWAY_SENT"; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + return "RST_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + return "CLOSE_FROM_API"; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + return "STREAM_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + return "TRANSPORT_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + return "SEND_SETTINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + return "BDP_ESTIMATOR_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + return "FLOW_CONTROL_UNSTALLED_BY_SETTING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + return "FLOW_CONTROL_UNSTALLED_BY_UPDATE"; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + return "APPLICATION_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + return "KEEPALIVE_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + return "TRANSPORT_FLOW_CONTROL_UNSTALLED"; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + return "PING_RESPONSE"; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + return "FORCE_RST_STREAM"; + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + static grpc_endpoint *chttp2_get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { return ((grpc_chttp2_transport *)t)->ep; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 0c4e2a91c0..55fb1a8343 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -25,6 +25,7 @@ extern grpc_tracer_flag grpc_http_trace; extern grpc_tracer_flag grpc_flowctl_trace; +extern grpc_tracer_flag grpc_trace_http2_stream_state; #ifndef NDEBUG extern grpc_tracer_flag grpc_trace_chttp2_refcount; diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 582fd7bfaa..81bd02ae70 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -117,7 +117,8 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, t->ping_acks, t->ping_ack_capacity * sizeof(*t->ping_acks)); } t->ping_acks[t->ping_ack_count++] = p->opaque_8bytes; - grpc_chttp2_initiate_write(exec_ctx, t, "ping response"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE); } } } diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index c94f7725bf..c9ab8d1b50 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -99,8 +99,10 @@ grpc_error *grpc_chttp2_window_update_parser_parse( grpc_chttp2_flowctl_recv_stream_update( &t->flow_control, &s->flow_control, received_update); if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "stream.read_flow_control"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE); } } } else { @@ -109,7 +111,9 @@ grpc_error *grpc_chttp2_window_update_parser_parse( received_update); bool is_zero = t->flow_control.remote_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_initiate_write(exec_ctx, t, "new_global_flow_control"); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); } } } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 048c9e4872..5ca215fcae 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1649,7 +1649,8 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index bed803a1b1..1aaeb35d76 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -79,6 +79,33 @@ typedef enum { GRPC_CHTTP2_PCL_COUNT /* must be last */ } grpc_chttp2_ping_closure_list; +typedef enum { + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE, + GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API, + GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM, +} grpc_chttp2_initiate_write_reason; + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason); + typedef struct { grpc_closure_list lists[GRPC_CHTTP2_PCL_COUNT]; uint64_t inflight_id; @@ -601,7 +628,8 @@ struct grpc_chttp2_stream { The actual call chain is documented in the implementation of this function. */ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason); + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason); typedef struct { /** are we writing? */ @@ -853,10 +881,9 @@ void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, /** add a ref to the stream and add it to the writable list; ref will be dropped in writing.c */ -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason); +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index 7cc85dea9c..47cd22d177 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -20,6 +20,27 @@ #include <grpc/support/log.h> +static char *stream_list_id_string(grpc_chttp2_stream_list_id id) { + switch (id) { + case GRPC_CHTTP2_LIST_WRITABLE: + return "writable"; + case GRPC_CHTTP2_LIST_WRITING: + return "writing"; + case GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT: + return "stalled_by_transport"; + case GRPC_CHTTP2_LIST_STALLED_BY_STREAM: + return "stalled_by_stream"; + case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: + return "waiting_for_concurrency"; + case STREAM_LIST_COUNT: + GPR_UNREACHABLE_CODE(return "unknown"); + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + +grpc_tracer_flag grpc_trace_http2_stream_state = + GRPC_TRACER_INITIALIZER(false, "http2_stream_state"); + /* core list management */ static bool stream_list_empty(grpc_chttp2_transport *t, @@ -44,6 +65,10 @@ static bool stream_list_pop(grpc_chttp2_transport *t, s->included[id] = 0; } *stream = s; + if (s && GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "%p[%d][%s]: pop from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); + } return s != 0; } @@ -62,6 +87,10 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, } else { t->lists[id].tail = s->links[id].prev; } + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "%p[%d][%s]: remove from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); + } } static bool stream_list_maybe_remove(grpc_chttp2_transport *t, @@ -90,6 +119,10 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, } t->lists[id].tail = s; s->included[id] = 1; + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "%p[%d][%s]: add to %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); + } } static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, @@ -150,17 +183,12 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by transport", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); - GRPC_FLOW_CONTROL_IF_TRACING(if (ret) gpr_log( - GPR_DEBUG, "stream %u un-stalled by transport", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, @@ -170,23 +198,15 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by stream", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - bool ret = stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", s->id)); - return ret; + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 6ea7c9ff9b..1244898f2f 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -201,9 +201,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (t->flow_control.remote_window > 0) { while (grpc_chttp2_list_pop_stalled_by_transport(t, &s)) { - if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s) && - stream_ref_if_not_destroyed(&s->refcount->refs)) { - grpc_chttp2_initiate_write(exec_ctx, t, "transport.read_flow_control"); + if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { + stream_ref_if_not_destroyed(&s->refcount->refs); } } } diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c index cd3e76a0b5..31739d07dd 100644 --- a/src/core/ext/transport/inproc/inproc_transport.c +++ b/src/core/ext/transport/inproc/inproc_transport.c @@ -1263,8 +1263,8 @@ grpc_channel *grpc_inproc_channel_create(grpc_server *server, grpc_arg default_authority_arg; default_authority_arg.type = GRPC_ARG_STRING; - default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY; - default_authority_arg.value.string = "inproc.authority"; + default_authority_arg.key = (char *)GRPC_ARG_DEFAULT_AUTHORITY; + default_authority_arg.value.string = (char *)"inproc.authority"; grpc_channel_args *client_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 9215707f1c..30248b3c60 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -243,7 +243,7 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm( GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -253,7 +253,7 @@ grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm( GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -308,7 +308,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( if (grpc_channel_args_get_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, "Tried to disable default compression algorithm '%s'. The " @@ -324,7 +324,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { @@ -349,7 +349,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, @@ -366,7 +366,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index ae1cac31f7..f0de80f0c0 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -281,7 +281,7 @@ grpc_channel_stack *grpc_channel_stack_from_top_element( /* Given the top element of a call stack, get the call stack itself */ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem); -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op); diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c index ec84c01811..1cfac23129 100644 --- a/src/core/lib/compression/compression.c +++ b/src/core/lib/compression/compression.c @@ -60,7 +60,7 @@ int grpc_stream_compression_algorithm_parse( } int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, - char **name) { + const char **name) { GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); switch (algorithm) { @@ -80,7 +80,7 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, } int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name) { + grpc_stream_compression_algorithm algorithm, const char **name) { GRPC_API_TRACE( "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 2c56765343..b3e1ee9b4e 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -50,6 +50,27 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "http2_writes_offloaded", "http2_writes_continued", "http2_partial_writes", + "http2_initiate_write_due_to_initial_write", + "http2_initiate_write_due_to_start_new_stream", + "http2_initiate_write_due_to_send_message", + "http2_initiate_write_due_to_send_initial_metadata", + "http2_initiate_write_due_to_send_trailing_metadata", + "http2_initiate_write_due_to_retry_send_ping", + "http2_initiate_write_due_to_continue_pings", + "http2_initiate_write_due_to_goaway_sent", + "http2_initiate_write_due_to_rst_stream", + "http2_initiate_write_due_to_close_from_api", + "http2_initiate_write_due_to_stream_flow_control", + "http2_initiate_write_due_to_transport_flow_control", + "http2_initiate_write_due_to_send_settings", + "http2_initiate_write_due_to_bdp_estimator_ping", + "http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "http2_initiate_write_due_to_flow_control_unstalled_by_update", + "http2_initiate_write_due_to_application_ping", + "http2_initiate_write_due_to_keepalive_ping", + "http2_initiate_write_due_to_transport_flow_control_unstalled", + "http2_initiate_write_due_to_ping_response", + "http2_initiate_write_due_to_force_rst_stream", "combiner_locks_initiated", "combiner_locks_scheduled_items", "combiner_locks_scheduled_final_items", @@ -92,6 +113,30 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "written", "Number of HTTP2 writes that were made knowing there was still more data " "to be written (we cap maximum write size to syscall_write)", + "Number of HTTP2 writes initiated due to 'initial_write'", + "Number of HTTP2 writes initiated due to 'start_new_stream'", + "Number of HTTP2 writes initiated due to 'send_message'", + "Number of HTTP2 writes initiated due to 'send_initial_metadata'", + "Number of HTTP2 writes initiated due to 'send_trailing_metadata'", + "Number of HTTP2 writes initiated due to 'retry_send_ping'", + "Number of HTTP2 writes initiated due to 'continue_pings'", + "Number of HTTP2 writes initiated due to 'goaway_sent'", + "Number of HTTP2 writes initiated due to 'rst_stream'", + "Number of HTTP2 writes initiated due to 'close_from_api'", + "Number of HTTP2 writes initiated due to 'stream_flow_control'", + "Number of HTTP2 writes initiated due to 'transport_flow_control'", + "Number of HTTP2 writes initiated due to 'send_settings'", + "Number of HTTP2 writes initiated due to 'bdp_estimator_ping'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_setting'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_update'", + "Number of HTTP2 writes initiated due to 'application_ping'", + "Number of HTTP2 writes initiated due to 'keepalive_ping'", + "Number of HTTP2 writes initiated due to " + "'transport_flow_control_unstalled'", + "Number of HTTP2 writes initiated due to 'ping_response'", + "Number of HTTP2 writes initiated due to 'force_rst_stream'", "Number of combiner lock entries by process (first items queued to a " "combiner)", "Number of items scheduled against combiner locks", diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index e93db32c6a..c9871c4a56 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -52,6 +52,27 @@ typedef enum { GRPC_STATS_COUNTER_HTTP2_WRITES_OFFLOADED, GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED, GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM, GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS, @@ -169,6 +190,95 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED) #define GRPC_STATS_INC_HTTP2_PARTIAL_WRITES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM) #define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED) diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 37dff9c5f4..84727fe6c4 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -117,6 +117,48 @@ - counter: http2_partial_writes doc: Number of HTTP2 writes that were made knowing there was still more data to be written (we cap maximum write size to syscall_write) +- counter: http2_initiate_write_due_to_initial_write + doc: Number of HTTP2 writes initiated due to 'initial_write' +- counter: http2_initiate_write_due_to_start_new_stream + doc: Number of HTTP2 writes initiated due to 'start_new_stream' +- counter: http2_initiate_write_due_to_send_message + doc: Number of HTTP2 writes initiated due to 'send_message' +- counter: http2_initiate_write_due_to_send_initial_metadata + doc: Number of HTTP2 writes initiated due to 'send_initial_metadata' +- counter: http2_initiate_write_due_to_send_trailing_metadata + doc: Number of HTTP2 writes initiated due to 'send_trailing_metadata' +- counter: http2_initiate_write_due_to_retry_send_ping + doc: Number of HTTP2 writes initiated due to 'retry_send_ping' +- counter: http2_initiate_write_due_to_continue_pings + doc: Number of HTTP2 writes initiated due to 'continue_pings' +- counter: http2_initiate_write_due_to_goaway_sent + doc: Number of HTTP2 writes initiated due to 'goaway_sent' +- counter: http2_initiate_write_due_to_rst_stream + doc: Number of HTTP2 writes initiated due to 'rst_stream' +- counter: http2_initiate_write_due_to_close_from_api + doc: Number of HTTP2 writes initiated due to 'close_from_api' +- counter: http2_initiate_write_due_to_stream_flow_control + doc: Number of HTTP2 writes initiated due to 'stream_flow_control' +- counter: http2_initiate_write_due_to_transport_flow_control + doc: Number of HTTP2 writes initiated due to 'transport_flow_control' +- counter: http2_initiate_write_due_to_send_settings + doc: Number of HTTP2 writes initiated due to 'send_settings' +- counter: http2_initiate_write_due_to_bdp_estimator_ping + doc: Number of HTTP2 writes initiated due to 'bdp_estimator_ping' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_setting + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_setting' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_update + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_update' +- counter: http2_initiate_write_due_to_application_ping + doc: Number of HTTP2 writes initiated due to 'application_ping' +- counter: http2_initiate_write_due_to_keepalive_ping + doc: Number of HTTP2 writes initiated due to 'keepalive_ping' +- counter: http2_initiate_write_due_to_transport_flow_control_unstalled + doc: Number of HTTP2 writes initiated due to 'transport_flow_control_unstalled' +- counter: http2_initiate_write_due_to_ping_response + doc: Number of HTTP2 writes initiated due to 'ping_response' +- counter: http2_initiate_write_due_to_force_rst_stream + doc: Number of HTTP2 writes initiated due to 'force_rst_stream' # combiner locks - counter: combiner_locks_initiated doc: Number of combiner lock entries by process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 5354769291..d21afbbfe4 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,5 +1,9 @@ client_calls_created_per_iteration:FLOAT, server_calls_created_per_iteration:FLOAT, +cqs_created_per_iteration:FLOAT, +client_channels_created_per_iteration:FLOAT, +client_subchannels_created_per_iteration:FLOAT, +server_channels_created_per_iteration:FLOAT, syscall_poll_per_iteration:FLOAT, syscall_wait_per_iteration:FLOAT, histogram_slow_lookups_per_iteration:FLOAT, @@ -21,6 +25,27 @@ http2_writes_begun_per_iteration:FLOAT, http2_writes_offloaded_per_iteration:FLOAT, http2_writes_continued_per_iteration:FLOAT, http2_partial_writes_per_iteration:FLOAT, +http2_initiate_write_due_to_initial_write_per_iteration:FLOAT, +http2_initiate_write_due_to_start_new_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_send_message_per_iteration:FLOAT, +http2_initiate_write_due_to_send_initial_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_send_trailing_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_retry_send_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_continue_pings_per_iteration:FLOAT, +http2_initiate_write_due_to_goaway_sent_per_iteration:FLOAT, +http2_initiate_write_due_to_rst_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_close_from_api_per_iteration:FLOAT, +http2_initiate_write_due_to_stream_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_send_settings_per_iteration:FLOAT, +http2_initiate_write_due_to_bdp_estimator_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_setting_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_update_per_iteration:FLOAT, +http2_initiate_write_due_to_application_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_keepalive_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_unstalled_per_iteration:FLOAT, +http2_initiate_write_due_to_ping_response_per_iteration:FLOAT, +http2_initiate_write_due_to_force_rst_stream_per_iteration:FLOAT, combiner_locks_initiated_per_iteration:FLOAT, combiner_locks_scheduled_items_per_iteration:FLOAT, combiner_locks_scheduled_final_items_per_iteration:FLOAT, diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index dd9e6a30fe..64f2e3fc33 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -35,7 +35,7 @@ typedef struct { #else bool value; #endif - char *name; + const char *name; } grpc_tracer_flag; #ifdef GRPC_THREADSAFE_TRACER diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 97c2886525..c553fa3981 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -43,7 +43,8 @@ static void httpcli_ssl_destroy(grpc_exec_ctx *exec_ctx, grpc_httpcli_ssl_channel_security_connector *c = (grpc_httpcli_ssl_channel_security_connector *)sc; if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_client_handshaker_factory_unref(c->handshaker_factory); + c->handshaker_factory = NULL; } if (c->secure_peer_name != NULL) gpr_free(c->secure_peer_name); gpr_free(sc); diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 7236e23cf7..00edefc6ae 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -167,7 +167,14 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, GPR_TIMER_BEGIN("grpc_closure_sched", 0); if (c != NULL) { #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; @@ -191,7 +198,14 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { while (c != NULL) { grpc_closure *next = c->next_data.next; #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index d5e00a8f64..aa05501537 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -641,7 +641,7 @@ static char *key_time(grpc_error_times which) { static char *fmt_time(gpr_timespec tm) { char *out; - char *pfx = "!!"; + const char *pfx = "!!"; switch (tm.clock_type) { case GPR_CLOCK_MONOTONIC: pfx = "@monotonic:"; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 6946a2cbf5..0d57833d85 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; -static const char *kick_state_string(kick_state_t st) { +static const char *kick_state_string(kick_state st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state_t st) { } struct grpc_pollset_worker { - kick_state_t kick_state; + kick_state state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; @@ -154,9 +154,9 @@ struct grpc_pollset_worker { grpc_closure_list schedule_on_end_work; }; -#define SET_KICK_STATE(worker, state) \ +#define SET_KICK_STATE(worker, kick_state) \ do { \ - (worker)->kick_state = (state); \ + (worker)->state = (kick_state); \ (worker)->kick_state_mutator = __LINE__; \ } while (false) @@ -508,7 +508,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - switch (worker->kick_state) { + switch (worker->state) { case KICKED: break; case UNKICKED: @@ -688,7 +688,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), is_reassigning); } if (pollset->seen_inactive) { @@ -708,12 +708,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, at this point is if it were "kicked specifically". Since the worker has not added itself to the pollset yet (by calling worker_insert()), it is not visible in the "kick any" path yet */ - if (worker->kick_state == UNKICKED) { + if (worker->state == UNKICKED) { pollset->seen_inactive = false; if (neighborhood->active_root == NULL) { neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ - if (worker->kick_state == UNKICKED && + if (worker->state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } @@ -733,19 +733,19 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker_insert(pollset, worker); pollset->begin_refs--; - if (worker->kick_state == UNKICKED && !pollset->kicked_without_poller) { + if (worker->state == UNKICKED && !pollset->kicked_without_poller) { GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (worker->kick_state == UNKICKED && !pollset->shutting_down) { + while (worker->state == UNKICKED && !pollset->shutting_down) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down); } if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && - worker->kick_state == UNKICKED) { + worker->state == UNKICKED) { /* If gpr_cv_wait returns true (i.e a timeout), pretend that the worker received a kick */ SET_KICK_STATE(worker, KICKED); @@ -758,7 +758,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_log(GPR_ERROR, "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d " "kicked_without_poller: %d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down, pollset->kicked_without_poller); } @@ -778,7 +778,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } GPR_TIMER_END("begin_worker", 0); - return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; + return worker->state == DESIGNATED_POLLER && !pollset->shutting_down; } static bool check_neighborhood_for_available_poller( @@ -795,7 +795,7 @@ static bool check_neighborhood_for_available_poller( grpc_pollset_worker *inspect_worker = inspect->root_worker; if (inspect_worker != NULL) { do { - switch (inspect_worker->kick_state) { + switch (inspect_worker->state) { case UNKICKED: if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { @@ -858,7 +858,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_list_move(&worker->schedule_on_end_work, &exec_ctx->closure_list); if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { - if (worker->next != worker && worker->next->kick_state == UNKICKED) { + if (worker->next != worker && worker->next->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, " .. choose next poller to be peer %p", worker); } @@ -993,14 +993,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_strvec_add(&log, tmp); if (pollset->root_worker != NULL) { gpr_asprintf(&tmp, " {kick_state=%s next=%p {kick_state=%s}}", - kick_state_string(pollset->root_worker->kick_state), + kick_state_string(pollset->root_worker->state), pollset->root_worker->next, - kick_state_string(pollset->root_worker->next->kick_state)); + kick_state_string(pollset->root_worker->next->state)); gpr_strvec_add(&log, tmp); } if (specific_worker != NULL) { gpr_asprintf(&tmp, " worker_kick_state=%s", - kick_state_string(specific_worker->kick_state)); + kick_state_string(specific_worker->state)); gpr_strvec_add(&log, tmp); } tmp = gpr_strvec_flatten(&log, NULL); @@ -1020,13 +1020,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } grpc_pollset_worker *next_worker = root_worker->next; - if (root_worker->kick_state == KICKED) { + if (root_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; - } else if (next_worker->kick_state == KICKED) { + } else if (next_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1043,7 +1043,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(root_worker, KICKED); ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; - } else if (next_worker->kick_state == UNKICKED) { + } else if (next_worker->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1051,8 +1051,8 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(next_worker, KICKED); gpr_cv_signal(&next_worker->cv); goto done; - } else if (next_worker->kick_state == DESIGNATED_POLLER) { - if (root_worker->kick_state != DESIGNATED_POLLER) { + } else if (next_worker->state == DESIGNATED_POLLER) { + if (root_worker->state != DESIGNATED_POLLER) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log( GPR_ERROR, @@ -1074,7 +1074,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { - GPR_ASSERT(next_worker->kick_state == KICKED); + GPR_ASSERT(next_worker->state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } @@ -1088,7 +1088,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, GPR_UNREACHABLE_CODE(goto done); } - if (specific_worker->kick_state == KICKED) { + if (specific_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index df69025f1a..51200fc064 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable_t { +typedef struct pollable { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable_t; +} pollable; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return "<invalid>"; } -static char *pollable_desc(pollable_t *p) { +static char *pollable_desc(pollable *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable_t *p) { return out; } -static pollable_t g_empty_pollable; +static pollable g_empty_pollable; -static void pollable_init(pollable_t *p, polling_obj_type type); -static void pollable_destroy(pollable_t *p); +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p); +static grpc_error *pollable_materialize(pollable *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable_t pollable; + pollable pollable_obj; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable_t *pollable; + pollable *pollable_obj; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable_t pollable; - pollable_t *current_pollable; + pollable pollable_obj; + pollable *current_pollable_obj; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -282,7 +282,7 @@ static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_fd *fd = (grpc_fd *)arg; /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); - pollable_destroy(&fd->pollable); + pollable_destroy(&fd->pollable_obj); gpr_mu_destroy(&fd->orphaned_mu); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; @@ -343,7 +343,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = (grpc_fd *)gpr_malloc(sizeof(grpc_fd)); } - pollable_init(&new_fd->pollable, PO_FD); + pollable_init(&new_fd->pollable_obj, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -385,7 +385,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = already_closed; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->pollable_obj.po.mu); gpr_mu_lock(&fd->orphaned_mu); fd->on_done_closure = on_done; @@ -411,7 +411,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_CLOSURE_SCHED(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->orphaned_mu); - gpr_mu_unlock(&fd->pollable.po.mu); + gpr_mu_unlock(&fd->pollable_obj.po.mu); UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable_t *p, polling_obj_type type) { +static void pollable_init(pollable *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable_t *p) { +static void pollable_destroy(pollable *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable_t *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p) { +static grpc_error *pollable_materialize(pollable *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable_t *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -557,30 +557,33 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error_unused) { grpc_error *error = GRPC_ERROR_NONE; grpc_pollset *pollset = (grpc_pollset *)arg; - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - if (worker->pollable != &pollset->pollable) { - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker->pollable_obj->po.mu); } if (worker->initialized_cv && worker != pollset->root_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_cv %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } worker->kicked = true; gpr_cv_signal(&worker->cv); } else { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_wakeup %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } - append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + append_error(&error, + grpc_wakeup_fd_wakeup(&worker->pollable_obj->wakeup), "pollset_shutdown"); } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); } worker = worker->links[PWL_POLLSET].next; @@ -588,7 +591,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, } pollset->kick_alls_pending--; pollset_maybe_finish_shutdown(exec_ctx, pollset); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("kick_all", error); } @@ -599,7 +602,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,24 +667,24 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable_t *p = pollset->current_pollable; - if (p != &pollset->pollable) { + pollable *p = pollset->current_pollable_obj; + if (p != &pollset->pollable_obj) { gpr_mu_lock(&p->po.mu); } grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); - if (p != &pollset->pollable) { + if (p != &pollset->pollable_obj) { gpr_mu_unlock(&p->po.mu); } return error; } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - pollable_init(&pollset->pollable, PO_POLLSET); - pollset->current_pollable = &g_empty_pollable; + pollable_init(&pollset->pollable_obj, PO_POLLSET); + pollset->current_pollable_obj = &g_empty_pollable; pollset->kicked_without_poller = false; pollset->shutdown_closure = NULL; pollset->root_worker = NULL; - *mu = &pollset->pollable.po.mu; + *mu = &pollset->pollable_obj.po.mu; } /* Convert a timespec to milliseconds: @@ -729,8 +732,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "fd_become_pollable"; - if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { - append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + if (append_error(&error, pollable_materialize(&fd->pollable_obj), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable_obj, fd), err_desc); } return error; } @@ -744,8 +747,8 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { - return p != &g_empty_pollable && p != &pollset->pollable; +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable_obj; } static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, @@ -791,9 +794,9 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - pollable_destroy(&pollset->pollable); - if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + pollable_destroy(&pollset->pollable_obj); + if (pollset_is_pollable_fd(pollset, pollset->current_pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable_obj, 2, "pollset_pollable"); } GRPC_LOG_IF_ERROR("pollset_process_events", @@ -801,7 +804,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable_t *p, gpr_timespec now, + pollable *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -883,68 +886,69 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = false; worker->kicked = false; worker->pollset = pollset; - worker->pollable = pollset->current_pollable; + worker->pollable_obj = pollset->current_pollable_obj; - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + REF_BY((grpc_fd *)worker->pollable_obj, 2, "one_poll"); } worker_insert(&pollset->root_worker, PWL_POLLSET, worker); - if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + if (!worker_insert(&worker->pollable_obj->root_worker, PWL_POLLABLE, + worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&pollset->pollable.po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&pollset->pollable_obj.po.mu); } if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, - worker->pollable, worker, + worker->pollable_obj, worker, poll_deadline_to_millis_timeout(deadline, *now)); } - while (do_poll && worker->pollable->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + while (do_poll && worker->pollable_obj->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable_obj->po.mu, deadline)) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } do_poll = false; } else if (worker->kicked) { if (GRPC_TRACER_ON(grpc_polling_trace)) { - gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, - worker); + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, + worker->pollable_obj, worker); } do_poll = false; } else if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); - gpr_mu_lock(&pollset->pollable.po.mu); - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + gpr_mu_lock(&worker->pollable_obj->po.mu); } *now = gpr_now(now->clock_type); } return do_poll && pollset->shutdown_closure == NULL && - pollset->current_pollable == worker->pollable; + pollset->current_pollable_obj == worker->pollable_obj; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { if (NEW_ROOT == - worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { - gpr_cv_signal(&worker->pollable->root_worker->cv); + worker_remove(&worker->pollable_obj->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable_obj->root_worker->cv); } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable_obj, 2, "one_poll"); } if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); @@ -972,41 +976,41 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } - if (pollset->current_pollable != &pollset->pollable) { - gpr_mu_lock(&pollset->current_pollable->po.mu); + if (pollset->current_pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&pollset->current_pollable_obj->po.mu); } if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - append_error(&error, pollable_materialize(worker.pollable), err_desc); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + append_error(&error, pollable_materialize(worker.pollable_obj), err_desc); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); if (pollset->event_cursor == pollset->event_count) { - append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable_obj, now, deadline), err_desc); } append_error(&error, pollset_process_events(exec_ctx, pollset, false), err_desc); - gpr_mu_lock(&pollset->pollable.po.mu); - if (worker.pollable != &pollset->pollable) { - gpr_mu_lock(&worker.pollable->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker.pollable_obj->po.mu); } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(exec_ctx, pollset, &worker, worker_hdl); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } if (grpc_exec_ctx_has_work(exec_ctx)) { - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); } return error; } @@ -1023,27 +1027,27 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, bool fd_locked) { static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; - if (pollset->current_pollable == &g_empty_pollable) { + if (pollset->current_pollable_obj == &g_empty_pollable) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable_t */ + /* empty pollable --> single fd pollable */ pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &fd->pollable; - if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + pollset->current_pollable_obj = &fd->pollable_obj; + if (!fd_locked) gpr_mu_lock(&fd->pollable_obj.po.mu); append_error(&error, fd_become_pollable_locked(fd), err_desc); - if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); + if (!fd_locked) gpr_mu_unlock(&fd->pollable_obj.po.mu); REF_BY(fd, 2, "pollset_pollable"); - } else if (pollset->current_pollable == &pollset->pollable) { + } else if (pollset->current_pollable_obj == &pollset->pollable_obj) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); } - append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + append_error(&error, pollable_add_fd(pollset->current_pollable_obj, fd), err_desc); - } else if (pollset->current_pollable != &fd->pollable) { - grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + } else if (pollset->current_pollable_obj != &fd->pollable_obj) { + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable_obj; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from fd %p to multipoller", @@ -1055,11 +1059,11 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read"); grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write"); pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &pollset->pollable; - if (append_error(&error, pollable_materialize(&pollset->pollable), + pollset->current_pollable_obj = &pollset->pollable_obj; + if (append_error(&error, pollable_materialize(&pollset->pollable_obj), err_desc)) { - pollable_add_fd(&pollset->pollable, had_fd); - pollable_add_fd(&pollset->pollable, fd); + pollable_add_fd(&pollset->pollable_obj, had_fd); + pollable_add_fd(&pollset->pollable_obj, fd); } GRPC_CLOSURE_SCHED(exec_ctx, GRPC_CLOSURE_CREATE(unref_fd_no_longer_poller, had_fd, @@ -1071,9 +1075,9 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -1095,7 +1099,7 @@ static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - po_join(exec_ctx, &pss->po, &fd->pollable.po); + po_join(exec_ctx, &pss->po, &fd->pollable_obj.po); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, @@ -1103,7 +1107,7 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - po_join(exec_ctx, &pss->po, &ps->pollable.po); + po_join(exec_ctx, &pss->po, &ps->pollable_obj.po); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c index 082e3b7947..60cfeebd47 100644 --- a/src/core/lib/iomgr/resolve_address_posix.c +++ b/src/core/lib/iomgr/resolve_address_posix.c @@ -85,7 +85,7 @@ static grpc_error *blocking_resolve_address_impl( if (s != 0) { /* Retry if well-known service name is recognized */ - char *svc[][2] = {{"http", "80"}, {"https", "443"}}; + const char *svc[][2] = {{"http", "80"}, {"https", "443"}}; for (i = 0; i < GPR_ARRAY_SIZE(svc); i++) { if (strcmp(port, svc[i][0]) == 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index a7568b995f..2a9e939d40 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -455,14 +455,14 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( typedef struct { grpc_channel_security_connector base; - tsi_ssl_client_handshaker_factory *handshaker_factory; + tsi_ssl_client_handshaker_factory *client_handshaker_factory; char *target_name; char *overridden_target_name; } grpc_ssl_channel_security_connector; typedef struct { grpc_server_security_connector base; - tsi_ssl_server_handshaker_factory *handshaker_factory; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; } grpc_ssl_server_security_connector; static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, @@ -470,9 +470,8 @@ static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, grpc_ssl_channel_security_connector *c = (grpc_ssl_channel_security_connector *)sc; grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); - if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_client_handshaker_factory_unref(c->client_handshaker_factory); + c->client_handshaker_factory = NULL; if (c->target_name != NULL) gpr_free(c->target_name); if (c->overridden_target_name != NULL) gpr_free(c->overridden_target_name); gpr_free(sc); @@ -482,9 +481,8 @@ static void ssl_server_destroy(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc) { grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; - if (c->handshaker_factory != NULL) { - tsi_ssl_server_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_server_handshaker_factory_unref(c->server_handshaker_factory); + c->server_handshaker_factory = NULL; gpr_free(sc); } @@ -496,7 +494,7 @@ static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker( - c->handshaker_factory, + c->client_handshaker_factory, c->overridden_target_name != NULL ? c->overridden_target_name : c->target_name, &tsi_hs); @@ -521,7 +519,7 @@ static void ssl_server_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_server_handshaker_factory_create_handshaker( - c->handshaker_factory, &tsi_hs); + c->server_handshaker_factory, &tsi_hs); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", tsi_result_to_string(result)); @@ -852,7 +850,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create( result = tsi_create_ssl_client_handshaker_factory( has_key_cert_pair ? &config->pem_key_cert_pair : NULL, pem_root_certs, ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->client_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -897,7 +895,7 @@ grpc_security_status grpc_ssl_server_security_connector_create( config->pem_root_certs, get_tsi_client_certificate_request_type( config->client_certificate_request), ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->server_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index 975d599523..3d19605617 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -137,7 +137,7 @@ static void on_peer_checked_inner(grpc_exec_ctx *exec_ctx, // Create zero-copy frame protector, if implemented. tsi_zero_copy_grpc_protector *zero_copy_protector = NULL; tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector( - h->handshaker_result, NULL, &zero_copy_protector); + exec_ctx, h->handshaker_result, NULL, &zero_copy_protector); if (result != TSI_OK && result != TSI_UNIMPLEMENTED) { error = grpc_set_tsi_error_result( GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c index 61d2346427..7755018693 100644 --- a/src/core/lib/support/log_linux.c +++ b/src/core/lib/support/log_linux.c @@ -57,7 +57,7 @@ void gpr_log(const char *file, int line, gpr_log_severity severity, } void gpr_default_log(gpr_log_func_args *args) { - char *final_slash; + const char *final_slash; char *prefix; const char *display_file; char time_buffer[64]; diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index 523e43445b..6b172df82f 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -276,7 +276,7 @@ static void add_string_to_split(const char *beg, const char *end, char ***strs, void gpr_string_split(const char *input, const char *sep, char ***strs, size_t *nstrs) { - char *next; + const char *next; *strs = NULL; *nstrs = 0; size_t capstrs = 0; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index de5aa7b6be..a41eb336d4 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call_t; +} parent_call; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call_t; +} child_call; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call_t* */ gpr_atm parent_call_atm; - child_call_t *child_call; + /* parent_call* */ gpr_atm parent_call_atm; + child_call *child; /* client or server call */ bool is_client; @@ -304,21 +304,21 @@ void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call_t *get_or_create_parent_call(grpc_call *call) { - parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_or_create_parent_call(grpc_call *call) { + parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call_t *get_parent_call(grpc_call *call) { - return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_parent_call(grpc_call *call) { + return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -377,24 +377,24 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; - if (args->parent_call != NULL) { - child_call_t *cc = call->child_call = - (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); - call->child_call->parent = args->parent_call; + if (args->parent != NULL) { + child_call *cc = call->child = + (child_call *)gpr_arena_alloc(arena, sizeof(child_call)); + call->child->parent = args->parent; - GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); + GRPC_CALL_INTERNAL_REF(args->parent, "child"); GPR_ASSERT(call->is_client); - GPR_ASSERT(!args->parent_call->is_client); + GPR_ASSERT(!args->parent->is_client); - parent_call_t *pc = get_or_create_parent_call(args->parent_call); + parent_call *pc = get_or_create_parent_call(args->parent); gpr_mu_lock(&pc->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( gpr_convert_clock_type(send_deadline, - args->parent_call->send_deadline.clock_type), - args->parent_call->send_deadline); + args->parent->send_deadline.clock_type), + args->parent->send_deadline); } /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with * GRPC_PROPAGATE_STATS_CONTEXT */ @@ -406,9 +406,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, "Census tracing propagation requested " "without Census context propagation")); } - grpc_call_context_set( - call, GRPC_CONTEXT_TRACING, - args->parent_call->context[GRPC_CONTEXT_TRACING].value, NULL); + grpc_call_context_set(call, GRPC_CONTEXT_TRACING, + args->parent->context[GRPC_CONTEXT_TRACING].value, + NULL); } else if (args->propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) { add_init_error(&error, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Census context propagation requested " @@ -416,7 +416,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; - if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { + if (gpr_atm_acq_load(&args->parent->received_final_op_atm)) { immediately_cancel = true; } } @@ -426,9 +426,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, cc->sibling_next = cc->sibling_prev = call; } else { cc->sibling_next = pc->first_child; - cc->sibling_prev = pc->first_child->child_call->sibling_prev; - cc->sibling_next->child_call->sibling_prev = - cc->sibling_prev->child_call->sibling_next = call; + cc->sibling_prev = pc->first_child->child->sibling_prev; + cc->sibling_next->child->sibling_prev = + cc->sibling_prev->child->sibling_next = call; } gpr_mu_unlock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call_t *pc = get_parent_call(c); + parent_call *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call_t *cc = c->child_call; + child_call *cc = c->child; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call_t *pc = get_parent_call(cc->parent); + parent_call *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -585,8 +585,8 @@ void grpc_call_unref(grpc_call *c) { pc->first_child = NULL; } } - cc->sibling_prev->child_call->sibling_next = cc->sibling_next; - cc->sibling_next->child_call->sibling_prev = cc->sibling_prev; + cc->sibling_prev->child->sibling_next = cc->sibling_next; + cc->sibling_next->child->sibling_prev = cc->sibling_prev; gpr_mu_unlock(&pc->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, cc->parent, "child"); } @@ -1309,14 +1309,14 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call_t *pc = get_parent_call(call); + parent_call *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); child = pc->first_child; if (child != NULL) { do { - next_child_call = child->child_call->sibling_next; + next_child_call = child->child->sibling_next; if (child->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); cancel_with_error(exec_ctx, child, STATUS_FROM_API_OVERRIDE, @@ -1511,7 +1511,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_stream_compression_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.", algo_name); @@ -1525,7 +1525,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->stream_encodings_accepted_by_peer, call->incoming_stream_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name( call->incoming_stream_compression_algorithm, &algo_name); gpr_log( @@ -1552,7 +1552,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); @@ -1568,7 +1568,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); gpr_log(GPR_ERROR, diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index d537637cbb..c680139cf6 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -37,7 +37,7 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx, typedef struct grpc_call_create_args { grpc_channel *channel; - grpc_call *parent_call; + grpc_call *parent; uint32_t propagation_mask; grpc_completion_queue *cq; diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 2f9b677c85..48962e5e45 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -282,7 +282,7 @@ static grpc_call *grpc_channel_create_call_internal( grpc_call_create_args args; memset(&args, 0, sizeof(args)); args.channel = channel; - args.parent_call = parent_call; + args.parent = parent_call; args.propagation_mask = propagation_mask; args.cq = cq; args.pollset_set_alternative = pollset_set_alternative; diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 6452f0894d..468360fbde 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -565,13 +565,13 @@ static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {} * true if the increment was successful; false if the counter is zero */ static bool atm_inc_if_nonzero(gpr_atm *counter) { while (true) { - gpr_atm count = gpr_atm_no_barrier_load(counter); + gpr_atm count = gpr_atm_acq_load(counter); /* If zero, we are done. If not, we must to a CAS (instead of an atomic * increment) to maintain the contract: do not increment the counter if it * is zero. */ if (count == 0) { return false; - } else if (gpr_atm_no_barrier_cas(counter, count, count + 1)) { + } else if (gpr_atm_full_cas(counter, count, count + 1)) { break; } } @@ -643,8 +643,12 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, /* Add the completion to the queue */ bool is_first = cq_event_queue_push(&cqd->queue, storage); gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); - bool will_definitely_shutdown = - gpr_atm_no_barrier_load(&cqd->pending_events) == 1; + + /* Since we do not hold the cq lock here, it is important to do an 'acquire' + load here (instead of a 'no_barrier' load) to match with the release store + (done via gpr_atm_full_fetch_add(pending_events, -1)) in cq_shutdown_next + */ + bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1; if (!will_definitely_shutdown) { /* Only kick if this is the first item queued */ @@ -888,7 +892,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } } - if (gpr_atm_no_barrier_load(&cqd->pending_events) == 0) { + if (gpr_atm_acq_load(&cqd->pending_events) == 0) { /* Before returning, check if the queue has any items left over (since gpr_mpscq_pop() can sometimes return NULL even if the queue is not empty. If so, keep retrying but do not return GRPC_QUEUE_SHUTDOWN */ @@ -934,7 +938,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } if (cq_event_queue_num_items(&cqd->queue) > 0 && - gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { + gpr_atm_acq_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); @@ -985,6 +989,9 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, return; } cqd->shutdown_called = true; + /* Doing a full_fetch_add (i.e acq/release) here to match with + * cq_begin_op_for_next and and cq_end_op_for_next functions which read/write + * on this counter without necessarily holding a lock on cq */ if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { cq_finish_shutdown_next(exec_ctx, cq); } diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 409a6c4103..858664715c 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -197,7 +197,7 @@ char *grpc_transport_op_string(grpc_transport_op *op) { return out; } -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { char *str = grpc_transport_stream_op_batch_string(op); diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index e7b3be3d86..64043fea08 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -493,7 +493,8 @@ static tsi_result fake_handshaker_result_extract_peer( } static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + void *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { *protector = tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size); diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 1fd65928f9..7ebf9dd96f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -67,7 +67,13 @@ /* --- Structure definitions. ---*/ +struct tsi_ssl_handshaker_factory { + const tsi_ssl_handshaker_factory_vtable *vtable; + gpr_refcount refcount; +}; + struct tsi_ssl_client_handshaker_factory { + tsi_ssl_handshaker_factory base; SSL_CTX *ssl_context; unsigned char *alpn_protocol_list; size_t alpn_protocol_list_length; @@ -77,6 +83,7 @@ struct tsi_ssl_server_handshaker_factory { /* Several contexts to support SNI. The tsi_peer array contains the subject names of the server certificates associated with the contexts at the same index. */ + tsi_ssl_handshaker_factory base; SSL_CTX **ssl_contexts; tsi_peer *ssl_context_x509_subject_names; size_t ssl_context_count; @@ -90,6 +97,7 @@ typedef struct { BIO *into_ssl; BIO *from_ssl; tsi_result result; + tsi_ssl_handshaker_factory *factory_ref; } tsi_ssl_handshaker; typedef struct { @@ -846,6 +854,47 @@ static const tsi_frame_protector_vtable frame_protector_vtable = { ssl_protector_destroy, }; +/* --- tsi_server_handshaker_factory methods implementation. --- */ + +static void tsi_ssl_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (self->vtable != NULL && self->vtable->destroy != NULL) { + self->vtable->destroy(self); + } + /* Note, we don't free(self) here because this object is always directly + * embedded in another object. If tsi_ssl_handshaker_factory_init allocates + * any memory, it should be free'd here. */ +} + +static tsi_ssl_handshaker_factory *tsi_ssl_handshaker_factory_ref( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return NULL; + gpr_refn(&self->refcount, 1); + return self; +} + +static void tsi_ssl_handshaker_factory_unref(tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (gpr_unref(&self->refcount)) { + tsi_ssl_handshaker_factory_destroy(self); + } +} + +static tsi_ssl_handshaker_factory_vtable handshaker_factory_vtable = {NULL}; + +/* Initializes a tsi_ssl_handshaker_factory object. Caller is responsible for + * allocating memory for the factory. */ +static void tsi_ssl_handshaker_factory_init( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + + factory->vtable = &handshaker_factory_vtable; + gpr_ref_init(&factory->refcount, 1); +} + /* --- tsi_handshaker methods implementation. ---*/ static tsi_result ssl_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, @@ -1013,6 +1062,7 @@ static tsi_result ssl_handshaker_create_frame_protector( static void ssl_handshaker_destroy(tsi_handshaker *self) { tsi_ssl_handshaker *impl = (tsi_ssl_handshaker *)self; SSL_free(impl->ssl); /* The BIO objects are owned by ssl */ + tsi_ssl_handshaker_factory_unref(impl->factory_ref); gpr_free(impl); } @@ -1030,6 +1080,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, const char *server_name_indication, + tsi_ssl_handshaker_factory *factory, tsi_handshaker **handshaker) { SSL *ssl = SSL_new(ctx); BIO *into_ssl = NULL; @@ -1085,6 +1136,8 @@ static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, impl->from_ssl = from_ssl; impl->result = TSI_HANDSHAKE_IN_PROGRESS; impl->base.vtable = &handshaker_vtable; + impl->factory_ref = tsi_ssl_handshaker_factory_ref(factory); + *handshaker = &impl->base; return TSI_OK; } @@ -1121,11 +1174,20 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker) { return create_tsi_ssl_handshaker(self->ssl_context, 1, server_name_indication, - handshaker); + &self->base, handshaker); } -void tsi_ssl_client_handshaker_factory_destroy( +void tsi_ssl_client_handshaker_factory_unref( tsi_ssl_client_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_client_handshaker_factory *self = + (tsi_ssl_client_handshaker_factory *)factory; if (self->ssl_context != NULL) SSL_CTX_free(self->ssl_context); if (self->alpn_protocol_list != NULL) gpr_free(self->alpn_protocol_list); gpr_free(self); @@ -1150,11 +1212,21 @@ tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( if (self->ssl_context_count == 0) return TSI_INVALID_ARGUMENT; /* Create the handshaker with the first context. We will switch if needed because of SNI in ssl_server_handshaker_factory_servername_callback. */ - return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, handshaker); + return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, &self->base, + handshaker); } -void tsi_ssl_server_handshaker_factory_destroy( +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_server_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_server_handshaker_factory *self = + (tsi_ssl_server_handshaker_factory *)factory; size_t i; for (i = 0; i < self->ssl_context_count; i++) { if (self->ssl_contexts[i] != NULL) { @@ -1263,6 +1335,9 @@ static int server_handshaker_factory_npn_advertised_callback( /* --- tsi_ssl_handshaker_factory constructors. --- */ +static tsi_ssl_handshaker_factory_vtable client_handshaker_factory_vtable = { + tsi_ssl_client_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_client_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pair, const char *pem_root_certs, const char *cipher_suites, @@ -1285,6 +1360,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &client_handshaker_factory_vtable; + impl->ssl_context = ssl_context; do { @@ -1322,7 +1400,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } } while (0); if (result != TSI_OK) { - tsi_ssl_client_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } SSL_CTX_set_verify(ssl_context, SSL_VERIFY_PEER, NULL); @@ -1332,6 +1410,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( return TSI_OK; } +static tsi_ssl_handshaker_factory_vtable server_handshaker_factory_vtable = { + tsi_ssl_server_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_server_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, size_t num_key_cert_pairs, const char *pem_client_root_certs, @@ -1364,12 +1445,15 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &server_handshaker_factory_vtable; + impl->ssl_contexts = gpr_zalloc(num_key_cert_pairs * sizeof(SSL_CTX *)); impl->ssl_context_x509_subject_names = gpr_zalloc(num_key_cert_pairs * sizeof(tsi_peer)); if (impl->ssl_contexts == NULL || impl->ssl_context_x509_subject_names == NULL) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return TSI_OUT_OF_RESOURCES; } impl->ssl_context_count = num_key_cert_pairs; @@ -1379,7 +1463,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( &impl->alpn_protocol_list, &impl->alpn_protocol_list_length); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } @@ -1451,10 +1535,11 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } while (0); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } + *factory = impl; return TSI_OK; } @@ -1501,3 +1586,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { return 0; /* Not found. */ } + +/* --- Testing support. --- */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable) { + GPR_ASSERT(factory != NULL); + GPR_ASSERT(factory->vtable != NULL); + + const tsi_ssl_handshaker_factory_vtable *orig_vtable = factory->vtable; + factory->vtable = new_vtable; + return orig_vtable; +} diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 177599930b..3abfdf5ed8 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -96,10 +96,10 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_client_handshaker_factory_destroy( - tsi_ssl_client_handshaker_factory *self); +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_client_handshaker_factory_unref( + tsi_ssl_client_handshaker_factory *factory); /* --- tsi_ssl_server_handshaker_factory object --- @@ -158,9 +158,9 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( tsi_ssl_server_handshaker_factory *self, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_server_handshaker_factory_destroy( +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self); /* Util that checks that an ssl peer matches a specific name. @@ -170,6 +170,29 @@ void tsi_ssl_server_handshaker_factory_destroy( - handle public suffix wildchar more strictly (e.g. *.co.uk) */ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); +/* --- Testing support. --- + + These functions and typedefs are not intended to be used outside of testing. + */ + +/* Base type of client and server handshaker factories. */ +typedef struct tsi_ssl_handshaker_factory tsi_ssl_handshaker_factory; + +/* Function pointer to handshaker_factory destructor. */ +typedef void (*tsi_ssl_handshaker_factory_destructor)( + tsi_ssl_handshaker_factory *factory); + +/* Virtual table for tsi_ssl_handshaker_factory. */ +typedef struct { + tsi_ssl_handshaker_factory_destructor destroy; +} tsi_ssl_handshaker_factory_vtable; + +/* Set destructor of handshaker_factory to new_destructor, returns previous + destructor. */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable); + #ifdef __cplusplus } #endif diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index b0d7039850..3bba38149c 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -84,11 +84,17 @@ struct tsi_handshaker { }; /* Base for tsi_handshaker_result implementations. - See transport_security_interface.h for documentation. */ + See transport_security_interface.h for documentation. + The exec_ctx parameter in create_zero_copy_grpc_protector is supposed to be + of type grpc_exec_ctx*, but we're using void* instead to avoid making the TSI + API depend on grpc. The create_zero_copy_grpc_protector() method is only used + in grpc, where we do need the exec_ctx passed through, but the API still + needs to compile in other applications, where grpc_exec_ctx is not defined. +*/ typedef struct { tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer); tsi_result (*create_zero_copy_grpc_protector)( - const tsi_handshaker_result *self, + void *exec_ctx, const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); tsi_result (*create_frame_protector)(const tsi_handshaker_result *self, diff --git a/src/core/tsi/transport_security_grpc.c b/src/core/tsi/transport_security_grpc.c index 773b35e717..affd995230 100644 --- a/src/core/tsi/transport_security_grpc.c +++ b/src/core/tsi/transport_security_grpc.c @@ -20,16 +20,18 @@ /* This method creates a tsi_zero_copy_grpc_protector object. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { - if (self == NULL || self->vtable == NULL || protector == NULL) { + if (exec_ctx == NULL || self == NULL || self->vtable == NULL || + protector == NULL) { return TSI_INVALID_ARGUMENT; } if (self->vtable->create_zero_copy_grpc_protector == NULL) { return TSI_UNIMPLEMENTED; } return self->vtable->create_zero_copy_grpc_protector( - self, max_output_protected_frame_size, protector); + exec_ctx, self, max_output_protected_frame_size, protector); } /* --- tsi_zero_copy_grpc_protector common implementation. --- diff --git a/src/core/tsi/transport_security_grpc.h b/src/core/tsi/transport_security_grpc.h index 375a758888..ca6755c12f 100644 --- a/src/core/tsi/transport_security_grpc.h +++ b/src/core/tsi/transport_security_grpc.h @@ -30,7 +30,8 @@ extern "C" { assuming there is no fatal error. The caller is responsible for destroying the protector. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); /* -- tsi_zero_copy_grpc_protector object -- */ diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc index 3af8bdc11a..40e95f3c05 100644 --- a/src/cpp/client/client_context.cc +++ b/src/cpp/client/client_context.cc @@ -96,7 +96,7 @@ void ClientContext::set_call(grpc_call* call, void ClientContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = nullptr; + const char* algorithm_name = nullptr; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index 66b1ef0e39..de2e449fe8 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -22,14 +22,29 @@ namespace grpc { +namespace { +std::unique_ptr<GenericClientAsyncReaderWriter> CallInternal( + ChannelInterface* channel, ClientContext* context, + const grpc::string& method, CompletionQueue* cq, bool start, void* tag) { + return std::unique_ptr<GenericClientAsyncReaderWriter>( + GenericClientAsyncReaderWriter::Create( + channel, cq, RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), + context, start, tag)); +} + +} // namespace + // begin a call to a named method std::unique_ptr<GenericClientAsyncReaderWriter> GenericStub::Call( ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag) { - return std::unique_ptr<GenericClientAsyncReaderWriter>( - GenericClientAsyncReaderWriter::Create( - channel_.get(), cq, - RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), context, tag)); + return CallInternal(channel_.get(), context, method, cq, true, tag); +} + +// setup a call to a named method +std::unique_ptr<GenericClientAsyncReaderWriter> GenericStub::PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq) { + return CallInternal(channel_.get(), context, method, cq, false, nullptr); } } // namespace grpc diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f89f5f1f03..f130aecd4b 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,10 +86,6 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } -void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { - SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); -} - void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 4913682f1d..d7876a000b 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -190,7 +190,7 @@ bool ServerContext::IsCancelled() const { void ServerContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = NULL; + const char* algorithm_name = NULL; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index abf326459c..bbcbd95be5 100755 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -15,7 +15,6 @@ <PackageTags>gRPC RPC Protocol HTTP/2 Auth OAuth2</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <NetStandardImplicitPackageVersion Condition=" '$(TargetFramework)' == 'netstandard1.5' ">1.6.0</NetStandardImplicitPackageVersion> <IncludeSymbols>true</IncludeSymbols> <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj index 9ad6fd0c61..4d6767fa98 100755 --- a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj @@ -15,7 +15,6 @@ <PackageTags>gRPC test testing</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <NetStandardImplicitPackageVersion Condition=" '$(TargetFramework)' == 'netstandard1.5' ">1.6.0</NetStandardImplicitPackageVersion> <IncludeSymbols>true</IncludeSymbols> <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 6df68fda58..18993a93e0 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.Core.Tests</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Core.Tests</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> @@ -21,7 +19,6 @@ <PackageReference Include="Newtonsoft.Json" Version="9.0.1" /> <PackageReference Include="NUnit" Version="3.6.0" /> <PackageReference Include="NUnitLite" Version="3.6.0" /> - <PackageReference Include="NUnit.ConsoleRunner" Version="3.6.0" /> <PackageReference Include="OpenCover" Version="4.6.519" /> <PackageReference Include="ReportGenerator" Version="2.4.4.0" /> </ItemGroup> diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index dde800aadd..d9950b2f20 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -14,7 +14,6 @@ <PackageTags>gRPC RPC Protocol HTTP/2</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <NetStandardImplicitPackageVersion Condition=" '$(TargetFramework)' == 'netstandard1.5' ">1.6.0</NetStandardImplicitPackageVersion> <IncludeSymbols>true</IncludeSymbols> <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> @@ -65,7 +64,7 @@ <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard1.5' "> <PackageReference Include="System.Runtime.Loader" Version="4.0.0" /> <PackageReference Include="System.Threading.Thread" Version="4.0.0" /> - <PackageReference Include="System.Threading.ThreadPool" Version="4.0.0" /> + <PackageReference Include="System.Threading.ThreadPool" Version="4.0.10" /> </ItemGroup> <Import Project="NativeDeps.csproj.include" /> diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj index 74deed6584..db4e3ef4e3 100755 --- a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj +++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj @@ -8,7 +8,6 @@ <AssemblyName>Grpc.Examples.MathClient</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Examples.MathClient</PackageId> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj index 1abf261498..b12b418d01 100755 --- a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj +++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj @@ -8,7 +8,6 @@ <AssemblyName>Grpc.Examples.MathServer</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Examples.MathServer</PackageId> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d2a13ed6e1..3ccc9adfaf 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.Examples.Tests</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Examples.Tests</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj index 491d313f17..baa3b4ce6c 100755 --- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj +++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj @@ -7,7 +7,6 @@ <TargetFrameworks>net45;netcoreapp1.0</TargetFrameworks> <AssemblyName>Grpc.Examples</AssemblyName> <PackageId>Grpc.Examples</PackageId> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 2ccf46b9b9..9da0539dcb 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.HealthCheck.Tests</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.HealthCheck.Tests</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 3eb90434f3..681719d124 100755 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -14,7 +14,6 @@ <PackageTags>gRPC health check</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <NetStandardImplicitPackageVersion Condition=" '$(TargetFramework)' == 'netstandard1.5' ">1.6.0</NetStandardImplicitPackageVersion> <IncludeSymbols>true</IncludeSymbols> <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index c67beea7cd..35713156ea 100755 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.IntegrationTesting.Client</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.IntegrationTesting.Client</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index e452257b1b..3ecefe3bc4 100755 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -9,8 +9,6 @@ <OutputType>Exe</OutputType> <PackageId>Grpc.IntegrationTesting.QpsWorker</PackageId> <ServerGarbageCollection>true</ServerGarbageCollection> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index a1fb316fdb..1092b2c21e 100755 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.IntegrationTesting.Server</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.IntegrationTesting.Server</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj index f64bea3d2b..22272547f6 100755 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.IntegrationTesting.StressClient</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.IntegrationTesting.StressClient</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index f5077fe0f7..c02c9844e3 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.IntegrationTesting</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.IntegrationTesting</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> @@ -31,10 +29,6 @@ <Reference Include="Microsoft.CSharp" /> </ItemGroup> - <ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp1.0' "> - <PackageReference Include="System.Linq.Expressions" Version="4.1.1" /> - </ItemGroup> - <ItemGroup> <Compile Include="..\Grpc.Core\Version.cs" /> </ItemGroup> diff --git a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj index 17797e1e1e..108357e4eb 100644 --- a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj +++ b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.Microbenchmarks</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Microbenchmarks</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index cf756c68ad..d368697124 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -8,8 +8,6 @@ <AssemblyName>Grpc.Reflection.Tests</AssemblyName> <OutputType>Exe</OutputType> <PackageId>Grpc.Reflection.Tests</PackageId> - <PackageTargetFallback Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">$(PackageTargetFallback);portable-net45</PackageTargetFallback> - <RuntimeFrameworkVersion Condition=" '$(TargetFramework)' == 'netcoreapp1.0' ">1.0.4</RuntimeFrameworkVersion> <TreatWarningsAsErrors>true</TreatWarningsAsErrors> </PropertyGroup> diff --git a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj index b77fd69aee..704eea5c17 100755 --- a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj +++ b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj @@ -14,7 +14,6 @@ <PackageTags>gRPC reflection</PackageTags> <PackageProjectUrl>https://github.com/grpc/grpc</PackageProjectUrl> <PackageLicenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</PackageLicenseUrl> - <NetStandardImplicitPackageVersion Condition=" '$(TargetFramework)' == 'netstandard1.5' ">1.6.0</NetStandardImplicitPackageVersion> <IncludeSymbols>true</IncludeSymbols> <IncludeSource>true</IncludeSource> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/src/csharp/doc/.gitignore b/src/csharp/doc/.gitignore new file mode 100644 index 0000000000..09ee235efc --- /dev/null +++ b/src/csharp/doc/.gitignore @@ -0,0 +1,2 @@ +html +obj diff --git a/src/csharp/doc/README.md b/src/csharp/doc/README.md index 585500b5ca..46cce013a1 100644 --- a/src/csharp/doc/README.md +++ b/src/csharp/doc/README.md @@ -1,2 +1,9 @@ +DocFX-generated C# API Reference +-------------------------------- -SandCastle project files to generate HTML reference documentation.
\ No newline at end of file +Install docfx based on instructions here: https://github.com/dotnet/docfx + +``` +# generate docfx documentation into ./html directory +$ docfx +``` diff --git a/src/csharp/doc/docfx.json b/src/csharp/doc/docfx.json new file mode 100644 index 0000000000..7219d0e7a6 --- /dev/null +++ b/src/csharp/doc/docfx.json @@ -0,0 +1,37 @@ +{ + "metadata": [ + { + "src": [ + { + "files": ["Grpc.Core/Grpc.Core.csproj", + "Grpc.Auth/Grpc.Auth.csproj", + "Grpc.Core.Testing/Grpc.Core.Testing.csproj", + "Grpc.HealthCheck/Grpc.HealthCheck.csproj", + "Grpc.Reflection/Grpc.HealthCheck.csproj"], + "exclude": [ "**/bin/**", "**/obj/**" ], + "cwd": ".." + } + ], + "properties": { "TargetFramework": "net45" }, + "dest": "obj/api" + } + ], + "build": { + "content": [ + { + "files": [ "**/*.yml" ], + "cwd": "obj/api", + "dest": "api" + }, + { + "files": [ "toc.yml"], + } + ], + "globalMetadata": { + "_appTitle": "gRPC C#", + "_enableSearch": true, + "_disableContribution": true + }, + "dest": "html" + } +} diff --git a/src/csharp/doc/grpc_csharp_public.shfbproj b/src/csharp/doc/grpc_csharp_public.shfbproj deleted file mode 100644 index fab953da35..0000000000 --- a/src/csharp/doc/grpc_csharp_public.shfbproj +++ /dev/null @@ -1,83 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> - <PropertyGroup> - <!-- The configuration and platform will be used to determine which assemblies to include from solution and - project documentation sources --> - <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> - <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> - <SchemaVersion>2.0</SchemaVersion> - <ProjectGuid>{77e3da09-fc92-486f-a90a-99ca788e8b59}</ProjectGuid> - <SHFBSchemaVersion>2015.6.5.0</SHFBSchemaVersion> - <!-- AssemblyName, Name, and RootNamespace are not used by SHFB but Visual Studio adds them anyway --> - <AssemblyName>Documentation</AssemblyName> - <RootNamespace>Documentation</RootNamespace> - <Name>Documentation</Name> - <!-- SHFB properties --> - <FrameworkVersion>.NET Framework 4.5</FrameworkVersion> - <OutputPath>..\..\..\doc\ref\csharp\html</OutputPath> - <Language>en-US</Language> - <DocumentationSources> - <DocumentationSource sourceFile="..\Grpc.Auth\Grpc.Auth.csproj" /> -<DocumentationSource sourceFile="..\Grpc.Core\Grpc.Core.csproj" /> -<DocumentationSource sourceFile="..\Grpc.HealthCheck\Grpc.HealthCheck.csproj" /> -<DocumentationSource sourceFile="..\Grpc.Reflection\Grpc.Reflection.csproj" /> -<DocumentationSource sourceFile="..\Grpc.Core.Testing\Grpc.Core.Testing.csproj" /></DocumentationSources> - <BuildAssemblerVerbosity>OnlyWarningsAndErrors</BuildAssemblerVerbosity> - <HelpFileFormat>Website</HelpFileFormat> - <IndentHtml>False</IndentHtml> - <KeepLogFile>True</KeepLogFile> - <DisableCodeBlockComponent>False</DisableCodeBlockComponent> - <CleanIntermediates>True</CleanIntermediates> - <HelpFileVersion>1.0.0.0</HelpFileVersion> - <MaximumGroupParts>2</MaximumGroupParts> - <NamespaceGrouping>False</NamespaceGrouping> - <SyntaxFilters>Standard</SyntaxFilters> - <SdkLinkTarget>Blank</SdkLinkTarget> - <RootNamespaceContainer>True</RootNamespaceContainer> - <PresentationStyle>VS2013</PresentationStyle> - <Preliminary>False</Preliminary> - <NamingMethod>MemberName</NamingMethod> - <HelpTitle>gRPC C#</HelpTitle> - <ContentPlacement>AboveNamespaces</ContentPlacement> - <HtmlHelpName>Documentation</HtmlHelpName> - <NamespaceSummaries> - <NamespaceSummaryItem name="Grpc.Auth" isDocumented="True">Provides OAuth2 based authentication for gRPC. <c>Grpc.Auth</c> currently consists of a set of very lightweight wrappers and uses C# <a href="https://www.nuget.org/packages/Google.Apis.Auth/">Google.Apis.Auth</a> library.</NamespaceSummaryItem> - <NamespaceSummaryItem name="Grpc.Core" isDocumented="True">Main namespace for gRPC C# functionality. Contains concepts representing both client side and server side gRPC logic. - -<seealso cref="Grpc.Core.Channel"/> -<seealso cref="Grpc.Core.Server"/></NamespaceSummaryItem> - <NamespaceSummaryItem name="Grpc.Core.Logging" isDocumented="True">Provides functionality to redirect gRPC logs to application-specified destination.</NamespaceSummaryItem> - <NamespaceSummaryItem name="Grpc.Core.Utils" isDocumented="True">Various utilities for gRPC C#.</NamespaceSummaryItem> - </NamespaceSummaries> - <MissingTags>Summary, Parameter, AutoDocumentCtors, Namespace, TypeParameter, AutoDocumentDispose</MissingTags> - </PropertyGroup> - <!-- There are no properties for these groups. AnyCPU needs to appear in order for Visual Studio to perform - the build. The others are optional common platform types that may appear. --> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|Win32' "> - </PropertyGroup> - <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|Win32' "> - </PropertyGroup> - <!-- Import the SHFB build targets --> - <Import Project="$(SHFBROOT)\SandcastleHelpFileBuilder.targets" /> - <!-- The pre-build and post-build event properties must appear *after* the targets file import in order to be - evaluated correctly. --> - <PropertyGroup> - <PreBuildEvent> - </PreBuildEvent> - <PostBuildEvent> - </PostBuildEvent> - <RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent> - </PropertyGroup> -</Project>
\ No newline at end of file diff --git a/src/csharp/doc/toc.yml b/src/csharp/doc/toc.yml new file mode 100644 index 0000000000..c3a1e415ab --- /dev/null +++ b/src/csharp/doc/toc.yml @@ -0,0 +1,3 @@ +- name: API Documentation + href: obj/api/ + homepage: obj/api/Grpc.Core.yml diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 4d90cfd384..df563ca36c 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,6 +170,13 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** + * The timeout for the RPC call in seconds. If set to 0, the call will not timeout. If set to + * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded if it is not completed + * within \a timeout seconds. A negative value is not allowed. + */ +@property NSTimeInterval timeout; + +/** * The container of the request headers of an RPC conforms to this protocol, which is a subset of * NSMutableDictionary's interface. It will become a NSMutableDictionary later on. * The keys of this container are the header names, which per the HTTP standard are case- diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 436c19e354..d6c3a3c165 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -423,7 +423,8 @@ static NSString * const kBearerPrefix = @"Bearer "; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host serverName:_serverName - path:_path]; + path:_path + timeout:_timeout]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); [self sendHeaders:_requestHeaders]; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index e2aa5bd036..d37182f754 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,5 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName + timeout:(NSTimeInterval)timeout completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 52dbc70b99..b78b14f2af 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,18 +182,28 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { + GPR_ASSERT(timeout >= 0); + if (timeout < 0) { + timeout = 0; + } grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); } grpc_slice path_slice = grpc_slice_from_copied_string(path.UTF8String); + gpr_timespec deadline_ms = timeout == 0 ? + gpr_inf_future(GPR_CLOCK_REALTIME) : + gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis((int64_t)(timeout * 1000), GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, path_slice, serverName ? &host_slice : NULL, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); + deadline_ms, NULL); if (serverName) { grpc_slice_unref(host_slice); } diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 0c1d715240..58171211b0 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,6 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 23794c1fed..f73e9cbc50 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,6 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. @@ -130,7 +131,10 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } channel = _channel; } - return [channel unmanagedCallWithPath:path serverName:serverName completionQueue:queue]; + return [channel unmanagedCallWithPath:path + serverName:serverName + timeout:timeout + completionQueue:queue]; } - (BOOL)setTLSPEMRootCerts:(nullable NSString *)pemRootCerts diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 64075591a3..1cd9da8f3e 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -76,7 +76,8 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path NS_DESIGNATED_INITIALIZER; + path:(NSString *)path + timeout:(NSTimeInterval)timeout NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index 87dc33af88..b0b1223b64 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -238,12 +238,13 @@ } - (instancetype)init { - return [self initWithHost:nil serverName:nil path:nil]; + return [self initWithHost:nil serverName:nil path:nil timeout:0]; } - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path { + path:(NSString *)path + timeout:(NSTimeInterval)timeout { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; @@ -255,7 +256,10 @@ // queue. Currently we use a singleton queue. _queue = [GRPCCompletionQueue completionQueue]; - _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path serverName:serverName completionQueue:_queue]; + _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path + serverName:serverName + timeout:timeout + completionQueue:_queue]; if (_call == NULL) { return nil; } diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 9afe507121..82ac2600fa 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -28,6 +28,7 @@ #import <RemoteTest/Messages.pbobjc.h> #import <RxLibrary/GRXWriteable.h> #import <RxLibrary/GRXWriter+Immediate.h> +#import <RxLibrary/GRXBufferedPipe.h> #define TEST_TIMEOUT 16 @@ -39,6 +40,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; static GRPCProtoMethod *kInexistentMethod; static GRPCProtoMethod *kEmptyCallMethod; static GRPCProtoMethod *kUnaryCallMethod; +static GRPCProtoMethod *kFullDuplexCallMethod; /** Observer class for testing that responseMetadata is KVO-compliant */ @interface PassthroughObserver : NSObject @@ -106,6 +108,9 @@ static GRPCProtoMethod *kUnaryCallMethod; kUnaryCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage service:kService method:@"UnaryCall"]; + kFullDuplexCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage + service:kService + method:@"FullDuplexCall"]; } - (void)testConnectionToRemoteServer { @@ -422,4 +427,26 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testTimeout { + __weak XCTestExpectation *completion = [self expectationWithDescription:@"RPC completed."]; + + GRXBufferedPipe *pipe = [GRXBufferedPipe pipe]; + GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + path:kFullDuplexCallMethod.HTTPPath + requestsWriter:pipe]; + + id<GRXWriteable> responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTAssert(0, @"Failure: response received; Expect: no response received."); + } completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Failure: no error received; Expect: receive deadline exceeded."); + XCTAssertEqual(errorOrNil.code, GRPCErrorCodeDeadlineExceeded); + [completion fulfill]; + }]; + + call.timeout = 0.001; + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 5b7a2d104a..608ae6884b 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -49,7 +49,8 @@ xcodebuild \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ | egrep -v "$XCODEBUILD_FILTER" \ - | egrep -v '^$' - + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -57,7 +58,8 @@ xcodebuild \ -scheme CoreCronetEnd2EndTests \ -destination name="iPhone 6" \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" @@ -65,7 +67,10 @@ xcodebuild \ -workspace Tests.xcworkspace \ -scheme CronetUnitTests \ -destination name="iPhone 6" \ - test | xcpretty + test \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -74,5 +79,6 @@ xcodebuild \ -destination name="iPhone 6" \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 28c30e5d35..237f430799 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -41,9 +41,8 @@ cdef class CompletionQueue: cdef object user_tag = None cdef Call operation_call = None cdef CallDetails request_call_details = None - cdef Metadata request_metadata = None + cdef object request_metadata = None cdef Operations batch_operations = None - cdef Operation batch_operation = None if event.type == GRPC_QUEUE_TIMEOUT: return Event( event.type, False, None, None, None, None, False, None) @@ -63,14 +62,8 @@ cdef class CompletionQueue: operation_call = tag.operation_call request_call_details = tag.request_call_details if tag.request_metadata is not None: - request_metadata = tag.request_metadata - request_metadata._claim_slice_ownership() + request_metadata = tuple(tag.request_metadata) batch_operations = tag.batch_operations - if tag.batch_operations is not None: - for op in batch_operations.operations: - batch_operation = <Operation>op - if batch_operation._received_metadata is not None: - batch_operation._received_metadata._claim_slice_ownership() if tag.is_new_request: # Stuff in the tag not explicitly handled by us needs to live through # the life of the call diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 98d7a9820d..57816f1cab 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -76,7 +76,7 @@ cdef class CredentialsMetadataPlugin: """ Args: plugin_callback (callable): Callback accepting a service URL (str/bytes) - and callback object (accepting a Metadata, + and callback object (accepting a MetadataArray, grpc_status_code, and a str/bytes error message). This argument when called should be non-blocking and eventually call the callback object with the appropriate status code/details and metadata (if @@ -129,8 +129,7 @@ cdef void plugin_get_metadata( def python_callback( Metadata metadata, grpc_status_code status, bytes error_details): - cb(user_data, metadata.c_metadata_array.metadata, - metadata.c_metadata_array.count, status, error_details) + cb(user_data, metadata.c_metadata, metadata.c_count, status, error_details) called_flag[0] = True cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state cdef AuthMetadataContext cy_context = AuthMetadataContext() @@ -139,8 +138,8 @@ cdef void plugin_get_metadata( self.plugin_callback(cy_context, python_callback) except Exception as error: if not called_flag[0]: - cb(user_data, Metadata([]).c_metadata_array.metadata, - 0, StatusCode.unknown, traceback.format_exc().encode()) + cb(user_data, NULL, 0, StatusCode.unknown, + traceback.format_exc().encode()) cdef void plugin_destroy_c_plugin_state(void *state) with gil: cpython.Py_DECREF(<CredentialsMetadataPlugin>state) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 5950bfa0e6..28cf114451 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -59,6 +59,7 @@ cdef extern from "grpc/grpc.h": grpc_slice grpc_slice_malloc(size_t length) nogil grpc_slice grpc_slice_from_copied_string(const char *source) nogil grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t len) nogil + grpc_slice grpc_slice_copy(grpc_slice s) nogil # Declare functions for function-like macros (because Cython)... void *grpc_slice_start_ptr "GRPC_SLICE_START_PTR" (grpc_slice s) nogil diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi index 8ace6aeb52..9c40ebf0c2 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi @@ -37,7 +37,7 @@ cdef class OperationTag: cdef Server shutting_down_server cdef Call operation_call cdef CallDetails request_call_details - cdef Metadata request_metadata + cdef MetadataArray request_metadata cdef Operations batch_operations cdef bint is_new_request @@ -51,7 +51,7 @@ cdef class Event: # For Server.request_call cdef readonly bint is_new_request cdef readonly CallDetails request_call_details - cdef readonly Metadata request_metadata + cdef readonly object request_metadata # For server calls cdef readonly Call operation_call @@ -92,15 +92,20 @@ cdef class Metadatum: cdef class Metadata: + cdef grpc_metadata *c_metadata + cdef readonly size_t c_count + + +cdef class MetadataArray: + cdef grpc_metadata_array c_metadata_array - cdef void _claim_slice_ownership(self) cdef class Operation: cdef grpc_op c_op cdef ByteBuffer _received_message - cdef Metadata _received_metadata + cdef MetadataArray _received_metadata cdef grpc_status_code _received_status_code cdef grpc_slice _status_details cdef int _received_cancelled diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 1b2ddd2469..0a2a6eee05 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -238,7 +238,7 @@ cdef class Event: def __cinit__(self, grpc_completion_type type, bint success, object tag, Call operation_call, CallDetails request_call_details, - Metadata request_metadata, + object request_metadata, bint is_new_request, Operations batch_operations): self.type = type @@ -437,48 +437,79 @@ cdef class Metadatum: cdef class _MetadataIterator: cdef size_t i - cdef Metadata metadata + cdef size_t _length + cdef object _metadatum_indexable - def __cinit__(self, Metadata metadata not None): + def __cinit__(self, length, metadatum_indexable): + self._length = length + self._metadatum_indexable = metadatum_indexable self.i = 0 - self.metadata = metadata def __iter__(self): return self def __next__(self): - if self.i < len(self.metadata): - result = self.metadata[self.i] + if self.i < self._length: + result = self._metadatum_indexable[self.i] self.i = self.i + 1 return result else: raise StopIteration +# TODO(https://github.com/grpc/grpc/issues/7950): Eliminate this; just use an +# ordinary sequence of pairs of bytestrings all the way down to the +# grpc_call_start_batch call. cdef class Metadata: + """Metadata being passed from application to core.""" def __cinit__(self, metadata_iterable): + metadata_sequence = tuple(metadata_iterable) + cdef size_t count = len(metadata_sequence) with nogil: grpc_init() - grpc_metadata_array_init(&self.c_metadata_array) - metadata = list(metadata_iterable) - for metadatum in metadata: - if not isinstance(metadatum, Metadatum): - raise TypeError("expected list of Metadatum") - self.c_metadata_array.count = len(metadata) - self.c_metadata_array.capacity = len(metadata) + self.c_metadata = <grpc_metadata *>gpr_malloc( + count * sizeof(grpc_metadata)) + self.c_count = count + for index, metadatum in enumerate(metadata_sequence): + self.c_metadata[index].key = grpc_slice_copy( + (<Metadatum>metadatum).c_metadata.key) + self.c_metadata[index].value = grpc_slice_copy( + (<Metadatum>metadatum).c_metadata.value) + + def __dealloc__(self): + with nogil: + for index in range(self.c_count): + grpc_slice_unref(self.c_metadata[index].key) + grpc_slice_unref(self.c_metadata[index].value) + gpr_free(self.c_metadata) + grpc_shutdown() + + def __len__(self): + return self.c_count + + def __getitem__(self, size_t index): + if index < self.c_count: + key = _slice_bytes(self.c_metadata[index].key) + value = _slice_bytes(self.c_metadata[index].value) + return Metadatum(key, value) + else: + raise IndexError() + + def __iter__(self): + return _MetadataIterator(self.c_count, self) + + +cdef class MetadataArray: + """Metadata being passed from core to application.""" + + def __cinit__(self): with nogil: - self.c_metadata_array.metadata = <grpc_metadata *>gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata) - ) - for i in range(self.c_metadata_array.count): - (<Metadatum>metadata[i])._copy_metadatum(&self.c_metadata_array.metadata[i]) + grpc_init() + grpc_metadata_array_init(&self.c_metadata_array) def __dealloc__(self): with nogil: - # this frees the allocated memory for the grpc_metadata_array (although - # it'd be nice if that were documented somewhere...) - # TODO(atash): document this in the C core grpc_metadata_array_destroy(&self.c_metadata_array) grpc_shutdown() @@ -493,21 +524,7 @@ cdef class Metadata: return Metadatum(key=key, value=value) def __iter__(self): - return _MetadataIterator(self) - - cdef void _claim_slice_ownership(self): - cdef grpc_metadata_array new_c_metadata_array - grpc_metadata_array_init(&new_c_metadata_array) - new_c_metadata_array.metadata = <grpc_metadata *>gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata)) - new_c_metadata_array.count = self.c_metadata_array.count - for i in range(self.c_metadata_array.count): - new_c_metadata_array.metadata[i].key = _copy_slice( - self.c_metadata_array.metadata[i].key) - new_c_metadata_array.metadata[i].value = _copy_slice( - self.c_metadata_array.metadata[i].value) - grpc_metadata_array_destroy(&self.c_metadata_array) - self.c_metadata_array = new_c_metadata_array + return _MetadataIterator(self.c_metadata_array.count, self) cdef class Operation: @@ -547,14 +564,13 @@ cdef class Operation: if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): raise TypeError("self must be an operation receiving metadata") - return self._received_metadata - - @property - def received_metadata_or_none(self): - if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and - self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): - return None - return self._received_metadata + # TODO(https://github.com/grpc/grpc/issues/7950): Drop the "all Cython + # objects must be legitimate for use from Python at any time" policy in + # place today, shift the policy toward "Operation objects are only usable + # while their calls are active", and move this making-a-copy-because-this- + # data-needs-to-live-much-longer-than-the-call-from-which-it-arose to the + # lowest Python layer. + return tuple(self._received_metadata) @property def received_status_code(self): @@ -601,9 +617,8 @@ def operation_send_initial_metadata(Metadata metadata, int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_SEND_INITIAL_METADATA op.c_op.flags = flags - op.c_op.data.send_initial_metadata.count = metadata.c_metadata_array.count - op.c_op.data.send_initial_metadata.metadata = ( - metadata.c_metadata_array.metadata) + op.c_op.data.send_initial_metadata.count = metadata.c_count + op.c_op.data.send_initial_metadata.metadata = metadata.c_metadata op.references.append(metadata) op.is_valid = True return op @@ -631,9 +646,8 @@ def operation_send_status_from_server( op.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER op.c_op.flags = flags op.c_op.data.send_status_from_server.trailing_metadata_count = ( - metadata.c_metadata_array.count) - op.c_op.data.send_status_from_server.trailing_metadata = ( - metadata.c_metadata_array.metadata) + metadata.c_count) + op.c_op.data.send_status_from_server.trailing_metadata = metadata.c_metadata op.c_op.data.send_status_from_server.status = code grpc_slice_unref(op._status_details) op._status_details = _slice_from_bytes(details) @@ -646,7 +660,7 @@ def operation_receive_initial_metadata(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_INITIAL_METADATA op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_initial_metadata.receive_initial_metadata = ( &op._received_metadata.c_metadata_array) op.is_valid = True @@ -669,7 +683,7 @@ def operation_receive_status_on_client(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_status_on_client.trailing_metadata = ( &op._received_metadata.c_metadata_array) op.c_op.data.receive_status_on_client.status = ( diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index dd276fd57b..b8db27469f 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -44,7 +44,7 @@ cdef class Server: cdef OperationTag operation_tag = OperationTag(tag) operation_tag.operation_call = Call() operation_tag.request_call_details = CallDetails() - operation_tag.request_metadata = Metadata([]) + operation_tag.request_metadata = MetadataArray() operation_tag.references.extend([self, call_queue, server_queue]) operation_tag.is_new_request = True operation_tag.batch_operations = Operations([]) diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 0299b4cca9..1f5e9c5130 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -34,7 +34,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index bed2311b59..9360550afb 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -35,7 +35,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index 917e11808e..994274500c 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -213,7 +213,7 @@ class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): raise NotImplementedError() -class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): +class Channel(six.with_metaclass(abc.ABCMeta, grpc.Channel)): """A grpc.Channel double with which to test a system that invokes RPCs.""" @abc.abstractmethod diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 41a75d46f6..592d08efc3 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.5.0.dev0' +VERSION='1.7.0.dev0' diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 162200112a..93f84572b7 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -67,55 +67,6 @@ class GatherProto(setuptools.Command): open(path, 'a').close() -class BuildProtoModules(setuptools.Command): - """Command to generate project *_pb2.py modules from proto files.""" - - description = 'build protobuf modules' - user_options = [ - ('include=', None, 'path patterns to include in protobuf generation'), - ('exclude=', None, 'path patterns to exclude from protobuf generation') - ] - - def initialize_options(self): - self.exclude = None - self.include = r'.*\.proto$' - - def finalize_options(self): - pass - - def run(self): - import grpc_tools.protoc as protoc - - include_regex = re.compile(self.include) - exclude_regex = re.compile(self.exclude) if self.exclude else None - paths = [] - for walk_root, directories, filenames in os.walk(PROTO_STEM): - for filename in filenames: - path = os.path.join(walk_root, filename) - if include_regex.match(path) and not ( - exclude_regex and exclude_regex.match(path)): - paths.append(path) - - # TODO(kpayson): It would be nice to do this in a batch command, - # but we currently have name conflicts in src/proto - for path in paths: - command = [ - 'grpc_tools.protoc', - '-I {}'.format(PROTO_STEM), - '--python_out={}'.format(PROTO_STEM), - '--grpc_python_out={}'.format(PROTO_STEM), - ] + [path] - if protoc.main(command) != 0: - sys.stderr.write( - 'warning: Command:\n{}\nFailed'.format(command)) - - # Generated proto directories dont include __init__.py, but - # these are needed for python package resolution - for walk_root, _, _ in os.walk(PROTO_STEM): - path = os.path.join(walk_root, '__init__.py') - open(path, 'a').close() - - class BuildPy(build_py.build_py): """Custom project build command.""" diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py b/src/python/grpcio_tests/tests/_sanity/__init__.py index 5772620b60..5772620b60 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py +++ b/src/python/grpcio_tests/tests/_sanity/__init__.py diff --git a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py index 19bc8801eb..b4079850ff 100644 --- a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py +++ b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py @@ -21,24 +21,25 @@ import six import tests -class Sanity(unittest.TestCase): +class SanityTest(unittest.TestCase): + + maxDiff = 32768 def testTestsJsonUpToDate(self): """Autodiscovers all test suites and checks that tests.json is up to date""" loader = tests.Loader() loader.loadTestsFromNames(['tests']) - test_suite_names = [ + test_suite_names = sorted({ test_case_class.id().rsplit('.', 1)[0] for test_case_class in tests._loader.iterate_suite_cases( loader.suite) - ] - test_suite_names = sorted(set(test_suite_names)) + }) tests_json_string = pkg_resources.resource_string('tests', 'tests.json') - if six.PY3: - tests_json_string = tests_json_string.decode() - tests_json = json.loads(tests_json_string) - self.assertListEqual(test_suite_names, tests_json) + tests_json = json.loads(tests_json_string.decode() + if six.PY3 else tests_json_string) + + self.assertSequenceEqual(tests_json, test_suite_names) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 71493bfec6..5b84001aab 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -33,7 +33,7 @@ from tests.unit.framework.common import test_constants import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +import tests.protoc_plugin.protos.service.test_service_pb2_grpc as service_pb2_grpc # Identifiers of entities we expect to find in the generated module. STUB_IDENTIFIER = 'TestServiceStub' @@ -138,7 +138,7 @@ def _CreateService(): """ servicer_methods = _ServicerMethods() - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): def UnaryCall(self, request, context): return servicer_methods.UnaryCall(request, context) @@ -157,11 +157,12 @@ def _CreateService(): server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(servicer_methods, server, stub) @@ -173,16 +174,17 @@ def _CreateIncompleteService(): servicer_methods implements none of the methods required of it. """ - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): pass server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(None, server, stub) @@ -223,10 +225,11 @@ class PythonPluginTest(unittest.TestCase): def testImportAttributes(self): # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(service_pb2_grpc, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) + getattr(service_pb2_grpc, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) def testUpDown(self): service = _CreateService() diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index 1aeb62a7c5..7868cdbfb3 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -12,22 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +import abc from concurrent import futures import contextlib -import distutils.spawn -import errno import importlib import os -import os.path +from os import path import pkgutil +import platform import shutil -import subprocess import sys import tempfile -import threading import unittest -import platform + +import six import grpc from grpc_tools import protoc @@ -37,292 +35,285 @@ _MESSAGES_IMPORT = b'import "messages.proto";' _SPLIT_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing.split;' _COMMON_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing;' +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + @contextlib.contextmanager -def _system_path(path): +def _system_path(path_insertion): old_system_path = sys.path[:] - sys.path = sys.path[0:1] + path + sys.path[1:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] yield sys.path = old_system_path -class DummySplitServicer(object): +# NOTE(nathaniel): https://twitter.com/exoplaneteer/status/677259364256747520 +# Life lesson "just always default to idempotence" reinforced. +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(proto_content, test_name_bytes, + messages_proto_relative_file_name_bytes): + package_substitution = (b'package grpc_protoc_plugin.invocation_testing.' + + test_name_bytes + b';') + common_namespace_substituted = proto_content.replace(_COMMON_NAMESPACE, + package_substitution) + split_namespace_substituted = common_namespace_substituted.replace( + _SPLIT_NAMESPACE, package_substitution) + message_import_replaced = split_namespace_substituted.replace( + _MESSAGES_IMPORT, + b'import "' + messages_proto_relative_file_name_bytes + b'";') + return message_import_replaced + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') - def __init__(self, request_class, response_class): - self.request_class = request_class - self.response_class = response_class + +class _Servicer(object): + + def __init__(self, response_class): + self._response_class = response_class def Call(self, request, context): - return self.response_class() + return self._response_class() -class SeparateTestMixin(object): +def _protoc(proto_path, python_out, grpc_python_out_flag, grpc_python_out, + absolute_proto_file_names): + args = [ + '', + '--proto_path={}'.format(proto_path), + ] + if python_out is not None: + args.append('--python_out={}'.format(python_out)) + if grpc_python_out is not None: + args.append('--grpc_python_out={}:{}'.format(grpc_python_out_flag, + grpc_python_out)) + args.extend(absolute_proto_file_names) + return protoc.main(args) - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -class CommonTestMixin(object): - - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameSeparateTest(unittest.TestCase, SeparateTestMixin): - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_separate.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_separate;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.grpc_python_out_directory, '__init__.py'), - 'w').write('') - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_separate_pb2' - self.pb2_grpc_import = 'same_separate_pb2_grpc' - self.should_find_services_in_pb2 = False +class _Mid2016ProtocStyle(object): - def tearDown(self): - shutil.rmtree(self.directory) + def name(self): + return 'Mid2016ProtocStyle' + def grpc_in_pb2_expected(self): + return True -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameCommonTest(unittest.TestCase, CommonTestMixin): + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_1_0', python_out, + absolute_proto_file_names),) - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_common.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_common;')) - - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_common_pb2' - self.pb2_grpc_import = 'same_common_pb2_grpc' - self.should_find_services_in_pb2 = True - def tearDown(self): - shutil.rmtree(self.directory) +class _SingleProtocExecutionProtocStyle(object): + def name(self): + return 'SingleProtocExecutionProtocStyle' -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitCommonTest(unittest.TestCase, CommonTestMixin): + def grpc_in_pb2_expected(self): + return False - def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_common_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_common_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_common_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_common_messages_pb2' - self.pb2_grpc_import = 'split_common_services_pb2_grpc' - self.should_find_services_in_pb2 = False + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_2_0', python_out, + absolute_proto_file_names),) + + +class _ProtoBeforeGrpcProtocStyle(object): + + def name(self): + return 'ProtoBeforeGrpcProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + return pb2_protoc_exit_code, pb2_grpc_protoc_exit_code, - def tearDown(self): - shutil.rmtree(self.directory) +class _GrpcBeforeProtoProtocStyle(object): -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): + def name(self): + return 'GrpcBeforeProtoProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code, + + +_PROTOC_STYLES = (_Mid2016ProtocStyle(), _SingleProtocExecutionProtocStyle(), + _ProtoBeforeGrpcProtocStyle(), _GrpcBeforeProtoProtocStyle(),) + + +@unittest.skipIf(platform.python_implementation() == 'PyPy', + 'Skip test if run with PyPy!') +class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)): def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_separate_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_separate_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_separate_messages_pb2' - self.pb2_grpc_import = 'split_separate_services_pb2_grpc' - self.should_find_services_in_pb2 = False + self._directory = tempfile.mkdtemp(suffix=self.NAME, dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + proto_directories_and_names = { + (self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.MESSAGES_PROTO_FILE_NAME,), + (self.SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.SERVICES_PROTO_FILE_NAME,), + } + messages_proto_relative_file_name_forward_slashes = '/'.join( + self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES + ( + self.MESSAGES_PROTO_FILE_NAME,)) + _create_directory_tree(self._proto_path, ( + relative_proto_directory_names + for relative_proto_directory_names, _ in proto_directories_and_names + )) + self._absolute_proto_file_names = set() + for relative_directory_names, file_name in proto_directories_and_names: + absolute_proto_file_name = path.join( + self._proto_path, *relative_directory_names + (file_name,)) + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos.invocation_testing', + path.join(*relative_directory_names + (file_name,))) + massaged_proto_content = _massage_proto_content( + raw_proto_content, + self.NAME.encode(), + messages_proto_relative_file_name_forward_slashes.encode()) + with open(absolute_proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._absolute_proto_file_names.add(absolute_proto_file_name) def tearDown(self): - shutil.rmtree(self.directory) + shutil.rmtree(self._directory) + + def _protoc(self): + protoc_exit_codes = self.PROTOC_STYLE.protoc( + self._proto_path, self._python_out, self._absolute_proto_file_names) + for protoc_exit_code in protoc_exit_codes: + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + generated_modules = {} + expected_generated_full_module_names = { + self.EXPECTED_MESSAGES_PB2, + self.EXPECTED_SERVICES_PB2, + self.EXPECTED_SERVICES_PB2_GRPC, + } + with _system_path([self._python_out]): + for full_module_name in expected_generated_full_module_names: + module = importlib.import_module(full_module_name) + generated_modules[full_module_name] = module + + self._messages_pb2 = generated_modules[self.EXPECTED_MESSAGES_PB2] + self._services_pb2 = generated_modules[self.EXPECTED_SERVICES_PB2] + self._services_pb2_grpc = generated_modules[ + self.EXPECTED_SERVICES_PB2_GRPC] + + def _services_modules(self): + if self.PROTOC_STYLE.grpc_in_pb2_expected(): + return self._services_pb2, self._services_pb2_grpc, + else: + return self._services_pb2_grpc, + + def test_imported_attributes(self): + self._protoc() + + self._messages_pb2.Request + self._messages_pb2.Response + self._services_pb2.DESCRIPTOR.services_by_name['TestService'] + for services_module in self._services_modules(): + services_module.TestServiceStub + services_module.TestServiceServicer + services_module.add_TestServiceServicer_to_server + + def test_call(self): + self._protoc() + + for services_module in self._services_modules(): + server = grpc.server( + futures.ThreadPoolExecutor( + max_workers=test_constants.POOL_SIZE)) + services_module.add_TestServiceServicer_to_server( + _Servicer(self._messages_pb2.Response), server) + port = server.add_insecure_port('[::]:0') + server.start() + channel = grpc.insecure_channel('localhost:{}'.format(port)) + stub = services_module.TestServiceStub(channel) + response = stub.Call(self._messages_pb2.Request()) + self.assertEqual(self._messages_pb2.Response(), response) + + +def _create_test_case_class(split_proto, protoc_style): + attributes = {} + + name = '{}{}'.format('SplitProto' if split_proto else 'SameProto', + protoc_style.name()) + attributes['NAME'] = name + + if split_proto: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_messages', 'sub',) + attributes['MESSAGES_PROTO_FILE_NAME'] = 'messages.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_services',) + attributes['SERVICES_PROTO_FILE_NAME'] = 'services.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'split_messages.sub.messages_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'split_services.services_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = ( + 'split_services.services_pb2_grpc') + else: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['MESSAGES_PROTO_FILE_NAME'] = 'same.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['SERVICES_PROTO_FILE_NAME'] = 'same.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = 'same_pb2_grpc' + + attributes['PROTOC_STYLE'] = protoc_style + + attributes['__module__'] = _Test.__module__ + + return type('{}Test'.format(name), (_Test,), attributes) + + +def _create_test_case_classes(): + for split_proto in (False, True,): + for protoc_style in _PROTOC_STYLES: + yield _create_test_case_class(split_proto, protoc_style) + + +def load_tests(loader, tests, pattern): + tests = tuple( + loader.loadTestsFromTestCase(test_case_class) + for test_case_class in _create_test_case_classes()) + return unittest.TestSuite(tests=tests) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py index 83f21ecbbb..424b153ff8 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py @@ -12,19 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import contextlib -import distutils.spawn -import errno -import itertools +import importlib import os -import pkg_resources +from os import path +import pkgutil import shutil -import subprocess import sys import tempfile import threading -import time import unittest from six import moves @@ -33,12 +29,22 @@ from grpc.beta import implementations from grpc.beta import interfaces from grpc.framework.foundation import future from grpc.framework.interfaces.face import face +from grpc_tools import protoc from tests.unit.framework.common import test_constants -import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 -import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 -import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + +_PROTO_FILES_PATH_COMPONENTS = ( + ('beta_grpc_plugin_test', 'payload', 'test_payload.proto',), + ('beta_grpc_plugin_test', 'requests', 'r', 'test_requests.proto',), + ('beta_grpc_plugin_test', 'responses', 'test_responses.proto',), + ('beta_grpc_plugin_test', 'service', 'test_service.proto',),) + +_PAYLOAD_PB2 = 'beta_grpc_plugin_test.payload.test_payload_pb2' +_REQUESTS_PB2 = 'beta_grpc_plugin_test.requests.r.test_requests_pb2' +_RESPONSES_PB2 = 'beta_grpc_plugin_test.responses.test_responses_pb2' +_SERVICE_PB2 = 'beta_grpc_plugin_test.service.test_service_pb2' # Identifiers of entities we expect to find in the generated module. SERVICER_IDENTIFIER = 'BetaTestServiceServicer' @@ -47,12 +53,50 @@ SERVER_FACTORY_IDENTIFIER = 'beta_create_TestService_server' STUB_FACTORY_IDENTIFIER = 'beta_create_TestService_stub' +@contextlib.contextmanager +def _system_path(path_insertion): + old_system_path = sys.path[:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] + yield + sys.path = old_system_path + + +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(raw_proto_content): + imports_substituted = raw_proto_content.replace( + b'import "tests/protoc_plugin/protos/', + b'import "beta_grpc_plugin_test/') + package_statement_substituted = imports_substituted.replace( + b'package grpc_protoc_plugin;', b'package beta_grpc_protoc_plugin;') + return package_statement_substituted + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') + + class _ServicerMethods(object): - def __init__(self): + def __init__(self, payload_pb2, responses_pb2): self._condition = threading.Condition() self._paused = False self._fail = False + self._payload_pb2 = payload_pb2 + self._responses_pb2 = responses_pb2 @contextlib.contextmanager def pause(self): # pylint: disable=invalid-name @@ -79,22 +123,22 @@ class _ServicerMethods(object): self._condition.wait() def UnaryCall(self, request, unused_rpc_context): - response = response_pb2.SimpleResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.SimpleResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * request.response_size self._control() return response def StreamingOutputCall(self, request, unused_rpc_context): for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response def StreamingInputCall(self, request_iter, unused_rpc_context): - response = response_pb2.StreamingInputCallResponse() + response = self._responses_pb2.StreamingInputCallResponse() aggregated_payload_size = 0 for request in request_iter: aggregated_payload_size += len(request.payload.payload_compressable) @@ -105,8 +149,8 @@ class _ServicerMethods(object): def FullDuplexCall(self, request_iter, unused_rpc_context): for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response @@ -115,8 +159,8 @@ class _ServicerMethods(object): responses = [] for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() responses.append(response) @@ -125,7 +169,7 @@ class _ServicerMethods(object): @contextlib.contextmanager -def _CreateService(): +def _CreateService(payload_pb2, responses_pb2, service_pb2): """Provides a servicer backend and a stub. The servicer is just the implementation of the actual servicer passed to the @@ -136,7 +180,7 @@ def _CreateService(): the service bound to the stub and and stub is the stub on which to invoke RPCs. """ - servicer_methods = _ServicerMethods() + servicer_methods = _ServicerMethods(payload_pb2, responses_pb2) class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): @@ -161,12 +205,12 @@ def _CreateService(): server.start() channel = implementations.insecure_channel('localhost', port) stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel) - yield (servicer_methods, stub) + yield servicer_methods, stub, server.stop(0) @contextlib.contextmanager -def _CreateIncompleteService(): +def _CreateIncompleteService(service_pb2): """Provides a servicer backend that fails to implement methods and its stub. The servicer is just the implementation of the actual servicer passed to the @@ -192,16 +236,16 @@ def _CreateIncompleteService(): server.stop(0) -def _streaming_input_request_iterator(): +def _streaming_input_request_iterator(payload_pb2, requests_pb2): for _ in range(3): - request = request_pb2.StreamingInputCallRequest() + request = requests_pb2.StreamingInputCallRequest() request.payload.payload_type = payload_pb2.COMPRESSABLE request.payload.payload_compressable = 'a' yield request -def _streaming_output_request(): - request = request_pb2.StreamingOutputCallRequest() +def _streaming_output_request(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() sizes = [1, 2, 3] request.response_parameters.add(size=sizes[0], interval_us=0) request.response_parameters.add(size=sizes[1], interval_us=0) @@ -209,11 +253,11 @@ def _streaming_output_request(): return request -def _full_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() +def _full_duplex_request_iterator(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -227,22 +271,78 @@ class PythonPluginTest(unittest.TestCase): methods and does not exist for response-streaming methods. """ + def setUp(self): + self._directory = tempfile.mkdtemp(dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + directories_path_components = { + proto_file_path_components[:-1] + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS + } + _create_directory_tree(self._proto_path, directories_path_components) + self._proto_file_names = set() + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS: + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos', + path.join(*proto_file_path_components[1:])) + massaged_proto_content = _massage_proto_content(raw_proto_content) + proto_file_name = path.join(self._proto_path, + *proto_file_path_components) + with open(proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._proto_file_names.add(proto_file_name) + + def tearDown(self): + shutil.rmtree(self._directory) + + def _protoc(self): + args = [ + '', + '--proto_path={}'.format(self._proto_path), + '--python_out={}'.format(self._python_out), + '--grpc_python_out=grpc_1_0:{}'.format(self._python_out), + ] + list(self._proto_file_names) + protoc_exit_code = protoc.main(args) + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + with _system_path([ + self._python_out, + ]): + self._payload_pb2 = importlib.import_module(_PAYLOAD_PB2) + self._requests_pb2 = importlib.import_module(_REQUESTS_PB2) + self._responses_pb2 = importlib.import_module(_RESPONSES_PB2) + self._service_pb2 = importlib.import_module(_SERVICE_PB2) + def testImportAttributes(self): + self._protoc() + # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(self._service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, STUB_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(self._service_pb2, STUB_FACTORY_IDENTIFIER, None)) def testUpDown(self): - with _CreateService(): - request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2): + self._requests_pb2.SimpleRequest(response_size=13) def testIncompleteServicer(self): - with _CreateIncompleteService() as (_, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateIncompleteService(self._service_pb2) as (_, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) try: stub.UnaryCall(request, test_constants.LONG_TIMEOUT) except face.AbortionError as error: @@ -250,15 +350,21 @@ class PythonPluginTest(unittest.TestCase): error.code) def testUnaryCall(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT) expected_response = methods.UnaryCall(request, 'not a real context!') self.assertEqual(expected_response, response) def testUnaryCallFuture(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) # Check that the call does not block waiting for the server to respond. with methods.pause(): response_future = stub.UnaryCall.future( @@ -268,8 +374,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testUnaryCallFutureExpired(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future( request, test_constants.SHORT_TIMEOUT) @@ -277,24 +386,33 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testUnaryCallFutureCancelled(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future(request, 1) response_future.cancel() self.assertTrue(response_future.cancelled()) def testUnaryCallFutureFailed(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.fail(): response_future = stub.UnaryCall.future( request, test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testStreamingOutputCall(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) expected_responses = methods.StreamingOutputCall( @@ -304,8 +422,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testStreamingOutputCallExpired(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.pause(): responses = stub.StreamingOutputCall( request, test_constants.SHORT_TIMEOUT) @@ -313,8 +434,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testStreamingOutputCallCancelled(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) next(responses) @@ -323,8 +447,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingOutputCallFailed(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.fail(): responses = stub.StreamingOutputCall(request, 1) self.assertIsNotNone(responses) @@ -332,30 +459,46 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingInputCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): response = stub.StreamingInputCall( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFuture(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response = response_future.result() expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFutureExpired(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.SHORT_TIMEOUT) with self.assertRaises(face.ExpirationError): response_future.result() @@ -363,10 +506,14 @@ class PythonPluginTest(unittest.TestCase): face.ExpirationError) def testStreamingInputCallFutureCancelled(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response_future.cancel() self.assertTrue(response_future.cancelled()) @@ -374,26 +521,38 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testStreamingInputCallFutureFailed(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testFullDuplexCall(self): - with _CreateService() as (methods, stub): - responses = stub.FullDuplexCall(_full_duplex_request_iterator(), - test_constants.LONG_TIMEOUT) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + responses = stub.FullDuplexCall( + _full_duplex_request_iterator(self._requests_pb2), + test_constants.LONG_TIMEOUT) expected_responses = methods.FullDuplexCall( - _full_duplex_request_iterator(), 'not a real RpcContext!') + _full_duplex_request_iterator(self._requests_pb2), + 'not a real RpcContext!') for expected_response, response in moves.zip_longest( expected_responses, responses): self.assertEqual(expected_response, response) def testFullDuplexCallExpired(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): responses = stub.FullDuplexCall(request_iterator, test_constants.SHORT_TIMEOUT) @@ -401,8 +560,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testFullDuplexCallCancelled(self): - with _CreateService() as (methods, stub): - request_iterator = _full_duplex_request_iterator() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request_iterator = _full_duplex_request_iterator(self._requests_pb2) responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) next(responses) @@ -411,8 +573,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testFullDuplexCallFailed(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) @@ -421,13 +586,16 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testHalfDuplexCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -441,6 +609,8 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testHalfDuplexCallWedged(self): + self._protoc() + condition = threading.Condition() wait_cell = [False] @@ -455,14 +625,15 @@ class PythonPluginTest(unittest.TestCase): condition.notify_all() def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request with condition: while wait_cell[0]: condition.wait() - with _CreateService() as (methods, stub): + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with wait(): responses = stub.HalfDuplexCall(half_duplex_request_iterator(), test_constants.SHORT_TIMEOUT) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto index 1b780c69ba..1b780c69ba 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto +++ b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto diff --git a/src/python/grpcio_tests/tests/qps/benchmark_server.py b/src/python/grpcio_tests/tests/qps/benchmark_server.py index 05101fdc6d..bb07844491 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_server.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_server.py @@ -13,10 +13,10 @@ # limitations under the License. from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import services_pb2 +from src.proto.grpc.testing import services_pb2_grpc -class BenchmarkServer(services_pb2.BenchmarkServiceServicer): +class BenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Synchronous Server implementation for the Benchmark service.""" def UnaryCall(self, request, context): @@ -29,7 +29,7 @@ class BenchmarkServer(services_pb2.BenchmarkServiceServicer): yield messages_pb2.SimpleResponse(payload=payload) -class GenericBenchmarkServer(services_pb2.BenchmarkServiceServicer): +class GenericBenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Generic Server implementation for the Benchmark service.""" def __init__(self, resp_size): diff --git a/src/python/grpcio_tests/tests/stress/metrics_server.py b/src/python/grpcio_tests/tests/stress/metrics_server.py index 11ab6c3f4e..33a74b4a38 100644 --- a/src/python/grpcio_tests/tests/stress/metrics_server.py +++ b/src/python/grpcio_tests/tests/stress/metrics_server.py @@ -16,11 +16,12 @@ import time from src.proto.grpc.testing import metrics_pb2 +from src.proto.grpc.testing import metrics_pb2_grpc GAUGE_NAME = 'python_overall_qps' -class MetricsServer(metrics_pb2.MetricsServiceServicer): +class MetricsServer(metrics_pb2_grpc.MetricsServiceServicer): def __init__(self, histogram): self._start_time = time.time() diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 4c078e6c22..8512d5b96f 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -1,12 +1,17 @@ [ + "_sanity._sanity_test.SanityTest", "health_check._health_servicer_test.HealthServicerTest", "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", "protoc_plugin._python_plugin_test.PythonPluginTest", - "protoc_plugin._split_definitions_test.SameCommonTest", - "protoc_plugin._split_definitions_test.SameSeparateTest", - "protoc_plugin._split_definitions_test.SplitCommonTest", - "protoc_plugin._split_definitions_test.SplitSeparateTest", + "protoc_plugin._split_definitions_test.SameProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoSingleProtocExecutionProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoSingleProtocExecutionProtocStyleTest", "protoc_plugin.beta_python_plugin_test.PythonPluginTest", "reflection._reflection_servicer_test.ReflectionServicerTest", "testing._client_test.ClientTest", @@ -41,7 +46,6 @@ "unit._reconnect_test.ReconnectTest", "unit._resource_exhausted_test.ResourceExhaustedTest", "unit._rpc_test.RPCTest", - "unit._sanity._sanity_test.Sanity", "unit._thread_cleanup_test.CleanupThreadTest", "unit.beta._beta_features_test.BetaFeaturesTest", "unit.beta._beta_features_test.ContextManagementAndLifecycleTest", diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index c5c848ae44..55a7eed1a4 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -164,10 +164,10 @@ extern census_record_values_type census_record_values_import; typedef int(*grpc_compression_algorithm_parse_type)(grpc_slice value, grpc_compression_algorithm *algorithm); extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_import; #define grpc_compression_algorithm_parse grpc_compression_algorithm_parse_import -typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name); +typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, const char **name); extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import; #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import -typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, char **name); +typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, const char **name); extern grpc_stream_compression_algorithm_name_type grpc_stream_compression_algorithm_name_import; #define grpc_stream_compression_algorithm_name grpc_stream_compression_algorithm_name_import typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings); diff --git a/src/ruby/lib/grpc/google_rpc_status_utils.rb b/src/ruby/lib/grpc/google_rpc_status_utils.rb new file mode 100644 index 0000000000..fdadd6b76e --- /dev/null +++ b/src/ruby/lib/grpc/google_rpc_status_utils.rb @@ -0,0 +1,28 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative './grpc' +require 'google/rpc/status_pb' + +# GRPC contains the General RPC module. +module GRPC + # GoogleRpcStatusUtils provides utilities to convert between a + # GRPC::Core::Status and a deserialized Google::Rpc::Status proto + class GoogleRpcStatusUtils + def self.extract_google_rpc_status(status) + fail ArgumentError, 'bad type' unless status.is_a? Struct::Status + Google::Rpc::Status.decode(status.metadata['grpc-status-details-bin']) + end + end +end diff --git a/src/ruby/qps/proxy-worker.rb b/src/ruby/qps/proxy-worker.rb index d7a9f114bd..488610ae74 100755 --- a/src/ruby/qps/proxy-worker.rb +++ b/src/ruby/qps/proxy-worker.rb @@ -41,7 +41,7 @@ class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Servi @histogram = Histogram.new(@histres, @histmax) @start_time = Time.now # TODO(vjpai): Support multiple client channels by spawning off a PHP client per channel - command = "php " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget puts "Starting command: " + command @php_pid = spawn(command) end diff --git a/src/ruby/spec/google_rpc_status_utils_spec.rb b/src/ruby/spec/google_rpc_status_utils_spec.rb new file mode 100644 index 0000000000..fe221c30dd --- /dev/null +++ b/src/ruby/spec/google_rpc_status_utils_spec.rb @@ -0,0 +1,223 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'grpc' +require_relative '../lib/grpc/google_rpc_status_utils' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require 'google/protobuf/well_known_types' + +include GRPC::Core + +describe 'conversion from a status struct to a google protobuf status' do + it 'fails if the input is not a status struct' do + begin + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status('string') + rescue => e + exception = e + end + expect(exception.is_a?(ArgumentError)).to be true + expect(exception.message.include?('bad type')).to be true + end + + it 'fails with some error if the header key is missing' do + status = Struct::Status.new(1, 'details', key: 'val') + expect(status.metadata.nil?).to be false + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'fails with some error if the header key fails to deserialize' do + status = Struct::Status.new(1, 'details', + 'grpc-status-details-bin' => 'string_val') + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'silently ignores erroneous mismatch between messages in '\ + 'status struct and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'proto message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'struct message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'silently ignores erroneous mismatch between codes in status struct '\ + 'and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(2, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'can succesfully convert a status struct into a google protobuf status '\ + 'when there are no rpcstatus details' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details).to eq([]) + end + + it 'can succesfully convert a status struct into a google protobuf '\ + 'status when there are multiple rpcstatus details' do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + proto = Google::Rpc::Status.new(code: 1, + message: 'matching message', + details: [ + simple_request_any, + simple_response_any, + payload_any + ]) + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details[0].unpack( + Grpc::Testing::SimpleRequest)).to eq(simple_request) + expect(out.details[1].unpack( + Grpc::Testing::SimpleResponse)).to eq(simple_response) + expect(out.details[2].unpack( + Grpc::Testing::Payload)).to eq(payload) + end +end + +# Test message +class EchoMsg + def self.marshal(_o) + '' + end + + def self.unmarshal(_o) + EchoMsg.new + end +end + +# A test service that fills in the "reserved" grpc-status-details-bin trailer, +# for client-side testing of GoogleRpcStatus protobuf extraction from trailers. +class GoogleRpcStatusTestService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + + def initialize(encoded_rpc_status) + @encoded_rpc_status = encoded_rpc_status + end + + def an_rpc(_, _) + # TODO: create a server-side utility API for sending a google rpc status. + # Applications are not expected to set the grpc-status-details-bin + # ("grpc"-fixed and reserved for library use) manually. + # Doing so here is only for testing of the client-side api for extracting + # a google rpc status, which is useful + # when the interacting with a server that does fill in this trailer. + fail GRPC::Unknown.new('test message', + 'grpc-status-details-bin' => @encoded_rpc_status) + end +end + +GoogleRpcStatusTestStub = GoogleRpcStatusTestService.rpc_stub_class + +describe 'receving a google rpc status from a remote endpoint' do + def start_server(encoded_rpc_status) + @srv = GRPC::RpcServer.new(pool_size: 1) + @server_port = @srv.add_http2_port('localhost:0', + :this_port_is_insecure) + @srv.handle(GoogleRpcStatusTestService.new(encoded_rpc_status)) + @server_thd = Thread.new { @srv.run } + @srv.wait_till_running + end + + def stop_server + expect(@srv.stopped?).to be(false) + @srv.stop + @server_thd.join + expect(@srv.stopped?).to be(true) + end + + before(:each) do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + @expected_proto = Google::Rpc::Status.new( + code: StatusCodes::UNKNOWN, + message: 'test message', + details: [simple_request_any, simple_response_any, payload_any]) + start_server(Google::Rpc::Status.encode(@expected_proto)) + end + + after(:each) do + stop_server + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from a BadStatus exception' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + begin + stub.an_rpc(EchoMsg.new) + rescue GRPC::BadStatus => e + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + e.to_status) + end + expect(rpc_status).to eq(@expected_proto) + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from the op view of a call' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + op = stub.an_rpc(EchoMsg.new, return_op: true) + begin + op.execute + rescue GRPC::BadStatus => e + status_from_exception = e.to_status + end + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + op.status) + expect(rpc_status).to eq(@expected_proto) + # "to_status" on the bad status should give the same result + # as "status" on the "op view". + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + status_from_exception)).to eq(rpc_status) + end +end diff --git a/templates/config.m4.template b/templates/config.m4.template index f91893c2bd..cd93fbd0fb 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -14,7 +14,7 @@ LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index e62e5b2721..215d5f9df9 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -31,6 +31,7 @@ s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' diff --git a/templates/package.xml.template b/templates/package.xml.template index 15da704a47..f10f75b8c0 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ <email>grpc-packages@google.com</email> <active>yes</active> </lead> - <date>2017-05-22</date> + <date>2017-08-24</date> <time>16:06:07</time> <version> <release>${settings.php_version.php()}</release> @@ -27,6 +27,9 @@ - Channel are now by default persistent #11878 - Some bug fixes from 1.4 branch #12109, #12123 - Fixed hang bug when fork() was used #11814 + - License changed to Apache 2.0 + - Added support for php_namespace option in codegen plugin #11886 + - Updated gRPC C Core library version 1.6 </notes> <contents> <dir baseinstalldir="/" name="/"> diff --git a/templates/src/python/grpcio_testing/grpc_version.py.template b/templates/src/python/grpcio_testing/grpc_version.py.template new file mode 100644 index 0000000000..74db811d60 --- /dev/null +++ b/templates/src/python/grpcio_testing/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2017 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! + + VERSION='${settings.python_version.pep440()}' diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include new file mode 100644 index 0000000000..6fa91c741a --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -0,0 +1,92 @@ +<%def name="resolver_component_tests(tests)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +% for test in tests: +$FLAGS_test_bin_path \\ + + --target_name='${test['target_name']}' \\ + + --expected_addrs='${test['expected_addrs']}' \\ + + --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + + --expected_lb_policy='${test['expected_lb_policy']}' \\ + + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +% endfor +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE</%def> diff --git a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template new file mode 100644 index 0000000000..86772dd141 --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="resolver_component_tests_defs.include" import="*"/>\ + ${resolver_component_tests(resolver_component_test_cases)} diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include index 9237e7dace..6e19e65157 100644 --- a/templates/tools/dockerfile/apt_get_basic.include +++ b/templates/tools/dockerfile/apt_get_basic.include @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y ${'\\'} bzip2 ${'\\'} ccache ${'\\'} curl ${'\\'} + dnsutils ${'\\'} gcc ${'\\'} gcc-multilib ${'\\'} git ${'\\'} diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include index 2c12981418..94b854ad21 100644 --- a/templates/tools/dockerfile/python_deps.include +++ b/templates/tools/dockerfile/python_deps.include @@ -11,4 +11,4 @@ RUN apt-get update && apt-get install -y ${'\\'} # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/test/core/compression/algorithm_test.c b/test/core/compression/algorithm_test.c index e37c8c3010..a11e6e90ac 100644 --- a/test/core/compression/algorithm_test.c +++ b/test/core/compression/algorithm_test.c @@ -35,7 +35,7 @@ static void test_algorithm_mesh(void) { gpr_log(GPR_DEBUG, "test_algorithm_mesh"); for (i = 0; i < GRPC_COMPRESS_ALGORITHMS_COUNT; i++) { - char *name; + const char *name; grpc_compression_algorithm parsed; grpc_slice mdstr; grpc_mdelem mdelem; diff --git a/test/core/compression/compression_test.c b/test/core/compression/compression_test.c index 4b6026ec77..326a800300 100644 --- a/test/core/compression/compression_test.c +++ b/test/core/compression/compression_test.c @@ -57,7 +57,7 @@ static void test_compression_algorithm_parse(void) { static void test_compression_algorithm_name(void) { int success; - char *name; + const char *name; size_t i; const char *valid_names[] = {"identity", "gzip", "deflate"}; const grpc_compression_algorithm valid_algorithms[] = { diff --git a/test/core/compression/message_compress_test.c b/test/core/compression/message_compress_test.c index fd692fe5a5..f7f4893dee 100644 --- a/test/core/compression/message_compress_test.c +++ b/test/core/compression/message_compress_test.c @@ -49,7 +49,7 @@ static void assert_passthrough(grpc_slice value, grpc_slice_buffer output; grpc_slice final; int was_compressed; - char *algorithm_name; + const char *algorithm_name; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algorithm_name) != 0); gpr_log( diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 429717a7be..639af15454 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; gpr_asprintf(&expected_details, "Compression algorithm '%s' is disabled.", diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 11e7999a1c..5a69091a57 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT( grpc_stream_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; diff --git a/test/core/tsi/ssl_transport_security_test.c b/test/core/tsi/ssl_transport_security_test.c index 364dfa1b73..2399b054b1 100644 --- a/test/core/tsi/ssl_transport_security_test.c +++ b/test/core/tsi/ssl_transport_security_test.c @@ -23,7 +23,9 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/transport/security_connector.h" #include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" #include "src/core/tsi/transport_security_adapter.h" +#include "src/core/tsi/transport_security_interface.h" #include "test/core/tsi/transport_security_test_lib.h" #include "test/core/util/test_config.h" @@ -312,10 +314,10 @@ static void ssl_test_destruct(tsi_test_fixture *fixture) { key_cert_lib->bad_client_pem_key_cert_pair); gpr_free(key_cert_lib->root_cert); gpr_free(key_cert_lib); - /* Destroy others. */ - tsi_ssl_server_handshaker_factory_destroy( + /* Unreference others. */ + tsi_ssl_server_handshaker_factory_unref( ssl_fixture->server_handshaker_factory); - tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_client_handshaker_factory_unref( ssl_fixture->client_handshaker_factory); } @@ -536,6 +538,118 @@ void ssl_tsi_test_do_round_trip_odd_buffer_size() { } } +static const tsi_ssl_handshaker_factory_vtable *original_vtable; +static bool handshaker_factory_destructor_called; + +static void ssl_tsi_test_handshaker_factory_destructor( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + handshaker_factory_destructor_called = true; + if (original_vtable != NULL && original_vtable->destroy != NULL) { + original_vtable->destroy(factory); + } +} + +static tsi_ssl_handshaker_factory_vtable test_handshaker_factory_vtable = { + ssl_tsi_test_handshaker_factory_destructor}; + +void test_tsi_ssl_client_handshaker_factory_refcounting() { + int i; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "client.pem"); + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)client_handshaker_factory, + &test_handshaker_factory_vtable); + + tsi_handshaker *handshaker[3]; + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_client_handshaker_factory_create_handshaker( + client_handshaker_factory, "google.com", &handshaker[i]) == + TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + gpr_free((void *)cert_chain); +} + +void test_tsi_ssl_server_handshaker_factory_refcounting() { + int i; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; + tsi_handshaker *handshaker[3]; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.pem"); + tsi_ssl_pem_key_cert_pair cert_pair; + + cert_pair.cert_chain = cert_chain; + cert_pair.private_key = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.key"); + + GPR_ASSERT(tsi_create_ssl_server_handshaker_factory( + &cert_pair, 1, cert_chain, 0, NULL, NULL, 0, + &server_handshaker_factory) == TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)server_handshaker_factory, + &test_handshaker_factory_vtable); + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_server_handshaker_factory_create_handshaker( + server_handshaker_factory, &handshaker[i]) == TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_server_handshaker_factory_unref(server_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + ssl_test_pem_key_cert_pair_destroy(cert_pair); +} + +/* Attempting to create a handshaker factory with invalid parameters should fail + * but not crash. */ +void test_tsi_ssl_client_handshaker_factory_bad_params() { + const char *cert_chain = "This is not a valid PEM file."; + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_INVALID_ARGUMENT); + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); +} + +void ssl_tsi_test_handshaker_factory_internals() { + test_tsi_ssl_client_handshaker_factory_refcounting(); + test_tsi_ssl_server_handshaker_factory_refcounting(); + test_tsi_ssl_client_handshaker_factory_bad_params(); +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); @@ -553,6 +667,7 @@ int main(int argc, char **argv) { ssl_tsi_test_do_handshake_alpn_client_server_ok(); ssl_tsi_test_do_round_trip_for_all_configs(); ssl_tsi_test_do_round_trip_odd_buffer_size(); + ssl_tsi_test_handshaker_factory_internals(); grpc_shutdown(); return 0; } diff --git a/test/cpp/codegen/compiler_test_golden b/test/cpp/codegen/compiler_test_golden index b43c27f3f7..3d664e8825 100644 --- a/test/cpp/codegen/compiler_test_golden +++ b/test/cpp/codegen/compiler_test_golden @@ -65,6 +65,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } // MethodA1 trailing comment 1 // MethodA2 detached leading comment 1 // @@ -76,6 +79,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } // MethodA2 trailing comment 1 // Method A3 leading comment 1 std::unique_ptr< ::grpc::ClientReaderInterface< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { @@ -84,6 +90,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } // Method A3 trailing comment 1 // Method A4 leading comment 1 std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { @@ -92,15 +101,22 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } // Method A4 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientWriterInterface< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) = 0; virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderInterface< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) = 0; virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) = 0; virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -109,34 +125,50 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>> MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response) { return std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>>(MethodA2Raw(context, response)); } std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { return std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>>(MethodA3Raw(context, request)); } std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { return std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(MethodA4Raw(context)); } std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientWriter< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) override; ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReader< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) override; ::grpc::ClientAsyncReader< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReader< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) override; ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodA1_; const ::grpc::RpcMethod rpcmethod_MethodA2_; const ::grpc::RpcMethod rpcmethod_MethodA3_; @@ -372,9 +404,13 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } // MethodB1 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -383,10 +419,14 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodB1_; }; static std::unique_ptr<Stub> NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions()); diff --git a/test/cpp/codegen/compiler_test_mock_golden b/test/cpp/codegen/compiler_test_mock_golden index 8e4b4d5911..f97c2dd83a 100644 --- a/test/cpp/codegen/compiler_test_mock_golden +++ b/test/cpp/codegen/compiler_test_mock_golden @@ -15,18 +15,23 @@ class MockServiceAStub : public ServiceA::StubInterface { public: MOCK_METHOD3(MethodA1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA2Raw, ::grpc::ClientWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response)); MOCK_METHOD4(AsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA3Raw, ::grpc::ClientReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request)); MOCK_METHOD4(AsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD1(MethodA4Raw, ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context)); MOCK_METHOD3(AsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD2(PrepareAsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq)); }; class MockServiceBStub : public ServiceB::StubInterface { public: MOCK_METHOD3(MethodB1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); }; } // namespace grpc diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 17a094f7a2..570a3d1067 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,9 +368,8 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub(int fallback_timeout = 0) { + void ResetStub() { ChannelArguments args; - args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -442,10 +441,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector<int> GetBackendPorts(const size_t start_index = 0) const { + const std::vector<int> GetBackendPorts() const { std::vector<int> backend_ports; - for (size_t i = start_index; i < backend_servers_.size(); ++i) { - backend_ports.push_back(backend_servers_[i].port_); + for (const auto& bs : backend_servers_) { + backend_ports.push_back(bs.port_); } return backend_ports; } @@ -616,71 +615,6 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } -TEST_F(SingleBalancerTest, Fallback) { - const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); - const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); - const size_t kNumBackendInResolution = backends_.size() / 2; - - ResetStub(kFallbackTimeoutMs); - std::vector<AddressData> addresses; - addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); - } - SetNextResolution(addresses); - - // Send non-empty serverlist only after kServerlistDelayMs - ScheduleResponseForBalancer( - 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(kNumBackendInResolution /* start_index */), {}), - kServerlistDelayMs); - - // The first request. The client will block while it's still trying to - // contact the balancer. - gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); - CheckRpcSendOk(kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); - - // Fallback is used: each backend returned by the resolver should have - // gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - - // Wait until update has been processed, as signaled by the backend returned - // by the balancer receiving a request. - do { - CheckRpcSendOk(1); - } while ( - backend_servers_[kNumBackendInResolution].service_->request_count() == 0); - for (size_t i = 0; i < backends_.size(); ++i) { - backend_servers_[i].service_->ResetCounters(); - } - - // Send out the second request. - gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); - CheckRpcSendOk(backends_.size() - kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); - - // Serverlist is used: each backend returned by the balancer should - // have gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - - balancers_[0]->NotifyDoneWithServerlists(); - // The balancer got a single request. - EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); - // and sent a single response. - EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); -} - TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( diff --git a/test/cpp/naming/BUILD b/test/cpp/naming/BUILD new file mode 100644 index 0000000000..24c3d1a443 --- /dev/null +++ b/test/cpp/naming/BUILD @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], + features = [ + "-layering_check", + "-parse_headers", + ], +) + +licenses(["notice"]) # Apache v2 + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_py_binary") + +load(":generate_resolver_component_tests.bzl", "generate_resolver_component_tests") + +# Meant to be invoked only through the top-level shell script driver. +grpc_sh_binary( + name = "resolver_component_tests_runner", + srcs = [ + "resolver_component_tests_runner.sh", + ], +) + +grpc_py_binary( + name = "test_dns_server", + srcs = ["test_dns_server.py"], + data = [ + "resolver_test_record_groups.yaml", + ], + deps = [ + "twisted", + "yaml", + ] +) + +generate_resolver_component_tests() diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py new file mode 100755 index 0000000000..3a51fef7a0 --- /dev/null +++ b/test/cpp/naming/gen_build_yaml.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Generates the appropriate build.json data for all the naming tests.""" + + +import yaml +import collections +import hashlib +import json + +_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353' + +def _append_zone_name(name, zone_name): + return '%s.%s' % (name, zone_name) + +def _build_expected_addrs_cmd_arg(expected_addrs): + out = [] + for addr in expected_addrs: + out.append('%s,%s' % (addr['address'], str(addr['is_balancer']))) + return ';'.join(out) + +def main(): + resolver_component_data = '' + with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: + resolver_component_data = yaml.load(f) + + json = { + 'resolver_component_test_cases': [ + { + 'target_name': _append_zone_name(test_case['record_to_resolve'], + resolver_component_data['resolver_component_tests_common_zone_name']), + 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), + 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), + 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), + } for test_case in resolver_component_data['resolver_component_tests'] + ], + 'targets': [ + { + 'name': 'resolver_component_test' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': False, + 'src': ['test/cpp/naming/resolver_component_test.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util' + unsecure_build_config_suffix, + 'grpc_test_util' + unsecure_build_config_suffix, + 'gpr_test_util', + 'grpc++' + unsecure_build_config_suffix, + 'grpc' + unsecure_build_config_suffix, + 'gpr', + 'grpc++_test_config', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + [ + { + 'name': 'resolver_component_tests_runner_invoker' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': True, + 'src': ['test/cpp/naming/resolver_component_tests_runner_invoker.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util', + 'grpc_test_util', + 'gpr_test_util', + 'grpc++', + 'grpc', + 'gpr', + 'grpc++_test_config', + ], + 'args': [ + '--test_bin_name=resolver_component_test%s' % unsecure_build_config_suffix, + '--running_under_bazel=false', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + } + + print(yaml.dump(json)) + +if __name__ == '__main__': + main() diff --git a/test/cpp/naming/generate_resolver_component_tests.bzl b/test/cpp/naming/generate_resolver_component_tests.bzl new file mode 100755 index 0000000000..118d9452d9 --- /dev/null +++ b/test/cpp/naming/generate_resolver_component_tests.bzl @@ -0,0 +1,64 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_cc_test", "grpc_cc_binary") + +def generate_resolver_component_tests(): + for unsecure_build_config_suffix in ['_unsecure', '']: + # meant to be invoked only through the top-level shell script driver + grpc_cc_binary( + name = "resolver_component_test%s" % unsecure_build_config_suffix, + testonly = 1, + srcs = [ + "resolver_component_test.cc", + ], + external_deps = [ + "gmock", + ], + deps = [ + "//test/cpp/util:test_util%s" % unsecure_build_config_suffix, + "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix, + "//test/core/util:gpr_test_util", + "//:grpc++%s" % unsecure_build_config_suffix, + "//:grpc%s" % unsecure_build_config_suffix, + "//:gpr", + "//test/cpp/util:test_config", + ], + ) + grpc_cc_test( + name = "resolver_component_tests_runner_invoker%s" % unsecure_build_config_suffix, + srcs = [ + "resolver_component_tests_runner_invoker.cc", + ], + deps = [ + "//test/cpp/util:test_util", + "//test/core/util:grpc_test_util", + "//test/core/util:gpr_test_util", + "//:grpc++", + "//:grpc", + "//:gpr", + "//test/cpp/util:test_config", + ], + data = [ + ":resolver_component_tests_runner", + ":resolver_component_test%s" % unsecure_build_config_suffix, + ":test_dns_server", + "resolver_test_record_groups.yaml", # include the transitive dependency so that the dns sever py binary can locate this + ], + args = [ + "--test_bin_name=resolver_component_test%s" % unsecure_build_config_suffix, + "--running_under_bazel=true", + ] + ) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc new file mode 100644 index 0000000000..0857fb6a32 --- /dev/null +++ b/test/cpp/naming/resolver_component_test.cc @@ -0,0 +1,323 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <grpc/grpc.h> +#include <grpc/support/alloc.h> +#include <grpc/support/host_port.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/sync.h> +#include <grpc/support/time.h> +#include <string.h> + +#include <gflags/gflags.h> +#include <gmock/gmock.h> +#include <vector> + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/ext/filters/client_channel/resolver_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/env.h" +#include "src/core/lib/support/string.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +} + +using std::vector; +using grpc::SubProcess; +using testing::UnorderedElementsAreArray; + +// Hack copied from "test/cpp/end2end/server_crash_test_client.cc"! +// In some distros, gflags is in the namespace google, and in some others, +// in gflags. This hack is enabling us to find both. +namespace google {} +namespace gflags {} +using namespace google; +using namespace gflags; + +DEFINE_string(target_name, "", "Target name to resolve."); +DEFINE_string(expected_addrs, "", + "Comma-separated list of expected " + "'<ip0:port0>,<is_balancer0>;<ip1:port1>,<is_balancer1>;...' " + "addresses of " + "backend and/or balancers. 'is_balancer' should be bool, i.e. " + "true or false."); +DEFINE_string(expected_chosen_service_config, "", + "Expected service config json string that gets chosen (no " + "whitespace). Empty for none."); +DEFINE_string( + local_dns_server_address, "", + "Optional. This address is placed as the uri authority if present."); +DEFINE_string(expected_lb_policy, "", + "Expected lb policy name that appears in resolver result channel " + "arg. Empty for none."); + +namespace { + +class GrpcLBAddress final { + public: + GrpcLBAddress(std::string address, bool is_balancer) + : is_balancer(is_balancer), address(address) {} + + bool operator==(const GrpcLBAddress &other) const { + return this->is_balancer == other.is_balancer && + this->address == other.address; + } + + bool operator!=(const GrpcLBAddress &other) const { + return !(*this == other); + } + + bool is_balancer; + std::string address; +}; + +vector<GrpcLBAddress> ParseExpectedAddrs(std::string expected_addrs) { + std::vector<GrpcLBAddress> out; + while (expected_addrs.size() != 0) { + // get the next <ip>,<port> (v4 or v6) + size_t next_comma = expected_addrs.find(","); + if (next_comma == std::string::npos) { + gpr_log( + GPR_ERROR, + "Missing ','. Expected_addrs arg should be a semi-colon-separated " + "list of " + "<ip-port>,<bool> pairs. Left-to-be-parsed arg is |%s|", + expected_addrs.c_str()); + abort(); + } + std::string next_addr = expected_addrs.substr(0, next_comma); + expected_addrs = expected_addrs.substr(next_comma + 1, std::string::npos); + // get the next is_balancer 'bool' associated with this address + size_t next_semicolon = expected_addrs.find(";"); + bool is_balancer = + gpr_is_true(expected_addrs.substr(0, next_semicolon).c_str()); + out.emplace_back(GrpcLBAddress(next_addr, is_balancer)); + if (next_semicolon == std::string::npos) { + break; + } + expected_addrs = + expected_addrs.substr(next_semicolon + 1, std::string::npos); + } + if (out.size() == 0) { + gpr_log(GPR_ERROR, + "expected_addrs arg should be a comma-separated list of " + "<ip-port>,<bool> pairs"); + abort(); + } + return out; +} + +gpr_timespec TestDeadline(void) { + return grpc_timeout_seconds_to_deadline(100); +} + +struct ArgsStruct { + gpr_event ev; + gpr_atm done_atm; + gpr_mu *mu; + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + grpc_combiner *lock; + grpc_channel_args *channel_args; + vector<GrpcLBAddress> expected_addrs; + std::string expected_service_config_string; + std::string expected_lb_policy; +}; + +void ArgsInit(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + gpr_event_init(&args->ev); + args->pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); + grpc_pollset_init(args->pollset, &args->mu); + args->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(exec_ctx, args->pollset_set, args->pollset); + args->lock = grpc_combiner_create(); + gpr_atm_rel_store(&args->done_atm, 0); + args->channel_args = NULL; +} + +void DoNothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void ArgsFinish(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + GPR_ASSERT(gpr_event_wait(&args->ev, TestDeadline())); + grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); + grpc_closure DoNothing_cb; + GRPC_CLOSURE_INIT(&DoNothing_cb, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(exec_ctx, args->pollset, &DoNothing_cb); + // exec_ctx needs to be flushed before calling grpc_pollset_destroy() + grpc_channel_args_destroy(exec_ctx, args->channel_args); + grpc_exec_ctx_flush(exec_ctx); + grpc_pollset_destroy(exec_ctx, args->pollset); + gpr_free(args->pollset); + GRPC_COMBINER_UNREF(exec_ctx, args->lock, NULL); +} + +gpr_timespec NSecondDeadline(int seconds) { + return gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(seconds, GPR_TIMESPAN)); +} + +void PollPollsetUntilRequestDone(ArgsStruct *args) { + gpr_timespec deadline = NSecondDeadline(10); + while (true) { + bool done = gpr_atm_acq_load(&args->done_atm) != 0; + if (done) { + break; + } + gpr_timespec time_left = + gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME)); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64 ".%09d", done, + time_left.tv_sec, time_left.tv_nsec); + GPR_ASSERT(gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) >= 0); + grpc_pollset_worker *worker = NULL; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(&exec_ctx, args->pollset, &worker, + gpr_now(GPR_CLOCK_REALTIME), NSecondDeadline(1))); + gpr_mu_unlock(args->mu); + grpc_exec_ctx_finish(&exec_ctx); + } + gpr_event_set(&args->ev, (void *)1); +} + +void CheckServiceConfigResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *service_config_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_SERVICE_CONFIG); + if (args->expected_service_config_string != "") { + GPR_ASSERT(service_config_arg != NULL); + GPR_ASSERT(service_config_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(service_config_arg->value.string, + args->expected_service_config_string); + } else { + GPR_ASSERT(service_config_arg == NULL); + } +} + +void CheckLBPolicyResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *lb_policy_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_POLICY_NAME); + if (args->expected_lb_policy != "") { + GPR_ASSERT(lb_policy_arg != NULL); + GPR_ASSERT(lb_policy_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(lb_policy_arg->value.string, args->expected_lb_policy); + } else { + GPR_ASSERT(lb_policy_arg == NULL); + } +} + +void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, + grpc_error *err) { + ArgsStruct *args = (ArgsStruct *)argsp; + grpc_channel_args *channel_args = args->channel_args; + const grpc_arg *channel_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); + GPR_ASSERT(channel_arg != NULL); + GPR_ASSERT(channel_arg->type == GRPC_ARG_POINTER); + grpc_lb_addresses *addresses = + (grpc_lb_addresses *)channel_arg->value.pointer.p; + gpr_log(GPR_INFO, "num addrs found: %" PRIdPTR ". expected %" PRIdPTR, + addresses->num_addresses, args->expected_addrs.size()); + GPR_ASSERT(addresses->num_addresses == args->expected_addrs.size()); + std::vector<GrpcLBAddress> found_lb_addrs; + for (size_t i = 0; i < addresses->num_addresses; i++) { + grpc_lb_address addr = addresses->addresses[i]; + char *str; + grpc_sockaddr_to_string(&str, &addr.address, 1 /* normalize */); + gpr_log(GPR_INFO, "%s", str); + found_lb_addrs.emplace_back( + GrpcLBAddress(std::string(str), addr.is_balancer)); + gpr_free(str); + } + if (args->expected_addrs.size() != found_lb_addrs.size()) { + gpr_log(GPR_DEBUG, "found lb addrs size is: %" PRIdPTR + ". expected addrs size is %" PRIdPTR, + found_lb_addrs.size(), args->expected_addrs.size()); + abort(); + } + EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); + CheckServiceConfigResultLocked(channel_args, args); + CheckLBPolicyResultLocked(channel_args, args); + gpr_atm_rel_store(&args->done_atm, 1); + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + gpr_mu_unlock(args->mu); +} + +TEST(ResolverComponentTest, TestResolvesRelevantRecords) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + ArgsStruct args; + ArgsInit(&exec_ctx, &args); + args.expected_addrs = ParseExpectedAddrs(FLAGS_expected_addrs); + args.expected_service_config_string = FLAGS_expected_chosen_service_config; + args.expected_lb_policy = FLAGS_expected_lb_policy; + // maybe build the address with an authority + char *whole_uri = NULL; + GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); + // create resolver and resolve + grpc_resolver *resolver = grpc_resolver_create(&exec_ctx, whole_uri, NULL, + args.pollset_set, args.lock); + gpr_free(whole_uri); + grpc_closure on_resolver_result_changed; + GRPC_CLOSURE_INIT(&on_resolver_result_changed, CheckResolverResultLocked, + (void *)&args, grpc_combiner_scheduler(args.lock)); + grpc_resolver_next_locked(&exec_ctx, resolver, &args.channel_args, + &on_resolver_result_changed); + grpc_exec_ctx_flush(&exec_ctx); + PollPollsetUntilRequestDone(&args); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, NULL); + ArgsFinish(&exec_ctx, &args); + grpc_exec_ctx_finish(&exec_ctx); +} + +} // namespace + +int main(int argc, char **argv) { + grpc_init(); + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_target_name == "") { + gpr_log(GPR_ERROR, "Missing target_name param."); + abort(); + } + if (FLAGS_local_dns_server_address != "") { + gpr_log(GPR_INFO, "Specifying authority in uris to: %s", + FLAGS_local_dns_server_address.c_str()); + } + auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; +} diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh new file mode 100755 index 0000000000..83b03b67a3 --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_component_tests_runner_invoker.cc b/test/cpp/naming/resolver_component_tests_runner_invoker.cc new file mode 100644 index 0000000000..b14391284d --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner_invoker.cc @@ -0,0 +1,189 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <grpc/grpc.h> +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <signal.h> +#include <string.h> +#include <unistd.h> + +#include <gflags/gflags.h> +#include <string> +#include <thread> +#include <vector> + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/lib/support/env.h" +#include "test/core/util/port.h" +} + +DEFINE_bool( + running_under_bazel, false, + "True if this test is running under bazel. " + "False indicates that this test is running under run_tests.py. " + "Child process test binaries are located differently based on this flag. "); + +DEFINE_string(test_bin_name, "", + "Name, without the preceding path, of the test binary"); + +DEFINE_string(grpc_test_directory_relative_to_test_srcdir, "/__main__", + "This flag only applies if runner_under_bazel is true. This " + "flag is ignored if runner_under_bazel is false. " + "Directory of the <repo-root>/test directory relative to bazel's " + "TEST_SRCDIR environment variable"); + +using grpc::SubProcess; + +static volatile sig_atomic_t abort_wait_for_child = 0; + +static void sighandler(int sig) { abort_wait_for_child = 1; } + +static void register_sighandler() { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = sighandler; + sigaction(SIGINT, &act, NULL); + sigaction(SIGTERM, &act, NULL); +} + +namespace { + +const int kTestTimeoutSeconds = 60 * 2; + +void RunSigHandlingThread(SubProcess *test_driver, gpr_mu *test_driver_mu, + gpr_cv *test_driver_cv, int *test_driver_done) { + gpr_timespec overall_deadline = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(kTestTimeoutSeconds, GPR_TIMESPAN)); + while (true) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (gpr_time_cmp(now, overall_deadline) > 0 || abort_wait_for_child) break; + gpr_mu_lock(test_driver_mu); + if (*test_driver_done) { + gpr_mu_unlock(test_driver_mu); + return; + } + gpr_timespec wait_deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)); + gpr_cv_wait(test_driver_cv, test_driver_mu, wait_deadline); + gpr_mu_unlock(test_driver_mu); + } + gpr_log(GPR_DEBUG, + "Test timeout reached or received signal. Interrupting test driver " + "child process."); + test_driver->Interrupt(); + return; +} +} + +namespace grpc { + +namespace testing { + +void InvokeResolverComponentTestsRunner(std::string test_runner_bin_path, + std::string test_bin_path, + std::string dns_server_bin_path, + std::string records_config_path) { + int test_dns_server_port = grpc_pick_unused_port_or_die(); + + SubProcess *test_driver = new SubProcess( + {test_runner_bin_path, "--test_bin_path=" + test_bin_path, + "--dns_server_bin_path=" + dns_server_bin_path, + "--records_config_path=" + records_config_path, + "--test_dns_server_port=" + std::to_string(test_dns_server_port)}); + gpr_mu test_driver_mu; + gpr_mu_init(&test_driver_mu); + gpr_cv test_driver_cv; + gpr_cv_init(&test_driver_cv); + int test_driver_done = 0; + register_sighandler(); + std::thread sig_handling_thread(RunSigHandlingThread, test_driver, + &test_driver_mu, &test_driver_cv, + &test_driver_done); + int status = test_driver->Join(); + if (WIFEXITED(status)) { + if (WEXITSTATUS(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner exited with code %d", + WEXITSTATUS(status)); + abort(); + } + } else if (WIFSIGNALED(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended from signal %d", + WTERMSIG(status)); + abort(); + } else { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended with unknown status %d", + status); + abort(); + } + gpr_mu_lock(&test_driver_mu); + test_driver_done = 1; + gpr_cv_signal(&test_driver_cv); + gpr_mu_unlock(&test_driver_mu); + sig_handling_thread.join(); + delete test_driver; + gpr_mu_destroy(&test_driver_mu); + gpr_cv_destroy(&test_driver_cv); +} + +} // namespace testing + +} // namespace grpc + +int main(int argc, char **argv) { + grpc::testing::InitTest(&argc, &argv, true); + grpc_init(); + GPR_ASSERT(FLAGS_test_bin_name != ""); + std::string my_bin = argv[0]; + if (FLAGS_running_under_bazel) { + GPR_ASSERT(FLAGS_grpc_test_directory_relative_to_test_srcdir != ""); + // Use bazel's TEST_SRCDIR environment variable to locate the "test data" + // binaries. + std::string const bin_dir = + gpr_getenv("TEST_SRCDIR") + + FLAGS_grpc_test_directory_relative_to_test_srcdir + + std::string("/test/cpp/naming"); + // Invoke bazel's executeable links to the .sh and .py scripts (don't use + // the .sh and .py suffixes) to make + // sure that we're using bazel's test environment. + grpc::testing::InvokeResolverComponentTestsRunner( + bin_dir + "/resolver_component_tests_runner", + bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/test_dns_server", + bin_dir + "/resolver_test_record_groups.yaml"); + } else { + // Get the current binary's directory relative to repo root to invoke the + // correct build config (asan/tsan/dbg, etc.). + std::string const bin_dir = my_bin.substr(0, my_bin.rfind('/')); + // Invoke the .sh and .py scripts directly where they are in source code. + grpc::testing::InvokeResolverComponentTestsRunner( + "test/cpp/naming/resolver_component_tests_runner.sh", + bin_dir + "/" + FLAGS_test_bin_name, + "test/cpp/naming/test_dns_server.py", + "test/cpp/naming/resolver_test_record_groups.yaml"); + } + grpc_shutdown(); + return 0; +} diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml new file mode 100644 index 0000000000..67c611d831 --- /dev/null +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -0,0 +1,155 @@ +resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp. +resolver_component_tests: +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-single-target + records: + _grpclb._tcp.srv-ipv4-single-target: + - {TTL: '2100', data: 0 0 1234 ipv4-single-target, type: SRV} + ipv4-single-target: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '1.2.3.5:1234', is_balancer: true} + - {address: '1.2.3.6:1234', is_balancer: true} + - {address: '1.2.3.7:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-multi-target + records: + _grpclb._tcp.srv-ipv4-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv4-multi-target, type: SRV} + ipv4-multi-target: + - {TTL: '2100', data: 1.2.3.5, type: A} + - {TTL: '2100', data: 1.2.3.6, type: A} + - {TTL: '2100', data: 1.2.3.7, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1001]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-single-target + records: + _grpclb._tcp.srv-ipv6-single-target: + - {TTL: '2100', data: 0 0 1234 ipv6-single-target, type: SRV} + ipv6-single-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1001', type: AAAA} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1003]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1004]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-multi-target + records: + _grpclb._tcp.srv-ipv6-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv6-multi-target, type: SRV} + ipv6-multi-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1003', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1004', type: AAAA} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: srv-ipv4-simple-service-config + records: + _grpclb._tcp.srv-ipv4-simple-service-config: + - {TTL: '2100', data: 0 0 1234 ipv4-simple-service-config, type: SRV} + ipv4-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-simple-service-config: + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-no-srv-simple-service-config + records: + ipv4-no-srv-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-no-config-for-cpp + records: + ipv4-no-config-for-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["python"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"PythonService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-cpp-config-has-zero-percentage + records: + ipv4-cpp-config-has-zero-percentage: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-second-language-is-cpp + records: + ipv4-second-language-is-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["go"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"GoService","waitForReady":true}]}]}},{"clientLanguage":["c++"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-config-with-percentages + records: + ipv4-config-with-percentages: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NeverPickedService","waitForReady":true}]}]}},{"percentage":100,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv4-has-backend-and-balancer, type: SRV} + balancer-for-ipv4-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1002]:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv6-has-backend-and-balancer, type: SRV} + balancer-for-ipv6-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + +resolver_component_tests_TODO: +- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 + bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, + e.g. via notes in RFC 2671)' +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + record_to_resolve: ipv4-config-causing-fallback-to-tcp + records: + ipv4-config-causing-fallback-to-tcp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py new file mode 100755 index 0000000000..9d4b89cffb --- /dev/null +++ b/test/cpp/naming/test_dns_server.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starts a local DNS server for use in tests""" + +import argparse +import sys +import yaml +import signal +import os + +import twisted +import twisted.internet +import twisted.internet.reactor +import twisted.internet.threads +import twisted.internet.defer +import twisted.internet.protocol +import twisted.names +import twisted.names.client +import twisted.names.dns +import twisted.names.server +from twisted.names import client, server, common, authority, dns +import argparse + +_SERVER_HEALTH_CHECK_RECORD_NAME = 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp' # missing end '.' for twisted syntax +_SERVER_HEALTH_CHECK_RECORD_DATA = '123.123.123.123' + +class NoFileAuthority(authority.FileAuthority): + def __init__(self, soa, records): + # skip FileAuthority + common.ResolverBase.__init__(self) + self.soa = soa + self.records = records + +def start_local_dns_server(args): + all_records = {} + def _push_record(name, r): + print('pushing record: |%s|' % name) + if all_records.get(name) is not None: + all_records[name].append(r) + return + all_records[name] = [r] + + def _maybe_split_up_txt_data(name, txt_data, r_ttl): + start = 0 + txt_data_list = [] + while len(txt_data[start:]) > 0: + next_read = len(txt_data[start:]) + if next_read > 255: + next_read = 255 + txt_data_list.append(txt_data[start:start+next_read]) + start += next_read + _push_record(name, dns.Record_TXT(*txt_data_list, ttl=r_ttl)) + + with open(args.records_config_path) as config: + test_records_config = yaml.load(config) + common_zone_name = test_records_config['resolver_component_tests_common_zone_name'] + for group in test_records_config['resolver_component_tests']: + for name in group['records'].keys(): + for record in group['records'][name]: + r_type = record['type'] + r_data = record['data'] + r_ttl = int(record['TTL']) + record_full_name = '%s.%s' % (name, common_zone_name) + assert record_full_name[-1] == '.' + record_full_name = record_full_name[:-1] + if r_type == 'A': + _push_record(record_full_name, dns.Record_A(r_data, ttl=r_ttl)) + if r_type == 'AAAA': + _push_record(record_full_name, dns.Record_AAAA(r_data, ttl=r_ttl)) + if r_type == 'SRV': + p, w, port, target = r_data.split(' ') + p = int(p) + w = int(w) + port = int(port) + target_full_name = '%s.%s' % (target, common_zone_name) + r_data = '%s %s %s %s' % (p, w, port, target_full_name) + _push_record(record_full_name, dns.Record_SRV(p, w, port, target_full_name, ttl=r_ttl)) + if r_type == 'TXT': + _maybe_split_up_txt_data(record_full_name, r_data, r_ttl) + # Server health check record + _push_record(_SERVER_HEALTH_CHECK_RECORD_NAME, dns.Record_A(_SERVER_HEALTH_CHECK_RECORD_DATA, ttl=0)) + soa_record = dns.Record_SOA(mname = common_zone_name) + test_domain_com = NoFileAuthority( + soa = (common_zone_name, soa_record), + records = all_records, + ) + server = twisted.names.server.DNSServerFactory( + authorities=[test_domain_com], verbose=2) + server.noisy = 2 + twisted.internet.reactor.listenTCP(args.port, server) + dns_proto = twisted.names.dns.DNSDatagramProtocol(server) + dns_proto.noisy = 2 + twisted.internet.reactor.listenUDP(args.port, dns_proto) + print('starting local dns server on 127.0.0.1:%s' % args.port) + print('starting twisted.internet.reactor') + twisted.internet.reactor.suggestThreadPoolSize(1) + twisted.internet.reactor.run() + +def _quit_on_signal(signum, _frame): + print('Received SIGNAL %d. Quitting with exit code 0' % signum) + twisted.internet.reactor.stop() + sys.stdout.flush() + sys.exit(0) + +def main(): + argp = argparse.ArgumentParser(description='Local DNS Server for resolver tests') + argp.add_argument('-p', '--port', default=None, type=int, + help='Port for DNS server to listen on for TCP and UDP.') + argp.add_argument('-r', '--records_config_path', default=None, type=str, + help=('Directory of resolver_test_record_groups.yaml file. ' + 'Defauls to path needed when the test is invoked as part of run_tests.py.')) + args = argp.parse_args() + signal.signal(signal.SIGALRM, _quit_on_signal) + signal.signal(signal.SIGTERM, _quit_on_signal) + signal.signal(signal.SIGINT, _quit_on_signal) + # Prevent zombies. Tests that use this server are short-lived. + signal.alarm(2 * 60) + start_local_dns_server(args) + +if __name__ == '__main__': + main() diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 912c871482..1a4438047d 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -56,11 +56,6 @@ class ClientRpcContext { } virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0; - void lock() { mu_.lock(); } - void unlock() { mu_.unlock(); } - - private: - std::mutex mu_; }; template <class RequestType, class ResponseType> @@ -73,7 +68,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req, + prepare_req, std::function<void(grpc::Status, ResponseType*, HistogramEntry*)> on_done) : context_(), stub_(stub), @@ -83,7 +78,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { next_state_(State::READY), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextUnaryImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -92,7 +87,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { switch (next_state_) { case State::READY: start_ = UsageTimer::Now(); - response_reader_ = start_req_(stub_, &context_, req_, cq_); + response_reader_ = prepare_req_(stub_, &context_, req_, cq_); + response_reader_->StartCall(); next_state_ = State::RESP_DONE; response_reader_->Finish(&response_, &status_, ClientRpcContext::tag(this)); @@ -111,8 +107,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextUnaryImpl(stub_, req_, next_issue_, - start_req_, callback_); - std::lock_guard<ClientRpcContext> lclone(*clone); + prepare_req_, callback_); clone->StartInternal(cq); } @@ -130,7 +125,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function<std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req_; + prepare_req_; grpc::Status status_; double start_; std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>> @@ -252,29 +247,13 @@ class AsyncClient : public ClientImpl<StubType, RequestType> { // this thread isn't supposed to shut down std::lock_guard<std::mutex> l(shutdown_state_[thread_idx]->mutex); if (shutdown_state_[thread_idx]->shutdown) { - // We want to delete the context. However, it is possible that - // another thread that just initiated an action on this - // context still has its lock even though the action on the - // context has completed. To delay for that, just grab the - // lock for serialization. Take a new scope. - { std::lock_guard<ClientRpcContext> lctx(*ctx); } delete ctx; return true; } - bool del = false; - - // Create a new scope for a lock_guard'ed region - { - std::lock_guard<ClientRpcContext> lctx(*ctx); - if (!ctx->RunNextState(ok, entry)) { - // The RPC and callback are done, so clone the ctx - // and kickstart the new one - ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); - // set the old version to delete - del = true; - } - } - if (del) { + if (!ctx->RunNextState(ok, entry)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); delete ctx; } return true; @@ -311,15 +290,15 @@ class AsyncUnaryClient final entry->set_status(s.error_code()); } static std::unique_ptr<grpc::ClientAsyncResponseReader<SimpleResponse>> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& request, CompletionQueue* cq) { - return stub->AsyncUnaryCall(ctx, request, cq); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + const SimpleRequest& request, CompletionQueue* cq) { + return stub->PrepareAsyncUnaryCall(ctx, request, cq); }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, std::function<gpr_timespec()> next_issue, const SimpleRequest& req) { return new ClientRpcContextUnaryImpl<SimpleRequest, SimpleResponse>( - stub, req, next_issue, AsyncUnaryClient::StartReq, + stub, req, next_issue, AsyncUnaryClient::PrepareReq, AsyncUnaryClient::CheckDone); } }; @@ -332,9 +311,8 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr< grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> - start_req, + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req, std::function<void(grpc::Status, ResponseType*)> on_done) : context_(), stub_(stub), @@ -344,7 +322,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingPingPongImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -407,8 +385,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingPingPongImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard<ClientRpcContext> lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -432,10 +409,10 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { State next_state_; std::function<void(grpc::Status, ResponseType*)> callback_; std::function<gpr_timespec()> next_issue_; - std::function<std::unique_ptr< - grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> - start_req_; + std::function< + std::unique_ptr<grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr<grpc::ClientAsyncReaderWriter<RequestType, ResponseType>> @@ -449,8 +426,9 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { cq_ = cq; messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -469,9 +447,9 @@ class AsyncStreamingPingPongClient final static void CheckDone(grpc::Status s, SimpleResponse* response) {} static std::unique_ptr< grpc::ClientAsyncReaderWriter<SimpleRequest, SimpleResponse>> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingCall(ctx, cq, tag); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingCall(ctx, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -479,7 +457,7 @@ class AsyncStreamingPingPongClient final const SimpleRequest& req) { return new ClientRpcContextStreamingPingPongImpl<SimpleRequest, SimpleResponse>( - stub, req, next_issue, AsyncStreamingPingPongClient::StartReq, + stub, req, next_issue, AsyncStreamingPingPongClient::PrepareReq, AsyncStreamingPingPongClient::CheckDone); } }; @@ -492,8 +470,8 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr<grpc::ClientAsyncWriter<RequestType>>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function<void(grpc::Status, ResponseType*)> on_done) : context_(), stub_(stub), @@ -503,7 +481,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromClientImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -546,8 +524,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromClientImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard<ClientRpcContext> lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -570,17 +547,17 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue_; std::function<std::unique_ptr<grpc::ClientAsyncWriter<RequestType>>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr<grpc::ClientAsyncWriter<RequestType>> stream_; void StartInternal(CompletionQueue* cq) { cq_ = cq; - stream_ = start_req_(stub_, &context_, &response_, cq, - ClientRpcContext::tag(this)); + stream_ = prepare_req_(stub_, &context_, &response_, cq); next_state_ = State::STREAM_IDLE; + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -597,10 +574,10 @@ class AsyncStreamingFromClientClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr<grpc::ClientAsyncWriter<SimpleRequest>> StartReq( + static std::unique_ptr<grpc::ClientAsyncWriter<SimpleRequest>> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - SimpleResponse* resp, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromClient(ctx, resp, cq, tag); + SimpleResponse* resp, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromClient(ctx, resp, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -608,7 +585,7 @@ class AsyncStreamingFromClientClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromClientImpl<SimpleRequest, SimpleResponse>( - stub, req, next_issue, AsyncStreamingFromClientClient::StartReq, + stub, req, next_issue, AsyncStreamingFromClientClient::PrepareReq, AsyncStreamingFromClientClient::CheckDone); } }; @@ -621,8 +598,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr<grpc::ClientAsyncReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function<void(grpc::Status, ResponseType*)> on_done) : context_(), stub_(stub), @@ -632,7 +609,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromServerImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -664,8 +641,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromServerImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard<ClientRpcContext> lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -682,8 +658,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue_; std::function<std::unique_ptr<grpc::ClientAsyncReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr<grpc::ClientAsyncReader<ResponseType>> stream_; @@ -691,9 +667,9 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { void StartInternal(CompletionQueue* cq) { // TODO(vjpai): Add support to rate-pace this cq_ = cq; + stream_ = prepare_req_(stub_, &context_, req_, cq); next_state_ = State::STREAM_IDLE; - stream_ = - start_req_(stub_, &context_, req_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -710,10 +686,10 @@ class AsyncStreamingFromServerClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr<grpc::ClientAsyncReader<SimpleResponse>> StartReq( + static std::unique_ptr<grpc::ClientAsyncReader<SimpleResponse>> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& req, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromServer(ctx, req, cq, tag); + const SimpleRequest& req, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromServer(ctx, req, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -721,7 +697,7 @@ class AsyncStreamingFromServerClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromServerImpl<SimpleRequest, SimpleResponse>( - stub, req, next_issue, AsyncStreamingFromServerClient::StartReq, + stub, req, next_issue, AsyncStreamingFromServerClient::PrepareReq, AsyncStreamingFromServerClient::CheckDone); } }; @@ -733,8 +709,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr<grpc::GenericClientAsyncReaderWriter>( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> - start_req, + const grpc::string& method_name, CompletionQueue*)> + prepare_req, std::function<void(grpc::Status, ByteBuffer*)> on_done) : context_(), stub_(stub), @@ -744,7 +720,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextGenericStreamingImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -807,8 +783,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextGenericStreamingImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard<ClientRpcContext> lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -834,8 +809,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function<gpr_timespec()> next_issue_; std::function<std::unique_ptr<grpc::GenericClientAsyncReaderWriter>( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr<grpc::GenericClientAsyncReaderWriter> stream_; @@ -850,9 +825,9 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { "/grpc.testing.BenchmarkService/StreamingCall"); messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, kMethodName, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, kMethodName, cq, - ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -874,17 +849,17 @@ class GenericAsyncStreamingClient final private: static void CheckDone(grpc::Status s, ByteBuffer* response) {} - static std::unique_ptr<grpc::GenericClientAsyncReaderWriter> StartReq( + static std::unique_ptr<grpc::GenericClientAsyncReaderWriter> PrepareReq( grpc::GenericStub* stub, grpc::ClientContext* ctx, - const grpc::string& method_name, CompletionQueue* cq, void* tag) { - auto stream = stub->Call(ctx, method_name, cq, tag); + const grpc::string& method_name, CompletionQueue* cq) { + auto stream = stub->PrepareCall(ctx, method_name, cq); return stream; }; static ClientRpcContext* SetupCtx(grpc::GenericStub* stub, std::function<gpr_timespec()> next_issue, const ByteBuffer& req) { return new ClientRpcContextGenericStreamingImpl( - stub, req, next_issue, GenericAsyncStreamingClient::StartReq, + stub, req, next_issue, GenericAsyncStreamingClient::PrepareReq, GenericAsyncStreamingClient::CheckDone); } }; diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 7649145f86..693c02fdb2 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -24,6 +24,7 @@ gen_build_yaml_dirs=" \ test/core/bad_client \ test/core/bad_ssl \ test/core/end2end \ + test/cpp/naming \ test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 5c0329bff0..e0e9226211 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -47,7 +47,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PY3 = sys.version_info.major == 3 diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index d13eecaa55..02ec4c278a 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -14,18 +14,15 @@ FROM debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index 71845b590b..758f314572 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -14,18 +14,15 @@ FROM 32bit/debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 6604caa42c..0f40f18e38 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -17,11 +17,16 @@ FROM ubuntu:16.04 RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean # make sure we have nuget 2.12+ (in case there's an older cached docker image) RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list -RUN apt-get update && apt-get install -y nuget +RUN apt-get update && apt-get install -y nuget && apt-get clean -RUN apt-get update && apt-get install -y unzip +# Prevent "Error: SendFailure (Error writing headers)" when fetching nuget packages +# See https://github.com/tianon/docker-brew-ubuntu-core/issues/86 +RUN apt-get update && apt-get install -y tzdata && apt-get clean + +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 4ccfbc43c3..dbf58023c5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index 824ecf4604..7cfe98cbc8 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index 5f7bedde73..febe2fa251 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index 2bfa87c4ea..3a516cbb62 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index 96391372ca..acb640a81d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 27a924343f..354b7bfdb5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 RUN pip install twisted h2==2.6.1 hyper diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index c59b12d155..4343d56cb4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index f859a46514..d6e229f84a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 42b1107c11..271c6e75e9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 3c744b434e..7bcada68e0 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile deleted file mode 100644 index 4ccfbc43c3..0000000000 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:jessie - -# Install Git and basic packages. -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - golang \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - perl \ - strace \ - python-dev \ - python-setuptools \ - python-yaml \ - telnet \ - unzip \ - wget \ - zip && apt-get clean - -#================ -# Build profiling -RUN apt-get update && apt-get install -y time && apt-get clean - -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install pip --upgrade -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 - -#================ -# C# dependencies - -# Update to a newer version of mono -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list - -# Install dependencies -RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ - mono-devel \ - ca-certificates-mono \ - nuget \ - && apt-get clean - -RUN nuget update -self - -# Install dotnet SDK based on https://www.microsoft.com/net/core#debian -RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003131 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 -RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet -# dotnet-dev-1.0.1 -RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 -RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet -RUN ln -s /opt/dotnet/dotnet /usr/local/bin - -# Trigger the population of the local package cache -ENV NUGET_XMLDOC_MODE skip -RUN mkdir warmup \ - && cd warmup \ - && dotnet new \ - && cd .. \ - && rm -rf warmup - -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - - -RUN mkdir /var/local/jenkins - -# Define the default command. -CMD ["bash"] diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 60dc7eb21a..40d46fcf58 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index f9622dd757..888a37baca 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 2d396ce2ff..319f1e1889 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 349a6eff83..61f005d9da 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index bccfdba7cd..f35247eccb 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 94c37b3f63..ce1badfeb1 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index b7373b5d9c..59fe4d8f93 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -122,7 +123,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install coverage for Python test coverage reporting RUN pip install coverage diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 31602e594e..103be8412b 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -76,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index e79305bb27..f6d426bcd6 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 245a642367..ae82a8d99f 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # PHP dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 5093b793b0..d5d781cd1a 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ff8f0231cb..3b4ad12b6d 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install dependencies for pyenv RUN apt-get update && apt-get install -y \ diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d77917c87f..3d879bb0c9 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 0f2f9ede58..44732a5ae4 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #======================== # Sanity test dependencies diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh index 76e4905272..3e59dc501a 100755 --- a/tools/gce/create_interop_worker.sh +++ b/tools/gce/create_interop_worker.sh @@ -33,7 +33,8 @@ gcloud compute instances create $INSTANCE_NAME \ --machine-type n1-standard-16 \ --image ubuntu-15-10 \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo + --scopes https://www.googleapis.com/auth/xapi.zoo \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh index 7f53732c05..4270f5cbfb 100755 --- a/tools/gce/create_linux_performance_worker.sh +++ b/tools/gce/create_linux_performance_worker.sh @@ -36,7 +36,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image-project ubuntu-os-cloud \ --image-family ubuntu-1704 \ --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index c96f3281fc..8bf44aa729 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -31,7 +31,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image=ubuntu-1510 \ --image-project=grpc-testing \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc index 1b7779caa8..dc3c75a6a1 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -19,4 +19,19 @@ ulimit -n 32768 ulimit -c unlimited +# Performance PR testing needs GH API key and PR metadata to comment results +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x + sudo apt-get install -y jq + export ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + + gsutil cp gs://grpc-testing-secrets/github_credentials/oauth_token.txt ~/ + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ~/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x +fi + +sudo pip install tabulate + git submodule update --init diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index ae91a6a6d2..bec529f85e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -53,7 +53,7 @@ pod repo update # needed by python # python brew install coreutils # we need grealpath pip install virtualenv --user python -pip install -U six tox setuptools --user python +pip install -U six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh new file mode 100755 index 0000000000..58ffcf336b --- /dev/null +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b $BENCHMARKS_TO_RUN || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 028704b3ff..bd1430b741 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -20,4 +20,14 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS +tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" + +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh new file mode 100755 index 0000000000..624031afa7 --- /dev/null +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="cli_transport_stalls_per_iteration cli_stream_stalls_per_iteration svr_transport_stalls_per_iteration svr_stream_stalls_per_iteration" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b bm_fullstack_trickle \ + -l 4 \ + -t $BENCHMARKS_TO_RUN \ + --no-counters \ + --pr_comment_name trickle || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg new file mode 100644 index 0000000000..577cb28ae5 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg new file mode 100644 index 0000000000..9e0b724b2e --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg new file mode 100644 index 0000000000..0fda74cf44 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg new file mode 100644 index 0000000000..199a8905d9 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/src/python/grpcio_tests/tests/unit/_sanity/__init__.py b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg index 5772620b60..9269c345f0 100644 --- a/src/python/grpcio_tests/tests/unit/_sanity/__init__.py +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -1,4 +1,4 @@ -# Copyright 2016 gRPC authors. +# Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg index 5772620b60..e86b3ab475 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -1,4 +1,4 @@ -# Copyright 2016 gRPC authors. +# Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 9a43e4869b..8e7fd54a62 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,6 +25,11 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + if [ "$FAILED" != "" ] then exit 1 diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 08d834f8b0..10627add25 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -17,8 +17,10 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% || goto :error -goto :EOF +python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +@rem Reveal leftover processes that might be left behind by the build +tasklist /V + +exit /b %RUNTESTS_EXITCODE% diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 12263282ae..2cc0dfceab 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -158,6 +158,7 @@ class PythonArtifact: return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_python.sh'], environ=environ, + timeout_seconds=60*60, use_workspace=True) def __str__(self): diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fb1be383cd..797ed51c7f 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -105,7 +105,9 @@ class CSharpDistribTest(object): use_workspace=True) elif self.platform == 'windows': if self.arch == 'x64': - environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', + # Use double leading / as the first occurence gets removed by msys bash + # when invoking the .bat file (side-effect of posix path conversion) + environ={'MSBUILD_EXTRA_ARGS': '//p:Platform=x64', 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} else: environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 0da13864f0..671d0f7b45 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -78,7 +78,7 @@ class CSharpPackage: if self.linux: return create_docker_jobspec( self.name, - 'tools/dockerfile/test/csharp_coreclr_x64', + 'tools/dockerfile/test/csharp_jessie_x64', 'src/csharp/build_packages_dotnetcli.sh') else: return create_jobspec(self.name, diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 9c6de888ed..a3a9d87519 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5631,6 +5631,86 @@ "deps": [ "gpr", "gpr_test_util", + "grpc++_test_config", + "grpc++_test_util_unsecure", + "grpc++_unsecure", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test_unsecure", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", "grpc", "grpc_test_util" ], diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 45d7145e8e..72dfbc0b98 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4406,6 +4406,52 @@ }, { "args": [ + "--test_bin_name=resolver_component_test_unsecure", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "--test_bin_name=resolver_component_test", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "third_party/boringssl/crypto/aes/aes_tests.txt" ], "boringssl": true, diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 30a94ca7cd..7994b544ac 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -22,6 +22,10 @@ def massage_qps_stats(scenario_result): del stats["coreStats"] stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_cqs_created"] = massage_qps_stats_helpers.counter(core_stats, "cqs_created") + stats["core_client_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_channels_created") + stats["core_client_subchannels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_subchannels_created") + stats["core_server_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "server_channels_created") stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") @@ -43,6 +47,27 @@ def massage_qps_stats(scenario_result): stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_initial_write") + stats["core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_start_new_stream") + stats["core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_message") + stats["core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_initial_metadata") + stats["core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_trailing_metadata") + stats["core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_retry_send_ping") + stats["core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_continue_pings") + stats["core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_goaway_sent") + stats["core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_rst_stream") + stats["core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_close_from_api") + stats["core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_stream_flow_control") + stats["core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control") + stats["core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_settings") + stats["core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_bdp_estimator_ping") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_setting") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_update") + stats["core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_application_ping") + stats["core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_keepalive_ping") + stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled") + stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response") + stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream") stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index 84b3cb7c0a..12a56dbec0 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -122,6 +122,26 @@ }, { "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_syscall_poll", "type": "INTEGER" }, @@ -227,6 +247,111 @@ }, { "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_combiner_locks_initiated", "type": "INTEGER" }, @@ -599,6 +724,26 @@ }, { "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_syscall_poll", "type": "INTEGER" }, @@ -704,6 +849,111 @@ }, { "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", "name": "core_combiner_locks_initiated", "type": "INTEGER" }, diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 4ad981237b..f99143fdd7 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -86,7 +86,6 @@ _WHITELIST_DICT = { '^test/distrib/php/': [_PHP_TEST_SUITE], '^test/distrib/python/': [_PYTHON_TEST_SUITE], '^test/distrib/ruby/': [_RUBY_TEST_SUITE], - '^tools/internal_ci/': [], '^vsprojects/': [_WINDOWS_TEST_SUITE], 'binding\.gyp$': [_NODE_TEST_SUITE], 'composer\.json$': [_PHP_TEST_SUITE], diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 80e79444ea..062c79a0de 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,10 +71,8 @@ def platform_string(): if platform_string() == 'windows': pass else: - have_alarm = False def alarm_handler(unused_signum, unused_frame): - global have_alarm - have_alarm = False + pass signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -367,10 +365,9 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time, clear_alarms): + stop_on_failure, add_env, quiet_success, max_time): self._running = set() self._check_cancelled = check_cancelled - self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -455,10 +452,7 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - global have_alarm - if not have_alarm: - have_alarm = True - signal.alarm(10) + signal.alarm(10) signal.pause() def cancelled(self): @@ -474,10 +468,7 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - # Clear the alarms when finished to avoid a race condition causing job - # failures. Don't do this when running multi-VM tests because clearing - # the alarms causes the test to stall - if platform_string() != 'windows' and self._clear_alarms: + if platform_string() != 'windows': signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -507,8 +498,7 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1, - clear_alarms=True): + max_time=-1): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -520,7 +510,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time, clear_alarms) + quiet_success, max_time) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 9b20fae78f..1bbab9e894 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) + [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_gra perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures @@ -556,7 +556,7 @@ def main(): jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) - scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b66c5f7f71..176c1097ee 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -80,7 +80,7 @@ def get_bqtest_data(limit=None): SELECT filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, - MAX(cpu_measured) as cpu + MAX(cpu_measured) + 0.01 as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, @@ -92,9 +92,7 @@ SELECT AND platform = '"""+platform_string()+"""' AND NOT REGEXP_MATCH(job_name, '.*portability.*') ) GROUP BY - filtered_test_name -HAVING - flaky OR cpu > 0""" + filtered_test_name""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) @@ -535,6 +533,7 @@ class PhpLanguage(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -547,7 +546,7 @@ class PhpLanguage(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] @@ -571,6 +570,7 @@ class Php7Language(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -583,7 +583,7 @@ class Php7Language(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] |