diff options
329 files changed, 6722 insertions, 4760 deletions
diff --git a/.clang-tidy b/.clang-tidy index fbf0b6541e..d217441792 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,6 +1,6 @@ --- -Checks: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*' -WarningsAsErrors: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*' +Checks: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*,bugprone-*' +WarningsAsErrors: 'modernize-use-nullptr,google-build-namespaces,google-build-explicit-make-pair,readability-function-size,performance-*,bugprone-*' CheckOptions: - key: readability-function-size.StatementThreshold value: '450' @@ -1 +1,2 @@ Google Inc. +WeWork Companies Inc. @@ -243,6 +243,7 @@ GRPCXX_PUBLIC_HDRS = [ "include/grpcpp/support/async_unary_call.h", "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", + "include/grpcpp/support/client_callback.h", "include/grpcpp/support/config.h", "include/grpcpp/support/proto_buffer_reader.h", "include/grpcpp/support/proto_buffer_writer.h", @@ -706,7 +707,6 @@ grpc_cc_library( "src/core/lib/iomgr/error.cc", "src/core/lib/iomgr/ev_epoll1_linux.cc", "src/core/lib/iomgr/ev_epollex_linux.cc", - "src/core/lib/iomgr/ev_epollsig_linux.cc", "src/core/lib/iomgr/ev_poll_posix.cc", "src/core/lib/iomgr/ev_posix.cc", "src/core/lib/iomgr/ev_windows.cc", @@ -858,7 +858,6 @@ grpc_cc_library( "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", - "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -1573,6 +1572,7 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/frame_window_update.cc", "src/core/ext/transport/chttp2/transport/hpack_encoder.cc", "src/core/ext/transport/chttp2/transport/hpack_parser.cc", + "src/core/ext/transport/chttp2/transport/hpack_mapping.cc", "src/core/ext/transport/chttp2/transport/hpack_table.cc", "src/core/ext/transport/chttp2/transport/http2_settings.cc", "src/core/ext/transport/chttp2/transport/huffsyms.cc", @@ -1597,6 +1597,7 @@ grpc_cc_library( "src/core/ext/transport/chttp2/transport/frame_window_update.h", "src/core/ext/transport/chttp2/transport/hpack_encoder.h", "src/core/ext/transport/chttp2/transport/hpack_parser.h", + "src/core/ext/transport/chttp2/transport/hpack_mapping.h", "src/core/ext/transport/chttp2/transport/hpack_table.h", "src/core/ext/transport/chttp2/transport/http2_settings.h", "src/core/ext/transport/chttp2/transport/huffsyms.h", @@ -1979,7 +1980,9 @@ grpc_cc_library( "include/grpcpp/impl/codegen/byte_buffer.h", "include/grpcpp/impl/codegen/call.h", "include/grpcpp/impl/codegen/call_hook.h", + "include/grpcpp/impl/codegen/callback_common.h", "include/grpcpp/impl/codegen/channel_interface.h", + "include/grpcpp/impl/codegen/client_callback.h", "include/grpcpp/impl/codegen/client_context.h", "include/grpcpp/impl/codegen/client_unary_call.h", "include/grpcpp/impl/codegen/completion_queue.h", diff --git a/BUILDING.md b/BUILDING.md index 81edb68e43..615b371db6 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -132,8 +132,8 @@ you will be able to browse and build the code. > @rem Run from grpc directory after cloning the repo with --recursive or updating submodules. > md .build > cd .build -> cmake .. -G "Visual Studio 14 2015" -DCMAKE_BUILD_TYPE=Release -> cmake --build . +> cmake .. -G "Visual Studio 14 2015" +> cmake --build . --config Release ``` ## cmake: Windows, Using Ninja (faster build, supports boringssl's assembly optimizations). diff --git a/CMakeLists.txt b/CMakeLists.txt index a21bb8b5fa..51788be108 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -253,9 +253,6 @@ add_dependencies(buildtests_c error_test) if(_gRPC_PLATFORM_LINUX) add_dependencies(buildtests_c ev_epollex_linux_test) endif() -if(_gRPC_PLATFORM_LINUX) -add_dependencies(buildtests_c ev_epollsig_linux_test) -endif() add_dependencies(buildtests_c fake_resolver_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c fake_transport_security_test) @@ -355,13 +352,11 @@ add_dependencies(buildtests_c no_server_test) add_dependencies(buildtests_c num_external_connectivity_watchers_test) add_dependencies(buildtests_c parse_address_test) add_dependencies(buildtests_c percent_encoding_test) -if(_gRPC_PLATFORM_LINUX) -add_dependencies(buildtests_c pollset_set_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c resolve_address_posix_test) endif() -add_dependencies(buildtests_c resolve_address_test) +add_dependencies(buildtests_c resolve_address_using_ares_resolver_test) +add_dependencies(buildtests_c resolve_address_using_native_resolver_test) add_dependencies(buildtests_c resource_quota_test) add_dependencies(buildtests_c secure_channel_create_test) add_dependencies(buildtests_c secure_endpoint_test) @@ -564,6 +559,7 @@ add_dependencies(buildtests_cxx check_gcp_environment_linux_test) add_dependencies(buildtests_cxx check_gcp_environment_windows_test) add_dependencies(buildtests_cxx chttp2_settings_timeout_test) add_dependencies(buildtests_cxx cli_call_test) +add_dependencies(buildtests_cxx client_callback_end2end_test) add_dependencies(buildtests_cxx client_channel_stress_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx client_crash_test) @@ -972,7 +968,6 @@ add_library(grpc src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -1103,6 +1098,7 @@ add_library(grpc src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -1380,7 +1376,6 @@ add_library(grpc_cronet src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -1514,6 +1509,7 @@ add_library(grpc_cronet src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -1774,7 +1770,6 @@ add_library(grpc_test_util src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -1927,6 +1922,7 @@ add_library(grpc_test_util src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -2084,7 +2080,6 @@ add_library(grpc_test_util_unsecure src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -2237,6 +2232,7 @@ add_library(grpc_test_util_unsecure src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -2373,7 +2369,6 @@ add_library(grpc_unsecure src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -2505,6 +2500,7 @@ add_library(grpc_unsecure src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -2917,6 +2913,7 @@ foreach(_hdr include/grpcpp/support/async_unary_call.h include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h + include/grpcpp/support/client_callback.h include/grpcpp/support/config.h include/grpcpp/support/proto_buffer_reader.h include/grpcpp/support/proto_buffer_writer.h @@ -3014,7 +3011,9 @@ foreach(_hdr include/grpcpp/impl/codegen/byte_buffer.h include/grpcpp/impl/codegen/call.h include/grpcpp/impl/codegen/call_hook.h + include/grpcpp/impl/codegen/callback_common.h include/grpcpp/impl/codegen/channel_interface.h + include/grpcpp/impl/codegen/client_callback.h include/grpcpp/impl/codegen/client_context.h include/grpcpp/impl/codegen/client_unary_call.h include/grpcpp/impl/codegen/completion_queue.h @@ -3171,6 +3170,7 @@ add_library(grpc++_cronet src/core/ext/transport/chttp2/transport/frame_settings.cc src/core/ext/transport/chttp2/transport/frame_window_update.cc src/core/ext/transport/chttp2/transport/hpack_encoder.cc + src/core/ext/transport/chttp2/transport/hpack_mapping.cc src/core/ext/transport/chttp2/transport/hpack_parser.cc src/core/ext/transport/chttp2/transport/hpack_table.cc src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -3215,7 +3215,6 @@ add_library(grpc++_cronet src/core/lib/iomgr/error.cc src/core/lib/iomgr/ev_epoll1_linux.cc src/core/lib/iomgr/ev_epollex_linux.cc - src/core/lib/iomgr/ev_epollsig_linux.cc src/core/lib/iomgr/ev_poll_posix.cc src/core/lib/iomgr/ev_posix.cc src/core/lib/iomgr/ev_windows.cc @@ -3486,6 +3485,7 @@ foreach(_hdr include/grpcpp/support/async_unary_call.h include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h + include/grpcpp/support/client_callback.h include/grpcpp/support/config.h include/grpcpp/support/proto_buffer_reader.h include/grpcpp/support/proto_buffer_writer.h @@ -3583,7 +3583,9 @@ foreach(_hdr include/grpcpp/impl/codegen/byte_buffer.h include/grpcpp/impl/codegen/call.h include/grpcpp/impl/codegen/call_hook.h + include/grpcpp/impl/codegen/callback_common.h include/grpcpp/impl/codegen/channel_interface.h + include/grpcpp/impl/codegen/client_callback.h include/grpcpp/impl/codegen/client_context.h include/grpcpp/impl/codegen/client_unary_call.h include/grpcpp/impl/codegen/completion_queue.h @@ -3993,7 +3995,9 @@ foreach(_hdr include/grpcpp/impl/codegen/byte_buffer.h include/grpcpp/impl/codegen/call.h include/grpcpp/impl/codegen/call_hook.h + include/grpcpp/impl/codegen/callback_common.h include/grpcpp/impl/codegen/channel_interface.h + include/grpcpp/impl/codegen/client_callback.h include/grpcpp/impl/codegen/client_context.h include/grpcpp/impl/codegen/client_unary_call.h include/grpcpp/impl/codegen/completion_queue.h @@ -4171,7 +4175,9 @@ foreach(_hdr include/grpcpp/impl/codegen/byte_buffer.h include/grpcpp/impl/codegen/call.h include/grpcpp/impl/codegen/call_hook.h + include/grpcpp/impl/codegen/callback_common.h include/grpcpp/impl/codegen/channel_interface.h + include/grpcpp/impl/codegen/client_callback.h include/grpcpp/impl/codegen/client_context.h include/grpcpp/impl/codegen/client_unary_call.h include/grpcpp/impl/codegen/completion_queue.h @@ -4392,6 +4398,7 @@ foreach(_hdr include/grpcpp/support/async_unary_call.h include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h + include/grpcpp/support/client_callback.h include/grpcpp/support/config.h include/grpcpp/support/proto_buffer_reader.h include/grpcpp/support/proto_buffer_writer.h @@ -4489,7 +4496,9 @@ foreach(_hdr include/grpcpp/impl/codegen/byte_buffer.h include/grpcpp/impl/codegen/call.h include/grpcpp/impl/codegen/call_hook.h + include/grpcpp/impl/codegen/callback_common.h include/grpcpp/impl/codegen/channel_interface.h + include/grpcpp/impl/codegen/client_callback.h include/grpcpp/impl/codegen/client_context.h include/grpcpp/impl/codegen/client_unary_call.h include/grpcpp/impl/codegen/completion_queue.h @@ -6375,37 +6384,6 @@ target_link_libraries(ev_epollex_linux_test endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX) - -add_executable(ev_epollsig_linux_test - test/core/iomgr/ev_epollsig_linux_test.cc -) - - -target_include_directories(ev_epollsig_linux_test - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${_gRPC_SSL_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} - PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} - PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} - PRIVATE ${_gRPC_CARES_INCLUDE_DIR} - PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} - PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} -) - -target_link_libraries(ev_epollsig_linux_test - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util - grpc - gpr_test_util - gpr -) - -endif() -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) add_executable(fake_resolver_test test/core/client_channel/resolvers/fake_resolver_test.cc @@ -8490,14 +8468,14 @@ target_link_libraries(percent_encoding_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_executable(pollset_set_test - test/core/iomgr/pollset_set_test.cc +add_executable(resolve_address_posix_test + test/core/iomgr/resolve_address_posix_test.cc ) -target_include_directories(pollset_set_test +target_include_directories(resolve_address_posix_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${_gRPC_SSL_INCLUDE_DIR} @@ -8510,7 +8488,7 @@ target_include_directories(pollset_set_test PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} ) -target_link_libraries(pollset_set_test +target_link_libraries(resolve_address_posix_test ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc @@ -8521,14 +8499,13 @@ target_link_libraries(pollset_set_test endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) -add_executable(resolve_address_posix_test - test/core/iomgr/resolve_address_posix_test.cc +add_executable(resolve_address_using_ares_resolver_test + test/core/iomgr/resolve_address_test.cc ) -target_include_directories(resolve_address_posix_test +target_include_directories(resolve_address_using_ares_resolver_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${_gRPC_SSL_INCLUDE_DIR} @@ -8541,7 +8518,7 @@ target_include_directories(resolve_address_posix_test PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} ) -target_link_libraries(resolve_address_posix_test +target_link_libraries(resolve_address_using_ares_resolver_test ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc @@ -8549,16 +8526,15 @@ target_link_libraries(resolve_address_posix_test gpr ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -add_executable(resolve_address_test +add_executable(resolve_address_using_native_resolver_test test/core/iomgr/resolve_address_test.cc ) -target_include_directories(resolve_address_test +target_include_directories(resolve_address_using_native_resolver_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${_gRPC_SSL_INCLUDE_DIR} @@ -8571,7 +8547,7 @@ target_include_directories(resolve_address_test PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} ) -target_link_libraries(resolve_address_test +target_link_libraries(resolve_address_using_native_resolver_test ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc @@ -11297,6 +11273,46 @@ target_link_libraries(cli_call_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(client_callback_end2end_test + test/cpp/end2end/client_callback_end2end_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(client_callback_end2end_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(client_callback_end2end_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(client_channel_stress_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/lb/v1/load_balancer.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc @@ -352,6 +352,7 @@ CXXFLAGS += -std=c++11 ifeq ($(SYSTEM),Darwin) CXXFLAGS += -stdlib=libc++ endif +CXXFLAGS += -Wnon-virtual-dtor CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT COREFLAGS += -fno-rtti -fno-exceptions LDFLAGS += -g @@ -997,7 +998,6 @@ dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test error_test: $(BINDIR)/$(CONFIG)/error_test ev_epollex_linux_test: $(BINDIR)/$(CONFIG)/ev_epollex_linux_test -ev_epollsig_linux_test: $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test fake_resolver_test: $(BINDIR)/$(CONFIG)/fake_resolver_test fake_transport_security_test: $(BINDIR)/$(CONFIG)/fake_transport_security_test fd_conservation_posix_test: $(BINDIR)/$(CONFIG)/fd_conservation_posix_test @@ -1079,9 +1079,9 @@ parse_address_test: $(BINDIR)/$(CONFIG)/parse_address_test percent_decode_fuzzer: $(BINDIR)/$(CONFIG)/percent_decode_fuzzer percent_encode_fuzzer: $(BINDIR)/$(CONFIG)/percent_encode_fuzzer percent_encoding_test: $(BINDIR)/$(CONFIG)/percent_encoding_test -pollset_set_test: $(BINDIR)/$(CONFIG)/pollset_set_test resolve_address_posix_test: $(BINDIR)/$(CONFIG)/resolve_address_posix_test -resolve_address_test: $(BINDIR)/$(CONFIG)/resolve_address_test +resolve_address_using_ares_resolver_test: $(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test +resolve_address_using_native_resolver_test: $(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test resource_quota_test: $(BINDIR)/$(CONFIG)/resource_quota_test secure_channel_create_test: $(BINDIR)/$(CONFIG)/secure_channel_create_test secure_endpoint_test: $(BINDIR)/$(CONFIG)/secure_endpoint_test @@ -1160,6 +1160,7 @@ check_gcp_environment_linux_test: $(BINDIR)/$(CONFIG)/check_gcp_environment_linu check_gcp_environment_windows_test: $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test chttp2_settings_timeout_test: $(BINDIR)/$(CONFIG)/chttp2_settings_timeout_test cli_call_test: $(BINDIR)/$(CONFIG)/cli_call_test +client_callback_end2end_test: $(BINDIR)/$(CONFIG)/client_callback_end2end_test client_channel_stress_test: $(BINDIR)/$(CONFIG)/client_channel_stress_test client_crash_test: $(BINDIR)/$(CONFIG)/client_crash_test client_crash_test_server: $(BINDIR)/$(CONFIG)/client_crash_test_server @@ -1452,7 +1453,6 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ $(BINDIR)/$(CONFIG)/error_test \ $(BINDIR)/$(CONFIG)/ev_epollex_linux_test \ - $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test \ $(BINDIR)/$(CONFIG)/fake_resolver_test \ $(BINDIR)/$(CONFIG)/fake_transport_security_test \ $(BINDIR)/$(CONFIG)/fd_conservation_posix_test \ @@ -1522,9 +1522,9 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/num_external_connectivity_watchers_test \ $(BINDIR)/$(CONFIG)/parse_address_test \ $(BINDIR)/$(CONFIG)/percent_encoding_test \ - $(BINDIR)/$(CONFIG)/pollset_set_test \ $(BINDIR)/$(CONFIG)/resolve_address_posix_test \ - $(BINDIR)/$(CONFIG)/resolve_address_test \ + $(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test \ + $(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test \ $(BINDIR)/$(CONFIG)/resource_quota_test \ $(BINDIR)/$(CONFIG)/secure_channel_create_test \ $(BINDIR)/$(CONFIG)/secure_endpoint_test \ @@ -1665,6 +1665,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test \ $(BINDIR)/$(CONFIG)/chttp2_settings_timeout_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ + $(BINDIR)/$(CONFIG)/client_callback_end2end_test \ $(BINDIR)/$(CONFIG)/client_channel_stress_test \ $(BINDIR)/$(CONFIG)/client_crash_test \ $(BINDIR)/$(CONFIG)/client_crash_test_server \ @@ -1845,6 +1846,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/check_gcp_environment_windows_test \ $(BINDIR)/$(CONFIG)/chttp2_settings_timeout_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ + $(BINDIR)/$(CONFIG)/client_callback_end2end_test \ $(BINDIR)/$(CONFIG)/client_channel_stress_test \ $(BINDIR)/$(CONFIG)/client_crash_test \ $(BINDIR)/$(CONFIG)/client_crash_test_server \ @@ -1986,8 +1988,6 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/error_test || ( echo test error_test failed ; exit 1 ) $(E) "[RUN] Testing ev_epollex_linux_test" $(Q) $(BINDIR)/$(CONFIG)/ev_epollex_linux_test || ( echo test ev_epollex_linux_test failed ; exit 1 ) - $(E) "[RUN] Testing ev_epollsig_linux_test" - $(Q) $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test || ( echo test ev_epollsig_linux_test failed ; exit 1 ) $(E) "[RUN] Testing fake_resolver_test" $(Q) $(BINDIR)/$(CONFIG)/fake_resolver_test || ( echo test fake_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing fake_transport_security_test" @@ -2114,12 +2114,12 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/parse_address_test || ( echo test parse_address_test failed ; exit 1 ) $(E) "[RUN] Testing percent_encoding_test" $(Q) $(BINDIR)/$(CONFIG)/percent_encoding_test || ( echo test percent_encoding_test failed ; exit 1 ) - $(E) "[RUN] Testing pollset_set_test" - $(Q) $(BINDIR)/$(CONFIG)/pollset_set_test || ( echo test pollset_set_test failed ; exit 1 ) $(E) "[RUN] Testing resolve_address_posix_test" $(Q) $(BINDIR)/$(CONFIG)/resolve_address_posix_test || ( echo test resolve_address_posix_test failed ; exit 1 ) - $(E) "[RUN] Testing resolve_address_test" - $(Q) $(BINDIR)/$(CONFIG)/resolve_address_test || ( echo test resolve_address_test failed ; exit 1 ) + $(E) "[RUN] Testing resolve_address_using_ares_resolver_test" + $(Q) $(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test || ( echo test resolve_address_using_ares_resolver_test failed ; exit 1 ) + $(E) "[RUN] Testing resolve_address_using_native_resolver_test" + $(Q) $(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test || ( echo test resolve_address_using_native_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing resource_quota_test" $(Q) $(BINDIR)/$(CONFIG)/resource_quota_test || ( echo test resource_quota_test failed ; exit 1 ) $(E) "[RUN] Testing secure_channel_create_test" @@ -2302,6 +2302,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/chttp2_settings_timeout_test || ( echo test chttp2_settings_timeout_test failed ; exit 1 ) $(E) "[RUN] Testing cli_call_test" $(Q) $(BINDIR)/$(CONFIG)/cli_call_test || ( echo test cli_call_test failed ; exit 1 ) + $(E) "[RUN] Testing client_callback_end2end_test" + $(Q) $(BINDIR)/$(CONFIG)/client_callback_end2end_test || ( echo test client_callback_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing client_channel_stress_test" $(Q) $(BINDIR)/$(CONFIG)/client_channel_stress_test || ( echo test client_channel_stress_test failed ; exit 1 ) $(E) "[RUN] Testing client_crash_test" @@ -3474,7 +3476,6 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -3605,6 +3606,7 @@ LIBGRPC_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -3881,7 +3883,6 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -4015,6 +4016,7 @@ LIBGRPC_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -4273,7 +4275,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -4426,6 +4427,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -4574,7 +4576,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -4727,6 +4728,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -4841,7 +4843,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -4973,6 +4974,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -5329,6 +5331,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ + include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ @@ -5426,7 +5429,9 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ + include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ + include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -5627,6 +5632,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ @@ -5671,7 +5677,6 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -5905,6 +5910,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ + include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ @@ -6002,7 +6008,9 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ + include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ + include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -6392,7 +6400,9 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ + include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ + include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -6546,7 +6556,9 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ + include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ + include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -6771,6 +6783,7 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ + include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ @@ -6868,7 +6881,9 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ + include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ + include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -11325,38 +11340,6 @@ endif endif -EV_EPOLLSIG_LINUX_TEST_SRC = \ - test/core/iomgr/ev_epollsig_linux_test.cc \ - -EV_EPOLLSIG_LINUX_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(EV_EPOLLSIG_LINUX_TEST_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/ev_epollsig_linux_test: openssl_dep_error - -else - - - -$(BINDIR)/$(CONFIG)/ev_epollsig_linux_test: $(EV_EPOLLSIG_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(EV_EPOLLSIG_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test - -endif - -$(OBJDIR)/$(CONFIG)/test/core/iomgr/ev_epollsig_linux_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a - -deps_ev_epollsig_linux_test: $(EV_EPOLLSIG_LINUX_TEST_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(EV_EPOLLSIG_LINUX_TEST_OBJS:.o=.dep) -endif -endif - - FAKE_RESOLVER_TEST_SRC = \ test/core/client_channel/resolvers/fake_resolver_test.cc \ @@ -13967,98 +13950,98 @@ endif endif -POLLSET_SET_TEST_SRC = \ - test/core/iomgr/pollset_set_test.cc \ +RESOLVE_ADDRESS_POSIX_TEST_SRC = \ + test/core/iomgr/resolve_address_posix_test.cc \ -POLLSET_SET_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(POLLSET_SET_TEST_SRC)))) +RESOLVE_ADDRESS_POSIX_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVE_ADDRESS_POSIX_TEST_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/pollset_set_test: openssl_dep_error +$(BINDIR)/$(CONFIG)/resolve_address_posix_test: openssl_dep_error else -$(BINDIR)/$(CONFIG)/pollset_set_test: $(POLLSET_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/resolve_address_posix_test: $(RESOLVE_ADDRESS_POSIX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(POLLSET_SET_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/pollset_set_test + $(Q) $(LD) $(LDFLAGS) $(RESOLVE_ADDRESS_POSIX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resolve_address_posix_test endif -$(OBJDIR)/$(CONFIG)/test/core/iomgr/pollset_set_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/core/iomgr/resolve_address_posix_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_pollset_set_test: $(POLLSET_SET_TEST_OBJS:.o=.dep) +deps_resolve_address_posix_test: $(RESOLVE_ADDRESS_POSIX_TEST_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(POLLSET_SET_TEST_OBJS:.o=.dep) +-include $(RESOLVE_ADDRESS_POSIX_TEST_OBJS:.o=.dep) endif endif -RESOLVE_ADDRESS_POSIX_TEST_SRC = \ - test/core/iomgr/resolve_address_posix_test.cc \ +RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_SRC = \ + test/core/iomgr/resolve_address_test.cc \ -RESOLVE_ADDRESS_POSIX_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVE_ADDRESS_POSIX_TEST_SRC)))) +RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/resolve_address_posix_test: openssl_dep_error +$(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test: openssl_dep_error else -$(BINDIR)/$(CONFIG)/resolve_address_posix_test: $(RESOLVE_ADDRESS_POSIX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test: $(RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(RESOLVE_ADDRESS_POSIX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resolve_address_posix_test + $(Q) $(LD) $(LDFLAGS) $(RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resolve_address_using_ares_resolver_test endif -$(OBJDIR)/$(CONFIG)/test/core/iomgr/resolve_address_posix_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/core/iomgr/resolve_address_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_resolve_address_posix_test: $(RESOLVE_ADDRESS_POSIX_TEST_OBJS:.o=.dep) +deps_resolve_address_using_ares_resolver_test: $(RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(RESOLVE_ADDRESS_POSIX_TEST_OBJS:.o=.dep) +-include $(RESOLVE_ADDRESS_USING_ARES_RESOLVER_TEST_OBJS:.o=.dep) endif endif -RESOLVE_ADDRESS_TEST_SRC = \ +RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_SRC = \ test/core/iomgr/resolve_address_test.cc \ -RESOLVE_ADDRESS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVE_ADDRESS_TEST_SRC)))) +RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/resolve_address_test: openssl_dep_error +$(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test: openssl_dep_error else -$(BINDIR)/$(CONFIG)/resolve_address_test: $(RESOLVE_ADDRESS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test: $(RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(RESOLVE_ADDRESS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resolve_address_test + $(Q) $(LD) $(LDFLAGS) $(RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resolve_address_using_native_resolver_test endif $(OBJDIR)/$(CONFIG)/test/core/iomgr/resolve_address_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_resolve_address_test: $(RESOLVE_ADDRESS_TEST_OBJS:.o=.dep) +deps_resolve_address_using_native_resolver_test: $(RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(RESOLVE_ADDRESS_TEST_OBJS:.o=.dep) +-include $(RESOLVE_ADDRESS_USING_NATIVE_RESOLVER_TEST_OBJS:.o=.dep) endif endif @@ -17076,6 +17059,49 @@ endif endif +CLIENT_CALLBACK_END2END_TEST_SRC = \ + test/cpp/end2end/client_callback_end2end_test.cc \ + +CLIENT_CALLBACK_END2END_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CLIENT_CALLBACK_END2END_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/client_callback_end2end_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/client_callback_end2end_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/client_callback_end2end_test: $(PROTOBUF_DEP) $(CLIENT_CALLBACK_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(CLIENT_CALLBACK_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/client_callback_end2end_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/end2end/client_callback_end2end_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_client_callback_end2end_test: $(CLIENT_CALLBACK_END2END_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CLIENT_CALLBACK_END2END_TEST_OBJS:.o=.dep) +endif +endif + + CLIENT_CHANNEL_STRESS_TEST_SRC = \ $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc \ test/cpp/client/client_channel_stress_test.cc \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index b3f97650e8..159ebd5d1f 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -24,7 +24,7 @@ # # The set of pollers to test against if a test exercises polling -POLLERS = ["epollex", "epollsig", "epoll1", "poll", "poll-cv"] +POLLERS = ["epollex", "epoll1", "poll", "poll-cv"] def if_not_windows(a): return select({ diff --git a/build.yaml b/build.yaml index a50a0a4ab6..f3895bb702 100644 --- a/build.yaml +++ b/build.yaml @@ -266,7 +266,6 @@ filegroups: - src/core/lib/iomgr/error.cc - src/core/lib/iomgr/ev_epoll1_linux.cc - src/core/lib/iomgr/ev_epollex_linux.cc - - src/core/lib/iomgr/ev_epollsig_linux.cc - src/core/lib/iomgr/ev_poll_posix.cc - src/core/lib/iomgr/ev_posix.cc - src/core/lib/iomgr/ev_windows.cc @@ -446,7 +445,6 @@ filegroups: - src/core/lib/iomgr/error_internal.h - src/core/lib/iomgr/ev_epoll1_linux.h - src/core/lib/iomgr/ev_epollex_linux.h - - src/core/lib/iomgr/ev_epollsig_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -936,6 +934,7 @@ filegroups: - src/core/ext/transport/chttp2/transport/frame_settings.h - src/core/ext/transport/chttp2/transport/frame_window_update.h - src/core/ext/transport/chttp2/transport/hpack_encoder.h + - src/core/ext/transport/chttp2/transport/hpack_mapping.h - src/core/ext/transport/chttp2/transport/hpack_parser.h - src/core/ext/transport/chttp2/transport/hpack_table.h - src/core/ext/transport/chttp2/transport/http2_settings.h @@ -957,6 +956,7 @@ filegroups: - src/core/ext/transport/chttp2/transport/frame_settings.cc - src/core/ext/transport/chttp2/transport/frame_window_update.cc - src/core/ext/transport/chttp2/transport/hpack_encoder.cc + - src/core/ext/transport/chttp2/transport/hpack_mapping.cc - src/core/ext/transport/chttp2/transport/hpack_parser.cc - src/core/ext/transport/chttp2/transport/hpack_table.cc - src/core/ext/transport/chttp2/transport/http2_settings.cc @@ -1169,7 +1169,9 @@ filegroups: - include/grpcpp/impl/codegen/byte_buffer.h - include/grpcpp/impl/codegen/call.h - include/grpcpp/impl/codegen/call_hook.h + - include/grpcpp/impl/codegen/callback_common.h - include/grpcpp/impl/codegen/channel_interface.h + - include/grpcpp/impl/codegen/client_callback.h - include/grpcpp/impl/codegen/client_context.h - include/grpcpp/impl/codegen/client_unary_call.h - include/grpcpp/impl/codegen/completion_queue.h @@ -1297,6 +1299,7 @@ filegroups: - include/grpcpp/support/async_unary_call.h - include/grpcpp/support/byte_buffer.h - include/grpcpp/support/channel_arguments.h + - include/grpcpp/support/client_callback.h - include/grpcpp/support/config.h - include/grpcpp/support/proto_buffer_reader.h - include/grpcpp/support/proto_buffer_writer.h @@ -2367,21 +2370,6 @@ targets: - uv platforms: - linux -- name: ev_epollsig_linux_test - cpu_cost: 3 - build: test - language: c - src: - - test/core/iomgr/ev_epollsig_linux_test.cc - deps: - - grpc_test_util - - grpc - - gpr_test_util - - gpr - exclude_iomgrs: - - uv - platforms: - - linux - name: fake_resolver_test build: test language: c @@ -3333,11 +3321,11 @@ targets: - gpr_test_util - gpr uses_polling: false -- name: pollset_set_test +- name: resolve_address_posix_test build: test language: c src: - - test/core/iomgr/pollset_set_test.cc + - test/core/iomgr/resolve_address_posix_test.cc deps: - grpc_test_util - grpc @@ -3346,24 +3334,22 @@ targets: exclude_iomgrs: - uv platforms: + - mac - linux -- name: resolve_address_posix_test + - posix +- name: resolve_address_using_ares_resolver_test build: test language: c src: - - test/core/iomgr/resolve_address_posix_test.cc + - test/core/iomgr/resolve_address_test.cc deps: - grpc_test_util - grpc - gpr_test_util - gpr - exclude_iomgrs: - - uv - platforms: - - mac - - linux - - posix -- name: resolve_address_test + args: + - --resolver=ares +- name: resolve_address_using_native_resolver_test build: test language: c src: @@ -3373,6 +3359,8 @@ targets: - grpc - gpr_test_util - gpr + args: + - --resolver=native - name: resource_quota_test cpu_cost: 30 build: test @@ -4465,6 +4453,20 @@ targets: - grpc - gpr_test_util - gpr +- name: client_callback_end2end_test + gtest: true + cpu_cost: 0.5 + build: test + language: c++ + src: + - test/cpp/end2end/client_callback_end2end_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: client_channel_stress_test gtest: false build: test @@ -5827,6 +5829,7 @@ defaults: COREFLAGS: -fno-rtti -fno-exceptions CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 -Wno-deprecated-declarations -Ithird_party/nanopb -DPB_FIELD_32BIT + CXXFLAGS: -Wnon-virtual-dtor LDFLAGS: -g zlib: CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration @@ -118,7 +118,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/error.cc \ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.cc \ - src/core/lib/iomgr/ev_epollsig_linux.cc \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_posix.cc \ src/core/lib/iomgr/ev_windows.cc \ @@ -249,6 +248,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/transport/chttp2/transport/frame_settings.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ + src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ src/core/ext/transport/chttp2/transport/http2_settings.cc \ diff --git a/config.w32 b/config.w32 index ad91ee40bd..6cdd2dac8b 100644 --- a/config.w32 +++ b/config.w32 @@ -93,7 +93,6 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\iomgr\\error.cc " + "src\\core\\lib\\iomgr\\ev_epoll1_linux.cc " + "src\\core\\lib\\iomgr\\ev_epollex_linux.cc " + - "src\\core\\lib\\iomgr\\ev_epollsig_linux.cc " + "src\\core\\lib\\iomgr\\ev_poll_posix.cc " + "src\\core\\lib\\iomgr\\ev_posix.cc " + "src\\core\\lib\\iomgr\\ev_windows.cc " + @@ -224,6 +223,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\transport\\chttp2\\transport\\frame_settings.cc " + "src\\core\\ext\\transport\\chttp2\\transport\\frame_window_update.cc " + "src\\core\\ext\\transport\\chttp2\\transport\\hpack_encoder.cc " + + "src\\core\\ext\\transport\\chttp2\\transport\\hpack_mapping.cc " + "src\\core\\ext\\transport\\chttp2\\transport\\hpack_parser.cc " + "src\\core\\ext\\transport\\chttp2\\transport\\hpack_table.cc " + "src\\core\\ext\\transport\\chttp2\\transport\\http2_settings.cc " + diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 3c33189196..1d6535d7ea 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -944,6 +944,11 @@ the experimental flag, `soak_iterations`. This tests puts stress on several gRPC components; the resolver, the load balancer, and the RPC hotpath. +#### long_lived_channel + +The client performs a number of large_unary RPCs over a single long-lived +channel with a fixed but configurable interval between each RPC. + ### TODO Tests #### High priority: diff --git a/etc/roots.pem b/etc/roots.pem index c22dfe69f8..3e6bbcd76e 100644 --- a/etc/roots.pem +++ b/etc/roots.pem @@ -4317,3 +4317,26 @@ JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R 8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= -----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- diff --git a/examples/csharp/Helloworld/Greeter/Greeter.csproj b/examples/csharp/Helloworld/Greeter/Greeter.csproj index 1ca821320c..eba262565d 100644 --- a/examples/csharp/Helloworld/Greeter/Greeter.csproj +++ b/examples/csharp/Helloworld/Greeter/Greeter.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>Greeter</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>Greeter</AssemblyName> <PackageId>Greeter</PackageId> diff --git a/examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj b/examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj index d1ed040411..24a89d58c5 100644 --- a/examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj +++ b/examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>GreeterClient</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>GreeterClient</AssemblyName> <OutputType>Exe</OutputType> diff --git a/examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj b/examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj index 159fbd8a23..9ea1fa3817 100644 --- a/examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj +++ b/examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>GreeterServer</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>GreeterServer</AssemblyName> <OutputType>Exe</OutputType> diff --git a/examples/csharp/Helloworld/README.md b/examples/csharp/Helloworld/README.md index e603179438..4871132426 100644 --- a/examples/csharp/Helloworld/README.md +++ b/examples/csharp/Helloworld/README.md @@ -12,7 +12,7 @@ which have been already added to the project for you. PREREQUISITES ------------- -- The [.NET Core SDK](https://www.microsoft.com/net/core) (version 2+ is recommended) +- The [.NET Core SDK 2.1+](https://www.microsoft.com/net/core) You can also build the example directly using Visual Studio 2017, but it's not a requirement. @@ -23,8 +23,6 @@ From the `examples/csharp/Helloworld` directory: - `dotnet build Greeter.sln` -(if you're using dotnet SDK 1.x you need to run `dotnet restore Greeter.sln` first) - Try it! ------- @@ -32,14 +30,14 @@ Try it! ``` > cd GreeterServer - > dotnet run -f netcoreapp1.0 + > dotnet run -f netcoreapp2.1 ``` - Run the client ``` > cd GreeterClient - > dotnet run -f netcoreapp1.0 + > dotnet run -f netcoreapp2.1 ``` Tutorial diff --git a/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj b/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj index e1c44ecf3c..86346d1e14 100644 --- a/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj +++ b/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>RouteGuide</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>RouteGuide</AssemblyName> <PackageId>RouteGuide</PackageId> diff --git a/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj b/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj index 96cc204ba3..c6dadf082b 100644 --- a/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj +++ b/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>RouteGuideClient</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>RouteGuideClient</AssemblyName> <OutputType>Exe</OutputType> diff --git a/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj b/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj index aa6315bf81..005c87ca0c 100644 --- a/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj +++ b/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj @@ -2,7 +2,7 @@ <PropertyGroup> <AssemblyTitle>RouteGuideServer</AssemblyTitle> - <TargetFrameworks>netcoreapp1.0</TargetFrameworks> + <TargetFrameworks>netcoreapp2.1</TargetFrameworks> <DebugType>portable</DebugType> <AssemblyName>RouteGuideServer</AssemblyName> <OutputType>Exe</OutputType> diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 581b9246bc..3ca132f7c3 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -111,6 +111,7 @@ Pod::Spec.new do |s| 'include/grpcpp/support/async_unary_call.h', 'include/grpcpp/support/byte_buffer.h', 'include/grpcpp/support/channel_arguments.h', + 'include/grpcpp/support/client_callback.h', 'include/grpcpp/support/config.h', 'include/grpcpp/support/proto_buffer_reader.h', 'include/grpcpp/support/proto_buffer_writer.h', @@ -127,7 +128,9 @@ Pod::Spec.new do |s| 'include/grpcpp/impl/codegen/byte_buffer.h', 'include/grpcpp/impl/codegen/call.h', 'include/grpcpp/impl/codegen/call_hook.h', + 'include/grpcpp/impl/codegen/callback_common.h', 'include/grpcpp/impl/codegen/channel_interface.h', + 'include/grpcpp/impl/codegen/client_callback.h', 'include/grpcpp/impl/codegen/client_context.h', 'include/grpcpp/impl/codegen/client_unary_call.h', 'include/grpcpp/impl/codegen/completion_queue.h', @@ -251,6 +254,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/frame_settings.h', 'src/core/ext/transport/chttp2/transport/frame_window_update.h', 'src/core/ext/transport/chttp2/transport/hpack_encoder.h', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.h', 'src/core/ext/transport/chttp2/transport/hpack_parser.h', 'src/core/ext/transport/chttp2/transport/hpack_table.h', 'src/core/ext/transport/chttp2/transport/http2_settings.h', @@ -392,7 +396,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll1_linux.h', 'src/core/lib/iomgr/ev_epollex_linux.h', - 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -582,7 +585,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll1_linux.h', 'src/core/lib/iomgr/ev_epollex_linux.h', - 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5a82a4200a..0c3fbe0a25 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -181,7 +181,7 @@ Pod::Spec.new do |s| ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL-GRPC', '0.0.1' + ss.dependency 'BoringSSL-GRPC', '0.0.2' ss.dependency 'nanopb', '~> 0.3' ss.compiler_flags = '-DGRPC_SHADOW_BORINGSSL_SYMBOLS' @@ -263,6 +263,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/frame_settings.h', 'src/core/ext/transport/chttp2/transport/frame_window_update.h', 'src/core/ext/transport/chttp2/transport/hpack_encoder.h', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.h', 'src/core/ext/transport/chttp2/transport/hpack_parser.h', 'src/core/ext/transport/chttp2/transport/hpack_table.h', 'src/core/ext/transport/chttp2/transport/http2_settings.h', @@ -404,7 +405,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll1_linux.h', 'src/core/lib/iomgr/ev_epollex_linux.h', - 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -550,7 +550,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -681,6 +680,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', @@ -866,6 +866,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/frame_settings.h', 'src/core/ext/transport/chttp2/transport/frame_window_update.h', 'src/core/ext/transport/chttp2/transport/hpack_encoder.h', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.h', 'src/core/ext/transport/chttp2/transport/hpack_parser.h', 'src/core/ext/transport/chttp2/transport/hpack_table.h', 'src/core/ext/transport/chttp2/transport/http2_settings.h', @@ -1007,7 +1008,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll1_linux.h', 'src/core/lib/iomgr/ev_epollex_linux.h', - 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -1307,5 +1307,6 @@ Pod::Spec.new do |s| s.prepare_command = <<-END_OF_COMMAND find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include <nanopb/\\1>;g' find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g' END_OF_COMMAND end @@ -74,7 +74,10 @@ EXPORTS grpc_resource_quota_set_max_threads grpc_resource_quota_arg_vtable grpc_channelz_get_top_channels + grpc_channelz_get_servers grpc_channelz_get_channel + grpc_channelz_get_subchannel + grpc_channelz_get_socket grpc_insecure_channel_create_from_fd grpc_server_add_insecure_channel_from_fd grpc_use_signal diff --git a/grpc.gemspec b/grpc.gemspec index c8e58faec9..5780dd7a07 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,6 +195,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/frame_settings.h ) s.files += %w( src/core/ext/transport/chttp2/transport/frame_window_update.h ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_encoder.h ) + s.files += %w( src/core/ext/transport/chttp2/transport/hpack_mapping.h ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_parser.h ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_table.h ) s.files += %w( src/core/ext/transport/chttp2/transport/http2_settings.h ) @@ -340,7 +341,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error_internal.h ) s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.h ) s.files += %w( src/core/lib/iomgr/ev_epollex_linux.h ) - s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -486,7 +486,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error.cc ) s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.cc ) s.files += %w( src/core/lib/iomgr/ev_epollex_linux.cc ) - s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.cc ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.cc ) s.files += %w( src/core/lib/iomgr/ev_posix.cc ) s.files += %w( src/core/lib/iomgr/ev_windows.cc ) @@ -617,6 +616,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/frame_settings.cc ) s.files += %w( src/core/ext/transport/chttp2/transport/frame_window_update.cc ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_encoder.cc ) + s.files += %w( src/core/ext/transport/chttp2/transport/hpack_mapping.cc ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_parser.cc ) s.files += %w( src/core/ext/transport/chttp2/transport/hpack_table.cc ) s.files += %w( src/core/ext/transport/chttp2/transport/http2_settings.cc ) @@ -310,7 +310,6 @@ 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -441,6 +440,7 @@ 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', @@ -672,7 +672,6 @@ 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -825,6 +824,7 @@ 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', @@ -907,7 +907,6 @@ 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -1060,6 +1059,7 @@ 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', @@ -1120,7 +1120,6 @@ 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -1252,6 +1251,7 @@ 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 4e50cd0bac..a9beee1c9e 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -111,7 +111,8 @@ GRPCAPI grpc_completion_queue* grpc_completion_queue_create_for_pluck( of GRPC_CQ_CALLBACK and grpc_cq_polling_type of GRPC_CQ_DEFAULT_POLLING. This function is experimental. */ GRPCAPI grpc_completion_queue* grpc_completion_queue_create_for_callback( - void* shutdown_callback, void* reserved); + grpc_experimental_completion_queue_functor* shutdown_callback, + void* reserved); /** Create a completion queue */ GRPCAPI grpc_completion_queue* grpc_completion_queue_create( @@ -499,10 +500,21 @@ GRPCAPI const grpc_arg_pointer_vtable* grpc_resource_quota_arg_vtable(void); The returned string is allocated and must be freed by the application. */ GRPCAPI char* grpc_channelz_get_top_channels(intptr_t start_channel_id); +/* Gets all servers that exist in the process. */ +GRPCAPI char* grpc_channelz_get_servers(intptr_t start_server_id); + /* Returns a single Channel, or else a NOT_FOUND code. The returned string is allocated and must be freed by the application. */ GRPCAPI char* grpc_channelz_get_channel(intptr_t channel_id); +/* Returns a single Subchannel, or else a NOT_FOUND code. The returned string + is allocated and must be freed by the application. */ +GRPCAPI char* grpc_channelz_get_subchannel(intptr_t subchannel_id); + +/* Returns a single Socket, or else a NOT_FOUND code. The returned string + is allocated and must be freed by the application. */ +GRPCAPI char* grpc_channelz_get_socket(intptr_t socket_id); + #ifdef __cplusplus } #endif diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 02d87a493a..de90971cc5 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -532,14 +532,14 @@ typedef struct grpc_alts_credentials_options grpc_alts_credentials_options; * It is used for experimental purpose for now and subject to change. */ GRPCAPI grpc_alts_credentials_options* -grpc_alts_credentials_client_options_create(); +grpc_alts_credentials_client_options_create(void); /** * This method creates a grpc ALTS credentials server options instance. * It is used for experimental purpose for now and subject to change. */ GRPCAPI grpc_alts_credentials_options* -grpc_alts_credentials_server_options_create(); +grpc_alts_credentials_server_options_create(void); /** * This method adds a target service account to grpc client's ALTS credentials diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index b5353c1dea..9ed5b3c1d4 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -342,6 +342,9 @@ typedef struct { "grpc.disable_client_authority_filter" /** If set to zero, disables use of http proxies. Enabled by default. */ #define GRPC_ARG_ENABLE_HTTP_PROXY "grpc.enable_http_proxy" +/** If set to non zero, surfaces the user agent string to the server. User + agent is surfaced by default. */ +#define GRPC_ARG_SURFACE_USER_AGENT "grpc.surface_user_agent" /** \} */ /** Result of a grpc call. If the caller satisfies the prerequisites of a @@ -657,6 +660,19 @@ typedef enum { GRPC_CQ_CALLBACK } grpc_cq_completion_type; +/** EXPERIMENTAL: Specifies an interface class to be used as a tag + for callback-based completion queues. This can be used directly, + as the first element of a struct in C, or as a base class in C++. + Its "run" value should be assigned to some non-member function, such as + a static method. */ +typedef struct grpc_experimental_completion_queue_functor { + /** The run member specifies a function that will be called when this + tag is extracted from the completion queue. Its arguments will be a + pointer to this functor and a boolean that indicates whether the + operation succeeded (non-zero) or failed (zero) */ + void (*functor_run)(struct grpc_experimental_completion_queue_functor*, int); +} grpc_experimental_completion_queue_functor; + /* The upgrade to version 2 is currently experimental. */ #define GRPC_CQ_CURRENT_VERSION 2 @@ -675,7 +691,7 @@ typedef struct grpc_completion_queue_attributes { /* EXPERIMENTAL: START OF VERSION 2 CQ ATTRIBUTES */ /** When creating a callbackable CQ, pass in a functor to get invoked when * shutdown is complete */ - void* cq_shutdown_cb; + grpc_experimental_completion_queue_functor* cq_shutdown_cb; /* END OF VERSION 2 CQ ATTRIBUTES */ } grpc_completion_queue_attributes; diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 8d9bd83285..b2028a6305 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -174,6 +174,7 @@ #ifdef __GLIBC__ #define GPR_POSIX_CRASH_HANDLER 1 #define GPR_LINUX_PTHREAD_NAME 1 +#include <linux/version.h> #else /* musl libc */ #define GPR_MUSL_LIBC_COMPAT 1 #endif diff --git a/include/grpcpp/alarm.h b/include/grpcpp/alarm.h index f484610a6e..365feb4eb9 100644 --- a/include/grpcpp/alarm.h +++ b/include/grpcpp/alarm.h @@ -21,6 +21,8 @@ #ifndef GRPCPP_ALARM_H #define GRPCPP_ALARM_H +#include <functional> + #include <grpc/grpc.h> #include <grpcpp/impl/codegen/completion_queue.h> #include <grpcpp/impl/codegen/completion_queue_tag.h> @@ -76,8 +78,33 @@ class Alarm : private GrpcLibraryCodegen { /// has already fired has no effect. void Cancel(); + /// NOTE: class experimental_type is not part of the public API of this class + /// TODO(vjpai): Move these contents to the public API of Alarm when + /// they are no longer experimental + class experimental_type { + public: + explicit experimental_type(Alarm* alarm) : alarm_(alarm) {} + + /// Set an alarm to invoke callback \a f. The argument to the callback + /// states whether the alarm expired at \a deadline (true) or was cancelled + /// (false) + template <typename T> + void Set(const T& deadline, std::function<void(bool)> f) { + alarm_->SetInternal(TimePoint<T>(deadline).raw_time(), std::move(f)); + } + + private: + Alarm* alarm_; + }; + + /// NOTE: The function experimental() is not stable public API. It is a view + /// to the experimental components of this class. It may be changed or removed + /// at any time. + experimental_type experimental() { return experimental_type(this); } + private: void SetInternal(CompletionQueue* cq, gpr_timespec deadline, void* tag); + void SetInternal(gpr_timespec deadline, std::function<void(bool)> f); internal::CompletionQueueTag* alarm_; }; diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index fed02bf7bc..f1dba5b8ad 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -78,8 +78,19 @@ class Channel final : public ChannelInterface, bool WaitForStateChangeImpl(grpc_connectivity_state last_observed, gpr_timespec deadline) override; + CompletionQueue* CallbackCQ() override; + const grpc::string host_; grpc_channel* const c_channel_; // owned + + // mu_ protects callback_cq_ (the per-channel callbackable completion queue) + std::mutex mu_; + + // callback_cq_ references the callbackable completion queue associated + // with this channel (if any). It is set on the first call to CallbackCQ(). + // It is _not owned_ by the channel; ownership belongs with its internal + // shutdown callback tag (invoked when the CQ is fully shutdown). + CompletionQueue* callback_cq_ = nullptr; }; } // namespace grpc diff --git a/include/grpcpp/generic/generic_stub.h b/include/grpcpp/generic/generic_stub.h index 92405a43fa..d509d9a520 100644 --- a/include/grpcpp/generic/generic_stub.h +++ b/include/grpcpp/generic/generic_stub.h @@ -19,9 +19,12 @@ #ifndef GRPCPP_GENERIC_GENERIC_STUB_H #define GRPCPP_GENERIC_GENERIC_STUB_H +#include <functional> + #include <grpcpp/support/async_stream.h> #include <grpcpp/support/async_unary_call.h> #include <grpcpp/support/byte_buffer.h> +#include <grpcpp/support/status.h> namespace grpc { @@ -62,6 +65,26 @@ class GenericStub final { ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag); + /// NOTE: class experimental_type is not part of the public API of this class + /// TODO(vjpai): Move these contents to the public API of GenericStub when + /// they are no longer experimental + class experimental_type { + public: + explicit experimental_type(GenericStub* stub) : stub_(stub) {} + + void UnaryCall(ClientContext* context, const grpc::string& method, + const ByteBuffer* request, ByteBuffer* response, + std::function<void(Status)> on_completion); + + private: + GenericStub* stub_; + }; + + /// NOTE: The function experimental() is not stable public API. It is a view + /// to the experimental components of this class. It may be changed or removed + /// at any time. + experimental_type experimental() { return experimental_type(this); } + private: std::shared_ptr<ChannelInterface> channel_; }; diff --git a/include/grpcpp/impl/codegen/async_stream.h b/include/grpcpp/impl/codegen/async_stream.h index b2134590c3..6e58fd0eef 100644 --- a/include/grpcpp/impl/codegen/async_stream.h +++ b/include/grpcpp/impl/codegen/async_stream.h @@ -64,7 +64,7 @@ class ClientAsyncStreamingInterface { /// earlier call to \a AsyncReaderInterface::Read that yielded a failed /// result, e.g. cq->Next(&read_tag, &ok) filled in 'ok' with 'false'). /// - /// This function will return when either: + /// The tag will be returned when either: /// - all incoming messages have been read and the server has returned /// a status. /// - the server has returned a non-OK status. @@ -114,6 +114,9 @@ class AsyncWriterInterface { /// queue BEFORE calling Write again. /// This is thread-safe with respect to \a AsyncReaderInterface::Read /// + /// gRPC doesn't take ownership or a reference to \a msg, so it is safe to + /// to deallocate once Write returns. + /// /// \param[in] msg The message to be written. /// \param[in] tag The tag identifying the operation. virtual void Write(const W& msg, void* tag) = 0; @@ -127,6 +130,9 @@ class AsyncWriterInterface { /// WriteOptions \a options is used to set the write options of this message. /// This is thread-safe with respect to \a AsyncReaderInterface::Read /// + /// gRPC doesn't take ownership or a reference to \a msg, so it is safe to + /// to deallocate once Write returns. + /// /// \param[in] msg The message to be written. /// \param[in] options The WriteOptions to be used to write this message. /// \param[in] tag The tag identifying the operation. @@ -144,6 +150,9 @@ class AsyncWriterInterface { /// the flow control window size. If \a msg size is larger than the window /// size, it will be sent on wire without buffering. /// + /// gRPC doesn't take ownership or a reference to \a msg, so it is safe to + /// to deallocate once Write returns. + /// /// \param[in] msg The message to be written. /// \param[in] options The WriteOptions to be used to write this message. /// \param[in] tag The tag identifying the operation. @@ -195,6 +204,13 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface<R> { assert(size == sizeof(ClientAsyncReader)); } + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + void StartCall(void* tag) override { assert(!started_); started_ = true; @@ -336,6 +352,13 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface<W> { assert(size == sizeof(ClientAsyncWriter)); } + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + void StartCall(void* tag) override { assert(!started_); started_ = true; @@ -496,6 +519,13 @@ class ClientAsyncReaderWriter final assert(size == sizeof(ClientAsyncReaderWriter)); } + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + void StartCall(void* tag) override { assert(!started_); started_ = true; @@ -630,6 +660,9 @@ class ServerAsyncReaderInterface /// metadata (if not sent already), response message, and status, or if /// some failure occurred when trying to do so. /// + /// gRPC doesn't take ownership or a reference to \a msg or \a status, so it + /// is safe to to deallocate once Finish returns. + /// /// \param[in] tag Tag identifying this request. /// \param[in] status To be sent to the client as the result of this call. /// \param[in] msg To be sent to the client as the response for this call. @@ -650,6 +683,9 @@ class ServerAsyncReaderInterface /// metadata (if not sent already), and status, or if some failure occurred /// when trying to do so. /// + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once FinishWithError returns. + /// /// \param[in] tag Tag identifying this request. /// \param[in] status To be sent to the client as the result of this call. /// - Note: \a status must have a non-OK code. @@ -697,6 +733,9 @@ class ServerAsyncReader final : public ServerAsyncReaderInterface<W, R> { /// initial and trailing metadata. /// /// Note: \a msg is not sent if \a status has a non-OK code. + /// + /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it + /// is safe to to deallocate once Finish returns. void Finish(const W& msg, const Status& status, void* tag) override { finish_ops_.set_output_tag(tag); if (!ctx_->sent_initial_metadata_) { @@ -723,6 +762,9 @@ class ServerAsyncReader final : public ServerAsyncReaderInterface<W, R> { /// - also sends initial metadata if not alreay sent. /// - uses the \a ServerContext associated with this call to send possible /// initial and trailing metadata. + /// + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once FinishWithError returns. void FinishWithError(const Status& status, void* tag) override { GPR_CODEGEN_ASSERT(!status.ok()); finish_ops_.set_output_tag(tag); @@ -773,6 +815,9 @@ class ServerAsyncWriterInterface /// metadata (if not sent already), response message, and status, or if /// some failure occurred when trying to do so. /// + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once Finish returns. + /// /// \param[in] tag Tag identifying this request. /// \param[in] status To be sent to the client as the result of this call. virtual void Finish(const Status& status, void* tag) = 0; @@ -784,6 +829,9 @@ class ServerAsyncWriterInterface /// WriteAndFinish is equivalent of performing WriteLast and Finish /// in a single step. /// + /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it + /// is safe to to deallocate once WriteAndFinish returns. + /// /// \param[in] msg The message to be written. /// \param[in] options The WriteOptions to be used to write this message. /// \param[in] status The Status that server returns to client. @@ -847,6 +895,9 @@ class ServerAsyncWriter final : public ServerAsyncWriterInterface<W> { /// for sending trailing (and initial) metadata to the client. /// /// Note: \a status must have an OK code. + /// + /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it + /// is safe to to deallocate once WriteAndFinish returns. void WriteAndFinish(const W& msg, WriteOptions options, const Status& status, void* tag) override { write_ops_.set_output_tag(tag); @@ -865,6 +916,9 @@ class ServerAsyncWriter final : public ServerAsyncWriterInterface<W> { /// /// Note: there are no restrictions are the code of /// \a status,it may be non-OK + /// + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once Finish returns. void Finish(const Status& status, void* tag) override { finish_ops_.set_output_tag(tag); EnsureInitialMetadataSent(&finish_ops_); @@ -924,6 +978,9 @@ class ServerAsyncReaderWriterInterface /// metadata (if not sent already), response message, and status, or if some /// failure occurred when trying to do so. /// + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once Finish returns. + /// /// \param[in] tag Tag identifying this request. /// \param[in] status To be sent to the client as the result of this call. virtual void Finish(const Status& status, void* tag) = 0; @@ -935,6 +992,9 @@ class ServerAsyncReaderWriterInterface /// WriteAndFinish is equivalent of performing WriteLast and Finish in a /// single step. /// + /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it + /// is safe to to deallocate once WriteAndFinish returns. + /// /// \param[in] msg The message to be written. /// \param[in] options The WriteOptions to be used to write this message. /// \param[in] status The Status that server returns to client. @@ -1006,6 +1066,9 @@ class ServerAsyncReaderWriter final /// for sending trailing (and initial) metadata to the client. /// /// Note: \a status must have an OK code. + // + /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it + /// is safe to to deallocate once WriteAndFinish returns. void WriteAndFinish(const W& msg, WriteOptions options, const Status& status, void* tag) override { write_ops_.set_output_tag(tag); @@ -1024,6 +1087,9 @@ class ServerAsyncReaderWriter final /// /// Note: there are no restrictions are the code of \a status, /// it may be non-OK + // + /// gRPC doesn't take ownership or a reference to \a status, so it is safe to + /// to deallocate once Finish returns. void Finish(const Status& status, void* tag) override { finish_ops_.set_output_tag(tag); EnsureInitialMetadataSent(&finish_ops_); diff --git a/include/grpcpp/impl/codegen/call.h b/include/grpcpp/impl/codegen/call.h index a5e930aaa5..7cadea0055 100644 --- a/include/grpcpp/impl/codegen/call.h +++ b/include/grpcpp/impl/codegen/call.h @@ -50,8 +50,6 @@ namespace internal { class Call; class CallHook; -const char kBinaryErrorDetailsKey[] = "grpc-status-details-bin"; - // TODO(yangg) if the map is changed before we send, the pointers will be a // mess. Make sure it does not happen. inline grpc_metadata* FillMetadataArray( @@ -531,7 +529,6 @@ class CallOpRecvInitialMetadata { void FinishOp(bool* status) { if (metadata_map_ == nullptr) return; - metadata_map_->FillMap(); metadata_map_ = nullptr; } @@ -566,13 +563,7 @@ class CallOpClientRecvStatus { void FinishOp(bool* status) { if (recv_status_ == nullptr) return; - metadata_map_->FillMap(); - grpc::string binary_error_details; - auto iter = metadata_map_->map()->find(kBinaryErrorDetailsKey); - if (iter != metadata_map_->map()->end()) { - binary_error_details = - grpc::string(iter->second.begin(), iter->second.length()); - } + grpc::string binary_error_details = metadata_map_->GetBinaryErrorDetails(); *recv_status_ = Status(static_cast<StatusCode>(status_code_), GRPC_SLICE_IS_EMPTY(error_message_) @@ -608,6 +599,11 @@ class CallOpSetInterface : public CompletionQueueTag { /// Fills in grpc_op, starting from ops[*nops] and moving /// upwards. virtual void FillOps(grpc_call* call, grpc_op* ops, size_t* nops) = 0; + + /// Get the tag to be used at the core completion queue. Generally, the + /// value of cq_tag will be "this". However, it can be overridden if we + /// want core to process the tag differently (e.g., as a core callback) + virtual void* cq_tag() = 0; }; /// Primary implementation of CallOpSetInterface. @@ -627,7 +623,7 @@ class CallOpSet : public CallOpSetInterface, public Op5, public Op6 { public: - CallOpSet() : return_tag_(this), call_(nullptr) {} + CallOpSet() : cq_tag_(this), return_tag_(this), call_(nullptr) {} void FillOps(grpc_call* call, grpc_op* ops, size_t* nops) override { this->Op1::AddOp(ops, nops); this->Op2::AddOp(ops, nops); @@ -654,7 +650,16 @@ class CallOpSet : public CallOpSetInterface, void set_output_tag(void* return_tag) { return_tag_ = return_tag; } + void* cq_tag() override { return cq_tag_; } + + /// set_cq_tag is used to provide a different core CQ tag than "this". + /// This is used for callback-based tags, where the core tag is the core + /// callback function. It does not change the use or behavior of any other + /// function (such as FinalizeResult) + void set_cq_tag(void* cq_tag) { cq_tag_ = cq_tag; } + private: + void* cq_tag_; void* return_tag_; grpc_call* call_; }; diff --git a/include/grpcpp/impl/codegen/callback_common.h b/include/grpcpp/impl/codegen/callback_common.h new file mode 100644 index 0000000000..ca2f867d04 --- /dev/null +++ b/include/grpcpp/impl/codegen/callback_common.h @@ -0,0 +1,164 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_IMPL_CODEGEN_CALLBACK_COMMON_H +#define GRPCPP_IMPL_CODEGEN_CALLBACK_COMMON_H + +#include <functional> + +#include <grpc/impl/codegen/grpc_types.h> +#include <grpcpp/impl/codegen/call.h> +#include <grpcpp/impl/codegen/channel_interface.h> +#include <grpcpp/impl/codegen/config.h> +#include <grpcpp/impl/codegen/core_codegen_interface.h> +#include <grpcpp/impl/codegen/status.h> + +namespace grpc { +namespace internal { + +/// An exception-safe way of invoking a user-specified callback function +template <class Func, class Arg> +void CatchingCallback(Func&& func, Arg&& arg) { +#if GRPC_ALLOW_EXCEPTIONS + try { + func(arg); + } catch (...) { + // nothing to return or change here, just don't crash the library + } +#else // GRPC_ALLOW_EXCEPTIONS + func(arg); +#endif // GRPC_ALLOW_EXCEPTIONS +} + +// The contract on these tags is that they are single-shot. They must be +// constructed and then fired at exactly one point. There is no expectation +// that they can be reused without reconstruction. + +class CallbackWithStatusTag + : public grpc_experimental_completion_queue_functor { + public: + // always allocated against a call arena, no memory free required + static void operator delete(void* ptr, std::size_t size) { + assert(size == sizeof(CallbackWithStatusTag)); + } + + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + + CallbackWithStatusTag(grpc_call* call, std::function<void(Status)> f, + CompletionQueueTag* ops) + : call_(call), func_(std::move(f)), ops_(ops) { + g_core_codegen_interface->grpc_call_ref(call); + functor_run = &CallbackWithStatusTag::StaticRun; + } + ~CallbackWithStatusTag() {} + Status* status_ptr() { return &status_; } + + // force_run can not be performed on a tag if operations using this tag + // have been sent to PerformOpsOnCall. It is intended for error conditions + // that are detected before the operations are internally processed. + void force_run(Status s) { + status_ = std::move(s); + Run(true); + } + + private: + grpc_call* call_; + std::function<void(Status)> func_; + CompletionQueueTag* ops_; + Status status_; + + static void StaticRun(grpc_experimental_completion_queue_functor* cb, + int ok) { + static_cast<CallbackWithStatusTag*>(cb)->Run(static_cast<bool>(ok)); + } + void Run(bool ok) { + void* ignored = ops_; + + GPR_CODEGEN_ASSERT(ops_->FinalizeResult(&ignored, &ok)); + GPR_CODEGEN_ASSERT(ignored == ops_); + + // Last use of func_ or status_, so ok to move them out + CatchingCallback(std::move(func_), std::move(status_)); + + func_ = nullptr; // reset to clear this out for sure + status_ = Status(); // reset to clear this out for sure + g_core_codegen_interface->grpc_call_unref(call_); + } +}; + +class CallbackWithSuccessTag + : public grpc_experimental_completion_queue_functor { + public: + // always allocated against a call arena, no memory free required + static void operator delete(void* ptr, std::size_t size) { + assert(size == sizeof(CallbackWithSuccessTag)); + } + + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + + CallbackWithSuccessTag(grpc_call* call, std::function<void(bool)> f, + CompletionQueueTag* ops) + : call_(call), func_(std::move(f)), ops_(ops) { + g_core_codegen_interface->grpc_call_ref(call); + functor_run = &CallbackWithSuccessTag::StaticRun; + } + + CompletionQueueTag* ops() { return ops_; } + + // force_run can not be performed on a tag if operations using this tag + // have been sent to PerformOpsOnCall. It is intended for error conditions + // that are detected before the operations are internally processed. + void force_run(bool ok) { Run(ok); } + + private: + grpc_call* call_; + std::function<void(bool)> func_; + CompletionQueueTag* ops_; + + static void StaticRun(grpc_experimental_completion_queue_functor* cb, + int ok) { + static_cast<CallbackWithSuccessTag*>(cb)->Run(static_cast<bool>(ok)); + } + void Run(bool ok) { + void* ignored = ops_; + bool new_ok = ok; + GPR_CODEGEN_ASSERT(ops_->FinalizeResult(&ignored, &new_ok)); + GPR_CODEGEN_ASSERT(ignored == ops_); + + // Last use of func_, so ok to move it out for rvalue call above + CatchingCallback(std::move(func_), ok); + + func_ = nullptr; // reset to clear this out for sure + g_core_codegen_interface->grpc_call_unref(call_); + } +}; + +} // namespace internal +} // namespace grpc + +#endif // GRPCPP_IMPL_CODEGEN_CALLBACK_COMMON_H diff --git a/include/grpcpp/impl/codegen/channel_interface.h b/include/grpcpp/impl/codegen/channel_interface.h index ec1c6c25d8..b257acc1ab 100644 --- a/include/grpcpp/impl/codegen/channel_interface.h +++ b/include/grpcpp/impl/codegen/channel_interface.h @@ -41,6 +41,8 @@ class CallOpSetInterface; class RpcMethod; template <class InputMessage, class OutputMessage> class BlockingUnaryCallImpl; +template <class InputMessage, class OutputMessage> +class CallbackUnaryCallImpl; template <class R> class ClientAsyncReaderFactory; template <class W> @@ -103,6 +105,8 @@ class ChannelInterface { friend class ::grpc::internal::ClientAsyncResponseReaderFactory; template <class InputMessage, class OutputMessage> friend class ::grpc::internal::BlockingUnaryCallImpl; + template <class InputMessage, class OutputMessage> + friend class ::grpc::internal::CallbackUnaryCallImpl; friend class ::grpc::internal::RpcMethod; virtual internal::Call CreateCall(const internal::RpcMethod& method, ClientContext* context, @@ -115,6 +119,16 @@ class ChannelInterface { CompletionQueue* cq, void* tag) = 0; virtual bool WaitForStateChangeImpl(grpc_connectivity_state last_observed, gpr_timespec deadline) = 0; + + // EXPERIMENTAL + // A method to get the callbackable completion queue associated with this + // channel. If the return value is nullptr, this channel doesn't support + // callback operations. + // TODO(vjpai): Consider a better default like using a global CQ + // Returns nullptr (rather than being pure) since this is a new method + // and adding a new pure method to an interface would be a breaking change + // (even though this is private and non-API) + virtual CompletionQueue* CallbackCQ() { return nullptr; } }; } // namespace grpc diff --git a/include/grpcpp/impl/codegen/client_callback.h b/include/grpcpp/impl/codegen/client_callback.h new file mode 100644 index 0000000000..4d4faea063 --- /dev/null +++ b/include/grpcpp/impl/codegen/client_callback.h @@ -0,0 +1,95 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_IMPL_CODEGEN_CLIENT_CALLBACK_H +#define GRPCPP_IMPL_CODEGEN_CLIENT_CALLBACK_H + +#include <functional> + +#include <grpcpp/impl/codegen/call.h> +#include <grpcpp/impl/codegen/callback_common.h> +#include <grpcpp/impl/codegen/channel_interface.h> +#include <grpcpp/impl/codegen/config.h> +#include <grpcpp/impl/codegen/core_codegen_interface.h> +#include <grpcpp/impl/codegen/status.h> + +namespace grpc { + +class Channel; +class ClientContext; +class CompletionQueue; + +namespace internal { +class RpcMethod; + +/// Perform a callback-based unary call +/// TODO(vjpai): Combine as much as possible with the blocking unary call code +template <class InputMessage, class OutputMessage> +void CallbackUnaryCall(ChannelInterface* channel, const RpcMethod& method, + ClientContext* context, const InputMessage* request, + OutputMessage* result, + std::function<void(Status)> on_completion) { + CallbackUnaryCallImpl<InputMessage, OutputMessage> x( + channel, method, context, request, result, on_completion); +} + +template <class InputMessage, class OutputMessage> +class CallbackUnaryCallImpl { + public: + CallbackUnaryCallImpl(ChannelInterface* channel, const RpcMethod& method, + ClientContext* context, const InputMessage* request, + OutputMessage* result, + std::function<void(Status)> on_completion) { + CompletionQueue* cq = channel->CallbackCQ(); + GPR_CODEGEN_ASSERT(cq != nullptr); + Call call(channel->CreateCall(method, context, cq)); + + using FullCallOpSet = + CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, + CallOpRecvInitialMetadata, CallOpRecvMessage<OutputMessage>, + CallOpClientSendClose, CallOpClientRecvStatus>; + + auto* ops = new (g_core_codegen_interface->grpc_call_arena_alloc( + call.call(), sizeof(FullCallOpSet))) FullCallOpSet; + + auto* tag = new (g_core_codegen_interface->grpc_call_arena_alloc( + call.call(), sizeof(CallbackWithStatusTag))) + CallbackWithStatusTag(call.call(), on_completion, ops); + + // TODO(vjpai): Unify code with sync API as much as possible + Status s = ops->SendMessage(*request); + if (!s.ok()) { + tag->force_run(s); + return; + } + ops->SendInitialMetadata(context->send_initial_metadata_, + context->initial_metadata_flags()); + ops->RecvInitialMetadata(context); + ops->RecvMessage(result); + ops->AllowNoMessage(); + ops->ClientSendClose(); + ops->ClientRecvStatus(context, tag->status_ptr()); + ops->set_cq_tag(tag); + call.PerformOps(ops); + } +}; + +} // namespace internal +} // namespace grpc + +#endif // GRPCPP_IMPL_CODEGEN_CLIENT_CALLBACK_H diff --git a/include/grpcpp/impl/codegen/client_context.h b/include/grpcpp/impl/codegen/client_context.h index 9dda4c7fac..46635a541a 100644 --- a/include/grpcpp/impl/codegen/client_context.h +++ b/include/grpcpp/impl/codegen/client_context.h @@ -68,6 +68,8 @@ class CallOpClientRecvStatus; class CallOpRecvInitialMetadata; template <class InputMessage, class OutputMessage> class BlockingUnaryCallImpl; +template <class InputMessage, class OutputMessage> +class CallbackUnaryCallImpl; } // namespace internal template <class R> @@ -389,6 +391,8 @@ class ClientContext { friend class ::grpc::ClientAsyncResponseReader; template <class InputMessage, class OutputMessage> friend class ::grpc::internal::BlockingUnaryCallImpl; + template <class InputMessage, class OutputMessage> + friend class ::grpc::internal::CallbackUnaryCallImpl; // Used by friend class CallOpClientRecvStatus void set_debug_error_string(const grpc::string& debug_error_string) { @@ -425,8 +429,8 @@ class ClientContext { mutable std::shared_ptr<const AuthContext> auth_context_; struct census_context* census_context_; std::multimap<grpc::string, grpc::string> send_initial_metadata_; - internal::MetadataMap recv_initial_metadata_; - internal::MetadataMap trailing_metadata_; + mutable internal::MetadataMap recv_initial_metadata_; + mutable internal::MetadataMap trailing_metadata_; grpc_call* propagate_from_call_; PropagationOptions propagation_options_; diff --git a/include/grpcpp/impl/codegen/completion_queue.h b/include/grpcpp/impl/codegen/completion_queue.h index 3f7d4fb765..f52f9a53be 100644 --- a/include/grpcpp/impl/codegen/completion_queue.h +++ b/include/grpcpp/impl/codegen/completion_queue.h @@ -274,6 +274,9 @@ class CompletionQueue : private GrpcLibraryCodegen { template <class InputMessage, class OutputMessage> friend class ::grpc::internal::BlockingUnaryCallImpl; + // Friends that need access to constructor for callback CQ + friend class ::grpc::Channel; + /// EXPERIMENTAL /// Creates a Thread Local cache to store the first event /// On this completion queue queued from this thread. Once diff --git a/include/grpcpp/impl/codegen/completion_queue_tag.h b/include/grpcpp/impl/codegen/completion_queue_tag.h index ffb642c56b..304386a9ec 100644 --- a/include/grpcpp/impl/codegen/completion_queue_tag.h +++ b/include/grpcpp/impl/codegen/completion_queue_tag.h @@ -26,10 +26,25 @@ namespace internal { class CompletionQueueTag { public: virtual ~CompletionQueueTag() {} - /// Called prior to returning from Next(), return value is the status of the - /// operation (return status is the default thing to do). If this function - /// returns false, the tag is dropped and not returned from the completion - /// queue + + /// FinalizeResult must be called before informing user code that the + /// operation bound to the underlying core completion queue tag has + /// completed. In practice, this means: + /// + /// 1. For the sync API - before returning from Pluck + /// 2. For the CQ-based async API - before returning from Next + /// 3. For the callback-based API - before invoking the user callback + /// + /// This is the method that translates from core-side tag/status to + /// C++ API-observable tag/status. + /// + /// The return value is the status of the operation (returning status is the + /// general behavior of this function). If this function returns false, the + /// tag is dropped and not returned from the completion queue: this concept is + /// for events that are observed at core but not requested by the user + /// application (e.g., server shutdown, for server unimplemented method + /// responses, or for cases where a server-side RPC doesn't have a completion + /// notification registered using AsyncNotifyWhenDone) virtual bool FinalizeResult(void** tag, bool* status) = 0; }; } // namespace internal diff --git a/include/grpcpp/impl/codegen/metadata_map.h b/include/grpcpp/impl/codegen/metadata_map.h index 0866539d88..5e062a50f8 100644 --- a/include/grpcpp/impl/codegen/metadata_map.h +++ b/include/grpcpp/impl/codegen/metadata_map.h @@ -19,11 +19,15 @@ #ifndef GRPCPP_IMPL_CODEGEN_METADATA_MAP_H #define GRPCPP_IMPL_CODEGEN_METADATA_MAP_H +#include <grpc/impl/codegen/log.h> #include <grpcpp/impl/codegen/slice.h> namespace grpc { namespace internal { + +const char kBinaryErrorDetailsKey[] = "grpc-status-details-bin"; + class MetadataMap { public: MetadataMap() { memset(&arr_, 0, sizeof(arr_)); } @@ -32,24 +36,54 @@ class MetadataMap { g_core_codegen_interface->grpc_metadata_array_destroy(&arr_); } - void FillMap() { - for (size_t i = 0; i < arr_.count; i++) { - // TODO(yangg) handle duplicates? - map_.insert(std::pair<grpc::string_ref, grpc::string_ref>( - StringRefFromSlice(&arr_.metadata[i].key), - StringRefFromSlice(&arr_.metadata[i].value))); + grpc::string GetBinaryErrorDetails() { + // if filled_, extract from the multimap for O(log(n)) + if (filled_) { + auto iter = map_.find(kBinaryErrorDetailsKey); + if (iter != map_.end()) { + return grpc::string(iter->second.begin(), iter->second.length()); + } + } + // if not yet filled, take the O(n) lookup to avoid allocating the + // multimap until it is requested. + // TODO(ncteisen): plumb this through core as a first class object, just + // like code and message. + else { + for (size_t i = 0; i < arr_.count; i++) { + if (strncmp(reinterpret_cast<const char*>( + GRPC_SLICE_START_PTR(arr_.metadata[i].key)), + kBinaryErrorDetailsKey, + GRPC_SLICE_LENGTH(arr_.metadata[i].key)) == 0) { + return grpc::string(reinterpret_cast<const char*>( + GRPC_SLICE_START_PTR(arr_.metadata[i].value)), + GRPC_SLICE_LENGTH(arr_.metadata[i].value)); + } + } } + return grpc::string(); } - std::multimap<grpc::string_ref, grpc::string_ref>* map() { return &map_; } - const std::multimap<grpc::string_ref, grpc::string_ref>* map() const { + std::multimap<grpc::string_ref, grpc::string_ref>* map() { + FillMap(); return &map_; } grpc_metadata_array* arr() { return &arr_; } private: + bool filled_ = false; grpc_metadata_array arr_; std::multimap<grpc::string_ref, grpc::string_ref> map_; + + void FillMap() { + if (filled_) return; + filled_ = true; + for (size_t i = 0; i < arr_.count; i++) { + // TODO(yangg) handle duplicates? + map_.insert(std::pair<grpc::string_ref, grpc::string_ref>( + StringRefFromSlice(&arr_.metadata[i].key), + StringRefFromSlice(&arr_.metadata[i].value))); + } + } }; } // namespace internal diff --git a/include/grpcpp/impl/codegen/server_context.h b/include/grpcpp/impl/codegen/server_context.h index 6314364db6..b58f029de9 100644 --- a/include/grpcpp/impl/codegen/server_context.h +++ b/include/grpcpp/impl/codegen/server_context.h @@ -294,7 +294,7 @@ class ServerContext { CompletionQueue* cq_; bool sent_initial_metadata_; mutable std::shared_ptr<const AuthContext> auth_context_; - internal::MetadataMap client_metadata_; + mutable internal::MetadataMap client_metadata_; std::multimap<grpc::string, grpc::string> initial_metadata_; std::multimap<grpc::string, grpc::string> trailing_metadata_; diff --git a/include/grpcpp/support/client_callback.h b/include/grpcpp/support/client_callback.h new file mode 100644 index 0000000000..063fdc4f85 --- /dev/null +++ b/include/grpcpp/support/client_callback.h @@ -0,0 +1,24 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_SUPPORT_CLIENT_CALLBACK_H +#define GRPCPP_SUPPORT_CLIENT_CALLBACK_H + +#include <grpcpp/impl/codegen/client_callback.h> + +#endif // GRPCPP_SUPPORT_CLIENT_CALLBACK_H diff --git a/package.xml b/package.xml index 9c0e078e92..fddc676d51 100644 --- a/package.xml +++ b/package.xml @@ -200,6 +200,7 @@ <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_settings.h" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_window_update.h" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_encoder.h" role="src" /> + <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_mapping.h" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_parser.h" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_table.h" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/http2_settings.h" role="src" /> @@ -345,7 +346,6 @@ <file baseinstalldir="/" name="src/core/lib/iomgr/error_internal.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll1_linux.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollex_linux.h" role="src" /> - <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollsig_linux.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_poll_posix.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_posix.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/exec_ctx.h" role="src" /> @@ -491,7 +491,6 @@ <file baseinstalldir="/" name="src/core/lib/iomgr/error.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epoll1_linux.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollex_linux.cc" role="src" /> - <file baseinstalldir="/" name="src/core/lib/iomgr/ev_epollsig_linux.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_poll_posix.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_posix.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/ev_windows.cc" role="src" /> @@ -622,6 +621,7 @@ <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_settings.cc" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_window_update.cc" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_encoder.cc" role="src" /> + <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_mapping.cc" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_parser.cc" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_table.cc" role="src" /> <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/http2_settings.cc" role="src" /> diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index 1e0c36451b..56716493dc 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -128,6 +128,7 @@ grpc::string GetHeaderIncludes(grpc_generator::File* file, ""); } static const char* headers_strs[] = { + "functional", "grpcpp/impl/codegen/async_generic_service.h", "grpcpp/impl/codegen/async_stream.h", "grpcpp/impl/codegen/async_unary_call.h", @@ -547,6 +548,116 @@ void PrintHeaderClientMethod(grpc_generator::Printer* printer, } } +void PrintHeaderClientMethodCallbackInterfacesStart( + grpc_generator::Printer* printer, + std::map<grpc::string, grpc::string>* vars) { + // This declares the interface for the callback-based API. The components + // are pure; even though this is new (post-1.0) API, it can be pure because + // it is an entirely new interface that happens to be scoped within + // StubInterface, not new additions to StubInterface itself + printer->Print("class experimental_async_interface {\n"); + // All methods in this new interface are public. There is no need for private + // "Raw" methods since the callback-based API returns unowned raw pointers + printer->Print(" public:\n"); + printer->Indent(); + printer->Print("virtual ~experimental_async_interface() {}\n"); +} + +void PrintHeaderClientMethodCallbackInterfaces( + grpc_generator::Printer* printer, const grpc_generator::Method* method, + std::map<grpc::string, grpc::string>* vars, bool is_public) { + // Reserve is_public for future expansion + assert(is_public); + + (*vars)["Method"] = method->name(); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + + if (method->NoStreaming()) { + printer->Print(*vars, + "virtual void $Method$(::grpc::ClientContext* context, " + "const $Request$* request, $Response$* response, " + "std::function<void(::grpc::Status)>) = 0;\n"); + } else if (ClientOnlyStreaming(method)) { + // TODO(vjpai): Add support for client-side streaming + } else if (ServerOnlyStreaming(method)) { + // TODO(vjpai): Add support for server-side streaming + } else if (method->BidiStreaming()) { + // TODO(vjpai): Add support for bidi streaming + } +} + +void PrintHeaderClientMethodCallbackInterfacesEnd( + grpc_generator::Printer* printer, + std::map<grpc::string, grpc::string>* vars) { + printer->Outdent(); + printer->Print("};\n"); + + // Declare a function to give the async stub contents. It can't be pure + // since this is a new API in StubInterface, but it is meaningless by default + // (since any stub that wants to use it must have its own implementation of + // the callback functions therein), so make the default return value nullptr. + // Intentionally include the word "class" to avoid possible shadowing. + printer->Print( + "virtual class experimental_async_interface* experimental_async() { " + "return nullptr; }\n"); +} + +void PrintHeaderClientMethodCallbackStart( + grpc_generator::Printer* printer, + std::map<grpc::string, grpc::string>* vars) { + // This declares the stub entry for the callback-based API. + printer->Print("class experimental_async final :\n"); + printer->Print(" public StubInterface::experimental_async_interface {\n"); + printer->Print(" public:\n"); + printer->Indent(); +} + +void PrintHeaderClientMethodCallback(grpc_generator::Printer* printer, + const grpc_generator::Method* method, + std::map<grpc::string, grpc::string>* vars, + bool is_public) { + // Reserve is_public for future expansion + assert(is_public); + + (*vars)["Method"] = method->name(); + (*vars)["Request"] = method->input_type_name(); + (*vars)["Response"] = method->output_type_name(); + + if (method->NoStreaming()) { + printer->Print(*vars, + "void $Method$(::grpc::ClientContext* context, " + "const $Request$* request, $Response$* response, " + "std::function<void(::grpc::Status)>) override;\n"); + } else if (ClientOnlyStreaming(method)) { + // TODO(vjpai): Add support for client-side streaming + } else if (ServerOnlyStreaming(method)) { + // TODO(vjpai): Add support for server-side streaming + } else if (method->BidiStreaming()) { + // TODO(vjpai): Add support for bidi streaming + } +} + +void PrintHeaderClientMethodCallbackEnd( + grpc_generator::Printer* printer, + std::map<grpc::string, grpc::string>* vars) { + printer->Outdent(); + printer->Print(" private:\n"); + printer->Indent(); + printer->Print("friend class Stub;\n"); + printer->Print("explicit experimental_async(Stub* stub): stub_(stub) { }\n"); + // include a function with a dummy use of stub_ to avoid an unused + // private member warning for service with no methods + printer->Print("Stub* stub() { return stub_; }\n"); + printer->Print("Stub* stub_;\n"); + printer->Outdent(); + printer->Print("};\n"); + + printer->Print( + "class experimental_async_interface* experimental_async() override { " + "return &async_stub_; }\n"); +} + void PrintHeaderClientMethodData(grpc_generator::Printer* printer, const grpc_generator::Method* method, std::map<grpc::string, grpc::string>* vars) { @@ -951,6 +1062,14 @@ void PrintHeaderService(grpc_generator::Printer* printer, true); printer->Print(service->method(i)->GetTrailingComments("//").c_str()); } + PrintHeaderClientMethodCallbackInterfacesStart(printer, vars); + for (int i = 0; i < service->method_count(); ++i) { + printer->Print(service->method(i)->GetLeadingComments("//").c_str()); + PrintHeaderClientMethodCallbackInterfaces(printer, service->method(i).get(), + vars, true); + printer->Print(service->method(i)->GetTrailingComments("//").c_str()); + } + PrintHeaderClientMethodCallbackInterfacesEnd(printer, vars); printer->Outdent(); printer->Print("private:\n"); printer->Indent(); @@ -970,10 +1089,17 @@ void PrintHeaderService(grpc_generator::Printer* printer, for (int i = 0; i < service->method_count(); ++i) { PrintHeaderClientMethod(printer, service->method(i).get(), vars, true); } + PrintHeaderClientMethodCallbackStart(printer, vars); + for (int i = 0; i < service->method_count(); ++i) { + PrintHeaderClientMethodCallback(printer, service->method(i).get(), vars, + true); + } + PrintHeaderClientMethodCallbackEnd(printer, vars); printer->Outdent(); printer->Print("\n private:\n"); printer->Indent(); printer->Print("std::shared_ptr< ::grpc::ChannelInterface> channel_;\n"); + printer->Print("class experimental_async async_stub_{this};\n"); for (int i = 0; i < service->method_count(); ++i) { PrintHeaderClientMethod(printer, service->method(i).get(), vars, false); } @@ -1199,10 +1325,12 @@ grpc::string GetSourceIncludes(grpc_generator::File* file, std::map<grpc::string, grpc::string> vars; static const char* headers_strs[] = { + "functional", "grpcpp/impl/codegen/async_stream.h", "grpcpp/impl/codegen/async_unary_call.h", "grpcpp/impl/codegen/channel_interface.h", "grpcpp/impl/codegen/client_unary_call.h", + "grpcpp/impl/codegen/client_callback.h", "grpcpp/impl/codegen/method_handler_impl.h", "grpcpp/impl/codegen/rpc_service_method.h", "grpcpp/impl/codegen/service_type.h", @@ -1247,6 +1375,17 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, " return ::grpc::internal::BlockingUnaryCall" "(channel_.get(), rpcmethod_$Method$_, " "context, request, response);\n}\n\n"); + + printer->Print(*vars, + "void $ns$$Service$::Stub::experimental_async::$Method$(" + "::grpc::ClientContext* context, " + "const $Request$* request, $Response$* response, " + "std::function<void(::grpc::Status)> f) {\n"); + printer->Print(*vars, + " return ::grpc::internal::CallbackUnaryCall" + "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, " + "context, request, response, std::move(f));\n}\n\n"); + for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; (*vars)["AsyncStart"] = async_prefix.start; @@ -1277,6 +1416,9 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "rpcmethod_$Method$_, " "context, response);\n" "}\n\n"); + + // TODO(vjpai): Add callback version + for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; (*vars)["AsyncStart"] = async_prefix.start; @@ -1308,6 +1450,9 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "rpcmethod_$Method$_, " "context, request);\n" "}\n\n"); + + // TODO(vjpai): Add callback version + for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; (*vars)["AsyncStart"] = async_prefix.start; @@ -1339,6 +1484,9 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "rpcmethod_$Method$_, " "context);\n" "}\n\n"); + + // TODO(vjpai): Add callback version + for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; (*vars)["AsyncStart"] = async_prefix.start; diff --git a/src/compiler/ruby_generator.cc b/src/compiler/ruby_generator.cc index c7af9c38fa..e39d8be5d4 100644 --- a/src/compiler/ruby_generator.cc +++ b/src/compiler/ruby_generator.cc @@ -160,12 +160,20 @@ grpc::string GetServices(const FileDescriptor* file) { return output; } + std::string package_name; + + if (file->options().has_ruby_package()) { + package_name = file->options().ruby_package(); + } else { + package_name = file->package(); + } + // Write out a file header. std::map<grpc::string, grpc::string> header_comment_vars = ListToDict({ "file.name", file->name(), "file.package", - file->package(), + package_name, }); out.Print("# Generated by the protocol buffer compiler. DO NOT EDIT!\n"); out.Print(header_comment_vars, @@ -190,7 +198,7 @@ grpc::string GetServices(const FileDescriptor* file) { // Write out services within the modules out.Print("\n"); - std::vector<grpc::string> modules = Split(file->package(), '.'); + std::vector<grpc::string> modules = Split(package_name, '.'); for (size_t i = 0; i < modules.size(); ++i) { std::map<grpc::string, grpc::string> module_vars = ListToDict({ "module.name", diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index d2bf4f388d..388736b60a 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -457,7 +457,6 @@ get_service_config_from_resolver_result_locked(channel_data* chand) { grpc_uri* uri = grpc_uri_parse(server_uri, true); GPR_ASSERT(uri->path[0] != '\0'); service_config_parsing_state parsing_state; - memset(&parsing_state, 0, sizeof(parsing_state)); parsing_state.server_name = uri->path[0] == '/' ? uri->path + 1 : uri->path; service_config->ParseGlobalParams(parse_retry_throttle_params, @@ -934,6 +933,11 @@ typedef struct client_channel_call_data { grpc_closure pick_closure; grpc_closure pick_cancel_closure; + // state needed to support channelz interception of recv trailing metadata. + grpc_closure recv_trailing_metadata_ready_channelz; + grpc_closure* original_recv_trailing_metadata; + grpc_metadata_batch* recv_trailing_metadata; + grpc_polling_entity* pollent; bool pollent_added_to_interested_parties; @@ -995,6 +999,8 @@ static void start_internal_recv_trailing_metadata(grpc_call_element* elem); static void on_complete(void* arg, grpc_error* error); static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored); static void start_pick_locked(void* arg, grpc_error* ignored); +static void maybe_intercept_recv_trailing_metadata_for_channelz( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch); // // send op data caching @@ -1293,6 +1299,7 @@ static void pending_batches_resume(grpc_call_element* elem) { pending_batch* pending = &calld->pending_batches[i]; grpc_transport_stream_op_batch* batch = pending->batch; if (batch != nullptr) { + maybe_intercept_recv_trailing_metadata_for_channelz(elem, batch); batch->handler_private.extra_arg = calld->subchannel_call; GRPC_CLOSURE_INIT(&batch->handler_private.closure, resume_pending_batch_in_call_combiner, batch, @@ -1778,23 +1785,22 @@ static void recv_message_ready(void* arg, grpc_error* error) { // recv_trailing_metadata handling // -// Sets *status and *server_pushback_md based on batch_data and error. -static void get_call_status(subchannel_batch_data* batch_data, - grpc_error* error, grpc_status_code* status, +// Sets *status and *server_pushback_md based on md_batch and error. +// Only sets *server_pushback_md if server_pushback_md != nullptr. +static void get_call_status(grpc_call_element* elem, + grpc_metadata_batch* md_batch, grpc_error* error, + grpc_status_code* status, grpc_mdelem** server_pushback_md) { - grpc_call_element* elem = batch_data->elem; call_data* calld = static_cast<call_data*>(elem->call_data); if (error != GRPC_ERROR_NONE) { grpc_error_get_status(error, calld->deadline, status, nullptr, nullptr, nullptr); } else { - grpc_metadata_batch* md_batch = - batch_data->batch.payload->recv_trailing_metadata - .recv_trailing_metadata; GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr); *status = grpc_get_status_code_from_metadata(md_batch->idx.named.grpc_status->md); - if (md_batch->idx.named.grpc_retry_pushback_ms != nullptr) { + if (server_pushback_md != nullptr && + md_batch->idx.named.grpc_retry_pushback_ms != nullptr) { *server_pushback_md = &md_batch->idx.named.grpc_retry_pushback_ms->md; } } @@ -1967,8 +1973,19 @@ static void recv_trailing_metadata_ready(void* arg, grpc_error* error) { // Get the call's status and check for server pushback metadata. grpc_status_code status = GRPC_STATUS_OK; grpc_mdelem* server_pushback_md = nullptr; - get_call_status(batch_data, GRPC_ERROR_REF(error), &status, + grpc_metadata_batch* md_batch = + batch_data->batch.payload->recv_trailing_metadata.recv_trailing_metadata; + get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, &server_pushback_md); + grpc_core::channelz::SubchannelNode* channelz_subchannel = + calld->pick.connected_subchannel->channelz_subchannel(); + if (channelz_subchannel != nullptr) { + if (status == GRPC_STATUS_OK) { + channelz_subchannel->RecordCallSucceeded(); + } else { + channelz_subchannel->RecordCallFailed(); + } + } if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: call finished, status=%s", chand, calld, grpc_status_code_to_string(status)); @@ -2573,6 +2590,69 @@ static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored) { } // +// Channelz +// + +static void recv_trailing_metadata_ready_channelz(void* arg, + grpc_error* error) { + grpc_call_element* elem = static_cast<grpc_call_element*>(arg); + channel_data* chand = static_cast<channel_data*>(elem->channel_data); + call_data* calld = static_cast<call_data*>(elem->call_data); + if (grpc_client_channel_trace.enabled()) { + gpr_log(GPR_INFO, + "chand=%p calld=%p: got recv_trailing_metadata_ready_channelz, " + "error=%s", + chand, calld, grpc_error_string(error)); + } + GPR_ASSERT(calld->recv_trailing_metadata != nullptr); + grpc_status_code status = GRPC_STATUS_OK; + grpc_metadata_batch* md_batch = calld->recv_trailing_metadata; + get_call_status(elem, md_batch, GRPC_ERROR_REF(error), &status, nullptr); + grpc_core::channelz::SubchannelNode* channelz_subchannel = + calld->pick.connected_subchannel->channelz_subchannel(); + GPR_ASSERT(channelz_subchannel != nullptr); + if (status == GRPC_STATUS_OK) { + channelz_subchannel->RecordCallSucceeded(); + } else { + channelz_subchannel->RecordCallFailed(); + } + calld->recv_trailing_metadata = nullptr; + GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata, error); +} + +// If channelz is enabled, intercept recv_trailing so that we may check the +// status and associate it to a subchannel. +// Returns true if callback was intercepted, false otherwise. +static void maybe_intercept_recv_trailing_metadata_for_channelz( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { + call_data* calld = static_cast<call_data*>(elem->call_data); + // only intercept payloads with recv trailing. + if (!batch->recv_trailing_metadata) { + return; + } + // only add interceptor is channelz is enabled. + if (calld->pick.connected_subchannel->channelz_subchannel() == nullptr) { + return; + } + if (grpc_client_channel_trace.enabled()) { + gpr_log(GPR_INFO, + "calld=%p batch=%p: intercepting recv trailing for channelz", calld, + batch); + } + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready_channelz, + recv_trailing_metadata_ready_channelz, elem, + grpc_schedule_on_exec_ctx); + // save some state needed for the interception callback. + GPR_ASSERT(calld->recv_trailing_metadata == nullptr); + calld->recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata; + calld->original_recv_trailing_metadata = + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &calld->recv_trailing_metadata_ready_channelz; +} + +// // LB pick // @@ -2601,6 +2681,11 @@ static void create_subchannel_call(grpc_call_element* elem, grpc_error* error) { new_error = grpc_error_add_child(new_error, error); pending_batches_fail(elem, new_error, true /* yield_call_combiner */); } else { + grpc_core::channelz::SubchannelNode* channelz_subchannel = + calld->pick.connected_subchannel->channelz_subchannel(); + if (channelz_subchannel != nullptr) { + channelz_subchannel->RecordCallStarted(); + } if (parent_data_size > 0) { subchannel_call_retry_state* retry_state = static_cast<subchannel_call_retry_state*>( diff --git a/src/core/ext/filters/client_channel/client_channel_channelz.cc b/src/core/ext/filters/client_channel/client_channel_channelz.cc index 86c765df52..7e8f59bcd3 100644 --- a/src/core/ext/filters/client_channel/client_channel_channelz.cc +++ b/src/core/ext/filters/client_channel/client_channel_channelz.cc @@ -20,10 +20,13 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/ext/filters/client_channel/client_channel_channelz.h" +#include "src/core/lib/channel/channelz_registry.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/connectivity_state.h" +#include <grpc/support/string_util.h> + namespace grpc_core { namespace channelz { namespace { @@ -109,5 +112,62 @@ RefCountedPtr<ChannelNode> ClientChannelNode::MakeClientChannelNode( is_top_level_channel); } +SubchannelNode::SubchannelNode(grpc_subchannel* subchannel, + size_t channel_tracer_max_nodes) + : BaseNode(EntityType::kSubchannel), + subchannel_(subchannel), + target_( + UniquePtr<char>(gpr_strdup(grpc_subchannel_get_target(subchannel_)))), + trace_(channel_tracer_max_nodes) {} + +SubchannelNode::~SubchannelNode() {} + +void SubchannelNode::PopulateConnectivityState(grpc_json* json) { + grpc_connectivity_state state; + if (subchannel_ == nullptr) { + state = GRPC_CHANNEL_SHUTDOWN; + } else { + state = grpc_subchannel_check_connectivity(subchannel_, nullptr); + } + json = grpc_json_create_child(nullptr, json, "state", nullptr, + GRPC_JSON_OBJECT, false); + grpc_json_create_child(nullptr, json, "state", + grpc_connectivity_state_name(state), GRPC_JSON_STRING, + false); +} + +grpc_json* SubchannelNode::RenderJson() { + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + json_iterator = grpc_json_create_child(json_iterator, json, "ref", nullptr, + GRPC_JSON_OBJECT, false); + json = json_iterator; + json_iterator = nullptr; + json_iterator = grpc_json_add_number_string_child(json, json_iterator, + "subchannelId", uuid()); + // reset json iterators to top level object + json = top_level_json; + json_iterator = nullptr; + // create and fill the data child. + grpc_json* data = grpc_json_create_child(json_iterator, json, "data", nullptr, + GRPC_JSON_OBJECT, false); + json = data; + json_iterator = nullptr; + PopulateConnectivityState(json); + GPR_ASSERT(target_.get() != nullptr); + grpc_json_create_child(nullptr, json, "target", target_.get(), + GRPC_JSON_STRING, false); + // fill in the channel trace if applicable + grpc_json* trace_json = trace_.RenderJson(); + if (trace_json != nullptr) { + trace_json->key = "trace"; // this object is named trace in channelz.proto + grpc_json_link_child(json, trace_json, nullptr); + } + // ask CallCountingHelper to populate trace and call count data. + call_counter_.PopulateCallCounts(json); + return top_level_json; +} + } // namespace channelz } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/client_channel_channelz.h b/src/core/ext/filters/client_channel/client_channel_channelz.h index 6f27b5c8b7..8ce331e529 100644 --- a/src/core/ext/filters/client_channel/client_channel_channelz.h +++ b/src/core/ext/filters/client_channel/client_channel_channelz.h @@ -23,9 +23,12 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/channel_trace.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/gprpp/inlined_vector.h" +typedef struct grpc_subchannel grpc_subchannel; + namespace grpc_core { // TODO(ncteisen), this only contains the uuids of the children for now, @@ -43,28 +46,59 @@ class ClientChannelNode : public ChannelNode { grpc_channel* channel, size_t channel_tracer_max_nodes, bool is_top_level_channel); - // Override this functionality since client_channels have a notion of - // channel connectivity. - void PopulateConnectivityState(grpc_json* json) override; + ClientChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, + bool is_top_level_channel); + virtual ~ClientChannelNode() {} - // Override this functionality since client_channels have subchannels + // Overriding template methods from ChannelNode to render information that + // only ClientChannelNode knows about. + void PopulateConnectivityState(grpc_json* json) override; void PopulateChildRefs(grpc_json* json) override; // Helper to create a channel arg to ensure this type of ChannelNode is // created. static grpc_arg CreateChannelArg(); - protected: - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW - ClientChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, - bool is_top_level_channel); - virtual ~ClientChannelNode() {} - private: grpc_channel_element* client_channel_; }; +// Handles channelz bookkeeping for sockets +class SubchannelNode : public BaseNode { + public: + SubchannelNode(grpc_subchannel* subchannel, size_t channel_tracer_max_nodes); + ~SubchannelNode() override; + + void MarkSubchannelDestroyed() { + GPR_ASSERT(subchannel_ != nullptr); + subchannel_ = nullptr; + } + + grpc_json* RenderJson() override; + + // proxy methods to composed classes. + void AddTraceEvent(ChannelTrace::Severity severity, grpc_slice data) { + trace_.AddTraceEvent(severity, data); + } + void AddTraceEventWithReference(ChannelTrace::Severity severity, + grpc_slice data, + RefCountedPtr<BaseNode> referenced_channel) { + trace_.AddTraceEventWithReference(severity, data, + std::move(referenced_channel)); + } + void RecordCallStarted() { call_counter_.RecordCallStarted(); } + void RecordCallFailed() { call_counter_.RecordCallFailed(); } + void RecordCallSucceeded() { call_counter_.RecordCallSucceeded(); } + + private: + grpc_subchannel* subchannel_; + UniquePtr<char> target_; + CallCountingHelper call_counter_; + ChannelTrace trace_; + + void PopulateConnectivityState(grpc_json* json); +}; + } // namespace channelz } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 25b0149393..0061190016 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -852,10 +852,12 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked( } } else { // No valid initial response or serverlist found. + char* response_slice_str = + grpc_dump_slice(response_slice, GPR_DUMP_ASCII | GPR_DUMP_HEX); gpr_log(GPR_ERROR, "[grpclb %p] Invalid LB response received: '%s'. Ignoring.", - grpclb_policy, - grpc_dump_slice(response_slice, GPR_DUMP_ASCII | GPR_DUMP_HEX)); + grpclb_policy, response_slice_str); + gpr_free(response_slice_str); } grpc_slice_unref_internal(response_slice); if (!grpclb_policy->shutting_down_) { @@ -1265,7 +1267,7 @@ void GrpcLb::FillChildRefsForChannelz(ChildRefsList* child_subchannels, grpc_core::channelz::ChannelNode* channel_node = grpc_channel_get_channelz_node(lb_channel_); if (channel_node != nullptr) { - child_channels->push_back(channel_node->channel_uuid()); + child_channels->push_back(channel_node->uuid()); } } } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 602d6e92f9..ed8cc60ea1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -71,11 +71,12 @@ class PickFirst : public LoadBalancingPolicy { : public SubchannelData<PickFirstSubchannelList, PickFirstSubchannelData> { public: - PickFirstSubchannelData(PickFirstSubchannelList* subchannel_list, - const grpc_lb_user_data_vtable* user_data_vtable, - const grpc_lb_address& address, - grpc_subchannel* subchannel, - grpc_combiner* combiner) + PickFirstSubchannelData( + SubchannelList<PickFirstSubchannelList, PickFirstSubchannelData>* + subchannel_list, + const grpc_lb_user_data_vtable* user_data_vtable, + const grpc_lb_address& address, grpc_subchannel* subchannel, + grpc_combiner* combiner) : SubchannelData(subchannel_list, user_data_vtable, address, subchannel, combiner) {} diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 4195c1e9d1..8dd5820bae 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -89,11 +89,12 @@ class RoundRobin : public LoadBalancingPolicy { : public SubchannelData<RoundRobinSubchannelList, RoundRobinSubchannelData> { public: - RoundRobinSubchannelData(RoundRobinSubchannelList* subchannel_list, - const grpc_lb_user_data_vtable* user_data_vtable, - const grpc_lb_address& address, - grpc_subchannel* subchannel, - grpc_combiner* combiner) + RoundRobinSubchannelData( + SubchannelList<RoundRobinSubchannelList, RoundRobinSubchannelData>* + subchannel_list, + const grpc_lb_user_data_vtable* user_data_vtable, + const grpc_lb_address& address, grpc_subchannel* subchannel, + grpc_combiner* combiner) : SubchannelData(subchannel_list, user_data_vtable, address, subchannel, combiner), user_data_vtable_(user_data_vtable), diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 91ddaec8b8..5e8682e056 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -65,6 +65,10 @@ class MySubchannelList namespace grpc_core { +// Forward declaration. +template <typename SubchannelListType, typename SubchannelDataType> +class SubchannelList; + // Stores data for a particular subchannel in a subchannel list. // Callers must create a subclass that implements the // ProcessConnectivityChangeLocked() method. @@ -72,7 +76,9 @@ template <typename SubchannelListType, typename SubchannelDataType> class SubchannelData { public: // Returns a pointer to the subchannel list containing this object. - SubchannelListType* subchannel_list() const { return subchannel_list_; } + SubchannelListType* subchannel_list() const { + return static_cast<SubchannelListType*>(subchannel_list_); + } // Returns the index into the subchannel list of this object. size_t Index() const { @@ -133,10 +139,11 @@ class SubchannelData { GRPC_ABSTRACT_BASE_CLASS protected: - SubchannelData(SubchannelListType* subchannel_list, - const grpc_lb_user_data_vtable* user_data_vtable, - const grpc_lb_address& address, grpc_subchannel* subchannel, - grpc_combiner* combiner); + SubchannelData( + SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list, + const grpc_lb_user_data_vtable* user_data_vtable, + const grpc_lb_address& address, grpc_subchannel* subchannel, + grpc_combiner* combiner); virtual ~SubchannelData(); @@ -161,7 +168,7 @@ class SubchannelData { static void OnConnectivityChangedLocked(void* arg, grpc_error* error); // Backpointer to owning subchannel list. Not owned. - SubchannelListType* subchannel_list_; + SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list_; // The subchannel and connected subchannel. grpc_subchannel* subchannel_; @@ -200,7 +207,7 @@ class SubchannelList grpc_core::channelz::SubchannelNode* subchannel_node = grpc_subchannel_get_channelz_node(subchannels_[i].subchannel()); if (subchannel_node != nullptr) { - refs_list->push_back(subchannel_node->subchannel_uuid()); + refs_list->push_back(subchannel_node->uuid()); } } } @@ -268,7 +275,7 @@ class SubchannelList template <typename SubchannelListType, typename SubchannelDataType> SubchannelData<SubchannelListType, SubchannelDataType>::SubchannelData( - SubchannelListType* subchannel_list, + SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list, const grpc_lb_user_data_vtable* user_data_vtable, const grpc_lb_address& address, grpc_subchannel* subchannel, grpc_combiner* combiner) @@ -532,8 +539,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList( address_uri); gpr_free(address_uri); } - subchannels_.emplace_back(static_cast<SubchannelListType*>(this), - addresses->user_data_vtable, + subchannels_.emplace_back(this, addresses->user_data_vtable, addresses->addresses[i], subchannel, combiner); } } diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index c07792d8a7..62bdbf2689 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -70,16 +70,14 @@ grpc_lb_addresses* grpc_lb_addresses_create( grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses); /** Sets the value of the address at index \a index of \a addresses. - * \a address is a socket address of length \a address_len. - * Takes ownership of \a balancer_name. */ + * \a address is a socket address of length \a address_len. */ void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, const void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data); /** Sets the value of the address at index \a index of \a addresses from \a uri. - * Returns true upon success, false otherwise. Takes ownership of \a - * balancer_name. */ + * Returns true upon success, false otherwise. */ bool grpc_lb_addresses_set_address_from_uri(grpc_lb_addresses* addresses, size_t index, const grpc_uri* uri, bool is_balancer, diff --git a/src/core/ext/filters/client_channel/parse_address.cc b/src/core/ext/filters/client_channel/parse_address.cc index b3900114ad..707beb8876 100644 --- a/src/core/ext/filters/client_channel/parse_address.cc +++ b/src/core/ext/filters/client_channel/parse_address.cc @@ -125,27 +125,41 @@ bool grpc_parse_ipv6_hostport(const char* hostport, grpc_resolved_address* addr, char* host_end = static_cast<char*>(gpr_memrchr(host, '%', strlen(host))); if (host_end != nullptr) { GPR_ASSERT(host_end >= host); - char host_without_scope[GRPC_INET6_ADDRSTRLEN]; + char host_without_scope[GRPC_INET6_ADDRSTRLEN + 1]; size_t host_without_scope_len = static_cast<size_t>(host_end - host); uint32_t sin6_scope_id = 0; + if (host_without_scope_len > GRPC_INET6_ADDRSTRLEN) { + if (log_errors) { + gpr_log( + GPR_ERROR, + "invalid ipv6 address length %zu. Length cannot be greater than " + "GRPC_INET6_ADDRSTRLEN i.e %d)", + host_without_scope_len, GRPC_INET6_ADDRSTRLEN); + } + goto done; + } strncpy(host_without_scope, host, host_without_scope_len); host_without_scope[host_without_scope_len] = '\0'; if (grpc_inet_pton(GRPC_AF_INET6, host_without_scope, &in6->sin6_addr) == 0) { - gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host_without_scope); + if (log_errors) { + gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host_without_scope); + } goto done; } if (gpr_parse_bytes_to_uint32(host_end + 1, strlen(host) - host_without_scope_len - 1, &sin6_scope_id) == 0) { - gpr_log(GPR_ERROR, "invalid ipv6 scope id: '%s'", host_end + 1); + if (log_errors) { + gpr_log(GPR_ERROR, "invalid ipv6 scope id: '%s'", host_end + 1); + } goto done; } // Handle "sin6_scope_id" being type "u_long". See grpc issue #10027. in6->sin6_scope_id = sin6_scope_id; } else { if (grpc_inet_pton(GRPC_AF_INET6, host, &in6->sin6_addr) == 0) { - gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host); + if (log_errors) gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host); goto done; } } @@ -190,3 +204,12 @@ bool grpc_parse_uri(const grpc_uri* uri, grpc_resolved_address* resolved_addr) { gpr_log(GPR_ERROR, "Can't parse scheme '%s'", uri->scheme); return false; } + +uint16_t grpc_strhtons(const char* port) { + if (strcmp(port, "http") == 0) { + return htons(80); + } else if (strcmp(port, "https") == 0) { + return htons(443); + } + return htons(static_cast<unsigned short>(atoi(port))); +} diff --git a/src/core/ext/filters/client_channel/parse_address.h b/src/core/ext/filters/client_channel/parse_address.h index 9a88b66edc..c2af0e6c49 100644 --- a/src/core/ext/filters/client_channel/parse_address.h +++ b/src/core/ext/filters/client_channel/parse_address.h @@ -47,4 +47,7 @@ bool grpc_parse_ipv4_hostport(const char* hostport, grpc_resolved_address* addr, bool grpc_parse_ipv6_hostport(const char* hostport, grpc_resolved_address* addr, bool log_errors); +/* Converts named or numeric port to a uint16 suitable for use in a sockaddr. */ +uint16_t grpc_strhtons(const char* port); + #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_PARSE_ADDRESS_H */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index 485998f5e4..4c795c34c8 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -87,15 +87,6 @@ typedef struct grpc_ares_hostbyname_request { static void do_basic_init(void) { gpr_mu_init(&g_init_mu); } -static uint16_t strhtons(const char* port) { - if (strcmp(port, "http") == 0) { - return htons(80); - } else if (strcmp(port, "https") == 0) { - return htons(443); - } - return htons(static_cast<unsigned short>(atoi(port))); -} - static void log_address_sorting_list(grpc_lb_addresses* lb_addrs, const char* input_output_str) { for (size_t i = 0; i < lb_addrs->num_addresses; i++) { @@ -139,12 +130,6 @@ void grpc_cares_wrapper_address_sorting_sort(grpc_lb_addresses* lb_addrs) { } } -/* Allow tests to access grpc_ares_wrapper_address_sorting_sort */ -void grpc_cares_wrapper_test_only_address_sorting_sort( - grpc_lb_addresses* lb_addrs) { - grpc_cares_wrapper_address_sorting_sort(lb_addrs); -} - static void grpc_ares_request_ref_locked(grpc_ares_request* r) { r->pending_queries++; } @@ -371,7 +356,8 @@ done: grpc_ares_request_unref_locked(r); } -static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( +static grpc_ares_request* +grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( const char* dns_server, const char* name, const char* default_port, grpc_pollset_set* interested_parties, grpc_closure* on_done, grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, @@ -454,12 +440,12 @@ static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( } r->pending_queries = 1; if (grpc_ares_query_ipv6()) { - hr = create_hostbyname_request_locked(r, host, strhtons(port), + hr = create_hostbyname_request_locked(r, host, grpc_strhtons(port), false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_locked, hr); } - hr = create_hostbyname_request_locked(r, host, strhtons(port), + hr = create_hostbyname_request_locked(r, host, grpc_strhtons(port), false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_locked, hr); @@ -494,6 +480,79 @@ error_cleanup: return nullptr; } +static bool inner_resolve_as_ip_literal_locked(const char* name, + const char* default_port, + grpc_lb_addresses** addrs, + char** host, char** port, + char** hostport) { + gpr_split_host_port(name, host, port); + if (*host == nullptr) { + gpr_log(GPR_ERROR, + "Failed to parse %s to host:port while attempting to resolve as ip " + "literal.", + name); + return false; + } + if (*port == nullptr) { + if (default_port == nullptr) { + gpr_log(GPR_ERROR, + "No port or default port for %s while attempting to resolve as " + "ip literal.", + name); + return false; + } + *port = gpr_strdup(default_port); + } + grpc_resolved_address addr; + GPR_ASSERT(gpr_join_host_port(hostport, *host, atoi(*port))); + if (grpc_parse_ipv4_hostport(*hostport, &addr, false /* log errors */) || + grpc_parse_ipv6_hostport(*hostport, &addr, false /* log errors */)) { + GPR_ASSERT(*addrs == nullptr); + *addrs = grpc_lb_addresses_create(1, nullptr); + grpc_lb_addresses_set_address( + *addrs, 0, addr.addr, addr.len, false /* is_balancer */, + nullptr /* balancer_name */, nullptr /* user_data */); + return true; + } + return false; +} + +static bool resolve_as_ip_literal_locked(const char* name, + const char* default_port, + grpc_lb_addresses** addrs) { + char* host = nullptr; + char* port = nullptr; + char* hostport = nullptr; + bool out = inner_resolve_as_ip_literal_locked(name, default_port, addrs, + &host, &port, &hostport); + gpr_free(host); + gpr_free(port); + gpr_free(hostport); + return out; +} + +static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( + const char* dns_server, const char* name, const char* default_port, + grpc_pollset_set* interested_parties, grpc_closure* on_done, + grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, + grpc_combiner* combiner) { + // Early out if the target is an ipv4 or ipv6 literal. + if (resolve_as_ip_literal_locked(name, default_port, addrs)) { + GRPC_CLOSURE_SCHED(on_done, GRPC_ERROR_NONE); + return nullptr; + } + // Early out if the target is localhost and we're on Windows. + if (grpc_ares_maybe_resolve_localhost_manually_locked(name, default_port, + addrs)) { + GRPC_CLOSURE_SCHED(on_done, GRPC_ERROR_NONE); + return nullptr; + } + // Look up name using c-ares lib. + return grpc_dns_lookup_ares_continue_after_check_localhost_and_ip_literals_locked( + dns_server, name, default_port, interested_parties, on_done, addrs, + check_grpclb, service_config_json, combiner); +} + grpc_ares_request* (*grpc_dns_lookup_ares_locked)( const char* dns_server, const char* name, const char* default_port, grpc_pollset_set* interested_parties, grpc_closure* on_done, @@ -502,7 +561,9 @@ grpc_ares_request* (*grpc_dns_lookup_ares_locked)( void grpc_cancel_ares_request(grpc_ares_request* r) { if (grpc_dns_lookup_ares_locked == grpc_dns_lookup_ares_locked_impl) { - grpc_ares_ev_driver_shutdown_locked(r->ev_driver); + if (r != nullptr) { + grpc_ares_ev_driver_shutdown_locked(r->ev_driver); + } } } diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index ca5779e1d7..1bc457d4cf 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -81,9 +81,15 @@ void grpc_ares_complete_request_locked(grpc_ares_request* request); /* E.g., return false if ipv6 is known to not be available. */ bool grpc_ares_query_ipv6(); -/* Exposed only for testing */ -void grpc_cares_wrapper_test_only_address_sorting_sort( - grpc_lb_addresses* lb_addrs); +/* Maybe (depending on the current platform) checks if "name" matches + * "localhost" and if so fills in addrs with the correct sockaddr structures. + * Returns a bool indicating whether or not such an action was performed. + * See https://github.com/grpc/grpc/issues/15158. */ +bool grpc_ares_maybe_resolve_localhost_manually_locked( + const char* name, const char* default_port, grpc_lb_addresses** addrs); + +/* Sorts destinations in lb_addrs according to RFC 6724. */ +void grpc_cares_wrapper_address_sorting_sort(grpc_lb_addresses* lb_addrs); #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_WRAPPER_H \ */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc index 23c0fec74f..639eec2323 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc @@ -26,4 +26,9 @@ bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } +bool grpc_ares_maybe_resolve_localhost_manually_locked( + const char* name, const char* default_port, grpc_lb_addresses** addrs) { + return false; +} + #endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc index ee827e284e..7e34784691 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc @@ -21,9 +21,79 @@ #include "src/core/lib/iomgr/port.h" #if GRPC_ARES == 1 && defined(GPR_WINDOWS) +#include <grpc/support/string_util.h> + +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/lib/gpr/host_port.h" +#include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/socket_windows.h" bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } +static bool inner_maybe_resolve_localhost_manually_locked( + const char* name, const char* default_port, grpc_lb_addresses** addrs, + char** host, char** port) { + gpr_split_host_port(name, host, port); + if (*host == nullptr) { + gpr_log(GPR_ERROR, + "Failed to parse %s into host:port during Windows localhost " + "resolution check.", + name); + return false; + } + if (*port == nullptr) { + if (default_port == nullptr) { + gpr_log(GPR_ERROR, + "No port or default port for %s during Windows localhost " + "resolution check.", + name); + return false; + } + *port = gpr_strdup(default_port); + } + if (gpr_stricmp(*host, "localhost") == 0) { + GPR_ASSERT(*addrs == nullptr); + *addrs = grpc_lb_addresses_create(2, nullptr); + uint16_t numeric_port = grpc_strhtons(*port); + // Append the ipv6 loopback address. + struct sockaddr_in6 ipv6_loopback_addr; + memset(&ipv6_loopback_addr, 0, sizeof(ipv6_loopback_addr)); + ((char*)&ipv6_loopback_addr.sin6_addr)[15] = 1; + ipv6_loopback_addr.sin6_family = AF_INET6; + ipv6_loopback_addr.sin6_port = numeric_port; + grpc_lb_addresses_set_address( + *addrs, 0, &ipv6_loopback_addr, sizeof(ipv6_loopback_addr), + false /* is_balancer */, nullptr /* balancer_name */, + nullptr /* user_data */); + // Append the ipv4 loopback address. + struct sockaddr_in ipv4_loopback_addr; + memset(&ipv4_loopback_addr, 0, sizeof(ipv4_loopback_addr)); + ((char*)&ipv4_loopback_addr.sin_addr)[0] = 0x7f; + ((char*)&ipv4_loopback_addr.sin_addr)[3] = 0x01; + ipv4_loopback_addr.sin_family = AF_INET; + ipv4_loopback_addr.sin_port = numeric_port; + grpc_lb_addresses_set_address( + *addrs, 1, &ipv4_loopback_addr, sizeof(ipv4_loopback_addr), + false /* is_balancer */, nullptr /* balancer_name */, + nullptr /* user_data */); + // Let the address sorter figure out which one should be tried first. + grpc_cares_wrapper_address_sorting_sort(*addrs); + return true; + } + return false; +} + +bool grpc_ares_maybe_resolve_localhost_manually_locked( + const char* name, const char* default_port, grpc_lb_addresses** addrs) { + char* host = nullptr; + char* port = nullptr; + bool out = inner_maybe_resolve_localhost_manually_locked(name, default_port, + addrs, &host, &port); + gpr_free(host); + gpr_free(port); + return out; +} + #endif /* GRPC_ARES == 1 && defined(GPR_WINDOWS) */ diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 0e40f42e18..57d0b3759f 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -183,7 +183,13 @@ static void connection_destroy(void* arg, grpc_error* error) { static void subchannel_destroy(void* arg, grpc_error* error) { grpc_subchannel* c = static_cast<grpc_subchannel*>(arg); - c->channelz_subchannel.reset(); + if (c->channelz_subchannel != nullptr) { + c->channelz_subchannel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Subchannel destroyed")); + c->channelz_subchannel->MarkSubchannelDestroyed(); + c->channelz_subchannel.reset(); + } gpr_free((void*)c->filters); grpc_channel_args_destroy(c->args); grpc_connectivity_state_destroy(&c->state_tracker); @@ -383,9 +389,18 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, const grpc_arg* arg = grpc_channel_args_find(c->args, GRPC_ARG_ENABLE_CHANNELZ); bool channelz_enabled = grpc_channel_arg_get_bool(arg, false); + arg = grpc_channel_args_find(c->args, + GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE); + const grpc_integer_options options = {0, 0, INT_MAX}; + size_t channel_tracer_max_nodes = + (size_t)grpc_channel_arg_get_integer(arg, options); if (channelz_enabled) { c->channelz_subchannel = - grpc_core::MakeRefCounted<grpc_core::channelz::SubchannelNode>(); + grpc_core::MakeRefCounted<grpc_core::channelz::SubchannelNode>( + c, channel_tracer_max_nodes); + c->channelz_subchannel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Subchannel created")); } return grpc_subchannel_index_register(key, c); @@ -625,8 +640,8 @@ static bool publish_transport_locked(grpc_subchannel* c) { } /* publish */ - c->connected_subchannel.reset( - grpc_core::New<grpc_core::ConnectedSubchannel>(stk)); + c->connected_subchannel.reset(grpc_core::New<grpc_core::ConnectedSubchannel>( + stk, c->channelz_subchannel.get())); gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p", c->connected_subchannel.get(), c); @@ -770,6 +785,14 @@ void grpc_get_subchannel_address_arg(const grpc_channel_args* args, } } +const char* grpc_subchannel_get_target(grpc_subchannel* subchannel) { + const grpc_arg* addr_arg = + grpc_channel_args_find(subchannel->args, GRPC_ARG_SUBCHANNEL_ADDRESS); + const char* addr_str = grpc_channel_arg_get_string(addr_arg); + GPR_ASSERT(addr_str != nullptr); // Should have been set by LB policy. + return addr_str; +} + const char* grpc_get_subchannel_address_uri_arg(const grpc_channel_args* args) { const grpc_arg* addr_arg = grpc_channel_args_find(args, GRPC_ARG_SUBCHANNEL_ADDRESS); @@ -786,9 +809,12 @@ grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address* addr) { namespace grpc_core { -ConnectedSubchannel::ConnectedSubchannel(grpc_channel_stack* channel_stack) +ConnectedSubchannel::ConnectedSubchannel( + grpc_channel_stack* channel_stack, + channelz::SubchannelNode* channelz_subchannel) : RefCountedWithTracing<ConnectedSubchannel>(&grpc_trace_stream_refcount), - channel_stack_(channel_stack) {} + channel_stack_(channel_stack), + channelz_subchannel_(channelz_subchannel) {} ConnectedSubchannel::~ConnectedSubchannel() { GRPC_CHANNEL_STACK_UNREF(channel_stack_, "connected_subchannel_dtor"); diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index a135035d62..84febb5204 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -85,7 +85,8 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { size_t parent_data_size; }; - explicit ConnectedSubchannel(grpc_channel_stack* channel_stack); + explicit ConnectedSubchannel(grpc_channel_stack* channel_stack, + channelz::SubchannelNode* channelz_subchannel); ~ConnectedSubchannel(); grpc_channel_stack* channel_stack() { return channel_stack_; } @@ -94,9 +95,15 @@ class ConnectedSubchannel : public RefCountedWithTracing<ConnectedSubchannel> { grpc_closure* closure); void Ping(grpc_closure* on_initiate, grpc_closure* on_ack); grpc_error* CreateCall(const CallArgs& args, grpc_subchannel_call** call); + channelz::SubchannelNode* channelz_subchannel() { + return channelz_subchannel_; + } private: grpc_channel_stack* channel_stack_; + // backpointer to the channelz node in this connected subchannel's + // owning subchannel. + channelz::SubchannelNode* channelz_subchannel_; }; } // namespace grpc_core @@ -184,6 +191,8 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, void grpc_get_subchannel_address_arg(const grpc_channel_args* args, grpc_resolved_address* addr); +const char* grpc_subchannel_get_target(grpc_subchannel* subchannel); + /// Returns the URI string for the address to connect to. const char* grpc_get_subchannel_address_uri_arg(const grpc_channel_args* args); diff --git a/src/core/ext/filters/client_channel/subchannel_index.cc b/src/core/ext/filters/client_channel/subchannel_index.cc index cb02b1a748..1c23a6c4be 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.cc +++ b/src/core/ext/filters/client_channel/subchannel_index.cc @@ -73,7 +73,8 @@ static grpc_subchannel_key* subchannel_key_copy(grpc_subchannel_key* k) { int grpc_subchannel_key_compare(const grpc_subchannel_key* a, const grpc_subchannel_key* b) { - if (g_force_creation) return false; + // To pretend the keys are different, return a non-zero value. + if (GPR_UNLIKELY(g_force_creation)) return 1; int c = GPR_ICMP(a->args.filter_count, b->args.filter_count); if (c != 0) return c; if (a->args.filter_count > 0) { diff --git a/src/core/ext/filters/client_channel/subchannel_index.h b/src/core/ext/filters/client_channel/subchannel_index.h index a7dae9d47d..c135613d26 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.h +++ b/src/core/ext/filters/client_channel/subchannel_index.h @@ -65,13 +65,10 @@ void grpc_subchannel_index_ref(void); void grpc_subchannel_index_unref(void); /** \em TEST ONLY. - * If \a force_creation is true, all key comparisons will be false, resulting in + * If \a force_creation is true, all keys are regarded different, resulting in * new subchannels always being created. Otherwise, the keys will be compared as * usual. * - * This function is *not* threadsafe on purpose: it should *only* be used in - * test code. - * * Tests using this function \em MUST run tests with and without \a * force_creation set. */ void grpc_subchannel_index_test_only_set_force_creation(bool force_creation); diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index 1678051beb..cd459e47cd 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -51,12 +51,15 @@ struct call_data { grpc_linked_mdelem user_agent; // State for handling recv_initial_metadata ops. grpc_metadata_batch* recv_initial_metadata; + grpc_error* recv_initial_metadata_error; grpc_closure* original_recv_initial_metadata_ready; grpc_closure recv_initial_metadata_ready; // State for handling recv_trailing_metadata ops. grpc_metadata_batch* recv_trailing_metadata; grpc_closure* original_recv_trailing_metadata_ready; grpc_closure recv_trailing_metadata_ready; + grpc_error* recv_trailing_metadata_error; + bool seen_recv_trailing_metadata_ready; // State for handling send_message ops. grpc_transport_stream_op_batch* send_message_batch; size_t send_message_bytes_read; @@ -78,7 +81,12 @@ struct channel_data { static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem, grpc_metadata_batch* b) { if (b->idx.named.status != nullptr) { - if (grpc_mdelem_eq(b->idx.named.status->md, GRPC_MDELEM_STATUS_200)) { + /* If both gRPC status and HTTP status are provided in the response, we + * should prefer the gRPC status code, as mentioned in + * https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md. + */ + if (b->idx.named.grpc_status != nullptr || + grpc_mdelem_eq(b->idx.named.status->md, GRPC_MDELEM_STATUS_200)) { grpc_metadata_batch_remove(b, b->idx.named.status); } else { char* val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.status->md), @@ -147,21 +155,39 @@ static void recv_initial_metadata_ready(void* user_data, grpc_error* error) { call_data* calld = static_cast<call_data*>(elem->call_data); if (error == GRPC_ERROR_NONE) { error = client_filter_incoming_metadata(elem, calld->recv_initial_metadata); + calld->recv_initial_metadata_error = GRPC_ERROR_REF(error); } else { GRPC_ERROR_REF(error); } - GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, error); + grpc_closure* closure = calld->original_recv_initial_metadata_ready; + calld->original_recv_initial_metadata_ready = nullptr; + if (calld->seen_recv_trailing_metadata_ready) { + GRPC_CALL_COMBINER_START( + calld->call_combiner, &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_error, "continue recv_trailing_metadata"); + } + GRPC_CLOSURE_RUN(closure, error); } static void recv_trailing_metadata_ready(void* user_data, grpc_error* error) { grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); call_data* calld = static_cast<call_data*>(elem->call_data); + if (calld->original_recv_initial_metadata_ready != nullptr) { + calld->recv_trailing_metadata_error = GRPC_ERROR_REF(error); + calld->seen_recv_trailing_metadata_ready = true; + GRPC_CALL_COMBINER_STOP(calld->call_combiner, + "deferring recv_trailing_metadata_ready until " + "after recv_initial_metadata_ready"); + return; + } if (error == GRPC_ERROR_NONE) { error = client_filter_incoming_metadata(elem, calld->recv_trailing_metadata); } else { GRPC_ERROR_REF(error); } + error = grpc_error_add_child( + error, GRPC_ERROR_REF(calld->recv_initial_metadata_error)); GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, error); } @@ -434,7 +460,10 @@ static grpc_error* init_call_elem(grpc_call_element* elem, /* Destructor for call_data */ static void destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, - grpc_closure* ignored) {} + grpc_closure* ignored) { + call_data* calld = static_cast<call_data*>(elem->call_data); + GRPC_ERROR_UNREF(calld->recv_initial_metadata_error); +} static grpc_mdelem scheme_from_args(const grpc_channel_args* args) { unsigned i; diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index 3919447f26..436ea09d94 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -23,6 +23,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <string.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/b64.h" @@ -50,6 +51,7 @@ struct call_data { // State for intercepting recv_initial_metadata. grpc_closure recv_initial_metadata_ready; + grpc_error* recv_initial_metadata_ready_error; grpc_closure* original_recv_initial_metadata_ready; grpc_metadata_batch* recv_initial_metadata; uint32_t* recv_initial_metadata_flags; @@ -60,6 +62,16 @@ struct call_data { grpc_closure recv_message_ready; grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message; bool seen_recv_message_ready; + + // State for intercepting recv_trailing_metadata + grpc_closure recv_trailing_metadata_ready; + grpc_closure* original_recv_trailing_metadata_ready; + grpc_error* recv_trailing_metadata_ready_error; + bool seen_recv_trailing_metadata_ready; +}; + +struct channel_data { + bool surface_user_agent; }; } // namespace @@ -258,6 +270,11 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem, GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":authority"))); } + channel_data* chand = static_cast<channel_data*>(elem->channel_data); + if (!chand->surface_user_agent && b->idx.named.user_agent != nullptr) { + grpc_metadata_batch_remove(b, b->idx.named.user_agent); + } + return error; } @@ -267,6 +284,7 @@ static void hs_recv_initial_metadata_ready(void* user_data, grpc_error* err) { calld->seen_recv_initial_metadata_ready = true; if (err == GRPC_ERROR_NONE) { err = hs_filter_incoming_metadata(elem, calld->recv_initial_metadata); + calld->recv_initial_metadata_ready_error = GRPC_ERROR_REF(err); if (calld->seen_recv_message_ready) { // We've already seen the recv_message callback, but we previously // deferred it, so we need to return it here. @@ -286,6 +304,13 @@ static void hs_recv_initial_metadata_ready(void* user_data, grpc_error* err) { } else { GRPC_ERROR_REF(err); } + if (calld->seen_recv_trailing_metadata_ready) { + GRPC_CALL_COMBINER_START(calld->call_combiner, + &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_ready_error, + "resuming hs_recv_trailing_metadata_ready from " + "hs_recv_initial_metadata_ready"); + } GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, err); } @@ -313,6 +338,23 @@ static void hs_recv_message_ready(void* user_data, grpc_error* err) { } } +static void hs_recv_trailing_metadata_ready(void* user_data, grpc_error* err) { + grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); + call_data* calld = static_cast<call_data*>(elem->call_data); + if (!calld->seen_recv_initial_metadata_ready) { + calld->recv_trailing_metadata_ready_error = GRPC_ERROR_REF(err); + calld->seen_recv_trailing_metadata_ready = true; + GRPC_CALL_COMBINER_STOP(calld->call_combiner, + "deferring hs_recv_trailing_metadata_ready until " + "ater hs_recv_initial_metadata_ready"); + return; + } + err = grpc_error_add_child( + GRPC_ERROR_REF(err), + GRPC_ERROR_REF(calld->recv_initial_metadata_ready_error)); + GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, err); +} + static grpc_error* hs_mutate_op(grpc_call_element* elem, grpc_transport_stream_op_batch* op) { /* grab pointers to our data from the call element */ @@ -357,6 +399,13 @@ static grpc_error* hs_mutate_op(grpc_call_element* elem, op->payload->recv_message.recv_message_ready = &calld->recv_message_ready; } + if (op->recv_trailing_metadata) { + calld->original_recv_trailing_metadata_ready = + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &calld->recv_trailing_metadata_ready; + } + if (op->send_trailing_metadata) { grpc_error* error = hs_filter_outgoing_metadata( elem, op->payload->send_trailing_metadata.send_trailing_metadata); @@ -389,6 +438,9 @@ static grpc_error* hs_init_call_elem(grpc_call_element* elem, grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&calld->recv_message_ready, hs_recv_message_ready, elem, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready, + hs_recv_trailing_metadata_ready, elem, + grpc_schedule_on_exec_ctx); return GRPC_ERROR_NONE; } @@ -397,6 +449,7 @@ static void hs_destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, grpc_closure* ignored) { call_data* calld = static_cast<call_data*>(elem->call_data); + GRPC_ERROR_UNREF(calld->recv_initial_metadata_ready_error); if (calld->have_read_stream) { calld->read_stream->Orphan(); } @@ -405,7 +458,12 @@ static void hs_destroy_call_elem(grpc_call_element* elem, /* Constructor for channel_data */ static grpc_error* hs_init_channel_elem(grpc_channel_element* elem, grpc_channel_element_args* args) { + channel_data* chand = static_cast<channel_data*>(elem->channel_data); GPR_ASSERT(!args->is_last); + chand->surface_user_agent = grpc_channel_arg_get_bool( + grpc_channel_args_find(args->channel_args, + const_cast<char*>(GRPC_ARG_SURFACE_USER_AGENT)), + true); return GRPC_ERROR_NONE; } @@ -419,7 +477,7 @@ const grpc_channel_filter grpc_http_server_filter = { hs_init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, hs_destroy_call_elem, - 0, + sizeof(channel_data), hs_init_channel_elem, hs_destroy_channel_elem, grpc_channel_next_get_info, diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index c7fc3f2e62..2d3b16d992 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -99,10 +99,17 @@ struct call_data { // recv_message_ready up-call on transport_stream_op, and remember to // call our next_recv_message_ready member after handling it. grpc_closure recv_message_ready; + grpc_closure recv_trailing_metadata_ready; + // The error caused by a message that is too large, or GRPC_ERROR_NONE + grpc_error* error; // Used by recv_message_ready. grpc_core::OrphanablePtr<grpc_core::ByteStream>* recv_message; // Original recv_message_ready callback, invoked after our own. grpc_closure* next_recv_message_ready; + // Original recv_trailing_metadata callback, invoked after our own. + grpc_closure* original_recv_trailing_metadata_ready; + bool seen_recv_trailing_metadata; + grpc_error* recv_trailing_metadata_error; }; struct channel_data { @@ -130,18 +137,52 @@ static void recv_message_ready(void* user_data, grpc_error* error) { grpc_error* new_error = grpc_error_set_int( GRPC_ERROR_CREATE_FROM_COPIED_STRING(message_string), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED); + GRPC_ERROR_UNREF(calld->error); if (error == GRPC_ERROR_NONE) { error = new_error; } else { error = grpc_error_add_child(error, new_error); - GRPC_ERROR_UNREF(new_error); } + calld->error = GRPC_ERROR_REF(error); gpr_free(message_string); } else { GRPC_ERROR_REF(error); } // Invoke the next callback. - GRPC_CLOSURE_RUN(calld->next_recv_message_ready, error); + grpc_closure* closure = calld->next_recv_message_ready; + calld->next_recv_message_ready = nullptr; + if (calld->seen_recv_trailing_metadata) { + /* We might potentially see another RECV_MESSAGE op. In that case, we do not + * want to run the recv_trailing_metadata_ready closure again. The newer + * RECV_MESSAGE op cannot cause any errors since the transport has already + * invoked the recv_trailing_metadata_ready closure and all further + * RECV_MESSAGE ops will get null payloads. */ + calld->seen_recv_trailing_metadata = false; + GRPC_CALL_COMBINER_START(calld->call_combiner, + &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_error, + "continue recv_trailing_metadata_ready"); + } + GRPC_CLOSURE_RUN(closure, error); +} + +// Callback invoked on completion of recv_trailing_metadata +// Notifies the recv_trailing_metadata batch of any message size failures +static void recv_trailing_metadata_ready(void* user_data, grpc_error* error) { + grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); + call_data* calld = static_cast<call_data*>(elem->call_data); + if (calld->next_recv_message_ready != nullptr) { + calld->seen_recv_trailing_metadata = true; + calld->recv_trailing_metadata_error = GRPC_ERROR_REF(error); + GRPC_CALL_COMBINER_STOP(calld->call_combiner, + "deferring recv_trailing_metadata_ready until " + "after recv_message_ready"); + return; + } + error = + grpc_error_add_child(GRPC_ERROR_REF(error), GRPC_ERROR_REF(calld->error)); + // Invoke the next callback. + GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, error); } // Start transport stream op. @@ -172,6 +213,13 @@ static void start_transport_stream_op_batch( calld->recv_message = op->payload->recv_message.recv_message; op->payload->recv_message.recv_message_ready = &calld->recv_message_ready; } + // Inject callback for receiving trailing metadata. + if (op->recv_trailing_metadata) { + calld->original_recv_trailing_metadata_ready = + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &calld->recv_trailing_metadata_ready; + } // Chain to the next filter. grpc_call_next_op(elem, op); } @@ -183,8 +231,13 @@ static grpc_error* init_call_elem(grpc_call_element* elem, call_data* calld = static_cast<call_data*>(elem->call_data); calld->call_combiner = args->call_combiner; calld->next_recv_message_ready = nullptr; + calld->original_recv_trailing_metadata_ready = nullptr; + calld->error = GRPC_ERROR_NONE; GRPC_CLOSURE_INIT(&calld->recv_message_ready, recv_message_ready, elem, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready, + recv_trailing_metadata_ready, elem, + grpc_schedule_on_exec_ctx); // Get max sizes from channel data, then merge in per-method config values. // Note: Per-method config is only available on the client, so we // apply the max request size to the send limit and the max response @@ -213,7 +266,10 @@ static grpc_error* init_call_elem(grpc_call_element* elem, // Destructor for call_data. static void destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, - grpc_closure* ignored) {} + grpc_closure* ignored) { + call_data* calld = (call_data*)elem->call_data; + GRPC_ERROR_UNREF(calld->error); +} static int default_size(const grpc_channel_args* args, int without_minimal_stack) { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 027a57d606..d3232f4d26 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -157,6 +157,10 @@ bool g_flow_control_enabled = true; static void destruct_transport(grpc_chttp2_transport* t) { size_t i; + if (t->channelz_socket != nullptr) { + t->channelz_socket.reset(); + } + grpc_endpoint_destroy(t->ep); grpc_slice_buffer_destroy_internal(&t->qbuf); @@ -230,35 +234,169 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { gpr_ref(&t->refs); } static const grpc_transport_vtable* get_vtable(void); -static void init_transport(grpc_chttp2_transport* t, - const grpc_channel_args* channel_args, - grpc_endpoint* ep, bool is_client) { +/* Returns whether bdp is enabled */ +static bool read_channel_args(grpc_chttp2_transport* t, + const grpc_channel_args* channel_args, + bool is_client) { + bool enable_bdp = true; size_t i; int j; - GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == - GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); - - t->base.vtable = get_vtable(); - t->ep = ep; - /* one ref is for destroy */ - gpr_ref_init(&t->refs, 1); - t->combiner = grpc_combiner_create(); - t->peer_string = grpc_endpoint_get_peer(ep); - t->endpoint_reading = 1; - t->next_stream_id = is_client ? 1 : 2; - t->is_client = is_client; - t->deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; - t->is_first_frame = true; - grpc_connectivity_state_init( - &t->channel_callback.state_tracker, GRPC_CHANNEL_READY, - is_client ? "client_transport" : "server_transport"); - - grpc_slice_buffer_init(&t->qbuf); - - grpc_slice_buffer_init(&t->outbuf); - grpc_chttp2_hpack_compressor_init(&t->hpack_compressor); + for (i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER)) { + const grpc_integer_options options = {-1, 0, INT_MAX}; + const int value = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + if (value >= 0) { + if ((t->next_stream_id & 1) != (value & 1)) { + gpr_log(GPR_ERROR, "%s: low bit must be %d on %s", + GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER, t->next_stream_id & 1, + is_client ? "client" : "server"); + } else { + t->next_stream_id = static_cast<uint32_t>(value); + } + } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER)) { + const grpc_integer_options options = {-1, 0, INT_MAX}; + const int value = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + if (value >= 0) { + grpc_chttp2_hpack_compressor_set_max_usable_size( + &t->hpack_compressor, static_cast<uint32_t>(value)); + } + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { + t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( + &channel_args->args[i], + {g_default_max_pings_without_data, 0, INT_MAX}); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { + t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( + &channel_args->args[i], {g_default_max_ping_strikes, 0, INT_MAX}); + } else if (0 == + strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_sent_ping_interval_without_data = + grpc_channel_arg_get_integer( + &channel_args->args[i], + grpc_integer_options{ + g_default_min_sent_ping_interval_without_data_ms, 0, + INT_MAX}); + } else if (0 == + strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_recv_ping_interval_without_data = + grpc_channel_arg_get_integer( + &channel_args->args[i], + grpc_integer_options{ + g_default_min_recv_ping_interval_without_data_ms, 0, + INT_MAX}); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { + t->write_buffer_size = static_cast<uint32_t>(grpc_channel_arg_get_integer( + &channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE})); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { + enable_bdp = grpc_channel_arg_get_bool(&channel_args->args[i], true); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { + const int value = grpc_channel_arg_get_integer( + &channel_args->args[i], + grpc_integer_options{t->is_client + ? g_default_client_keepalive_time_ms + : g_default_server_keepalive_time_ms, + 1, INT_MAX}); + t->keepalive_time = value == INT_MAX ? GRPC_MILLIS_INF_FUTURE : value; + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_KEEPALIVE_TIMEOUT_MS)) { + const int value = grpc_channel_arg_get_integer( + &channel_args->args[i], + grpc_integer_options{t->is_client + ? g_default_client_keepalive_timeout_ms + : g_default_server_keepalive_timeout_ms, + 0, INT_MAX}); + t->keepalive_timeout = value == INT_MAX ? GRPC_MILLIS_INF_FUTURE : value; + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)) { + t->keepalive_permit_without_calls = static_cast<uint32_t>( + grpc_channel_arg_get_integer(&channel_args->args[i], {0, 0, 1})); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_OPTIMIZATION_TARGET)) { + if (channel_args->args[i].type != GRPC_ARG_STRING) { + gpr_log(GPR_ERROR, "%s should be a string", + GRPC_ARG_OPTIMIZATION_TARGET); + } else if (0 == strcmp(channel_args->args[i].value.string, "blend")) { + t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY; + } else if (0 == strcmp(channel_args->args[i].value.string, "latency")) { + t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY; + } else if (0 == + strcmp(channel_args->args[i].value.string, "throughput")) { + t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_THROUGHPUT; + } else { + gpr_log(GPR_ERROR, "%s value '%s' unknown, assuming 'blend'", + GRPC_ARG_OPTIMIZATION_TARGET, + channel_args->args[i].value.string); + } + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_ENABLE_CHANNELZ)) { + t->channelz_socket = + grpc_core::MakeRefCounted<grpc_core::channelz::SocketNode>(); + } else { + static const struct { + const char* channel_arg_name; + grpc_chttp2_setting_id setting_id; + grpc_integer_options integer_options; + bool availability[2] /* server, client */; + } settings_map[] = {{GRPC_ARG_MAX_CONCURRENT_STREAMS, + GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, + {-1, 0, INT32_MAX}, + {true, false}}, + {GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER, + GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, + {-1, 0, INT32_MAX}, + {true, true}}, + {GRPC_ARG_MAX_METADATA_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, + {-1, 0, INT32_MAX}, + {true, true}}, + {GRPC_ARG_HTTP2_MAX_FRAME_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, + {-1, 16384, 16777215}, + {true, true}}, + {GRPC_ARG_HTTP2_ENABLE_TRUE_BINARY, + GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, + {1, 0, 1}, + {true, true}}, + {GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES, + GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, + {-1, 5, INT32_MAX}, + {true, true}}}; + for (j = 0; j < static_cast<int> GPR_ARRAY_SIZE(settings_map); j++) { + if (0 == strcmp(channel_args->args[i].key, + settings_map[j].channel_arg_name)) { + if (!settings_map[j].availability[is_client]) { + gpr_log(GPR_DEBUG, "%s is not available on %s", + settings_map[j].channel_arg_name, + is_client ? "clients" : "servers"); + } else { + int value = grpc_channel_arg_get_integer( + &channel_args->args[i], settings_map[j].integer_options); + if (value >= 0) { + queue_setting_update(t, settings_map[j].setting_id, + static_cast<uint32_t>(value)); + } + } + break; + } + } + } + } + return enable_bdp; +} +static void init_transport_closures(grpc_chttp2_transport* t) { GRPC_CLOSURE_INIT(&t->read_action_locked, read_action_locked, t, grpc_combiner_scheduler(t->combiner)); GRPC_CLOSURE_INIT(&t->benign_reclaimer_locked, benign_reclaimer_locked, t, @@ -286,6 +424,79 @@ static void init_transport(grpc_chttp2_transport* t, GRPC_CLOSURE_INIT(&t->keepalive_watchdog_fired_locked, keepalive_watchdog_fired_locked, t, grpc_combiner_scheduler(t->combiner)); +} + +static void init_transport_keepalive_settings(grpc_chttp2_transport* t) { + if (t->is_client) { + t->keepalive_time = g_default_client_keepalive_time_ms == INT_MAX + ? GRPC_MILLIS_INF_FUTURE + : g_default_client_keepalive_time_ms; + t->keepalive_timeout = g_default_client_keepalive_timeout_ms == INT_MAX + ? GRPC_MILLIS_INF_FUTURE + : g_default_client_keepalive_timeout_ms; + t->keepalive_permit_without_calls = + g_default_client_keepalive_permit_without_calls; + } else { + t->keepalive_time = g_default_server_keepalive_time_ms == INT_MAX + ? GRPC_MILLIS_INF_FUTURE + : g_default_server_keepalive_time_ms; + t->keepalive_timeout = g_default_server_keepalive_timeout_ms == INT_MAX + ? GRPC_MILLIS_INF_FUTURE + : g_default_server_keepalive_timeout_ms; + t->keepalive_permit_without_calls = + g_default_server_keepalive_permit_without_calls; + } +} + +static void configure_transport_ping_policy(grpc_chttp2_transport* t) { + t->ping_policy.max_pings_without_data = g_default_max_pings_without_data; + t->ping_policy.min_sent_ping_interval_without_data = + g_default_min_sent_ping_interval_without_data_ms; + t->ping_policy.max_ping_strikes = g_default_max_ping_strikes; + t->ping_policy.min_recv_ping_interval_without_data = + g_default_min_recv_ping_interval_without_data_ms; +} + +static void init_keepalive_pings_if_enabled(grpc_chttp2_transport* t) { + if (t->keepalive_time != GRPC_MILLIS_INF_FUTURE) { + t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING; + GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); + grpc_timer_init(&t->keepalive_ping_timer, + grpc_core::ExecCtx::Get()->Now() + t->keepalive_time, + &t->init_keepalive_ping_locked); + } else { + /* Use GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED to indicate there are no + inflight keeaplive timers */ + t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED; + } +} + +static void init_transport(grpc_chttp2_transport* t, + const grpc_channel_args* channel_args, + grpc_endpoint* ep, bool is_client) { + GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == + GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); + + t->base.vtable = get_vtable(); + t->ep = ep; + /* one ref is for destroy */ + gpr_ref_init(&t->refs, 1); + t->combiner = grpc_combiner_create(); + t->peer_string = grpc_endpoint_get_peer(ep); + t->endpoint_reading = 1; + t->next_stream_id = is_client ? 1 : 2; + t->is_client = is_client; + t->deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; + t->is_first_frame = true; + grpc_connectivity_state_init( + &t->channel_callback.state_tracker, GRPC_CHANNEL_READY, + is_client ? "client_transport" : "server_transport"); + + grpc_slice_buffer_init(&t->qbuf); + grpc_slice_buffer_init(&t->outbuf); + grpc_chttp2_hpack_compressor_init(&t->hpack_compressor); + + init_transport_closures(t); t->goaway_error = GRPC_ERROR_NONE; grpc_chttp2_goaway_parser_init(&t->goaway_parser); @@ -301,6 +512,8 @@ static void init_transport(grpc_chttp2_transport* t, grpc_chttp2_stream_map_init(&t->stream_map, 8); /* copy in initial settings to all setting sets */ + size_t i; + int j; for (i = 0; i < GRPC_CHTTP2_NUM_SETTINGS; i++) { for (j = 0; j < GRPC_NUM_SETTING_SETS; j++) { t->settings[j][i] = grpc_chttp2_settings_parameters[i].default_value; @@ -328,191 +541,14 @@ static void init_transport(grpc_chttp2_transport* t, queue_setting_update(t, GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, 1); - t->ping_policy.max_pings_without_data = g_default_max_pings_without_data; - t->ping_policy.min_sent_ping_interval_without_data = - g_default_min_sent_ping_interval_without_data_ms; - t->ping_policy.max_ping_strikes = g_default_max_ping_strikes; - t->ping_policy.min_recv_ping_interval_without_data = - g_default_min_recv_ping_interval_without_data_ms; - - /* Keepalive setting */ - if (t->is_client) { - t->keepalive_time = g_default_client_keepalive_time_ms == INT_MAX - ? GRPC_MILLIS_INF_FUTURE - : g_default_client_keepalive_time_ms; - t->keepalive_timeout = g_default_client_keepalive_timeout_ms == INT_MAX - ? GRPC_MILLIS_INF_FUTURE - : g_default_client_keepalive_timeout_ms; - t->keepalive_permit_without_calls = - g_default_client_keepalive_permit_without_calls; - } else { - t->keepalive_time = g_default_server_keepalive_time_ms == INT_MAX - ? GRPC_MILLIS_INF_FUTURE - : g_default_server_keepalive_time_ms; - t->keepalive_timeout = g_default_server_keepalive_timeout_ms == INT_MAX - ? GRPC_MILLIS_INF_FUTURE - : g_default_server_keepalive_timeout_ms; - t->keepalive_permit_without_calls = - g_default_server_keepalive_permit_without_calls; - } + configure_transport_ping_policy(t); + init_transport_keepalive_settings(t); t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY; bool enable_bdp = true; - if (channel_args) { - for (i = 0; i < channel_args->num_args; i++) { - if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER)) { - const grpc_integer_options options = {-1, 0, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - if ((t->next_stream_id & 1) != (value & 1)) { - gpr_log(GPR_ERROR, "%s: low bit must be %d on %s", - GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER, - t->next_stream_id & 1, is_client ? "client" : "server"); - } else { - t->next_stream_id = static_cast<uint32_t>(value); - } - } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER)) { - const grpc_integer_options options = {-1, 0, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - grpc_chttp2_hpack_compressor_set_max_usable_size( - &t->hpack_compressor, static_cast<uint32_t>(value)); - } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { - t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( - &channel_args->args[i], - {g_default_max_pings_without_data, 0, INT_MAX}); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { - t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( - &channel_args->args[i], {g_default_max_ping_strikes, 0, INT_MAX}); - } else if (0 == - strcmp( - channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { - t->ping_policy.min_sent_ping_interval_without_data = - grpc_channel_arg_get_integer( - &channel_args->args[i], - grpc_integer_options{ - g_default_min_sent_ping_interval_without_data_ms, 0, - INT_MAX}); - } else if (0 == - strcmp( - channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { - t->ping_policy.min_recv_ping_interval_without_data = - grpc_channel_arg_get_integer( - &channel_args->args[i], - grpc_integer_options{ - g_default_min_recv_ping_interval_without_data_ms, 0, - INT_MAX}); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { - t->write_buffer_size = - static_cast<uint32_t>(grpc_channel_arg_get_integer( - &channel_args->args[i], {0, 0, MAX_WRITE_BUFFER_SIZE})); - } else if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_BDP_PROBE)) { - enable_bdp = grpc_channel_arg_get_bool(&channel_args->args[i], true); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_KEEPALIVE_TIME_MS)) { - const int value = grpc_channel_arg_get_integer( - &channel_args->args[i], - grpc_integer_options{t->is_client - ? g_default_client_keepalive_time_ms - : g_default_server_keepalive_time_ms, - 1, INT_MAX}); - t->keepalive_time = value == INT_MAX ? GRPC_MILLIS_INF_FUTURE : value; - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_KEEPALIVE_TIMEOUT_MS)) { - const int value = grpc_channel_arg_get_integer( - &channel_args->args[i], - grpc_integer_options{t->is_client - ? g_default_client_keepalive_timeout_ms - : g_default_server_keepalive_timeout_ms, - 0, INT_MAX}); - t->keepalive_timeout = - value == INT_MAX ? GRPC_MILLIS_INF_FUTURE : value; - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)) { - t->keepalive_permit_without_calls = static_cast<uint32_t>( - grpc_channel_arg_get_integer(&channel_args->args[i], {0, 0, 1})); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_OPTIMIZATION_TARGET)) { - if (channel_args->args[i].type != GRPC_ARG_STRING) { - gpr_log(GPR_ERROR, "%s should be a string", - GRPC_ARG_OPTIMIZATION_TARGET); - } else if (0 == strcmp(channel_args->args[i].value.string, "blend")) { - t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY; - } else if (0 == strcmp(channel_args->args[i].value.string, "latency")) { - t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY; - } else if (0 == - strcmp(channel_args->args[i].value.string, "throughput")) { - t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_THROUGHPUT; - } else { - gpr_log(GPR_ERROR, "%s value '%s' unknown, assuming 'blend'", - GRPC_ARG_OPTIMIZATION_TARGET, - channel_args->args[i].value.string); - } - } else { - static const struct { - const char* channel_arg_name; - grpc_chttp2_setting_id setting_id; - grpc_integer_options integer_options; - bool availability[2] /* server, client */; - } settings_map[] = { - {GRPC_ARG_MAX_CONCURRENT_STREAMS, - GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, - {-1, 0, INT32_MAX}, - {true, false}}, - {GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER, - GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, - {-1, 0, INT32_MAX}, - {true, true}}, - {GRPC_ARG_MAX_METADATA_SIZE, - GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, - {-1, 0, INT32_MAX}, - {true, true}}, - {GRPC_ARG_HTTP2_MAX_FRAME_SIZE, - GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, - {-1, 16384, 16777215}, - {true, true}}, - {GRPC_ARG_HTTP2_ENABLE_TRUE_BINARY, - GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, - {1, 0, 1}, - {true, true}}, - {GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES, - GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, - {-1, 5, INT32_MAX}, - {true, true}}}; - for (j = 0; j < static_cast<int> GPR_ARRAY_SIZE(settings_map); j++) { - if (0 == strcmp(channel_args->args[i].key, - settings_map[j].channel_arg_name)) { - if (!settings_map[j].availability[is_client]) { - gpr_log(GPR_DEBUG, "%s is not available on %s", - settings_map[j].channel_arg_name, - is_client ? "clients" : "servers"); - } else { - int value = grpc_channel_arg_get_integer( - &channel_args->args[i], settings_map[j].integer_options); - if (value >= 0) { - queue_setting_update(t, settings_map[j].setting_id, - static_cast<uint32_t>(value)); - } - } - break; - } - } - } - } + enable_bdp = read_channel_args(t, channel_args, is_client); } if (g_flow_control_enabled) { @@ -531,23 +567,11 @@ static void init_transport(grpc_chttp2_transport* t, t->ping_recv_state.last_ping_recv_time = GRPC_MILLIS_INF_PAST; t->ping_recv_state.ping_strikes = 0; - /* Start keepalive pings */ - if (t->keepalive_time != GRPC_MILLIS_INF_FUTURE) { - t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING; - GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); - grpc_timer_init(&t->keepalive_ping_timer, - grpc_core::ExecCtx::Get()->Now() + t->keepalive_time, - &t->init_keepalive_ping_locked); - } else { - /* Use GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED to indicate there are no - inflight keeaplive timers */ - t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED; - } + init_keepalive_pings_if_enabled(t); if (enable_bdp) { GRPC_CHTTP2_REF_TRANSPORT(t, "bdp_ping"); schedule_bdp_ping_locked(t); - grpc_chttp2_act_on_flowctl_action(t->flow_control->PeriodicUpdate(), t, nullptr); } @@ -704,6 +728,14 @@ static void destroy_stream_locked(void* sp, grpc_error* error) { grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(sp); grpc_chttp2_transport* t = s->t; + if (t->channelz_socket != nullptr) { + if ((t->is_client && s->eos_received) || (!t->is_client && s->eos_sent)) { + t->channelz_socket->RecordStreamSucceeded(); + } else { + t->channelz_socket->RecordStreamFailed(); + } + } + GPR_ASSERT((s->write_closed && s->read_closed) || s->id == 0); if (s->id != 0) { GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, s->id) == nullptr); @@ -1391,6 +1423,9 @@ static void perform_stream_op_locked(void* stream_op, } if (op->send_initial_metadata) { + if (t->is_client && t->channelz_socket != nullptr) { + t->channelz_socket->RecordStreamStartedFromLocal(); + } GRPC_STATS_INC_HTTP2_OP_SEND_INITIAL_METADATA(); GPR_ASSERT(s->send_initial_metadata_finished == nullptr); on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; @@ -1476,6 +1511,7 @@ static void perform_stream_op_locked(void* stream_op, if (op->send_message) { GRPC_STATS_INC_HTTP2_OP_SEND_MESSAGE(); + t->num_messages_in_next_write++; GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE( op->payload->send_message.send_message->length()); on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; @@ -2691,6 +2727,9 @@ static void start_keepalive_ping_locked(void* arg, grpc_error* error) { if (error != GRPC_ERROR_NONE) { return; } + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordKeepaliveSent(); + } GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive watchdog"); grpc_timer_init(&t->keepalive_watchdog_timer, grpc_core::ExecCtx::Get()->Now() + t->keepalive_timeout, @@ -2887,17 +2926,20 @@ bool Chttp2IncomingByteStream::Next(size_t max_size_hint, } } +void Chttp2IncomingByteStream::MaybeCreateStreamDecompressionCtx() { + if (!stream_->stream_decompression_ctx) { + stream_->stream_decompression_ctx = grpc_stream_compression_context_create( + stream_->stream_decompression_method); + } +} + grpc_error* Chttp2IncomingByteStream::Pull(grpc_slice* slice) { GPR_TIMER_SCOPE("incoming_byte_stream_pull", 0); grpc_error* error; if (stream_->unprocessed_incoming_frames_buffer.length > 0) { if (!stream_->unprocessed_incoming_frames_decompressed) { bool end_of_context; - if (!stream_->stream_decompression_ctx) { - stream_->stream_decompression_ctx = - grpc_stream_compression_context_create( - stream_->stream_decompression_method); - } + MaybeCreateStreamDecompressionCtx(); if (!grpc_stream_decompress(stream_->stream_decompression_ctx, &stream_->unprocessed_incoming_frames_buffer, &stream_->decompressed_data_buffer, nullptr, diff --git a/src/core/ext/transport/chttp2/transport/frame_data.cc b/src/core/ext/transport/chttp2/transport/frame_data.cc index f8f06f6789..933b32c03c 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.cc +++ b/src/core/ext/transport/chttp2/transport/frame_data.cc @@ -62,6 +62,7 @@ grpc_error* grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser* parser, if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) { s->received_last_frame = true; + s->eos_received = true; } else { s->received_last_frame = false; } @@ -191,6 +192,9 @@ grpc_error* grpc_deframe_unprocessed_incoming_frames( GPR_ASSERT(stream_out != nullptr); GPR_ASSERT(p->parsing_frame == nullptr); p->frame_size |= (static_cast<uint32_t>(*cur)); + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordMessageReceived(); + } p->state = GRPC_CHTTP2_DATA_FRAME; ++cur; message_flags = 0; diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc index 0eaf63f133..920d52770f 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc @@ -525,7 +525,7 @@ static void hpack_enc(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem, /* should this elem be in the table? */ size_t decoder_space_usage = - grpc_mdelem_get_size_in_hpack_table(elem, st->use_true_binary_metadata); + grpc_chttp2_get_size_in_hpack_table(elem, st->use_true_binary_metadata); bool should_add_elem = elem_interned && decoder_space_usage < MAX_DECODER_SPACE_USAGE && c->filter_elems[HASH_FRAGMENT_1(elem_hash)] >= @@ -688,11 +688,22 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c, emit_advertise_table_size_change(c, &st); } for (size_t i = 0; i < extra_headers_size; ++i) { - hpack_enc(c, *extra_headers[i], &st); + grpc_mdelem md = *extra_headers[i]; + uint8_t static_index = grpc_chttp2_get_static_hpack_table_index(md); + if (static_index) { + emit_indexed(c, static_index, &st); + } else { + hpack_enc(c, md, &st); + } } grpc_metadata_batch_assert_ok(metadata); for (grpc_linked_mdelem* l = metadata->list.head; l; l = l->next) { - hpack_enc(c, l->md, &st); + uint8_t static_index = grpc_chttp2_get_static_hpack_table_index(l->md); + if (static_index) { + emit_indexed(c, static_index, &st); + } else { + hpack_enc(c, l->md, &st); + } } grpc_millis deadline = metadata->deadline; if (deadline != GRPC_MILLIS_INF_FUTURE) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_mapping.cc b/src/core/ext/transport/chttp2/transport/hpack_mapping.cc new file mode 100644 index 0000000000..fd529f0fd4 --- /dev/null +++ b/src/core/ext/transport/chttp2/transport/hpack_mapping.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * WARNING: Auto-generated code. + * + * To make changes to this file, change + * tools/codegen/core/gen_static_metadata.py, and then re-run it. + * + * This file contains the mapping from the index of each metadata element in the + * grpc static metadata table to the index of that element in the hpack static + * metadata table. If the element is not contained in the static hpack table, + * then the returned index is 0. + */ + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/transport/chttp2/transport/hpack_mapping.h" + +const uint8_t grpc_hpack_static_mdelem_indices[GRPC_STATIC_MDELEM_COUNT] = { + 0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 13, 6, 7, 0, 1, 2, 0, 4, + 5, 9, 10, 11, 12, 14, 15, 0, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 0, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 0, 0, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; diff --git a/src/core/ext/transport/chttp2/transport/hpack_mapping.h b/src/core/ext/transport/chttp2/transport/hpack_mapping.h new file mode 100644 index 0000000000..ebcd65bd9f --- /dev/null +++ b/src/core/ext/transport/chttp2/transport/hpack_mapping.h @@ -0,0 +1,38 @@ +/* + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * WARNING: Auto-generated code. + * + * To make changes to this file, change + * tools/codegen/core/gen_static_metadata.py, and then re-run it. + * + * This file contains the mapping from the index of each metadata element in the + * grpc static metadata table to the index of that element in the hpack static + * metadata table. If the element is not contained in the static hpack table, + * then the returned index is 0. + */ + +#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H +#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H + +#include <grpc/support/port_platform.h> + +#include "src/core/lib/transport/static_metadata.h" + +extern const uint8_t grpc_hpack_static_mdelem_indices[GRPC_STATIC_MDELEM_COUNT]; + +#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H */ diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.cc b/src/core/ext/transport/chttp2/transport/hpack_table.cc index 7929258356..117679ab75 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_table.cc @@ -27,8 +27,10 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/ext/transport/chttp2/transport/hpack_mapping.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/murmur_hash.h" +#include "src/core/lib/transport/static_metadata.h" extern grpc_core::TraceFlag grpc_http_trace; @@ -366,3 +368,30 @@ grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( return r; } + +static size_t get_base64_encoded_size(size_t raw_length) { + static const uint8_t tail_xtra[3] = {0, 2, 3}; + return raw_length / 3 * 4 + tail_xtra[raw_length % 3]; +} + +size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem, + bool use_true_binary_metadata) { + size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); + size_t value_len = GRPC_SLICE_LENGTH(GRPC_MDVALUE(elem)); + if (grpc_is_binary_header(GRPC_MDKEY(elem))) { + return overhead_and_key + (use_true_binary_metadata + ? value_len + 1 + : get_base64_encoded_size(value_len)); + } else { + return overhead_and_key + value_len; + } +} + +uint8_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md) { + if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) { + return grpc_hpack_static_mdelem_indices[GRPC_MDELEM_DATA(md) - + grpc_static_mdelem_table]; + } else { + return 0; + } +} diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h index 98026a4ba4..a0ffc6fab7 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.h +++ b/src/core/ext/transport/chttp2/transport/hpack_table.h @@ -83,6 +83,15 @@ grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, /* add a table entry to the index */ grpc_error* grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, grpc_mdelem md) GRPC_MUST_USE_RESULT; + +size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem, + bool use_true_binary_metadata); + +/* Returns the static hpack table index that corresponds to /a elem. Returns 0 + if /a elem is not statically stored or if it is not in the static hpack + table */ +uint8_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md); + /* Find a key/value pair in the table... returns the index in the table of the most similar entry, or 0 if the value was not found */ typedef struct { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index ca6e715978..ff26dd9255 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -36,6 +36,7 @@ #include "src/core/ext/transport/chttp2/transport/hpack_parser.h" #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/ext/transport/chttp2/transport/stream_map.h" +#include "src/core/lib/channel/channelz.h" #include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/iomgr/combiner.h" @@ -246,6 +247,8 @@ class Chttp2IncomingByteStream : public ByteStream { static void NextLocked(void* arg, grpc_error* error_ignored); static void OrphanLocked(void* arg, grpc_error* error_ignored); + void MaybeCreateStreamDecompressionCtx(); + grpc_chttp2_transport* transport_; // Immutable. grpc_chttp2_stream* stream_; // Immutable. @@ -469,6 +472,9 @@ struct grpc_chttp2_transport { bool keepalive_permit_without_calls; /** keep-alive state machine state */ grpc_chttp2_keepalive_state keepalive_state; + + grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode> channelz_socket; + uint32_t num_messages_in_next_write; }; typedef enum { @@ -532,6 +538,10 @@ struct grpc_chttp2_stream { /** Has trailing metadata been received. */ bool received_trailing_metadata; + /* have we sent or received the EOS bit? */ + bool eos_received; + bool eos_sent; + /** the error that resulted in this stream being read-closed */ grpc_error* read_closed_error; /** the error that resulted in this stream being write-closed */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index 1e491d2ef8..f532b084c9 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -409,67 +409,81 @@ static void on_initial_header(void* tp, grpc_mdelem md) { gpr_free(value); } - if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && - !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { - /* TODO(ctiller): check for a status like " 0" */ - s->seen_error = true; - } + if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) { + // We don't use grpc_mdelem_eq here to avoid executing additional + // instructions. The reasoning is if the payload is not equal, we already + // know that the metadata elements are not equal because the md is + // confirmed to be static. If we had used grpc_mdelem_eq here, then if the + // payloads are not equal, grpc_mdelem_eq executes more instructions to + // determine if they're equal or not. + if (md.payload == GRPC_MDELEM_GRPC_STATUS_1.payload || + md.payload == GRPC_MDELEM_GRPC_STATUS_2.payload) { + s->seen_error = true; + } + } else { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && + !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { + /* TODO(ctiller): check for a status like " 0" */ + s->seen_error = true; + } - if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) { - grpc_millis* cached_timeout = - static_cast<grpc_millis*>(grpc_mdelem_get_user_data(md, free_timeout)); - grpc_millis timeout; - if (cached_timeout != nullptr) { - timeout = *cached_timeout; - } else { - if (GPR_UNLIKELY( - !grpc_http2_decode_timeout(GRPC_MDVALUE(md), &timeout))) { - char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md)); - gpr_log(GPR_ERROR, "Ignoring bad timeout value '%s'", val); - gpr_free(val); - timeout = GRPC_MILLIS_INF_FUTURE; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) { + grpc_millis* cached_timeout = static_cast<grpc_millis*>( + grpc_mdelem_get_user_data(md, free_timeout)); + grpc_millis timeout; + if (cached_timeout != nullptr) { + timeout = *cached_timeout; + } else { + if (GPR_UNLIKELY( + !grpc_http2_decode_timeout(GRPC_MDVALUE(md), &timeout))) { + char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md)); + gpr_log(GPR_ERROR, "Ignoring bad timeout value '%s'", val); + gpr_free(val); + timeout = GRPC_MILLIS_INF_FUTURE; + } + if (GRPC_MDELEM_IS_INTERNED(md)) { + /* store the result */ + cached_timeout = + static_cast<grpc_millis*>(gpr_malloc(sizeof(grpc_millis))); + *cached_timeout = timeout; + grpc_mdelem_set_user_data(md, free_timeout, cached_timeout); + } } - if (GRPC_MDELEM_IS_INTERNED(md)) { - /* store the result */ - cached_timeout = - static_cast<grpc_millis*>(gpr_malloc(sizeof(grpc_millis))); - *cached_timeout = timeout; - grpc_mdelem_set_user_data(md, free_timeout, cached_timeout); + if (timeout != GRPC_MILLIS_INF_FUTURE) { + grpc_chttp2_incoming_metadata_buffer_set_deadline( + &s->metadata_buffer[0], grpc_core::ExecCtx::Get()->Now() + timeout); } + GRPC_MDELEM_UNREF(md); + return; } - if (timeout != GRPC_MILLIS_INF_FUTURE) { - grpc_chttp2_incoming_metadata_buffer_set_deadline( - &s->metadata_buffer[0], grpc_core::ExecCtx::Get()->Now() + timeout); - } + } + + const size_t new_size = s->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); + const size_t metadata_size_limit = + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (new_size > metadata_size_limit) { + gpr_log(GPR_DEBUG, + "received initial metadata size exceeds limit (%" PRIuPTR + " vs. %" PRIuPTR ")", + new_size, metadata_size_limit); + grpc_chttp2_cancel_stream( + t, s, + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "received initial metadata size exceeds limit"), + GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_RESOURCE_EXHAUSTED)); + grpc_chttp2_parsing_become_skip_parser(t); + s->seen_error = true; GRPC_MDELEM_UNREF(md); } else { - const size_t new_size = s->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); - const size_t metadata_size_limit = - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; - if (new_size > metadata_size_limit) { - gpr_log(GPR_DEBUG, - "received initial metadata size exceeds limit (%" PRIuPTR - " vs. %" PRIuPTR ")", - new_size, metadata_size_limit); - grpc_chttp2_cancel_stream( - t, s, - grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "received initial metadata size exceeds limit"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); + grpc_error* error = + grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[0], md); + if (error != GRPC_ERROR_NONE) { + grpc_chttp2_cancel_stream(t, s, error); grpc_chttp2_parsing_become_skip_parser(t); s->seen_error = true; GRPC_MDELEM_UNREF(md); - } else { - grpc_error* error = - grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[0], md); - if (error != GRPC_ERROR_NONE) { - grpc_chttp2_cancel_stream(t, s, error); - grpc_chttp2_parsing_become_skip_parser(t); - s->seen_error = true; - GRPC_MDELEM_UNREF(md); - } } } } @@ -491,8 +505,19 @@ static void on_trailing_header(void* tp, grpc_mdelem md) { gpr_free(value); } - if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && - !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { + if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) { + // We don't use grpc_mdelem_eq here to avoid executing additional + // instructions. The reasoning is if the payload is not equal, we already + // know that the metadata elements are not equal because the md is + // confirmed to be static. If we had used grpc_mdelem_eq here, then if the + // payloads are not equal, grpc_mdelem_eq executes more instructions to + // determine if they're equal or not. + if (md.payload == GRPC_MDELEM_GRPC_STATUS_1.payload || + md.payload == GRPC_MDELEM_GRPC_STATUS_2.payload) { + s->seen_error = true; + } + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && + !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { /* TODO(ctiller): check for a status like " 0" */ s->seen_error = true; } @@ -598,6 +623,9 @@ static grpc_error* init_header_frame_parser(grpc_chttp2_transport* t, gpr_log(GPR_ERROR, "grpc_chttp2_stream not accepted")); return init_skip_frame_parser(t, 1); } + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordStreamStartedFromRemote(); + } } else { t->incoming_stream = s; } @@ -611,6 +639,9 @@ static grpc_error* init_header_frame_parser(grpc_chttp2_transport* t, } t->parser = grpc_chttp2_header_parser_parse; t->parser_data = &t->hpack_parser; + if (t->header_eof) { + s->eos_received = true; + } switch (s->header_frames_received) { case 0: if (t->is_client && t->header_eof) { diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 8b73b01dea..d533989444 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -569,6 +569,7 @@ class StreamWriteContext { void SentLastFrame() { s_->send_trailing_metadata = nullptr; s_->sent_trailing_metadata = true; + s_->eos_sent = true; if (!t_->is_client && !s_->read_closed) { grpc_slice_buffer_add( @@ -632,6 +633,11 @@ void grpc_chttp2_end_write(grpc_chttp2_transport* t, grpc_error* error) { GPR_TIMER_SCOPE("grpc_chttp2_end_write", 0); grpc_chttp2_stream* s; + if (t->channelz_socket != nullptr) { + t->channelz_socket->RecordMessagesSent(t->num_messages_in_next_write); + } + t->num_messages_in_next_write = 0; + while (grpc_chttp2_list_pop_writing_stream(t, &s)) { if (s->sending_bytes != 0) { update_list(t, s, static_cast<int64_t>(s->sending_bytes), diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index 4a252d972d..81e2634e3a 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -1287,7 +1287,7 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { grpc_error* error = GRPC_ERROR_NONE; if (stream_state->state_op_done[OP_CANCEL_ERROR]) { error = GRPC_ERROR_REF(stream_state->cancel_error); - } else if (stream_state->state_op_done[OP_FAILED]) { + } else if (stream_state->state_callback_received[OP_FAILED]) { error = make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."); } else if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( diff --git a/src/core/lib/channel/channel_trace.cc b/src/core/lib/channel/channel_trace.cc index b3443310ac..cfb2faba51 100644 --- a/src/core/lib/channel/channel_trace.cc +++ b/src/core/lib/channel/channel_trace.cc @@ -41,16 +41,14 @@ namespace grpc_core { namespace channelz { -ChannelTrace::TraceEvent::TraceEvent( - Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_channel, ReferencedType type) +ChannelTrace::TraceEvent::TraceEvent(Severity severity, grpc_slice data, + RefCountedPtr<BaseNode> referenced_entity) : severity_(severity), data_(data), timestamp_(grpc_millis_to_timespec(grpc_core::ExecCtx::Get()->Now(), GPR_CLOCK_REALTIME)), next_(nullptr), - referenced_channel_(std::move(referenced_channel)), - referenced_type_(type) {} + referenced_entity_(std::move(referenced_entity)) {} ChannelTrace::TraceEvent::TraceEvent(Severity severity, grpc_slice data) : severity_(severity), @@ -110,23 +108,13 @@ void ChannelTrace::AddTraceEvent(Severity severity, grpc_slice data) { AddTraceEventHelper(New<TraceEvent>(severity, data)); } -void ChannelTrace::AddTraceEventReferencingChannel( - Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_channel) { - if (max_list_size_ == 0) return; // tracing is disabled if max_events == 0 - // create and fill up the new event - AddTraceEventHelper(New<TraceEvent>( - severity, data, std::move(referenced_channel), ReferencedType::Channel)); -} - -void ChannelTrace::AddTraceEventReferencingSubchannel( +void ChannelTrace::AddTraceEventWithReference( Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_subchannel) { + RefCountedPtr<BaseNode> referenced_entity) { if (max_list_size_ == 0) return; // tracing is disabled if max_events == 0 // create and fill up the new event - AddTraceEventHelper(New<TraceEvent>(severity, data, - std::move(referenced_subchannel), - ReferencedType::Subchannel)); + AddTraceEventHelper( + New<TraceEvent>(severity, data, std::move(referenced_entity))); } namespace { @@ -157,19 +145,18 @@ void ChannelTrace::TraceEvent::RenderTraceEvent(grpc_json* json) const { json_iterator = grpc_json_create_child(json_iterator, json, "timestamp", gpr_format_timespec(timestamp_), GRPC_JSON_STRING, true); - if (referenced_channel_ != nullptr) { + if (referenced_entity_ != nullptr) { + const bool is_channel = + (referenced_entity_->type() == BaseNode::EntityType::kTopLevelChannel || + referenced_entity_->type() == BaseNode::EntityType::kInternalChannel); char* uuid_str; - gpr_asprintf(&uuid_str, "%" PRIdPTR, referenced_channel_->channel_uuid()); + gpr_asprintf(&uuid_str, "%" PRIdPTR, referenced_entity_->uuid()); grpc_json* child_ref = grpc_json_create_child( - json_iterator, json, - (referenced_type_ == ReferencedType::Channel) ? "channelRef" - : "subchannelRef", + json_iterator, json, is_channel ? "channelRef" : "subchannelRef", nullptr, GRPC_JSON_OBJECT, false); json_iterator = grpc_json_create_child( - nullptr, child_ref, - (referenced_type_ == ReferencedType::Channel) ? "channelId" - : "subchannelId", - uuid_str, GRPC_JSON_STRING, true); + nullptr, child_ref, is_channel ? "channelId" : "subchannelId", uuid_str, + GRPC_JSON_STRING, true); json_iterator = child_ref; } } @@ -178,24 +165,26 @@ grpc_json* ChannelTrace::RenderJson() const { if (!max_list_size_) return nullptr; // tracing is disabled if max_events == 0 grpc_json* json = grpc_json_create(GRPC_JSON_OBJECT); - char* num_events_logged_str; - gpr_asprintf(&num_events_logged_str, "%" PRId64, num_events_logged_); grpc_json* json_iterator = nullptr; - json_iterator = - grpc_json_create_child(json_iterator, json, "numEventsLogged", - num_events_logged_str, GRPC_JSON_STRING, true); + if (num_events_logged_ > 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "numEventsLogged", num_events_logged_); + } json_iterator = grpc_json_create_child( json_iterator, json, "creationTimestamp", gpr_format_timespec(time_created_), GRPC_JSON_STRING, true); - grpc_json* events = grpc_json_create_child(json_iterator, json, "events", - nullptr, GRPC_JSON_ARRAY, false); - json_iterator = nullptr; - TraceEvent* it = head_trace_; - while (it != nullptr) { - json_iterator = grpc_json_create_child(json_iterator, events, nullptr, - nullptr, GRPC_JSON_OBJECT, false); - it->RenderTraceEvent(json_iterator); - it = it->next(); + // only add in the event list if it is non-empty. + if (num_events_logged_ > 0) { + grpc_json* events = grpc_json_create_child(json_iterator, json, "events", + nullptr, GRPC_JSON_ARRAY, false); + json_iterator = nullptr; + TraceEvent* it = head_trace_; + while (it != nullptr) { + json_iterator = grpc_json_create_child(json_iterator, events, nullptr, + nullptr, GRPC_JSON_OBJECT, false); + it->RenderTraceEvent(json_iterator); + it = it->next(); + } } return json; } diff --git a/src/core/lib/channel/channel_trace.h b/src/core/lib/channel/channel_trace.h index 596af7402f..94fea20b45 100644 --- a/src/core/lib/channel/channel_trace.h +++ b/src/core/lib/channel/channel_trace.h @@ -30,7 +30,7 @@ namespace grpc_core { namespace channelz { -class ChannelNode; +class BaseNode; // Object used to hold live data for a channel. This data is exposed via the // channelz service: @@ -55,35 +55,28 @@ class ChannelTrace { void AddTraceEvent(Severity severity, grpc_slice data); // Adds a new trace event to the tracing object. This trace event refers to a - // an event on a child of the channel. For example, if this channel has - // created a new subchannel, then it would record that with a TraceEvent - // referencing the new subchannel. + // an event that concerns a different channelz entity. For example, if this + // channel has created a new subchannel, then it would record that with + // a TraceEvent referencing the new subchannel. // // TODO(ncteisen): as this call is used more and more throughout the gRPC // stack, determine if it makes more sense to accept a char* instead of a // slice. - void AddTraceEventReferencingChannel( - Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_channel); - void AddTraceEventReferencingSubchannel( - Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_subchannel); + void AddTraceEventWithReference(Severity severity, grpc_slice data, + RefCountedPtr<BaseNode> referenced_entity); // Creates and returns the raw grpc_json object, so a parent channelz // object may incorporate the json before rendering. grpc_json* RenderJson() const; private: - // Types of objects that can be references by trace events. - enum class ReferencedType { Channel, Subchannel }; // Private class to encapsulate all the data and bookkeeping needed for a // a trace event. class TraceEvent { public: - // Constructor for a TraceEvent that references a different channel. + // Constructor for a TraceEvent that references a channel. TraceEvent(Severity severity, grpc_slice data, - RefCountedPtr<ChannelNode> referenced_channel, - ReferencedType type); + RefCountedPtr<BaseNode> referenced_entity_); // Constructor for a TraceEvent that does not reverence a different // channel. @@ -105,10 +98,7 @@ class ChannelTrace { gpr_timespec timestamp_; TraceEvent* next_; // the tracer object for the (sub)channel that this trace event refers to. - RefCountedPtr<ChannelNode> referenced_channel_; - // the type that the referenced tracer points to. Unused if this trace - // does not point to any channel or subchannel - ReferencedType referenced_type_; + RefCountedPtr<BaseNode> referenced_entity_; }; // TraceEvent // Internal helper to add and link in a trace event diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index 9d6002ed8a..339c827525 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -41,33 +41,64 @@ namespace grpc_core { namespace channelz { -ChannelNode::ChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, - bool is_top_level_channel) - : channel_(channel), - target_(nullptr), - channel_uuid_(-1), - is_top_level_channel_(is_top_level_channel) { - trace_.Init(channel_tracer_max_nodes); - target_ = UniquePtr<char>(grpc_channel_get_target(channel_)); - channel_uuid_ = ChannelzRegistry::RegisterChannelNode(this); +BaseNode::BaseNode(EntityType type) + : type_(type), uuid_(ChannelzRegistry::Register(this)) {} + +BaseNode::~BaseNode() { ChannelzRegistry::Unregister(uuid_); } + +char* BaseNode::RenderJsonString() { + grpc_json* json = RenderJson(); + GPR_ASSERT(json != nullptr); + char* json_str = grpc_json_dump_to_string(json, 0); + grpc_json_destroy(json); + return json_str; +} + +CallCountingHelper::CallCountingHelper() { gpr_atm_no_barrier_store(&last_call_started_millis_, (gpr_atm)ExecCtx::Get()->Now()); } -ChannelNode::~ChannelNode() { - trace_.Destroy(); - ChannelzRegistry::UnregisterChannelNode(channel_uuid_); -} +CallCountingHelper::~CallCountingHelper() {} -void ChannelNode::RecordCallStarted() { - gpr_atm_no_barrier_fetch_add(&calls_started_, (gpr_atm)1); +void CallCountingHelper::RecordCallStarted() { + gpr_atm_no_barrier_fetch_add(&calls_started_, static_cast<gpr_atm>(1)); gpr_atm_no_barrier_store(&last_call_started_millis_, (gpr_atm)ExecCtx::Get()->Now()); } -void ChannelNode::PopulateConnectivityState(grpc_json* json) {} +void CallCountingHelper::PopulateCallCounts(grpc_json* json) { + grpc_json* json_iterator = nullptr; + if (calls_started_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "callsStarted", calls_started_); + } + if (calls_succeeded_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "callsSucceeded", calls_succeeded_); + } + if (calls_failed_) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "callsFailed", calls_failed_); + } + if (calls_started_ != 0) { + gpr_timespec ts = + grpc_millis_to_timespec(last_call_started_millis_, GPR_CLOCK_REALTIME); + json_iterator = + grpc_json_create_child(json_iterator, json, "lastCallStartedTimestamp", + gpr_format_timespec(ts), GRPC_JSON_STRING, true); + } +} + +ChannelNode::ChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, + bool is_top_level_channel) + : BaseNode(is_top_level_channel ? EntityType::kTopLevelChannel + : EntityType::kInternalChannel), + channel_(channel), + target_(UniquePtr<char>(grpc_channel_get_target(channel_))), + trace_(channel_tracer_max_nodes) {} -void ChannelNode::PopulateChildRefs(grpc_json* json) {} +ChannelNode::~ChannelNode() {} grpc_json* ChannelNode::RenderJson() { // We need to track these three json objects to build our object @@ -80,7 +111,7 @@ grpc_json* ChannelNode::RenderJson() { json = json_iterator; json_iterator = nullptr; json_iterator = grpc_json_add_number_string_child(json, json_iterator, - "channelId", channel_uuid_); + "channelId", uuid()); // reset json iterators to top level object json = top_level_json; json_iterator = nullptr; @@ -89,51 +120,28 @@ grpc_json* ChannelNode::RenderJson() { GRPC_JSON_OBJECT, false); json = data; json_iterator = nullptr; + // template method. Child classes may override this to add their specific + // functionality. PopulateConnectivityState(json); + // populate the target. GPR_ASSERT(target_.get() != nullptr); - json_iterator = grpc_json_create_child( - json_iterator, json, "target", target_.get(), GRPC_JSON_STRING, false); + grpc_json_create_child(nullptr, json, "target", target_.get(), + GRPC_JSON_STRING, false); // fill in the channel trace if applicable - grpc_json* trace = trace_->RenderJson(); - if (trace != nullptr) { - // we manually link up and fill the child since it was created for us in - // ChannelTrace::RenderJson - trace->key = "trace"; // this object is named trace in channelz.proto - json_iterator = grpc_json_link_child(json, trace, json_iterator); + grpc_json* trace_json = trace_.RenderJson(); + if (trace_json != nullptr) { + trace_json->key = "trace"; // this object is named trace in channelz.proto + grpc_json_link_child(json, trace_json, nullptr); } - // reset the parent to be the data object. - json = data; - json_iterator = nullptr; - if (calls_started_ != 0) { - json_iterator = grpc_json_add_number_string_child( - json, json_iterator, "callsStarted", calls_started_); - } - if (calls_succeeded_ != 0) { - json_iterator = grpc_json_add_number_string_child( - json, json_iterator, "callsSucceeded", calls_succeeded_); - } - if (calls_failed_) { - json_iterator = grpc_json_add_number_string_child( - json, json_iterator, "callsFailed", calls_failed_); - } - gpr_timespec ts = - grpc_millis_to_timespec(last_call_started_millis_, GPR_CLOCK_REALTIME); - json_iterator = - grpc_json_create_child(json_iterator, json, "lastCallStartedTimestamp", - gpr_format_timespec(ts), GRPC_JSON_STRING, true); + // ask CallCountingHelper to populate trace and call count data. + call_counter_.PopulateCallCounts(json); json = top_level_json; - json_iterator = nullptr; + // template method. Child classes may override this to add their specific + // functionality. PopulateChildRefs(json); return top_level_json; } -char* ChannelNode::RenderJsonString() { - grpc_json* json = RenderJson(); - char* json_str = grpc_json_dump_to_string(json, 0); - grpc_json_destroy(json); - return json_str; -} - RefCountedPtr<ChannelNode> ChannelNode::MakeChannelNode( grpc_channel* channel, size_t channel_tracer_max_nodes, bool is_top_level_channel) { @@ -141,12 +149,137 @@ RefCountedPtr<ChannelNode> ChannelNode::MakeChannelNode( channel, channel_tracer_max_nodes, is_top_level_channel); } -SubchannelNode::SubchannelNode() { - subchannel_uuid_ = ChannelzRegistry::RegisterSubchannelNode(this); +ServerNode::ServerNode(size_t channel_tracer_max_nodes) + : BaseNode(EntityType::kServer), trace_(channel_tracer_max_nodes) {} + +ServerNode::~ServerNode() {} + +grpc_json* ServerNode::RenderJson() { + // We need to track these three json objects to build our object + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + // create and fill the ref child + json_iterator = grpc_json_create_child(json_iterator, json, "ref", nullptr, + GRPC_JSON_OBJECT, false); + json = json_iterator; + json_iterator = nullptr; + json_iterator = grpc_json_add_number_string_child(json, json_iterator, + "serverId", uuid()); + // reset json iterators to top level object + json = top_level_json; + json_iterator = nullptr; + // create and fill the data child. + grpc_json* data = grpc_json_create_child(json_iterator, json, "data", nullptr, + GRPC_JSON_OBJECT, false); + json = data; + json_iterator = nullptr; + // fill in the channel trace if applicable + grpc_json* trace_json = trace_.RenderJson(); + if (trace_json != nullptr) { + trace_json->key = "trace"; // this object is named trace in channelz.proto + grpc_json_link_child(json, trace_json, nullptr); + } + // ask CallCountingHelper to populate trace and call count data. + call_counter_.PopulateCallCounts(json); + return top_level_json; +} + +SocketNode::SocketNode() : BaseNode(EntityType::kSocket) {} + +void SocketNode::RecordStreamStartedFromLocal() { + gpr_atm_no_barrier_fetch_add(&streams_started_, static_cast<gpr_atm>(1)); + gpr_atm_no_barrier_store(&last_local_stream_created_millis_, + (gpr_atm)ExecCtx::Get()->Now()); +} + +void SocketNode::RecordStreamStartedFromRemote() { + gpr_atm_no_barrier_fetch_add(&streams_started_, static_cast<gpr_atm>(1)); + gpr_atm_no_barrier_store(&last_remote_stream_created_millis_, + (gpr_atm)ExecCtx::Get()->Now()); } -SubchannelNode::~SubchannelNode() { - ChannelzRegistry::UnregisterSubchannelNode(subchannel_uuid_); +void SocketNode::RecordMessagesSent(uint32_t num_sent) { + gpr_atm_no_barrier_fetch_add(&messages_sent_, static_cast<gpr_atm>(num_sent)); + gpr_atm_no_barrier_store(&last_message_sent_millis_, + (gpr_atm)ExecCtx::Get()->Now()); +} + +void SocketNode::RecordMessageReceived() { + gpr_atm_no_barrier_fetch_add(&messages_received_, static_cast<gpr_atm>(1)); + gpr_atm_no_barrier_store(&last_message_received_millis_, + (gpr_atm)ExecCtx::Get()->Now()); +} + +grpc_json* SocketNode::RenderJson() { + // We need to track these three json objects to build our object + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + // create and fill the ref child + json_iterator = grpc_json_create_child(json_iterator, json, "ref", nullptr, + GRPC_JSON_OBJECT, false); + json = json_iterator; + json_iterator = nullptr; + json_iterator = grpc_json_add_number_string_child(json, json_iterator, + "socketId", uuid()); + // reset json iterators to top level object + json = top_level_json; + json_iterator = nullptr; + // create and fill the data child. + grpc_json* data = grpc_json_create_child(json_iterator, json, "data", nullptr, + GRPC_JSON_OBJECT, false); + json = data; + json_iterator = nullptr; + gpr_timespec ts; + if (streams_started_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "streamsStarted", streams_started_); + if (last_local_stream_created_millis_ != 0) { + ts = grpc_millis_to_timespec(last_local_stream_created_millis_, + GPR_CLOCK_REALTIME); + json_iterator = grpc_json_create_child( + json_iterator, json, "lastLocalStreamCreatedTimestamp", + gpr_format_timespec(ts), GRPC_JSON_STRING, true); + } + if (last_remote_stream_created_millis_ != 0) { + ts = grpc_millis_to_timespec(last_remote_stream_created_millis_, + GPR_CLOCK_REALTIME); + json_iterator = grpc_json_create_child( + json_iterator, json, "lastRemoteStreamCreatedTimestamp", + gpr_format_timespec(ts), GRPC_JSON_STRING, true); + } + } + if (streams_succeeded_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "streamsSucceeded", streams_succeeded_); + } + if (streams_failed_) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "streamsFailed", streams_failed_); + } + if (messages_sent_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "messagesSent", messages_sent_); + ts = grpc_millis_to_timespec(last_message_sent_millis_, GPR_CLOCK_REALTIME); + json_iterator = + grpc_json_create_child(json_iterator, json, "lastMessageSentTimestamp", + gpr_format_timespec(ts), GRPC_JSON_STRING, true); + } + if (messages_received_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "messagesReceived", messages_received_); + ts = grpc_millis_to_timespec(last_message_received_millis_, + GPR_CLOCK_REALTIME); + json_iterator = grpc_json_create_child( + json_iterator, json, "lastMessageReceivedTimestamp", + gpr_format_timespec(ts), GRPC_JSON_STRING, true); + } + if (keepalives_sent_ != 0) { + json_iterator = grpc_json_add_number_string_child( + json, json_iterator, "keepAlivesSent", keepalives_sent_); + } + return top_level_json; } } // namespace channelz diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index 07eb73d626..b7ae101238 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -43,34 +43,101 @@ namespace grpc_core { namespace channelz { namespace testing { +class CallCountingHelperPeer; class ChannelNodePeer; -} +} // namespace testing -class ChannelNode : public RefCounted<ChannelNode> { +// base class for all channelz entities +class BaseNode : public RefCounted<BaseNode> { public: - static RefCountedPtr<ChannelNode> MakeChannelNode( - grpc_channel* channel, size_t channel_tracer_max_nodes, - bool is_top_level_channel); + // There are only four high level channelz entities. However, to support + // GetTopChannelsRequest, we split the Channel entity into two different + // types. All children of BaseNode must be one of these types. + enum class EntityType { + kTopLevelChannel, + kInternalChannel, + kSubchannel, + kServer, + kSocket, + }; + + explicit BaseNode(EntityType type); + virtual ~BaseNode(); + + // All children must implement this function. + virtual grpc_json* RenderJson() GRPC_ABSTRACT; + + // Renders the json and returns allocated string that must be freed by the + // caller. + char* RenderJsonString(); + + EntityType type() const { return type_; } + intptr_t uuid() const { return uuid_; } + + private: + const EntityType type_; + const intptr_t uuid_; +}; + +// This class is a helper class for channelz entities that deal with Channels, +// Subchannels, and Servers, since those have similar proto definitions. +// This class has the ability to: +// - track calls_{started,succeeded,failed} +// - track last_call_started_timestamp +// - perform rendering of the above items +class CallCountingHelper { + public: + CallCountingHelper(); + ~CallCountingHelper(); void RecordCallStarted(); void RecordCallFailed() { - gpr_atm_no_barrier_fetch_add(&calls_failed_, (gpr_atm(1))); + gpr_atm_no_barrier_fetch_add(&calls_failed_, static_cast<gpr_atm>(1)); } void RecordCallSucceeded() { - gpr_atm_no_barrier_fetch_add(&calls_succeeded_, (gpr_atm(1))); + gpr_atm_no_barrier_fetch_add(&calls_succeeded_, static_cast<gpr_atm>(1)); } - grpc_json* RenderJson(); - char* RenderJsonString(); + // Common rendering of the call count data and last_call_started_timestamp. + void PopulateCallCounts(grpc_json* json); + + private: + // testing peer friend. + friend class testing::CallCountingHelperPeer; - // helper for getting and populating connectivity state. It is virtual - // because it allows the client_channel specific code to live in ext/ - // instead of lib/ - virtual void PopulateConnectivityState(grpc_json* json); + gpr_atm calls_started_ = 0; + gpr_atm calls_succeeded_ = 0; + gpr_atm calls_failed_ = 0; + gpr_atm last_call_started_millis_ = 0; +}; - virtual void PopulateChildRefs(grpc_json* json); +// Handles channelz bookkeeping for channels +class ChannelNode : public BaseNode { + public: + static RefCountedPtr<ChannelNode> MakeChannelNode( + grpc_channel* channel, size_t channel_tracer_max_nodes, + bool is_top_level_channel); - ChannelTrace* trace() { return trace_.get(); } + ChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, + bool is_top_level_channel); + ~ChannelNode() override; + + grpc_json* RenderJson() override; + + // template methods. RenderJSON uses these methods to render its JSON + // representation. These are virtual so that children classes may provide + // their specific mechanism for populating these parts of the channelz + // object. + // + // ChannelNode does not have a notion of connectivity state or child refs, + // so it leaves these implementations blank. + // + // This is utilizing the template method design pattern. + // + // TODO(ncteisen): remove these template methods in favor of manual traversal + // and mutation of the grpc_json object. + virtual void PopulateConnectivityState(grpc_json* json) {} + virtual void PopulateChildRefs(grpc_json* json) {} void MarkChannelDestroyed() { GPR_ASSERT(channel_ != nullptr); @@ -79,47 +146,90 @@ class ChannelNode : public RefCounted<ChannelNode> { bool ChannelIsDestroyed() { return channel_ == nullptr; } - intptr_t channel_uuid() { return channel_uuid_; } - bool is_top_level_channel() { return is_top_level_channel_; } - - protected: - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW - ChannelNode(grpc_channel* channel, size_t channel_tracer_max_nodes, - bool is_top_level_channel); - virtual ~ChannelNode(); + // proxy methods to composed classes. + void AddTraceEvent(ChannelTrace::Severity severity, grpc_slice data) { + trace_.AddTraceEvent(severity, data); + } + void AddTraceEventWithReference(ChannelTrace::Severity severity, + grpc_slice data, + RefCountedPtr<BaseNode> referenced_channel) { + trace_.AddTraceEventWithReference(severity, data, + std::move(referenced_channel)); + } + void RecordCallStarted() { call_counter_.RecordCallStarted(); } + void RecordCallFailed() { call_counter_.RecordCallFailed(); } + void RecordCallSucceeded() { call_counter_.RecordCallSucceeded(); } private: - // testing peer friend. + // to allow the channel trace test to access trace_. friend class testing::ChannelNodePeer; - grpc_channel* channel_ = nullptr; UniquePtr<char> target_; - gpr_atm calls_started_ = 0; - gpr_atm calls_succeeded_ = 0; - gpr_atm calls_failed_ = 0; - gpr_atm last_call_started_millis_ = 0; - intptr_t channel_uuid_; - bool is_top_level_channel_ = true; - ManualConstructor<ChannelTrace> trace_; + CallCountingHelper call_counter_; + ChannelTrace trace_; }; -// Placeholds channelz class for subchannels. All this can do now is track its -// uuid (this information is needed by the parent channelz class). -// TODO(ncteisen): build this out to support the GetSubchannel channelz request. -class SubchannelNode : public RefCounted<SubchannelNode> { +// Handles channelz bookkeeping for servers +class ServerNode : public BaseNode { public: - SubchannelNode(); - virtual ~SubchannelNode(); + explicit ServerNode(size_t channel_tracer_max_nodes); + ~ServerNode() override; - intptr_t subchannel_uuid() { return subchannel_uuid_; } + grpc_json* RenderJson() override; - protected: - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE - GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW + // proxy methods to composed classes. + void AddTraceEvent(ChannelTrace::Severity severity, grpc_slice data) { + trace_.AddTraceEvent(severity, data); + } + void AddTraceEventWithReference(ChannelTrace::Severity severity, + grpc_slice data, + RefCountedPtr<BaseNode> referenced_channel) { + trace_.AddTraceEventWithReference(severity, data, + std::move(referenced_channel)); + } + void RecordCallStarted() { call_counter_.RecordCallStarted(); } + void RecordCallFailed() { call_counter_.RecordCallFailed(); } + void RecordCallSucceeded() { call_counter_.RecordCallSucceeded(); } + + private: + CallCountingHelper call_counter_; + ChannelTrace trace_; +}; + +// Handles channelz bookkeeping for sockets +class SocketNode : public BaseNode { + public: + SocketNode(); + ~SocketNode() override {} + + grpc_json* RenderJson() override; + + void RecordStreamStartedFromLocal(); + void RecordStreamStartedFromRemote(); + void RecordStreamSucceeded() { + gpr_atm_no_barrier_fetch_add(&streams_succeeded_, static_cast<gpr_atm>(1)); + } + void RecordStreamFailed() { + gpr_atm_no_barrier_fetch_add(&streams_failed_, static_cast<gpr_atm>(1)); + } + void RecordMessagesSent(uint32_t num_sent); + void RecordMessageReceived(); + void RecordKeepaliveSent() { + gpr_atm_no_barrier_fetch_add(&keepalives_sent_, static_cast<gpr_atm>(1)); + } private: - intptr_t subchannel_uuid_; + gpr_atm streams_started_ = 0; + gpr_atm streams_succeeded_ = 0; + gpr_atm streams_failed_ = 0; + gpr_atm messages_sent_ = 0; + gpr_atm messages_received_ = 0; + gpr_atm keepalives_sent_ = 0; + gpr_atm last_local_stream_created_millis_ = 0; + gpr_atm last_remote_stream_created_millis_ = 0; + gpr_atm last_message_sent_millis_ = 0; + gpr_atm last_message_received_millis_ = 0; + UniquePtr<char> peer_string_; }; // Creation functions diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index f79d2f0c17..841f1c6104 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -53,54 +53,46 @@ ChannelzRegistry::ChannelzRegistry() { gpr_mu_init(&mu_); } ChannelzRegistry::~ChannelzRegistry() { gpr_mu_destroy(&mu_); } -intptr_t ChannelzRegistry::InternalRegisterEntry(const RegistryEntry& entry) { +intptr_t ChannelzRegistry::InternalRegister(BaseNode* node) { MutexLock lock(&mu_); - entities_.push_back(entry); + entities_.push_back(node); intptr_t uuid = entities_.size(); return uuid; } -void ChannelzRegistry::InternalUnregisterEntry(intptr_t uuid, EntityType type) { +void ChannelzRegistry::InternalUnregister(intptr_t uuid) { GPR_ASSERT(uuid >= 1); MutexLock lock(&mu_); GPR_ASSERT(static_cast<size_t>(uuid) <= entities_.size()); - GPR_ASSERT(entities_[uuid - 1].type == type); - entities_[uuid - 1].object = nullptr; - entities_[uuid - 1].type = EntityType::kUnset; + entities_[uuid - 1] = nullptr; } -void* ChannelzRegistry::InternalGetEntry(intptr_t uuid, EntityType type) { +BaseNode* ChannelzRegistry::InternalGet(intptr_t uuid) { MutexLock lock(&mu_); if (uuid < 1 || uuid > static_cast<intptr_t>(entities_.size())) { return nullptr; } - if (entities_[uuid - 1].type == type) { - return entities_[uuid - 1].object; - } else { - return nullptr; - } + return entities_[uuid - 1]; } char* ChannelzRegistry::InternalGetTopChannels(intptr_t start_channel_id) { grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); grpc_json* json = top_level_json; grpc_json* json_iterator = nullptr; - InlinedVector<ChannelNode*, 10> top_level_channels; + InlinedVector<BaseNode*, 10> top_level_channels; // uuids index into entities one-off (idx 0 is really uuid 1, since 0 is // reserved). However, we want to support requests coming in with // start_channel_id=0, which signifies "give me everything." Hence this // funky looking line below. size_t start_idx = start_channel_id == 0 ? 0 : start_channel_id - 1; for (size_t i = start_idx; i < entities_.size(); ++i) { - if (entities_[i].type == EntityType::kChannelNode) { - ChannelNode* channel_node = - static_cast<ChannelNode*>(entities_[i].object); - if (channel_node->is_top_level_channel()) { - top_level_channels.push_back(channel_node); - } + if (entities_[i] != nullptr && + entities_[i]->type() == + grpc_core::channelz::BaseNode::EntityType::kTopLevelChannel) { + top_level_channels.push_back(entities_[i]); } } - if (top_level_channels.size() > 0) { + if (!top_level_channels.empty()) { // create list of channels grpc_json* array_parent = grpc_json_create_child( nullptr, json, "channel", nullptr, GRPC_JSON_ARRAY, false); @@ -120,6 +112,42 @@ char* ChannelzRegistry::InternalGetTopChannels(intptr_t start_channel_id) { return json_str; } +char* ChannelzRegistry::InternalGetServers(intptr_t start_server_id) { + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* json_iterator = nullptr; + InlinedVector<BaseNode*, 10> servers; + // uuids index into entities one-off (idx 0 is really uuid 1, since 0 is + // reserved). However, we want to support requests coming in with + // start_server_id=0, which signifies "give me everything." + size_t start_idx = start_server_id == 0 ? 0 : start_server_id - 1; + for (size_t i = start_idx; i < entities_.size(); ++i) { + if (entities_[i] != nullptr && + entities_[i]->type() == + grpc_core::channelz::BaseNode::EntityType::kServer) { + servers.push_back(entities_[i]); + } + } + if (!servers.empty()) { + // create list of servers + grpc_json* array_parent = grpc_json_create_child( + nullptr, json, "server", nullptr, GRPC_JSON_ARRAY, false); + for (size_t i = 0; i < servers.size(); ++i) { + grpc_json* server_json = servers[i]->RenderJson(); + json_iterator = + grpc_json_link_child(array_parent, server_json, json_iterator); + } + } + // For now we do not have any pagination rules. In the future we could + // pick a constant for max_channels_sent for a GetServers request. + // Tracking: https://github.com/grpc/grpc/issues/16019. + json_iterator = grpc_json_create_child(nullptr, json, "end", nullptr, + GRPC_JSON_TRUE, false); + char* json_str = grpc_json_dump_to_string(top_level_json, 0); + grpc_json_destroy(top_level_json); + return json_str; +} + } // namespace channelz } // namespace grpc_core @@ -128,10 +156,18 @@ char* grpc_channelz_get_top_channels(intptr_t start_channel_id) { start_channel_id); } +char* grpc_channelz_get_servers(intptr_t start_server_id) { + return grpc_core::channelz::ChannelzRegistry::GetServers(start_server_id); +} + char* grpc_channelz_get_channel(intptr_t channel_id) { - grpc_core::channelz::ChannelNode* channel_node = - grpc_core::channelz::ChannelzRegistry::GetChannelNode(channel_id); - if (channel_node == nullptr) { + grpc_core::channelz::BaseNode* channel_node = + grpc_core::channelz::ChannelzRegistry::Get(channel_id); + if (channel_node == nullptr || + (channel_node->type() != + grpc_core::channelz::BaseNode::EntityType::kTopLevelChannel && + channel_node->type() != + grpc_core::channelz::BaseNode::EntityType::kInternalChannel)) { return nullptr; } grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); @@ -143,3 +179,39 @@ char* grpc_channelz_get_channel(intptr_t channel_id) { grpc_json_destroy(top_level_json); return json_str; } + +char* grpc_channelz_get_subchannel(intptr_t subchannel_id) { + grpc_core::channelz::BaseNode* subchannel_node = + grpc_core::channelz::ChannelzRegistry::Get(subchannel_id); + if (subchannel_node == nullptr || + subchannel_node->type() != + grpc_core::channelz::BaseNode::EntityType::kSubchannel) { + return nullptr; + } + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* subchannel_json = subchannel_node->RenderJson(); + subchannel_json->key = "subchannel"; + grpc_json_link_child(json, subchannel_json, nullptr); + char* json_str = grpc_json_dump_to_string(top_level_json, 0); + grpc_json_destroy(top_level_json); + return json_str; +} + +char* grpc_channelz_get_socket(intptr_t socket_id) { + grpc_core::channelz::BaseNode* socket_node = + grpc_core::channelz::ChannelzRegistry::Get(socket_id); + if (socket_node == nullptr || + socket_node->type() != + grpc_core::channelz::BaseNode::EntityType::kSocket) { + return nullptr; + } + grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); + grpc_json* json = top_level_json; + grpc_json* socket_json = socket_node->RenderJson(); + socket_json->key = "socket"; + grpc_json_link_child(json, socket_json, nullptr); + char* json_str = grpc_json_dump_to_string(top_level_json, 0); + grpc_json_destroy(top_level_json); + return json_str; +} diff --git a/src/core/lib/channel/channelz_registry.h b/src/core/lib/channel/channelz_registry.h index 5d7c936726..d0d660600d 100644 --- a/src/core/lib/channel/channelz_registry.h +++ b/src/core/lib/channel/channelz_registry.h @@ -40,32 +40,11 @@ class ChannelzRegistry { // To be called in grpc_shutdown(); static void Shutdown(); - // Register/Unregister/Get for ChannelNode - static intptr_t RegisterChannelNode(ChannelNode* channel_node) { - RegistryEntry entry(channel_node, EntityType::kChannelNode); - return Default()->InternalRegisterEntry(entry); - } - static void UnregisterChannelNode(intptr_t uuid) { - Default()->InternalUnregisterEntry(uuid, EntityType::kChannelNode); - } - static ChannelNode* GetChannelNode(intptr_t uuid) { - void* gotten = Default()->InternalGetEntry(uuid, EntityType::kChannelNode); - return gotten == nullptr ? nullptr : static_cast<ChannelNode*>(gotten); - } - - // Register/Unregister/Get for SubchannelNode - static intptr_t RegisterSubchannelNode(SubchannelNode* channel_node) { - RegistryEntry entry(channel_node, EntityType::kSubchannelNode); - return Default()->InternalRegisterEntry(entry); - } - static void UnregisterSubchannelNode(intptr_t uuid) { - Default()->InternalUnregisterEntry(uuid, EntityType::kSubchannelNode); - } - static SubchannelNode* GetSubchannelNode(intptr_t uuid) { - void* gotten = - Default()->InternalGetEntry(uuid, EntityType::kSubchannelNode); - return gotten == nullptr ? nullptr : static_cast<SubchannelNode*>(gotten); + static intptr_t Register(BaseNode* node) { + return Default()->InternalRegister(node); } + static void Unregister(intptr_t uuid) { Default()->InternalUnregister(uuid); } + static BaseNode* Get(intptr_t uuid) { return Default()->InternalGet(uuid); } // Returns the allocated JSON string that represents the proto // GetTopChannelsResponse as per channelz.proto. @@ -73,20 +52,13 @@ class ChannelzRegistry { return Default()->InternalGetTopChannels(start_channel_id); } - private: - enum class EntityType { - kChannelNode, - kSubchannelNode, - kUnset, - }; - - struct RegistryEntry { - RegistryEntry(void* object_in, EntityType type_in) - : object(object_in), type(type_in) {} - void* object; - EntityType type; - }; + // Returns the allocated JSON string that represents the proto + // GetServersResponse as per channelz.proto. + static char* GetServers(intptr_t start_server_id) { + return Default()->InternalGetServers(start_server_id); + } + private: GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE @@ -97,21 +69,22 @@ class ChannelzRegistry { static ChannelzRegistry* Default(); // globally registers an Entry. Returns its unique uuid - intptr_t InternalRegisterEntry(const RegistryEntry& entry); + intptr_t InternalRegister(BaseNode* node); // globally unregisters the object that is associated to uuid. Also does // sanity check that an object doesn't try to unregister the wrong type. - void InternalUnregisterEntry(intptr_t uuid, EntityType type); + void InternalUnregister(intptr_t uuid); // if object with uuid has previously been registered as the correct type, // returns the void* associated with that uuid. Else returns nullptr. - void* InternalGetEntry(intptr_t uuid, EntityType type); + BaseNode* InternalGet(intptr_t uuid); char* InternalGetTopChannels(intptr_t start_channel_id); + char* InternalGetServers(intptr_t start_server_id); // protects entities_ and uuid_ gpr_mu mu_; - InlinedVector<RegistryEntry, 20> entities_; + InlinedVector<BaseNode*, 20> entities_; }; } // namespace channelz diff --git a/src/core/lib/iomgr/error.cc b/src/core/lib/iomgr/error.cc index 90ed34da11..146a539027 100644 --- a/src/core/lib/iomgr/error.cc +++ b/src/core/lib/iomgr/error.cc @@ -454,7 +454,7 @@ typedef struct { grpc_status_code code; const char* msg; } special_error_status_map; -static special_error_status_map error_status_map[] = { +static const special_error_status_map error_status_map[] = { {GRPC_ERROR_NONE, GRPC_STATUS_OK, ""}, {GRPC_ERROR_CANCELLED, GRPC_STATUS_CANCELLED, "Cancelled"}, {GRPC_ERROR_OOM, GRPC_STATUS_RESOURCE_EXHAUSTED, "Out of memory"}, @@ -463,15 +463,13 @@ static special_error_status_map error_status_map[] = { bool grpc_error_get_int(grpc_error* err, grpc_error_ints which, intptr_t* p) { GPR_TIMER_SCOPE("grpc_error_get_int", 0); if (grpc_error_is_special(err)) { - if (which == GRPC_ERROR_INT_GRPC_STATUS) { - for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { - if (error_status_map[i].error == err) { - if (p != nullptr) *p = error_status_map[i].code; - return true; - } + for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { + if (error_status_map[i].error == err) { + if (which != GRPC_ERROR_INT_GRPC_STATUS) return false; + if (p != nullptr) *p = error_status_map[i].code; + return true; } } - return false; } uint8_t slot = err->ints[which]; if (slot != UINT8_MAX) { @@ -492,15 +490,13 @@ grpc_error* grpc_error_set_str(grpc_error* src, grpc_error_strs which, bool grpc_error_get_str(grpc_error* err, grpc_error_strs which, grpc_slice* str) { if (grpc_error_is_special(err)) { - if (which == GRPC_ERROR_STR_GRPC_MESSAGE) { - for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { - if (error_status_map[i].error == err) { - *str = grpc_slice_from_static_string(error_status_map[i].msg); - return true; - } + for (size_t i = 0; i < GPR_ARRAY_SIZE(error_status_map); i++) { + if (error_status_map[i].error == err) { + if (which != GRPC_ERROR_STR_GRPC_MESSAGE) return false; + *str = grpc_slice_from_static_string(error_status_map[i].msg); + return true; } } - return false; } uint8_t slot = err->strs[which]; if (slot != UINT8_MAX) { @@ -513,9 +509,24 @@ bool grpc_error_get_str(grpc_error* err, grpc_error_strs which, grpc_error* grpc_error_add_child(grpc_error* src, grpc_error* child) { GPR_TIMER_SCOPE("grpc_error_add_child", 0); - grpc_error* new_err = copy_error_and_unref(src); - internal_add_error(&new_err, child); - return new_err; + if (src != GRPC_ERROR_NONE) { + if (child == GRPC_ERROR_NONE) { + /* \a child is empty. Simply return the ref to \a src */ + return src; + } else if (child != src) { + grpc_error* new_err = copy_error_and_unref(src); + internal_add_error(&new_err, child); + return new_err; + } else { + /* \a src and \a child are the same. Drop one of the references and return + * the other */ + GRPC_ERROR_UNREF(child); + return src; + } + } else { + /* \a src is empty. Simply return the ref to \a child */ + return child; + } } static const char* no_error_string = "\"No Error\""; diff --git a/src/core/lib/iomgr/error.h b/src/core/lib/iomgr/error.h index 27c4d22fd1..49f4029bc2 100644 --- a/src/core/lib/iomgr/error.h +++ b/src/core/lib/iomgr/error.h @@ -185,8 +185,16 @@ bool grpc_error_get_str(grpc_error* error, grpc_error_strs which, /// error occurring. Allows root causing high level errors from lower level /// errors that contributed to them. The src error takes ownership of the /// child error. +/// +/// Edge Conditions - +/// 1) If either of \a src or \a child is GRPC_ERROR_NONE, returns a reference +/// to the other argument. 2) If both \a src and \a child are GRPC_ERROR_NONE, +/// returns GRPC_ERROR_NONE. 3) If \a src and \a child point to the same error, +/// returns a single reference. (Note that, 2 references should have been +/// received to the error in this case.) grpc_error* grpc_error_add_child(grpc_error* src, grpc_error* child) GRPC_MUST_USE_RESULT; + grpc_error* grpc_os_error(const char* file, int line, int err, const char* call_name) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc deleted file mode 100644 index 5695ac795d..0000000000 --- a/src/core/lib/iomgr/ev_epollsig_linux.cc +++ /dev/null @@ -1,1743 +0,0 @@ -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include <grpc/support/port_platform.h> - -#include "src/core/lib/iomgr/port.h" - -#include <grpc/grpc_posix.h> -#include <grpc/support/log.h> - -/* This polling engine is only relevant on linux kernels supporting epoll() */ -#ifdef GRPC_LINUX_EPOLL_CREATE1 - -#include "src/core/lib/iomgr/ev_epollsig_linux.h" - -#include <assert.h> -#include <errno.h> -#include <limits.h> -#include <poll.h> -#include <pthread.h> -#include <signal.h> -#include <string.h> -#include <sys/epoll.h> -#include <sys/socket.h> -#include <unistd.h> - -#include <grpc/support/alloc.h> -#include <grpc/support/string_util.h> - -#include "src/core/lib/debug/stats.h" -#include "src/core/lib/gpr/tls.h" -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/iomgr/block_annotate.h" -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/iomgr/iomgr_internal.h" -#include "src/core/lib/iomgr/lockfree_event.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/iomgr/wakeup_fd_posix.h" -#include "src/core/lib/profiling/timers.h" - -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1) - -#define GRPC_POLLING_TRACE(...) \ - if (grpc_polling_trace.enabled()) { \ - gpr_log(GPR_INFO, __VA_ARGS__); \ - } - -static int grpc_wakeup_signal = -1; -static bool is_grpc_wakeup_signal_initialized = false; - -/* Implements the function defined in grpc_posix.h. This function might be - * called before even calling grpc_init() to set either a different signal to - * use. If signum == -1, then the use of signals is disabled */ -void grpc_use_signal(int signum) { - grpc_wakeup_signal = signum; - is_grpc_wakeup_signal_initialized = true; - - if (grpc_wakeup_signal < 0) { - gpr_log(GPR_INFO, - "Use of signals is disabled. Epoll engine will not be used"); - } else { - gpr_log(GPR_INFO, "epoll engine will be using signal: %d", - grpc_wakeup_signal); - } -} - -struct polling_island; - -typedef enum { - POLL_OBJ_FD, - POLL_OBJ_POLLSET, - POLL_OBJ_POLLSET_SET -} poll_obj_type; - -typedef struct poll_obj { -#ifndef NDEBUG - poll_obj_type obj_type; -#endif - gpr_mu mu; - struct polling_island* pi; -} poll_obj; - -const char* poll_obj_string(poll_obj_type po_type) { - switch (po_type) { - case POLL_OBJ_FD: - return "fd"; - case POLL_OBJ_POLLSET: - return "pollset"; - case POLL_OBJ_POLLSET_SET: - return "pollset_set"; - } - - GPR_UNREACHABLE_CODE(return "UNKNOWN"); -} - - /******************************************************************************* - * Fd Declarations - */ - -#define FD_FROM_PO(po) ((grpc_fd*)(po)) - -struct grpc_fd { - poll_obj po; - - int fd; - /* refst format: - bit 0 : 1=Active / 0=Orphaned - bits 1-n : refcount - Ref/Unref by two to avoid altering the orphaned bit */ - gpr_atm refst; - - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ - bool orphaned; - - grpc_core::ManualConstructor<grpc_core::LockfreeEvent> read_closure; - grpc_core::ManualConstructor<grpc_core::LockfreeEvent> write_closure; - grpc_core::ManualConstructor<grpc_core::LockfreeEvent> error_closure; - - struct grpc_fd* freelist_next; - grpc_closure* on_done_closure; - - grpc_iomgr_object iomgr_object; - - /* Do we need to track EPOLLERR events separately? */ - bool track_err; -}; - -/* Reference counting for fds */ -#ifndef NDEBUG -static void fd_ref(grpc_fd* fd, const char* reason, const char* file, int line); -static void fd_unref(grpc_fd* fd, const char* reason, const char* file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd* fd); -static void fd_unref(grpc_fd* fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - -static void fd_global_init(void); -static void fd_global_shutdown(void); - -/******************************************************************************* - * Polling island Declarations - */ - -#ifndef NDEBUG - -#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) -#define PI_UNREF(p, r) pi_unref_dbg((p), (r), __FILE__, __LINE__) - -#else - -#define PI_ADD_REF(p, r) pi_add_ref((p)) -#define PI_UNREF(p, r) pi_unref((p)) - -#endif - -/* This is also used as grpc_workqueue (by directly casing it) */ -typedef struct polling_island { - gpr_mu mu; - /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement - the refcount. - Once the ref count becomes zero, this structure is destroyed which means - we should ensure that there is never a scenario where a PI_ADD_REF() is - racing with a PI_UNREF() that just made the ref_count zero. */ - gpr_atm ref_count; - - /* Pointer to the polling_island this merged into. - * merged_to value is only set once in polling_island's lifetime (and that too - * only if the island is merged with another island). Because of this, we can - * use gpr_atm type here so that we can do atomic access on this and reduce - * lock contention on 'mu' mutex. - * - * Note that if this field is not NULL (i.e not 0), all the remaining fields - * (except mu and ref_count) are invalid and must be ignored. */ - gpr_atm merged_to; - - /* Number of threads currently polling on this island */ - gpr_atm poller_count; - - /* The fd of the underlying epoll set */ - int epoll_fd; - - /* The file descriptors in the epoll set */ - size_t fd_cnt; - size_t fd_capacity; - grpc_fd** fds; -} polling_island; - -/******************************************************************************* - * Pollset Declarations - */ -struct grpc_pollset_worker { - /* Thread id of this worker */ - pthread_t pt_id; - - /* Used to prevent a worker from getting kicked multiple times */ - gpr_atm is_kicked; - struct grpc_pollset_worker* next; - struct grpc_pollset_worker* prev; -}; - -struct grpc_pollset { - poll_obj po; - - grpc_pollset_worker root_worker; - bool kicked_without_pollers; - - bool shutting_down; /* Is the pollset shutting down ? */ - bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ - grpc_closure* shutdown_done; /* Called after after shutdown is complete */ -}; - -/******************************************************************************* - * Pollset-set Declarations - */ -struct grpc_pollset_set { - poll_obj po; -}; - -/******************************************************************************* - * Common helpers - */ - -static bool append_error(grpc_error** composite, grpc_error* error, - const char* desc) { - if (error == GRPC_ERROR_NONE) return true; - if (*composite == GRPC_ERROR_NONE) { - *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); - } - *composite = grpc_error_add_child(*composite, error); - return false; -} - -/******************************************************************************* - * Polling island Definitions - */ - -/* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the polling island merge operation where we need to wakeup all - the threads currently polling the smaller polling island (so that they can - start polling the new/merged polling island) - - NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the - threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ -static grpc_wakeup_fd polling_island_wakeup_fd; - -/* The polling island being polled right now. - See comments in workqueue_maybe_wakeup for why this is tracked. */ -static __thread polling_island* g_current_thread_polling_island; - -/* Forward declaration */ -static void polling_island_delete(polling_island* pi); - -#ifdef GRPC_TSAN -/* Currently TSAN may incorrectly flag data races between epoll_ctl and - epoll_wait for any grpc_fd structs that are added to the epoll set via - epoll_ctl and are returned (within a very short window) via epoll_wait(). - - To work-around this race, we establish a happens-before relation between - the code just-before epoll_ctl() and the code after epoll_wait() by using - this atomic */ -gpr_atm g_epoll_sync; -#endif /* defined(GRPC_TSAN) */ - -static void pi_add_ref(polling_island* pi); -static void pi_unref(polling_island* pi); - -#ifndef NDEBUG -static void pi_add_ref_dbg(polling_island* pi, const char* reason, - const char* file, int line) { - if (grpc_polling_trace.enabled()) { - gpr_atm old_cnt = gpr_atm_acq_load(&pi->ref_count); - gpr_log(GPR_INFO, - "Add ref pi: %p, old:%" PRIdPTR " -> new:%" PRIdPTR - " (%s) - (%s, %d)", - pi, old_cnt, old_cnt + 1, reason, file, line); - } - pi_add_ref(pi); -} - -static void pi_unref_dbg(polling_island* pi, const char* reason, - const char* file, int line) { - if (grpc_polling_trace.enabled()) { - gpr_atm old_cnt = gpr_atm_acq_load(&pi->ref_count); - gpr_log(GPR_INFO, - "Unref pi: %p, old:%" PRIdPTR " -> new:%" PRIdPTR - " (%s) - (%s, %d)", - pi, old_cnt, (old_cnt - 1), reason, file, line); - } - pi_unref(pi); -} -#endif - -static void pi_add_ref(polling_island* pi) { - gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); -} - -static void pi_unref(polling_island* pi) { - /* If ref count went to zero, delete the polling island. - Note that this deletion not be done under a lock. Once the ref count goes - to zero, we are guaranteed that no one else holds a reference to the - polling island (and that there is no racing pi_add_ref() call either). - - Also, if we are deleting the polling island and the merged_to field is - non-empty, we should remove a ref to the merged_to polling island - */ - if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island* next = (polling_island*)gpr_atm_acq_load(&pi->merged_to); - polling_island_delete(pi); - if (next != nullptr) { - PI_UNREF(next, "pi_delete"); /* Recursive call */ - } - } -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_add_fds_locked(polling_island* pi, grpc_fd** fds, - size_t fd_count, bool add_fd_refs, - grpc_error** error) { - int err; - size_t i; - struct epoll_event ev; - char* err_msg; - const char* err_desc = "polling_island_add_fds"; - -#ifdef GRPC_TSAN - /* See the definition of g_epoll_sync for more context */ - gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); -#endif /* defined(GRPC_TSAN) */ - - for (i = 0; i < fd_count; i++) { - ev.events = static_cast<uint32_t>(EPOLLIN | EPOLLOUT | EPOLLET); - /* Use the least significant bit of ev.data.ptr to store track_err to avoid - * synchronization issues when accessing it after receiving an event */ - ev.data.ptr = reinterpret_cast<void*>(reinterpret_cast<intptr_t>(fds[i]) | - (fds[i]->track_err ? 1 : 0)); - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); - - if (err < 0) { - if (errno != EEXIST) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - continue; - } - - if (pi->fd_cnt == pi->fd_capacity) { - pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); - pi->fds = static_cast<grpc_fd**>( - gpr_realloc(pi->fds, sizeof(grpc_fd*) * pi->fd_capacity)); - } - - pi->fds[pi->fd_cnt++] = fds[i]; - if (add_fd_refs) { - GRPC_FD_REF(fds[i], "polling_island"); - } - } -} - -/* The caller is expected to hold pi->mu before calling this */ -static void polling_island_add_wakeup_fd_locked(polling_island* pi, - grpc_wakeup_fd* wakeup_fd, - grpc_error** error) { - struct epoll_event ev; - int err; - char* err_msg; - const char* err_desc = "polling_island_add_wakeup_fd"; - - ev.events = static_cast<uint32_t>(EPOLLIN | EPOLLET); - ev.data.ptr = wakeup_fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, - GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); - if (err < 0 && errno != EEXIST) { - gpr_asprintf(&err_msg, - "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " - "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, - strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_all_fds_locked(polling_island* pi, - bool remove_fd_refs, - grpc_error** error) { - int err; - size_t i; - char* err_msg; - const char* err_desc = "polling_island_remove_fds"; - - for (i = 0; i < pi->fd_cnt; i++) { - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, nullptr); - if (err < 0 && errno != ENOENT) { - gpr_asprintf(&err_msg, - "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " - "error: %d (%s)", - pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - if (remove_fd_refs) { - GRPC_FD_UNREF(pi->fds[i], "polling_island"); - } - } - - pi->fd_cnt = 0; -} - -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_fd_locked(polling_island* pi, grpc_fd* fd, - grpc_error** error) { - int err; - size_t i; - char* err_msg; - const char* err_desc = "polling_island_remove_fd"; - - /* If fd is already closed, then it would have been automatically been removed - from the epoll set */ - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, nullptr); - if (err < 0 && errno != ENOENT) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - for (i = 0; i < pi->fd_cnt; i++) { - if (pi->fds[i] == fd) { - pi->fds[i] = pi->fds[--pi->fd_cnt]; - GRPC_FD_UNREF(fd, "polling_island"); - break; - } - } -} - -/* Might return NULL in case of an error */ -static polling_island* polling_island_create(grpc_fd* initial_fd, - grpc_error** error) { - polling_island* pi = nullptr; - const char* err_desc = "polling_island_create"; - - *error = GRPC_ERROR_NONE; - - pi = static_cast<polling_island*>(gpr_malloc(sizeof(*pi))); - gpr_mu_init(&pi->mu); - pi->fd_cnt = 0; - pi->fd_capacity = 0; - pi->fds = nullptr; - pi->epoll_fd = -1; - - gpr_atm_rel_store(&pi->ref_count, 0); - gpr_atm_rel_store(&pi->poller_count, 0); - gpr_atm_rel_store(&pi->merged_to, (gpr_atm) nullptr); - - pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - - if (pi->epoll_fd < 0) { - append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); - goto done; - } - - if (initial_fd != nullptr) { - polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); - } - -done: - if (*error != GRPC_ERROR_NONE) { - polling_island_delete(pi); - pi = nullptr; - } - return pi; -} - -static void polling_island_delete(polling_island* pi) { - GPR_ASSERT(pi->fd_cnt == 0); - - if (pi->epoll_fd >= 0) { - close(pi->epoll_fd); - } - gpr_mu_destroy(&pi->mu); - gpr_free(pi->fds); - gpr_free(pi); -} - -/* Attempts to gets the last polling island in the linked list (liked by the - * 'merged_to' field). Since this does not lock the polling island, there are no - * guarantees that the island returned is the last island */ -static polling_island* polling_island_maybe_get_latest(polling_island* pi) { - polling_island* next = (polling_island*)gpr_atm_acq_load(&pi->merged_to); - while (next != nullptr) { - pi = next; - next = (polling_island*)gpr_atm_acq_load(&pi->merged_to); - } - - return pi; -} - -/* Gets the lock on the *latest* polling island i.e the last polling island in - the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the - returned polling island's mu. - Usage: To lock/unlock polling island "pi", do the following: - polling_island *pi_latest = polling_island_lock(pi); - ... - ... critical section .. - ... - gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ -static polling_island* polling_island_lock(polling_island* pi) { - polling_island* next = nullptr; - - while (true) { - next = (polling_island*)gpr_atm_acq_load(&pi->merged_to); - if (next == nullptr) { - /* Looks like 'pi' is the last node in the linked list but unless we check - this by holding the pi->mu lock, we cannot be sure (i.e without the - pi->mu lock, we don't prevent island merges). - To be absolutely sure, check once more by holding the pi->mu lock */ - gpr_mu_lock(&pi->mu); - next = (polling_island*)gpr_atm_acq_load(&pi->merged_to); - if (next == nullptr) { - /* pi is infact the last node and we have the pi->mu lock. we're done */ - break; - } - - /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu - * isn't the lock we are interested in. Continue traversing the list */ - gpr_mu_unlock(&pi->mu); - } - - pi = next; - } - - return pi; -} - -/* Gets the lock on the *latest* polling islands in the linked lists pointed by - *p and *q (and also updates *p and *q to point to the latest polling islands) - - This function is needed because calling the following block of code to obtain - locks on polling islands (*p and *q) is prone to deadlocks. - { - polling_island_lock(*p, true); - polling_island_lock(*q, true); - } - - Usage/example: - polling_island *p1; - polling_island *p2; - .. - polling_island_lock_pair(&p1, &p2); - .. - .. Critical section with both p1 and p2 locked - .. - // Release locks: Always call polling_island_unlock_pair() to release locks - polling_island_unlock_pair(p1, p2); -*/ -static void polling_island_lock_pair(polling_island** p, polling_island** q) { - polling_island* pi_1 = *p; - polling_island* pi_2 = *q; - polling_island* next_1 = nullptr; - polling_island* next_2 = nullptr; - - /* The algorithm is simple: - - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and - keep updating pi_1 and pi_2) - - Then obtain locks on the islands by following a lock order rule of - locking polling_island with lower address first - Special case: Before obtaining the locks, check if pi_1 and pi_2 are - pointing to the same island. If that is the case, we can just call - polling_island_lock() - - After obtaining both the locks, double check that the polling islands - are still the last polling islands in their respective linked lists - (this is because there might have been polling island merges before - we got the lock) - - If the polling islands are the last islands, we are done. If not, - release the locks and continue the process from the first step */ - while (true) { - next_1 = (polling_island*)gpr_atm_acq_load(&pi_1->merged_to); - while (next_1 != nullptr) { - pi_1 = next_1; - next_1 = (polling_island*)gpr_atm_acq_load(&pi_1->merged_to); - } - - next_2 = (polling_island*)gpr_atm_acq_load(&pi_2->merged_to); - while (next_2 != nullptr) { - pi_2 = next_2; - next_2 = (polling_island*)gpr_atm_acq_load(&pi_2->merged_to); - } - - if (pi_1 == pi_2) { - pi_1 = pi_2 = polling_island_lock(pi_1); - break; - } - - if (pi_1 < pi_2) { - gpr_mu_lock(&pi_1->mu); - gpr_mu_lock(&pi_2->mu); - } else { - gpr_mu_lock(&pi_2->mu); - gpr_mu_lock(&pi_1->mu); - } - - next_1 = (polling_island*)gpr_atm_acq_load(&pi_1->merged_to); - next_2 = (polling_island*)gpr_atm_acq_load(&pi_2->merged_to); - if (next_1 == nullptr && next_2 == nullptr) { - break; - } - - gpr_mu_unlock(&pi_1->mu); - gpr_mu_unlock(&pi_2->mu); - } - - *p = pi_1; - *q = pi_2; -} - -static void polling_island_unlock_pair(polling_island* p, polling_island* q) { - if (p == q) { - gpr_mu_unlock(&p->mu); - } else { - gpr_mu_unlock(&p->mu); - gpr_mu_unlock(&q->mu); - } -} - -static polling_island* polling_island_merge(polling_island* p, - polling_island* q, - grpc_error** error) { - /* Get locks on both the polling islands */ - polling_island_lock_pair(&p, &q); - - if (p != q) { - /* Make sure that p points to the polling island with fewer fds than q */ - if (p->fd_cnt > q->fd_cnt) { - GPR_SWAP(polling_island*, p, q); - } - - /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q - Note that the refcounts on the fds being moved will not change here. - This is why the last param in the following two functions is 'false') */ - polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); - polling_island_remove_all_fds_locked(p, false, error); - - /* Wakeup all the pollers (if any) on p so that they pickup this change */ - polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); - - /* Add the 'merged_to' link from p --> q */ - gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); - PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ - } - /* else if p == q, nothing needs to be done */ - - polling_island_unlock_pair(p, q); - - /* Return the merged polling island (Note that no merge would have happened - if p == q which is ok) */ - return q; -} - -static grpc_error* polling_island_global_init() { - grpc_error* error = GRPC_ERROR_NONE; - - error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); - if (error == GRPC_ERROR_NONE) { - error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); - } - - return error; -} - -static void polling_island_global_shutdown() { - grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); -} - -/******************************************************************************* - * Fd Definitions - */ - -/* We need to keep a freelist not because of any concerns of malloc performance - * but instead so that implementations with multiple threads in (for example) - * epoll_wait deal with the race between pollset removal and incoming poll - * notifications. - * - * The problem is that the poller ultimately holds a reference to this - * object, so it is very difficult to know when is safe to free it, at least - * without some expensive synchronization. - * - * If we keep the object freelisted, in the worst case losing this race just - * becomes a spurious read notification on a reused fd. - */ - -/* The alarm system needs to be able to wakeup 'some poller' sometimes - * (specifically when a new alarm needs to be triggered earlier than the next - * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a - * case occurs. */ - -static grpc_fd* fd_freelist = nullptr; -static gpr_mu fd_freelist_mu; - -#ifndef NDEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd* fd, int n, const char* reason, const char* file, - int line) { - if (grpc_trace_fd_refcount.enabled()) { - gpr_log(GPR_DEBUG, - "FD %d %p ref %d %" PRIdPTR " -> %" PRIdPTR " [%s; %s:%d]", - fd->fd, fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); - } -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd* fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} - -#ifndef NDEBUG -static void unref_by(grpc_fd* fd, int n, const char* reason, const char* file, - int line) { - if (grpc_trace_fd_refcount.enabled()) { - gpr_log(GPR_DEBUG, - "FD %d %p unref %d %" PRIdPTR " -> %" PRIdPTR " [%s; %s:%d]", - fd->fd, fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); - } -#else -static void unref_by(grpc_fd* fd, int n) { -#endif - gpr_atm old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - fd->read_closure->DestroyEvent(); - fd->write_closure->DestroyEvent(); - fd->error_closure->DestroyEvent(); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); - } -} - -/* Increment refcount by two to avoid changing the orphan bit */ -#ifndef NDEBUG -static void fd_ref(grpc_fd* fd, const char* reason, const char* file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd* fd, const char* reason, const char* file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd* fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd* fd) { unref_by(fd, 2); } -#endif - -static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } - -static void fd_global_shutdown(void) { - gpr_mu_lock(&fd_freelist_mu); - gpr_mu_unlock(&fd_freelist_mu); - while (fd_freelist != nullptr) { - grpc_fd* fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); - gpr_free(fd); - } - gpr_mu_destroy(&fd_freelist_mu); -} - -static grpc_fd* fd_create(int fd, const char* name, bool track_err) { - grpc_fd* new_fd = nullptr; - - gpr_mu_lock(&fd_freelist_mu); - if (fd_freelist != nullptr) { - new_fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - } - gpr_mu_unlock(&fd_freelist_mu); - - if (new_fd == nullptr) { - new_fd = static_cast<grpc_fd*>(gpr_malloc(sizeof(grpc_fd))); - gpr_mu_init(&new_fd->po.mu); - new_fd->read_closure.Init(); - new_fd->write_closure.Init(); - new_fd->error_closure.Init(); - } - - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); - new_fd->po.pi = nullptr; -#ifndef NDEBUG - new_fd->po.obj_type = POLL_OBJ_FD; -#endif - - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); - new_fd->fd = fd; - new_fd->orphaned = false; - new_fd->read_closure->InitEvent(); - new_fd->write_closure->InitEvent(); - new_fd->error_closure->InitEvent(); - new_fd->track_err = track_err; - - new_fd->freelist_next = nullptr; - new_fd->on_done_closure = nullptr; - - gpr_mu_unlock(&new_fd->po.mu); - - char* fd_name; - gpr_asprintf(&fd_name, "%s fd=%d", name, fd); - grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); - gpr_free(fd_name); - return new_fd; -} - -static int fd_wrapped_fd(grpc_fd* fd) { - int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); - if (!fd->orphaned) { - ret_fd = fd->fd; - } - gpr_mu_unlock(&fd->po.mu); - - return ret_fd; -} - -static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, - const char* reason) { - grpc_error* error = GRPC_ERROR_NONE; - polling_island* unref_pi = nullptr; - - gpr_mu_lock(&fd->po.mu); - fd->on_done_closure = on_done; - - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->po.pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->po.pi to NULL (but remove the ref on the polling island - before doing this.) */ - if (fd->po.pi != nullptr) { - polling_island* pi_latest = polling_island_lock(fd->po.pi); - polling_island_remove_fd_locked(pi_latest, fd, &error); - gpr_mu_unlock(&pi_latest->mu); - - unref_pi = fd->po.pi; - fd->po.pi = nullptr; - } - - /* If release_fd is not NULL, we should be relinquishing control of the file - descriptor fd->fd (but we still own the grpc_fd structure). */ - if (release_fd != nullptr) { - *release_fd = fd->fd; - } else { - close(fd->fd); - } - - fd->orphaned = true; - - GRPC_CLOSURE_SCHED(fd->on_done_closure, GRPC_ERROR_REF(error)); - - gpr_mu_unlock(&fd->po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ - if (unref_pi != nullptr) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing - inside the lock can cause an eventual lock loop that makes TSAN very - unhappy. */ - PI_UNREF(unref_pi, "fd_orphan"); - } - if (error != GRPC_ERROR_NONE) { - const char* msg = grpc_error_string(error); - gpr_log(GPR_DEBUG, "fd_orphan: %s", msg); - } - GRPC_ERROR_UNREF(error); -} - -static bool fd_is_shutdown(grpc_fd* fd) { - return fd->read_closure->IsShutdown(); -} - -/* Might be called multiple times */ -static void fd_shutdown(grpc_fd* fd, grpc_error* why) { - if (fd->read_closure->SetShutdown(GRPC_ERROR_REF(why))) { - shutdown(fd->fd, SHUT_RDWR); - fd->write_closure->SetShutdown(GRPC_ERROR_REF(why)); - fd->error_closure->SetShutdown(GRPC_ERROR_REF(why)); - } - GRPC_ERROR_UNREF(why); -} - -static void fd_notify_on_read(grpc_fd* fd, grpc_closure* closure) { - fd->read_closure->NotifyOn(closure); -} - -static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) { - fd->write_closure->NotifyOn(closure); -} - -static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { - fd->error_closure->NotifyOn(closure); -} - -static void fd_become_readable(grpc_fd* fd) { fd->read_closure->SetReady(); } - -static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); } - -static void fd_has_errors(grpc_fd* fd) { fd->error_closure->SetReady(); } - -/******************************************************************************* - * Pollset Definitions - */ -GPR_TLS_DECL(g_current_thread_pollset); -GPR_TLS_DECL(g_current_thread_worker); -static __thread bool g_initialized_sigmask; -static __thread sigset_t g_orig_sigmask; - -static void sig_handler(int sig_num) { -#ifdef GRPC_EPOLL_DEBUG - gpr_log(GPR_INFO, "Received signal %d", sig_num); -#endif -} - -static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } - -/* Global state management */ -static grpc_error* pollset_global_init(void) { - gpr_tls_init(&g_current_thread_pollset); - gpr_tls_init(&g_current_thread_worker); - poller_kick_init(); - return GRPC_ERROR_NONE; -} - -static void pollset_global_shutdown(void) { - gpr_tls_destroy(&g_current_thread_pollset); - gpr_tls_destroy(&g_current_thread_worker); -} - -static grpc_error* pollset_worker_kick(grpc_pollset_worker* worker) { - grpc_error* err = GRPC_ERROR_NONE; - - /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(&worker->is_kicked, static_cast<gpr_atm>(0), - static_cast<gpr_atm>(1))) { - GRPC_POLLING_TRACE( - "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", - (void*)worker, (long int)worker->pt_id); - int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); - if (err_num != 0) { - err = GRPC_OS_ERROR(err_num, "pthread_kill"); - } - } - return err; -} - -/* Return 1 if the pollset has active threads in pollset_work (pollset must - * be locked) */ -static int pollset_has_workers(grpc_pollset* p) { - return p->root_worker.next != &p->root_worker; -} - -static void remove_worker(grpc_pollset* p, grpc_pollset_worker* worker) { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; -} - -static grpc_pollset_worker* pop_front_worker(grpc_pollset* p) { - if (pollset_has_workers(p)) { - grpc_pollset_worker* w = p->root_worker.next; - remove_worker(p, w); - return w; - } else { - return nullptr; - } -} - -static void push_back_worker(grpc_pollset* p, grpc_pollset_worker* worker) { - worker->next = &p->root_worker; - worker->prev = worker->next->prev; - worker->prev->next = worker->next->prev = worker; -} - -static void push_front_worker(grpc_pollset* p, grpc_pollset_worker* worker) { - worker->prev = &p->root_worker; - worker->next = worker->prev->next; - worker->prev->next = worker->next->prev = worker; -} - -/* p->mu must be held before calling this function */ -static grpc_error* pollset_kick(grpc_pollset* p, - grpc_pollset_worker* specific_worker) { - GPR_TIMER_SCOPE("pollset_kick", 0); - grpc_error* error = GRPC_ERROR_NONE; - GRPC_STATS_INC_POLLSET_KICK(); - const char* err_desc = "Kick Failure"; - grpc_pollset_worker* worker = specific_worker; - if (worker != nullptr) { - if (worker == GRPC_POLLSET_KICK_BROADCAST) { - if (pollset_has_workers(p)) { - GPR_TIMER_SCOPE("pollset_kick.broadcast", 0); - for (worker = p->root_worker.next; worker != &p->root_worker; - worker = worker->next) { - if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { - append_error(&error, pollset_worker_kick(worker), err_desc); - } - } - } else { - p->kicked_without_pollers = true; - } - } else { - GPR_TIMER_MARK("kicked_specifically", 0); - if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { - append_error(&error, pollset_worker_kick(worker), err_desc); - } - } - } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - /* Since worker == NULL, it means that we can kick "any" worker on this - pollset 'p'. If 'p' happens to be the same pollset this thread is - currently polling (i.e in pollset_work() function), then there is no need - to kick any other worker since the current thread can just absorb the - kick. This is the reason why we enter this case only when - g_current_thread_pollset is != p */ - - GPR_TIMER_MARK("kick_anonymous", 0); - worker = pop_front_worker(p); - if (worker != nullptr) { - GPR_TIMER_MARK("finally_kick", 0); - push_back_worker(p, worker); - append_error(&error, pollset_worker_kick(worker), err_desc); - } else { - GPR_TIMER_MARK("kicked_no_pollers", 0); - p->kicked_without_pollers = true; - } - } - - GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); - return error; -} - -static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) { - gpr_mu_init(&pollset->po.mu); - *mu = &pollset->po.mu; - pollset->po.pi = nullptr; -#ifndef NDEBUG - pollset->po.obj_type = POLL_OBJ_POLLSET; -#endif - - pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; - pollset->kicked_without_pollers = false; - - pollset->shutting_down = false; - pollset->finish_shutdown_called = false; - pollset->shutdown_done = nullptr; -} - -static int poll_deadline_to_millis_timeout(grpc_millis millis) { - if (millis == GRPC_MILLIS_INF_FUTURE) return -1; - grpc_millis delta = millis - grpc_core::ExecCtx::Get()->Now(); - if (delta > INT_MAX) - return INT_MAX; - else if (delta < 0) - return 0; - else - return static_cast<int>(delta); -} - -static void pollset_release_polling_island(grpc_pollset* ps, - const char* reason) { - if (ps->po.pi != nullptr) { - PI_UNREF(ps->po.pi, reason); - } - ps->po.pi = nullptr; -} - -static void finish_shutdown_locked(grpc_pollset* pollset) { - /* The pollset cannot have any workers if we are at this stage */ - GPR_ASSERT(!pollset_has_workers(pollset)); - - pollset->finish_shutdown_called = true; - - /* Release the ref and set pollset->po.pi to NULL */ - pollset_release_polling_island(pollset, "ps_shutdown"); - GRPC_CLOSURE_SCHED(pollset->shutdown_done, GRPC_ERROR_NONE); -} - -/* pollset->po.mu lock must be held by the caller before calling this */ -static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { - GPR_TIMER_SCOPE("pollset_shutdown", 0); - GPR_ASSERT(!pollset->shutting_down); - pollset->shutting_down = true; - pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); - - /* If the pollset has any workers, we cannot call finish_shutdown_locked() - because it would release the underlying polling island. In such a case, we - let the last worker call finish_shutdown_locked() from pollset_work() */ - if (!pollset_has_workers(pollset)) { - GPR_ASSERT(!pollset->finish_shutdown_called); - GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); - finish_shutdown_locked(pollset); - } -} - -/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other - * than destroying the mutexes, there is nothing special that needs to be done - * here */ -static void pollset_destroy(grpc_pollset* pollset) { - GPR_ASSERT(!pollset_has_workers(pollset)); - gpr_mu_destroy(&pollset->po.mu); -} - -#define GRPC_EPOLL_MAX_EVENTS 100 -/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ -static void pollset_work_and_unlock(grpc_pollset* pollset, - grpc_pollset_worker* worker, int timeout_ms, - sigset_t* sig_mask, grpc_error** error) { - GPR_TIMER_SCOPE("pollset_work_and_unlock", 0); - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; - int epoll_fd = -1; - int ep_rv; - polling_island* pi = nullptr; - char* err_msg; - const char* err_desc = "pollset_work_and_unlock"; - - /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the - latest polling island pointed by pollset->po.pi - - Since epoll_fd is immutable, we can read it without obtaining the polling - island lock. There is however a possibility that the polling island (from - which we got the epoll_fd) got merged with another island while we are - in this function. This is still okay because in such a case, we will wakeup - right-away from epoll_wait() and pick up the latest polling_island the next - this function (i.e pollset_work_and_unlock()) is called */ - - if (pollset->po.pi == nullptr) { - pollset->po.pi = polling_island_create(nullptr, error); - if (pollset->po.pi == nullptr) { - return; /* Fatal error. We cannot continue */ - } - - PI_ADD_REF(pollset->po.pi, "ps"); - GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", - (void*)pollset, (void*)pollset->po.pi); - } - - pi = polling_island_maybe_get_latest(pollset->po.pi); - epoll_fd = pi->epoll_fd; - - /* Update the pollset->po.pi since the island being pointed by - pollset->po.pi maybe older than the one pointed by pi) */ - if (pollset->po.pi != pi) { - /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the - polling island to be deleted */ - PI_ADD_REF(pi, "ps"); - PI_UNREF(pollset->po.pi, "ps"); - pollset->po.pi = pi; - } - - /* Add an extra ref so that the island does not get destroyed (which means - the epoll_fd won't be closed) while we are are doing an epoll_wait() on the - epoll_fd */ - PI_ADD_REF(pi, "ps_work"); - gpr_mu_unlock(&pollset->po.mu); - - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - g_current_thread_polling_island = pi; - - GRPC_SCHEDULING_START_BLOCKING_REGION; - GRPC_STATS_INC_SYSCALL_POLL(); - ep_rv = - epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", - (void*)pollset, (void*)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void* data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void*)pollset, (void*)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd* fd = reinterpret_cast<grpc_fd*>( - reinterpret_cast<intptr_t>(data_ptr) & ~static_cast<intptr_t>(1)); - bool track_err = - reinterpret_cast<intptr_t>(data_ptr) & ~static_cast<intptr_t>(1); - bool cancel = (ep_ev[i].events & EPOLLHUP) != 0; - bool error = (ep_ev[i].events & EPOLLERR) != 0; - bool read_ev = (ep_ev[i].events & (EPOLLIN | EPOLLPRI)) != 0; - bool write_ev = (ep_ev[i].events & EPOLLOUT) != 0; - bool err_fallback = error && !track_err; - - if (error && !err_fallback) { - fd_has_errors(fd); - } - if (read_ev || cancel || err_fallback) { - fd_become_readable(fd); - } - if (write_ev || cancel || err_fallback) { - fd_become_writable(fd); - } - } - } - - g_current_thread_polling_island = nullptr; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); - - GPR_ASSERT(pi != nullptr); - - /* Before leaving, release the extra ref we added to the polling island. It - is important to use "pi" here (i.e our old copy of pollset->po.pi - that we got before releasing the polling island lock). This is because - pollset->po.pi pointer might get udpated in other parts of the - code when there is an island merge while we are doing epoll_wait() above */ - PI_UNREF(pi, "ps_work"); -} - -/* pollset->po.mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->po.mu) - during the course of its execution but it will always re-acquire the lock and - ensure that it is held by the time the function returns */ -static grpc_error* pollset_work(grpc_pollset* pollset, - grpc_pollset_worker** worker_hdl, - grpc_millis deadline) { - GPR_TIMER_SCOPE("pollset_work", 0); - grpc_error* error = GRPC_ERROR_NONE; - int timeout_ms = poll_deadline_to_millis_timeout(deadline); - - sigset_t new_mask; - - grpc_pollset_worker worker; - worker.next = worker.prev = nullptr; - worker.pt_id = pthread_self(); - gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); - - if (worker_hdl) *worker_hdl = &worker; - - gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); - gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); - - if (pollset->kicked_without_pollers) { - /* If the pollset was kicked without pollers, pretend that the current - worker got the kick and skip polling. A kick indicates that there is some - work that needs attention like an event on the completion queue or an - alarm */ - GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); - pollset->kicked_without_pollers = 0; - } else if (!pollset->shutting_down) { - /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up - (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the - worker that there is some pending work that needs immediate attention - (like an event on the completion queue, or a polling island merge that - results in a new epoll-fd to wait on) and that the worker should not - spend time waiting in epoll_pwait(). - - A worker can be kicked anytime from the point it is added to the pollset - via push_front_worker() (or push_back_worker()) to the point it is - removed via remove_worker(). - If the worker is kicked before/during it calls epoll_pwait(), it should - immediately exit from epoll_wait(). If the worker is kicked after it - returns from epoll_wait(), then nothing really needs to be done. - - To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all - times *except* when it is in epoll_pwait(). This way, the worker never - misses acting on a kick */ - - if (!g_initialized_sigmask) { - sigemptyset(&new_mask); - sigaddset(&new_mask, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); - sigdelset(&g_orig_sigmask, grpc_wakeup_signal); - g_initialized_sigmask = true; - /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. - This is the mask used at all times *except during - epoll_wait()*" - g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and - this is the mask to use *during epoll_wait()* - - The new_mask is set on the worker before it is added to the pollset - (i.e before it can be kicked) */ - } - - push_front_worker(pollset, &worker); /* Add worker to pollset */ - - pollset_work_and_unlock(pollset, &worker, timeout_ms, &g_orig_sigmask, - &error); - grpc_core::ExecCtx::Get()->Flush(); - - gpr_mu_lock(&pollset->po.mu); - - /* Note: There is no need to reset worker.is_kicked to 0 since we are no - longer going to use this worker */ - remove_worker(pollset, &worker); - } - - /* If we are the last worker on the pollset (i.e pollset_has_workers() is - false at this point) and the pollset is shutting down, we may have to - finish the shutdown process by calling finish_shutdown_locked(). - See pollset_shutdown() for more details. - - Note: Continuing to access pollset here is safe; it is the caller's - responsibility to not destroy a pollset when it has outstanding calls to - pollset_work() */ - if (pollset->shutting_down && !pollset_has_workers(pollset) && - !pollset->finish_shutdown_called) { - GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); - finish_shutdown_locked(pollset); - - gpr_mu_unlock(&pollset->po.mu); - grpc_core::ExecCtx::Get()->Flush(); - gpr_mu_lock(&pollset->po.mu); - } - - if (worker_hdl) *worker_hdl = nullptr; - - gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); - gpr_tls_set(&g_current_thread_worker, (intptr_t)0); - - GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); - return error; -} - -static void add_poll_object(poll_obj* bag, poll_obj_type bag_type, - poll_obj* item, poll_obj_type item_type) { - GPR_TIMER_SCOPE("add_poll_object", 0); - -#ifndef NDEBUG - GPR_ASSERT(item->obj_type == item_type); - GPR_ASSERT(bag->obj_type == bag_type); -#endif - - grpc_error* error = GRPC_ERROR_NONE; - polling_island* pi_new = nullptr; - - gpr_mu_lock(&bag->mu); - gpr_mu_lock(&item->mu); - -retry: - /* - * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing - * 2) If item->pi and bag->pi are both NULL, create a new polling island (with - * a refcount of 2) and point item->pi and bag->pi to the new island - * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to - * the other's non-NULL pi - * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the - * polling islands and update item->pi and bag->pi to point to the new - * island - */ - - /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already - * orphaned */ - if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { - gpr_mu_unlock(&item->mu); - gpr_mu_unlock(&bag->mu); - return; - } - - if (item->pi == bag->pi) { - pi_new = item->pi; - if (pi_new == nullptr) { - /* GPR_ASSERT(item->pi == bag->pi == NULL) */ - - /* If we are adding an fd to a bag (i.e pollset or pollset_set), then - * we need to do some extra work to make TSAN happy */ - if (item_type == POLL_OBJ_FD) { - /* Unlock before creating a new polling island: the polling island will - create a workqueue which creates a file descriptor, and holding an fd - lock here can eventually cause a loop to appear to TSAN (making it - unhappy). We don't think it's a real loop (there's an epoch point - where that loop possibility disappears), but the advantages of - keeping TSAN happy outweigh any performance advantage we might have - by keeping the lock held. */ - gpr_mu_unlock(&item->mu); - pi_new = polling_island_create(FD_FROM_PO(item), &error); - gpr_mu_lock(&item->mu); - - /* Need to reverify any assumptions made between the initial lock and - getting to this branch: if they've changed, we need to throw away our - work and figure things out again. */ - if (item->pi != nullptr) { - GRPC_POLLING_TRACE( - "add_poll_object: Raced creating new polling island. pi_new: %p " - "(fd: %d, %s: %p)", - (void*)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), - (void*)bag); - /* No need to lock 'pi_new' here since this is a new polling island - and no one has a reference to it yet */ - polling_island_remove_all_fds_locked(pi_new, true, &error); - - /* Ref and unref so that the polling island gets deleted during unref - */ - PI_ADD_REF(pi_new, "dance_of_destruction"); - PI_UNREF(pi_new, "dance_of_destruction"); - goto retry; - } - } else { - pi_new = polling_island_create(nullptr, &error); - } - - GRPC_POLLING_TRACE( - "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " - "%s: %p)", - (void*)pi_new, poll_obj_string(item_type), (void*)item, - poll_obj_string(bag_type), (void*)bag); - } else { - GRPC_POLLING_TRACE( - "add_poll_object: Same polling island. pi: %p (%s, %s)", - (void*)pi_new, poll_obj_string(item_type), poll_obj_string(bag_type)); - } - } else if (item->pi == nullptr) { - /* GPR_ASSERT(bag->pi != NULL) */ - /* Make pi_new point to latest pi*/ - pi_new = polling_island_lock(bag->pi); - - if (item_type == POLL_OBJ_FD) { - grpc_fd* fd = FD_FROM_PO(item); - polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); - } - - gpr_mu_unlock(&pi_new->mu); - GRPC_POLLING_TRACE( - "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void*)pi_new, poll_obj_string(item_type), (void*)item, - poll_obj_string(bag_type), (void*)bag); - } else if (bag->pi == nullptr) { - /* GPR_ASSERT(item->pi != NULL) */ - /* Make pi_new to point to latest pi */ - pi_new = polling_island_lock(item->pi); - gpr_mu_unlock(&pi_new->mu); - GRPC_POLLING_TRACE( - "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void*)pi_new, poll_obj_string(item_type), (void*)item, - poll_obj_string(bag_type), (void*)bag); - } else { - pi_new = polling_island_merge(item->pi, bag->pi, &error); - GRPC_POLLING_TRACE( - "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " - "bag(%s): %p)", - (void*)pi_new, poll_obj_string(item_type), (void*)item, - poll_obj_string(bag_type), (void*)bag); - } - - /* At this point, pi_new is the polling island that both item->pi and bag->pi - MUST be pointing to */ - - if (item->pi != pi_new) { - PI_ADD_REF(pi_new, poll_obj_string(item_type)); - if (item->pi != nullptr) { - PI_UNREF(item->pi, poll_obj_string(item_type)); - } - item->pi = pi_new; - } - - if (bag->pi != pi_new) { - PI_ADD_REF(pi_new, poll_obj_string(bag_type)); - if (bag->pi != nullptr) { - PI_UNREF(bag->pi, poll_obj_string(bag_type)); - } - bag->pi = pi_new; - } - - gpr_mu_unlock(&item->mu); - gpr_mu_unlock(&bag->mu); - - GRPC_LOG_IF_ERROR("add_poll_object", error); -} - -static void pollset_add_fd(grpc_pollset* pollset, grpc_fd* fd) { - add_poll_object(&pollset->po, POLL_OBJ_POLLSET, &fd->po, POLL_OBJ_FD); -} - -/******************************************************************************* - * Pollset-set Definitions - */ - -static grpc_pollset_set* pollset_set_create(void) { - grpc_pollset_set* pss = - static_cast<grpc_pollset_set*>(gpr_malloc(sizeof(*pss))); - gpr_mu_init(&pss->po.mu); - pss->po.pi = nullptr; -#ifndef NDEBUG - pss->po.obj_type = POLL_OBJ_POLLSET_SET; -#endif - return pss; -} - -static void pollset_set_destroy(grpc_pollset_set* pss) { - gpr_mu_destroy(&pss->po.mu); - - if (pss->po.pi != nullptr) { - PI_UNREF(pss->po.pi, "pss_destroy"); - } - - gpr_free(pss); -} - -static void pollset_set_add_fd(grpc_pollset_set* pss, grpc_fd* fd) { - add_poll_object(&pss->po, POLL_OBJ_POLLSET_SET, &fd->po, POLL_OBJ_FD); -} - -static void pollset_set_del_fd(grpc_pollset_set* pss, grpc_fd* fd) { - /* Nothing to do */ -} - -static void pollset_set_add_pollset(grpc_pollset_set* pss, grpc_pollset* ps) { - add_poll_object(&pss->po, POLL_OBJ_POLLSET_SET, &ps->po, POLL_OBJ_POLLSET); -} - -static void pollset_set_del_pollset(grpc_pollset_set* pss, grpc_pollset* ps) { - /* Nothing to do */ -} - -static void pollset_set_add_pollset_set(grpc_pollset_set* bag, - grpc_pollset_set* item) { - add_poll_object(&bag->po, POLL_OBJ_POLLSET_SET, &item->po, - POLL_OBJ_POLLSET_SET); -} - -static void pollset_set_del_pollset_set(grpc_pollset_set* bag, - grpc_pollset_set* item) { - /* Nothing to do */ -} - -/* Test helper functions - * */ -void* grpc_fd_get_polling_island(grpc_fd* fd) { - polling_island* pi; - - gpr_mu_lock(&fd->po.mu); - pi = fd->po.pi; - gpr_mu_unlock(&fd->po.mu); - - return pi; -} - -void* grpc_pollset_get_polling_island(grpc_pollset* ps) { - polling_island* pi; - - gpr_mu_lock(&ps->po.mu); - pi = ps->po.pi; - gpr_mu_unlock(&ps->po.mu); - - return pi; -} - -bool grpc_are_polling_islands_equal(void* p, void* q) { - polling_island* p1 = static_cast<polling_island*>(p); - polling_island* p2 = static_cast<polling_island*>(q); - - /* Note: polling_island_lock_pair() may change p1 and p2 to point to the - latest polling islands in their respective linked lists */ - polling_island_lock_pair(&p1, &p2); - polling_island_unlock_pair(p1, p2); - - return p1 == p2; -} - -/******************************************************************************* - * Event engine binding - */ - -static void shutdown_engine(void) { - fd_global_shutdown(); - pollset_global_shutdown(); - polling_island_global_shutdown(); -} - -static const grpc_event_engine_vtable vtable = { - sizeof(grpc_pollset), - true, - - fd_create, - fd_wrapped_fd, - fd_orphan, - fd_shutdown, - fd_notify_on_read, - fd_notify_on_write, - fd_notify_on_error, - fd_become_readable, - fd_become_writable, - fd_has_errors, - fd_is_shutdown, - - pollset_init, - pollset_shutdown, - pollset_destroy, - pollset_work, - pollset_kick, - pollset_add_fd, - - pollset_set_create, - pollset_set_destroy, - pollset_set_add_pollset, - pollset_set_del_pollset, - pollset_set_add_pollset_set, - pollset_set_del_pollset_set, - pollset_set_add_fd, - pollset_set_del_fd, - - shutdown_engine, -}; - -/* It is possible that GLIBC has epoll but the underlying kernel doesn't. - * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epoll_available() { - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", - fd); - return false; - } - close(fd); - return true; -} - -const grpc_event_engine_vtable* grpc_init_epollsig_linux( - bool explicit_request) { - /* If use of signals is disabled, we cannot use epoll engine*/ - if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { - gpr_log(GPR_ERROR, "Skipping epollsig because use of signals is disabled."); - return nullptr; - } - - if (!grpc_has_wakeup_fd()) { - gpr_log(GPR_ERROR, "Skipping epollsig because of no wakeup fd."); - return nullptr; - } - - if (!is_epoll_available()) { - gpr_log(GPR_ERROR, "Skipping epollsig because epoll is unavailable."); - return nullptr; - } - - if (!is_grpc_wakeup_signal_initialized) { - if (explicit_request) { - grpc_use_signal(SIGRTMIN + 6); - } else { - gpr_log(GPR_ERROR, - "Skipping epollsig because uninitialized wakeup signal."); - return nullptr; - } - } - - fd_global_init(); - - if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { - return nullptr; - } - - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { - return nullptr; - } - - return &vtable; -} - -#else /* defined(GRPC_LINUX_EPOLL_CREATE1) */ -#if defined(GRPC_POSIX_SOCKET_EV_EPOLLSIG) -#include "src/core/lib/iomgr/ev_epollsig_linux.h" -/* If GRPC_LINUX_EPOLL_CREATE1 is not defined, it means - epoll_create1 is not available. Return NULL */ -const grpc_event_engine_vtable* grpc_init_epollsig_linux( - bool explicit_request) { - return nullptr; -} -#endif /* defined(GRPC_POSIX_SOCKET) */ - -void grpc_use_signal(int signum) {} -#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h deleted file mode 100644 index 2ba2f0a63b..0000000000 --- a/src/core/lib/iomgr/ev_epollsig_linux.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H -#define GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H - -#include <grpc/support/port_platform.h> - -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/iomgr/port.h" - -const grpc_event_engine_vtable* grpc_init_epollsig_linux(bool explicit_request); - -#ifdef GRPC_LINUX_EPOLL_CREATE1 -void* grpc_fd_get_polling_island(grpc_fd* fd); -void* grpc_pollset_get_polling_island(grpc_pollset* ps); -bool grpc_are_polling_islands_equal(void* p, void* q); -#endif /* defined(GRPC_LINUX_EPOLL_CREATE1) */ - -#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index d4377e2d50..764fb08142 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -35,7 +35,6 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/ev_epoll1_linux.h" #include "src/core/lib/iomgr/ev_epollex_linux.h" -#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" grpc_core::TraceFlag grpc_polling_trace(false, @@ -123,13 +122,13 @@ const grpc_event_engine_vtable* init_non_polling(bool explicit_request) { // environment variable if that variable is set (which should be a // comma-separated list of one or more event engine names) static event_engine_factory g_factories[] = { - {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, - {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, - {"epollex", grpc_init_epollex_linux}, {"epoll1", grpc_init_epoll1_linux}, - {"epollsig", grpc_init_epollsig_linux}, {"poll", grpc_init_poll_posix}, - {"poll-cv", grpc_init_poll_cv_posix}, {"none", init_non_polling}, - {ENGINE_TAIL_CUSTOM, nullptr}, {ENGINE_TAIL_CUSTOM, nullptr}, - {ENGINE_TAIL_CUSTOM, nullptr}, {ENGINE_TAIL_CUSTOM, nullptr}, + {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, + {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, + {"epollex", grpc_init_epollex_linux}, {"epoll1", grpc_init_epoll1_linux}, + {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, + {"none", init_non_polling}, {ENGINE_TAIL_CUSTOM, nullptr}, + {ENGINE_TAIL_CUSTOM, nullptr}, {ENGINE_TAIL_CUSTOM, nullptr}, + {ENGINE_TAIL_CUSTOM, nullptr}, }; static void add(const char* beg, const char* end, char*** ss, size_t* ns) { @@ -396,4 +395,6 @@ void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { g_event_engine->pollset_set_del_fd(pollset_set, fd); } +void grpc_use_signal(int signum) {} + #endif // GRPC_POSIX_SOCKET_EV diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index abf96662f5..c8046b21dc 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -82,6 +82,11 @@ #define GRPC_LINUX_SOCKETUTILS 1 #endif #endif +#ifdef LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37) +#define GRPC_HAVE_TCP_USER_TIMEOUT +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37) */ +#endif /* LINUX_VERSION_CODE */ #ifndef __GLIBC__ #define GRPC_LINUX_EPOLL 1 #define GRPC_LINUX_EPOLL_CREATE1 1 @@ -110,7 +115,6 @@ #define GRPC_POSIX_SOCKET_EV 1 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1 #define GRPC_POSIX_SOCKET_EV_EPOLLEX 1 -#define GRPC_POSIX_SOCKET_EV_EPOLLSIG 1 #define GRPC_POSIX_SOCKET_EV_POLL 1 #define GRPC_POSIX_SOCKET_RESOLVE_ADDRESS 1 #define GRPC_POSIX_SOCKET_SOCKADDR 1 @@ -178,7 +182,6 @@ #define GRPC_POSIX_SOCKET_ARES_EV_DRIVER 1 #define GRPC_POSIX_SOCKET_EV 1 #define GRPC_POSIX_SOCKET_EV_EPOLLEX 1 -#define GRPC_POSIX_SOCKET_EV_EPOLLSIG 1 #define GRPC_POSIX_SOCKET_EV_POLL 1 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1 #define GRPC_POSIX_SOCKET_IOMGR 1 diff --git a/src/core/lib/iomgr/socket_utils_common_posix.cc b/src/core/lib/iomgr/socket_utils_common_posix.cc index c4b991c94d..50674b0845 100644 --- a/src/core/lib/iomgr/socket_utils_common_posix.cc +++ b/src/core/lib/iomgr/socket_utils_common_posix.cc @@ -41,6 +41,7 @@ #include <grpc/support/log.h> #include <grpc/support/sync.h> +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gpr/host_port.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/sockaddr.h" @@ -222,6 +223,95 @@ grpc_error* grpc_set_socket_low_latency(int fd, int low_latency) { return GRPC_ERROR_NONE; } +/* The default values for TCP_USER_TIMEOUT are currently configured to be in + * line with the default values of KEEPALIVE_TIMEOUT as proposed in + * https://github.com/grpc/proposal/blob/master/A18-tcp-user-timeout.md */ +#define DEFAULT_CLIENT_TCP_USER_TIMEOUT_MS 20000 /* 20 seconds */ +#define DEFAULT_SERVER_TCP_USER_TIMEOUT_MS 20000 /* 20 seconds */ + +static int g_default_client_tcp_user_timeout_ms = + DEFAULT_CLIENT_TCP_USER_TIMEOUT_MS; +static int g_default_server_tcp_user_timeout_ms = + DEFAULT_SERVER_TCP_USER_TIMEOUT_MS; +static bool g_default_client_tcp_user_timeout_enabled = false; +static bool g_default_server_tcp_user_timeout_enabled = true; + +void config_default_tcp_user_timeout(bool enable, int timeout, bool is_client) { + if (is_client) { + g_default_client_tcp_user_timeout_enabled = enable; + if (timeout > 0) { + g_default_client_tcp_user_timeout_ms = timeout; + } + } else { + g_default_server_tcp_user_timeout_enabled = enable; + if (timeout > 0) { + g_default_server_tcp_user_timeout_ms = timeout; + } + } +} + +/* Set TCP_USER_TIMEOUT */ +grpc_error* grpc_set_socket_tcp_user_timeout( + int fd, const grpc_channel_args* channel_args, bool is_client) { +#ifdef GRPC_HAVE_TCP_USER_TIMEOUT + bool enable; + int timeout; + if (is_client) { + enable = g_default_client_tcp_user_timeout_enabled; + timeout = g_default_client_tcp_user_timeout_ms; + } else { + enable = g_default_server_tcp_user_timeout_enabled; + timeout = g_default_server_tcp_user_timeout_ms; + } + if (channel_args) { + for (unsigned int i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { + const int value = grpc_channel_arg_get_integer( + &channel_args->args[i], grpc_integer_options{0, 1, INT_MAX}); + /* Continue using default if value is 0 */ + if (value == 0) { + continue; + } + /* Disable if value is INT_MAX */ + enable = value != INT_MAX; + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_KEEPALIVE_TIMEOUT_MS)) { + const int value = grpc_channel_arg_get_integer( + &channel_args->args[i], grpc_integer_options{0, 1, INT_MAX}); + /* Continue using default if value is 0 */ + if (value == 0) { + continue; + } + timeout = value; + } + } + } + if (enable) { + extern grpc_core::TraceFlag grpc_tcp_trace; + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_INFO, "Enabling TCP_USER_TIMEOUT with a timeout of %d ms", + timeout); + } + int newval; + socklen_t len = sizeof(newval); + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &timeout, + sizeof(timeout))) { + return GRPC_OS_ERROR(errno, "setsockopt(TCP_USER_TIMEOUT)"); + } + if (0 != getsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &newval, &len)) { + return GRPC_OS_ERROR(errno, "getsockopt(TCP_USER_TIMEOUT)"); + } + if (newval != timeout) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Failed to set TCP_USER_TIMEOUT"); + } + } +#else + gpr_log(GPR_INFO, "TCP_USER_TIMEOUT not supported for this platform"); +#endif /* GRPC_HAVE_TCP_USER_TIMEOUT */ + return GRPC_ERROR_NONE; +} + /* set a socket using a grpc_socket_mutator */ grpc_error* grpc_set_socket_with_mutator(int fd, grpc_socket_mutator* mutator) { GPR_ASSERT(mutator); diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h index b3fd58a530..71a304dc4e 100644 --- a/src/core/lib/iomgr/socket_utils_posix.h +++ b/src/core/lib/iomgr/socket_utils_posix.h @@ -53,6 +53,13 @@ grpc_error* grpc_set_socket_low_latency(int fd, int low_latency); /* set SO_REUSEPORT */ grpc_error* grpc_set_socket_reuse_port(int fd, int reuse); +/* Configure the default values for TCP_USER_TIMEOUT */ +void config_default_tcp_user_timeout(bool enable, int timeout, bool is_client); + +/* Set TCP_USER_TIMEOUT */ +grpc_error* grpc_set_socket_tcp_user_timeout( + int fd, const grpc_channel_args* channel_args, bool is_client); + /* Returns true if this system can create AF_INET6 sockets bound to ::1. The value is probed once, and cached for the life of the process. diff --git a/src/core/lib/iomgr/tcp_client_posix.cc b/src/core/lib/iomgr/tcp_client_posix.cc index 9c989b7dfe..8553ed0db4 100644 --- a/src/core/lib/iomgr/tcp_client_posix.cc +++ b/src/core/lib/iomgr/tcp_client_posix.cc @@ -76,6 +76,9 @@ static grpc_error* prepare_socket(const grpc_resolved_address* addr, int fd, if (!grpc_is_unix_socket(addr)) { err = grpc_set_socket_low_latency(fd, 1); if (err != GRPC_ERROR_NONE) goto error; + err = grpc_set_socket_tcp_user_timeout(fd, channel_args, + true /* is_client */); + if (err != GRPC_ERROR_NONE) goto error; } err = grpc_set_socket_no_sigpipe_if_possible(fd); if (err != GRPC_ERROR_NONE) goto error; diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.cc b/src/core/lib/iomgr/tcp_server_utils_posix_common.cc index 9595c028ce..8d8d3f4273 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.cc @@ -166,6 +166,9 @@ grpc_error* grpc_tcp_server_prepare_socket(grpc_tcp_server* s, int fd, if (err != GRPC_ERROR_NONE) goto error; err = grpc_set_socket_reuse_addr(fd, 1); if (err != GRPC_ERROR_NONE) goto error; + err = grpc_set_socket_tcp_user_timeout(fd, s->channel_args, + false /* is_client */); + if (err != GRPC_ERROR_NONE) goto error; } err = grpc_set_socket_no_sigpipe_if_possible(fd); if (err != GRPC_ERROR_NONE) goto error; diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc index 008d37119a..aba5539199 100644 --- a/src/core/lib/iomgr/timer_generic.cc +++ b/src/core/lib/iomgr/timer_generic.cc @@ -256,7 +256,7 @@ static grpc_millis compute_min_deadline(timer_shard* shard) { static void timer_list_init() { uint32_t i; - g_num_shards = GPR_MIN(1, 2 * gpr_cpu_num_cores()); + g_num_shards = GPR_CLAMP(2 * gpr_cpu_num_cores(), 1, 32); g_shards = static_cast<timer_shard*>(gpr_zalloc(g_num_shards * sizeof(*g_shards))); g_shard_queue = static_cast<timer_shard**>( diff --git a/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc b/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc index 0a39c6c485..118d18d119 100644 --- a/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +++ b/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc @@ -72,7 +72,8 @@ static void target_service_account_destroy( static const grpc_alts_credentials_options_vtable vtable = { alts_client_options_copy, alts_client_options_destroy}; -grpc_alts_credentials_options* grpc_alts_credentials_client_options_create() { +grpc_alts_credentials_options* grpc_alts_credentials_client_options_create( + void) { auto client_options = static_cast<grpc_alts_credentials_client_options*>( gpr_zalloc(sizeof(grpc_alts_credentials_client_options))); client_options->base.vtable = &vtable; diff --git a/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc b/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc index 62aa7a620a..1a59c45675 100644 --- a/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc +++ b/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc @@ -36,7 +36,8 @@ static void alts_server_options_destroy( static const grpc_alts_credentials_options_vtable vtable = { alts_server_options_copy, alts_server_options_destroy}; -grpc_alts_credentials_options* grpc_alts_credentials_server_options_create() { +grpc_alts_credentials_options* grpc_alts_credentials_server_options_create( + void) { grpc_alts_credentials_server_options* server_options = static_cast<grpc_alts_credentials_server_options*>( gpr_zalloc(sizeof(*server_options))); diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 04b4c87c71..6246613e7b 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -59,8 +59,8 @@ static const char* installed_roots_path = /** Environment variable used as a flag to enable/disable loading system root certificates from the OS trust store. */ -#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR -#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#ifndef GRPC_NOT_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_NOT_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_NOT_USE_SYSTEM_SSL_ROOTS" #endif #ifndef TSI_OPENSSL_ALPN_SUPPORT @@ -1192,10 +1192,10 @@ const char* DefaultSslRootStore::GetPemRootCerts() { grpc_slice DefaultSslRootStore::ComputePemRootCerts() { grpc_slice result = grpc_empty_slice(); - char* use_system_roots_env_value = - gpr_getenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); - const bool use_system_roots = gpr_is_true(use_system_roots_env_value); - gpr_free(use_system_roots_env_value); + char* not_use_system_roots_env_value = + gpr_getenv(GRPC_NOT_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + const bool not_use_system_roots = gpr_is_true(not_use_system_roots_env_value); + gpr_free(not_use_system_roots_env_value); // First try to load the roots from the environment. char* default_root_certs_path = gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); @@ -1218,7 +1218,7 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() { gpr_free(pem_root_certs); } // Try loading roots from OS trust store if flag is enabled. - if (GRPC_SLICE_IS_EMPTY(result) && use_system_roots) { + if (GRPC_SLICE_IS_EMPTY(result) && !not_use_system_roots) { result = LoadSystemRootCerts(); } // Fallback to roots manually shipped with gRPC. diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc index 19cbb03b63..b99fc5e178 100644 --- a/src/core/lib/security/transport/server_auth_filter.cc +++ b/src/core/lib/security/transport/server_auth_filter.cc @@ -41,6 +41,11 @@ struct call_data { grpc_transport_stream_op_batch* recv_initial_metadata_batch; grpc_closure* original_recv_initial_metadata_ready; grpc_closure recv_initial_metadata_ready; + grpc_error* recv_initial_metadata_error; + grpc_closure recv_trailing_metadata_ready; + grpc_closure* original_recv_trailing_metadata_ready; + grpc_error* recv_trailing_metadata_error; + bool seen_recv_trailing_metadata_ready; grpc_metadata_array md; const grpc_metadata* consumed_md; size_t num_consumed_md; @@ -111,7 +116,16 @@ static void on_md_processing_done_inner(grpc_call_element* elem, batch->payload->recv_initial_metadata.recv_initial_metadata, remove_consumed_md, elem, "Response metadata filtering error"); } - GRPC_CLOSURE_SCHED(calld->original_recv_initial_metadata_ready, error); + calld->recv_initial_metadata_error = GRPC_ERROR_REF(error); + grpc_closure* closure = calld->original_recv_initial_metadata_ready; + calld->original_recv_initial_metadata_ready = nullptr; + if (calld->seen_recv_trailing_metadata_ready) { + GRPC_CALL_COMBINER_START(calld->call_combiner, + &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_error, + "continue recv_trailing_metadata_ready"); + } + GRPC_CLOSURE_SCHED(closure, error); } // Called from application code. @@ -180,8 +194,31 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) { return; } } - GRPC_CLOSURE_RUN(calld->original_recv_initial_metadata_ready, - GRPC_ERROR_REF(error)); + grpc_closure* closure = calld->original_recv_initial_metadata_ready; + calld->original_recv_initial_metadata_ready = nullptr; + if (calld->seen_recv_trailing_metadata_ready) { + GRPC_CALL_COMBINER_START(calld->call_combiner, + &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_error, + "continue recv_trailing_metadata_ready"); + } + GRPC_CLOSURE_RUN(closure, GRPC_ERROR_REF(error)); +} + +static void recv_trailing_metadata_ready(void* user_data, grpc_error* err) { + grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); + call_data* calld = static_cast<call_data*>(elem->call_data); + if (calld->original_recv_initial_metadata_ready != nullptr) { + calld->recv_trailing_metadata_error = GRPC_ERROR_REF(err); + calld->seen_recv_trailing_metadata_ready = true; + GRPC_CALL_COMBINER_STOP(calld->call_combiner, + "deferring recv_trailing_metadata_ready until " + "after recv_initial_metadata_ready"); + return; + } + err = grpc_error_add_child( + GRPC_ERROR_REF(err), GRPC_ERROR_REF(calld->recv_initial_metadata_error)); + GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, err); } static void auth_start_transport_stream_op_batch( @@ -195,6 +232,12 @@ static void auth_start_transport_stream_op_batch( batch->payload->recv_initial_metadata.recv_initial_metadata_ready = &calld->recv_initial_metadata_ready; } + if (batch->recv_trailing_metadata) { + calld->original_recv_trailing_metadata_ready = + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &calld->recv_trailing_metadata_ready; + } grpc_call_next_op(elem, batch); } @@ -208,6 +251,9 @@ static grpc_error* init_call_elem(grpc_call_element* elem, GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready, recv_initial_metadata_ready, elem, grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready, + recv_trailing_metadata_ready, elem, + grpc_schedule_on_exec_ctx); // Create server security context. Set its auth context from channel // data and save it in the call context. grpc_server_security_context* server_ctx = @@ -227,7 +273,10 @@ static grpc_error* init_call_elem(grpc_call_element* elem, /* Destructor for call_data */ static void destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, - grpc_closure* ignored) {} + grpc_closure* ignored) { + call_data* calld = static_cast<call_data*>(elem->call_data); + GRPC_ERROR_UNREF(calld->recv_initial_metadata_error); +} /* Constructor for channel_data */ static grpc_error* init_channel_elem(grpc_channel_element* elem, diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 2923a86646..a9349afa68 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -48,6 +48,7 @@ #include "src/core/lib/surface/call_test_only.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server.h" #include "src/core/lib/surface/validate_metadata.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata.h" @@ -71,46 +72,6 @@ // Used to create arena for the first call. #define ESTIMATED_MDELEM_COUNT 16 -/* Status data for a request can come from several sources; this - enumerates them all, and acts as a priority sorting for which - status to return to the application - earlier entries override - later ones */ -typedef enum { - /* Status came from the application layer overriding whatever - the wire says */ - STATUS_FROM_API_OVERRIDE = 0, - /* Status came from 'the wire' - or somewhere below the surface - layer */ - STATUS_FROM_WIRE, - /* Status was created by some internal channel stack operation: must come via - add_batch_error */ - STATUS_FROM_CORE, - /* Status was created by some surface error */ - STATUS_FROM_SURFACE, - /* Status came from the server sending status */ - STATUS_FROM_SERVER_STATUS, - STATUS_SOURCE_COUNT -} status_source; - -typedef struct { - bool is_set; - grpc_error* error; -} received_status; - -static gpr_atm pack_received_status(received_status r) { - return r.is_set ? (1 | (gpr_atm)r.error) : 0; -} - -static received_status unpack_received_status(gpr_atm atm) { - if ((atm & 1) == 0) { - return {false, GRPC_ERROR_NONE}; - } else { - return {true, (grpc_error*)(atm & ~static_cast<gpr_atm>(1))}; - } -} - -#define MAX_ERRORS_PER_BATCH 4 - typedef struct batch_control { grpc_call* call; /* Share memory for cq_completion and notify_tag as they are never needed @@ -135,10 +96,7 @@ typedef struct batch_control { grpc_closure start_batch; grpc_closure finish_batch; gpr_refcount steps_to_complete; - - grpc_error* errors[MAX_ERRORS_PER_BATCH]; - gpr_atm num_errors; - + gpr_atm batch_error; grpc_transport_stream_op_batch op; } batch_control; @@ -201,9 +159,6 @@ struct grpc_call { // A char* indicating the peer name. gpr_atm peer_string; - /* Packed received call statuses from various sources */ - gpr_atm status[STATUS_SOURCE_COUNT]; - /* Call data useful used for reporting. Only valid after the call has * completed */ grpc_call_final_info final_info; @@ -236,6 +191,7 @@ struct grpc_call { grpc_closure receiving_initial_metadata_ready; grpc_closure receiving_trailing_metadata_ready; uint32_t test_only_last_message_flags; + gpr_atm cancelled; grpc_closure release_call; @@ -247,8 +203,11 @@ struct grpc_call { } client; struct { int* cancelled; + // backpointer to owning server if this is a server side call. + grpc_server* server; } server; } final_op; + gpr_atm status_error; /* recv_state can contain one of the following values: RECV_NONE : : no initial metadata and messages received @@ -286,23 +245,15 @@ grpc_core::TraceFlag grpc_compression_trace(false, "compression"); static void execute_batch(grpc_call* call, grpc_transport_stream_op_batch* op, grpc_closure* start_batch_closure); -static void cancel_with_status(grpc_call* c, status_source source, - grpc_status_code status, + +static void cancel_with_status(grpc_call* c, grpc_status_code status, const char* description); -static void cancel_with_error(grpc_call* c, status_source source, - grpc_error* error); +static void cancel_with_error(grpc_call* c, grpc_error* error); static void destroy_call(void* call_stack, grpc_error* error); static void receiving_slice_ready(void* bctlp, grpc_error* error); -static void get_final_status( - grpc_call* call, void (*set_value)(grpc_status_code code, void* user_data), - void* set_value_user_data, grpc_slice* details, const char** error_string); -static void set_status_value_directly(grpc_status_code status, void* dest); -static void set_status_from_error(grpc_call* call, status_source source, - grpc_error* error); +static void set_final_status(grpc_call* call, grpc_error* error); static void process_data_after_md(batch_control* bctl); static void post_batch_completion(batch_control* bctl); -static void add_batch_error(batch_control* bctl, grpc_error* error, - bool has_cancelled); static void add_init_error(grpc_error** composite, grpc_error* new_err) { if (new_err == GRPC_ERROR_NONE) return; @@ -353,6 +304,7 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, gpr_arena_alloc(arena, GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call)) + channel_stack->call_stack_size)); gpr_ref_init(&call->ext_ref, 1); + gpr_atm_no_barrier_store(&call->cancelled, 0); call->arena = arena; grpc_call_combiner_init(&call->call_combiner); *out_call = call; @@ -362,14 +314,10 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, /* Always support no compression */ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_MESSAGE_COMPRESS_NONE); call->is_client = args->server_transport_data == nullptr; - if (call->is_client) { - GRPC_STATS_INC_CLIENT_CALLS_CREATED(); - } else { - GRPC_STATS_INC_SERVER_CALLS_CREATED(); - } call->stream_op_payload.context = call->context; grpc_slice path = grpc_empty_slice(); if (call->is_client) { + GRPC_STATS_INC_CLIENT_CALLS_CREATED(); GPR_ASSERT(args->add_initial_metadata_count < MAX_SEND_EXTRA_METADATA_COUNT); for (i = 0; i < args->add_initial_metadata_count; i++) { @@ -383,6 +331,8 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, call->send_extra_metadata_count = static_cast<int>(args->add_initial_metadata_count); } else { + GRPC_STATS_INC_SERVER_CALLS_CREATED(); + call->final_op.server.server = args->server; GPR_ASSERT(args->add_initial_metadata_count == 0); call->send_extra_metadata_count = 0; } @@ -464,10 +414,10 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, gpr_mu_unlock(&pc->child_list_mu); } if (error != GRPC_ERROR_NONE) { - cancel_with_error(call, STATUS_FROM_SURFACE, GRPC_ERROR_REF(error)); + cancel_with_error(call, GRPC_ERROR_REF(error)); } if (immediately_cancel) { - cancel_with_error(call, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); + cancel_with_error(call, GRPC_ERROR_CANCELLED); } if (args->cq != nullptr) { GPR_ASSERT(args->pollset_set_alternative == nullptr && @@ -486,10 +436,18 @@ grpc_error* grpc_call_create(const grpc_call_create_args* args, &call->pollent); } - grpc_core::channelz::ChannelNode* channelz_channel = - grpc_channel_get_channelz_node(call->channel); - if (channelz_channel != nullptr) { - channelz_channel->RecordCallStarted(); + if (call->is_client) { + grpc_core::channelz::ChannelNode* channelz_channel = + grpc_channel_get_channelz_node(call->channel); + if (channelz_channel != nullptr) { + channelz_channel->RecordCallStarted(); + } + } else { + grpc_core::channelz::ServerNode* channelz_server = + grpc_server_get_channelz_node(call->final_op.server.server); + if (channelz_server != nullptr) { + channelz_server->RecordCallStarted(); + } } grpc_slice_unref_internal(path); @@ -561,16 +519,15 @@ static void destroy_call(void* call, grpc_error* error) { GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); } - get_final_status(c, set_status_value_directly, &c->final_info.final_status, - nullptr, &(c->final_info.error_string)); + grpc_error* status_error = + reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&c->status_error)); + grpc_error_get_status(status_error, c->send_deadline, + &c->final_info.final_status, nullptr, nullptr, + &(c->final_info.error_string)); + GRPC_ERROR_UNREF(status_error); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - GRPC_ERROR_UNREF( - unpack_received_status(gpr_atm_acq_load(&c->status[i])).error); - } - grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); @@ -608,7 +565,7 @@ void grpc_call_unref(grpc_call* c) { bool cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) != 0 && gpr_atm_acq_load(&c->received_final_op_atm) == 0; if (cancel) { - cancel_with_error(c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); + cancel_with_error(c, GRPC_ERROR_CANCELLED); } else { // Unset the call combiner cancellation closure. This has the // effect of scheduling the previously set cancellation closure, if @@ -626,8 +583,7 @@ grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) { GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved)); GPR_ASSERT(!reserved); grpc_core::ExecCtx exec_ctx; - cancel_with_error(call, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); - + cancel_with_error(call, GRPC_ERROR_CANCELLED); return GRPC_CALL_OK; } @@ -681,8 +637,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call* c, "c=%p, status=%d, description=%s, reserved=%p)", 4, (c, (int)status, description, reserved)); GPR_ASSERT(reserved == nullptr); - cancel_with_status(c, STATUS_FROM_API_OVERRIDE, status, description); - + cancel_with_status(c, status, description); return GRPC_CALL_OK; } @@ -702,15 +657,17 @@ static void done_termination(void* arg, grpc_error* error) { gpr_free(state); } -static void cancel_with_error(grpc_call* c, status_source source, - grpc_error* error) { +static void cancel_with_error(grpc_call* c, grpc_error* error) { + if (!gpr_atm_rel_cas(&c->cancelled, 0, 1)) { + GRPC_ERROR_UNREF(error); + return; + } GRPC_CALL_INTERNAL_REF(c, "termination"); // Inform the call combiner of the cancellation, so that it can cancel // any in-flight asynchronous actions that may be holding the call // combiner. This ensures that the cancel_stream batch can be sent // down the filter stack in a timely manner. grpc_call_combiner_cancel(&c->call_combiner, GRPC_ERROR_REF(error)); - set_status_from_error(c, source, GRPC_ERROR_REF(error)); cancel_state* state = static_cast<cancel_state*>(gpr_malloc(sizeof(*state))); state->call = c; GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state, @@ -733,90 +690,47 @@ static grpc_error* error_from_status(grpc_status_code status, GRPC_ERROR_INT_GRPC_STATUS, status); } -static void cancel_with_status(grpc_call* c, status_source source, - grpc_status_code status, +static void cancel_with_status(grpc_call* c, grpc_status_code status, const char* description) { - cancel_with_error(c, source, error_from_status(status, description)); -} - -/******************************************************************************* - * FINAL STATUS CODE MANIPULATION - */ - -static bool get_final_status_from( - grpc_call* call, grpc_error* error, bool allow_ok_status, - void (*set_value)(grpc_status_code code, void* user_data), - void* set_value_user_data, grpc_slice* details, const char** error_string) { - grpc_status_code code; - grpc_slice slice = grpc_empty_slice(); - grpc_error_get_status(error, call->send_deadline, &code, &slice, nullptr, - error_string); - if (code == GRPC_STATUS_OK && !allow_ok_status) { - return false; - } - - set_value(code, set_value_user_data); - if (details != nullptr) { - *details = grpc_slice_ref_internal(slice); - } - return true; + cancel_with_error(c, error_from_status(status, description)); } -static void get_final_status( - grpc_call* call, void (*set_value)(grpc_status_code code, void* user_data), - void* set_value_user_data, grpc_slice* details, const char** error_string) { - int i; - received_status status[STATUS_SOURCE_COUNT]; - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - status[i] = unpack_received_status(gpr_atm_acq_load(&call->status[i])); - } +static void set_final_status(grpc_call* call, grpc_error* error) { if (grpc_call_error_trace.enabled()) { - gpr_log(GPR_INFO, "get_final_status %s", call->is_client ? "CLI" : "SVR"); - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - if (status[i].is_set) { - gpr_log(GPR_INFO, " %d: %s", i, grpc_error_string(status[i].error)); - } - } + gpr_log(GPR_DEBUG, "set_final_status %s", call->is_client ? "CLI" : "SVR"); + gpr_log(GPR_DEBUG, "%s", grpc_error_string(error)); } - /* first search through ignoring "OK" statuses: if something went wrong, - * ensure we report it */ - for (int allow_ok_status = 0; allow_ok_status < 2; allow_ok_status++) { - /* search for the best status we can present: ideally the error we use has a - clearly defined grpc-status, and we'll prefer that. */ - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - if (status[i].is_set && - grpc_error_has_clear_grpc_status(status[i].error)) { - if (get_final_status_from(call, status[i].error, allow_ok_status != 0, - set_value, set_value_user_data, details, - error_string)) { - return; - } + if (call->is_client) { + grpc_error_get_status(error, call->send_deadline, + call->final_op.client.status, + call->final_op.client.status_details, nullptr, + call->final_op.client.error_string); + // explicitly take a ref + grpc_slice_ref_internal(*call->final_op.client.status_details); + gpr_atm_rel_store(&call->status_error, reinterpret_cast<gpr_atm>(error)); + grpc_core::channelz::ChannelNode* channelz_channel = + grpc_channel_get_channelz_node(call->channel); + if (channelz_channel != nullptr) { + if (*call->final_op.client.status != GRPC_STATUS_OK) { + channelz_channel->RecordCallFailed(); + } else { + channelz_channel->RecordCallSucceeded(); } } - /* If no clearly defined status exists, search for 'anything' */ - for (i = 0; i < STATUS_SOURCE_COUNT; i++) { - if (status[i].is_set) { - if (get_final_status_from(call, status[i].error, allow_ok_status != 0, - set_value, set_value_user_data, details, - error_string)) { - return; - } + } else { + *call->final_op.server.cancelled = + error != GRPC_ERROR_NONE || + reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&call->status_error)) != + GRPC_ERROR_NONE; + grpc_core::channelz::ServerNode* channelz_server = + grpc_server_get_channelz_node(call->final_op.server.server); + if (channelz_server != nullptr) { + if (*call->final_op.server.cancelled) { + channelz_server->RecordCallFailed(); + } else { + channelz_server->RecordCallSucceeded(); } } - } - /* If nothing exists, set some default */ - if (call->is_client) { - set_value(GRPC_STATUS_UNKNOWN, set_value_user_data); - } else { - set_value(GRPC_STATUS_OK, set_value_user_data); - } -} - -static void set_status_from_error(grpc_call* call, status_source source, - grpc_error* error) { - if (!gpr_atm_rel_cas(&call->status[source], - pack_received_status({false, GRPC_ERROR_NONE}), - pack_received_status({true, error}))) { GRPC_ERROR_UNREF(error); } } @@ -1035,6 +949,7 @@ static grpc_stream_compression_algorithm decode_stream_compression( static void publish_app_metadata(grpc_call* call, grpc_metadata_batch* b, int is_trailing) { if (b->list.count == 0) return; + if (!call->is_client && is_trailing) return; if (is_trailing && call->buffered_metadata[1] == nullptr) return; GPR_TIMER_SCOPE("publish_app_metadata", 0); grpc_metadata_array* dest; @@ -1088,9 +1003,12 @@ static void recv_initial_filter(grpc_call* call, grpc_metadata_batch* b) { publish_app_metadata(call, b, false); } -static void recv_trailing_filter(void* args, grpc_metadata_batch* b) { +static void recv_trailing_filter(void* args, grpc_metadata_batch* b, + grpc_error* batch_error) { grpc_call* call = static_cast<grpc_call*>(args); - if (b->idx.named.grpc_status != nullptr) { + if (batch_error != GRPC_ERROR_NONE) { + set_final_status(call, batch_error); + } else if (b->idx.named.grpc_status != nullptr) { grpc_status_code status_code = grpc_get_status_code_from_metadata(b->idx.named.grpc_status->md); grpc_error* error = GRPC_ERROR_NONE; @@ -1108,8 +1026,18 @@ static void recv_trailing_filter(void* args, grpc_metadata_batch* b) { error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, grpc_empty_slice()); } - set_status_from_error(call, STATUS_FROM_WIRE, error); + set_final_status(call, GRPC_ERROR_REF(error)); grpc_metadata_batch_remove(b, b->idx.named.grpc_status); + GRPC_ERROR_UNREF(error); + } else if (!call->is_client) { + set_final_status(call, GRPC_ERROR_NONE); + } else { + gpr_log(GPR_DEBUG, + "Received trailing metadata with no error and no status"); + set_final_status( + call, grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("No status received"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNKNOWN)); } publish_app_metadata(call, b, true); } @@ -1124,14 +1052,6 @@ grpc_call_stack* grpc_call_get_call_stack(grpc_call* call) { * BATCH API IMPLEMENTATION */ -static void set_status_value_directly(grpc_status_code status, void* dest) { - *static_cast<grpc_status_code*>(dest) = status; -} - -static void set_cancelled_value(grpc_status_code status, void* dest) { - *static_cast<int*>(dest) = (status != GRPC_STATUS_OK); -} - static bool are_write_flags_valid(uint32_t flags) { /* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */ const uint32_t allowed_write_positions = @@ -1199,31 +1119,18 @@ static void finish_batch_completion(void* user_data, GRPC_CALL_INTERNAL_UNREF(call, "completion"); } -static grpc_error* consolidate_batch_errors(batch_control* bctl) { - size_t n = static_cast<size_t>(gpr_atm_acq_load(&bctl->num_errors)); - if (n == 0) { - return GRPC_ERROR_NONE; - } else if (n == 1) { - /* Skip creating a composite error in the case that only one error was - logged */ - grpc_error* e = bctl->errors[0]; - bctl->errors[0] = nullptr; - return e; - } else { - grpc_error* error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Call batch failed", bctl->errors, n); - for (size_t i = 0; i < n; i++) { - GRPC_ERROR_UNREF(bctl->errors[i]); - bctl->errors[i] = nullptr; - } - return error; - } +static void reset_batch_errors(batch_control* bctl) { + GRPC_ERROR_UNREF( + reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&bctl->batch_error))); + gpr_atm_rel_store(&bctl->batch_error, + reinterpret_cast<gpr_atm>(GRPC_ERROR_NONE)); } static void post_batch_completion(batch_control* bctl) { grpc_call* next_child_call; grpc_call* call = bctl->call; - grpc_error* error = consolidate_batch_errors(bctl); + grpc_error* error = GRPC_ERROR_REF( + reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&bctl->batch_error))); if (bctl->op.send_initial_metadata) { grpc_metadata_batch_destroy( @@ -1249,8 +1156,7 @@ static void post_batch_completion(batch_control* bctl) { next_child_call = child->child->sibling_next; if (child->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); - cancel_with_error(child, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_CANCELLED); + cancel_with_error(child, GRPC_ERROR_CANCELLED); GRPC_CALL_INTERNAL_UNREF(child, "propagate_cancel"); } child = next_child_call; @@ -1258,24 +1164,6 @@ static void post_batch_completion(batch_control* bctl) { } gpr_mu_unlock(&pc->child_list_mu); } - if (call->is_client) { - get_final_status(call, set_status_value_directly, - call->final_op.client.status, - call->final_op.client.status_details, - call->final_op.client.error_string); - } else { - get_final_status(call, set_cancelled_value, - call->final_op.server.cancelled, nullptr, nullptr); - } - grpc_core::channelz::ChannelNode* channelz_channel = - grpc_channel_get_channelz_node(call->channel); - if (channelz_channel != nullptr) { - if (*call->final_op.client.status != GRPC_STATUS_OK) { - channelz_channel->RecordCallFailed(); - } else { - channelz_channel->RecordCallSucceeded(); - } - } GRPC_ERROR_UNREF(error); error = GRPC_ERROR_NONE; } @@ -1284,9 +1172,10 @@ static void post_batch_completion(batch_control* bctl) { grpc_byte_buffer_destroy(*call->receiving_buffer); *call->receiving_buffer = nullptr; } + reset_batch_errors(bctl); if (bctl->completion_data.notify_tag.is_closure) { - /* unrefs bctl->error */ + /* unrefs error */ bctl->call = nullptr; /* This closure may be meant to be run within some combiner. Since we aren't * running in any combiner here, we need to use GRPC_CLOSURE_SCHED instead @@ -1296,7 +1185,7 @@ static void post_batch_completion(batch_control* bctl) { error); GRPC_CALL_INTERNAL_UNREF(call, "completion"); } else { - /* unrefs bctl->error */ + /* unrefs error */ grpc_cq_end_op(bctl->call->cq, bctl->completion_data.notify_tag.tag, error, finish_batch_completion, bctl, &bctl->completion_data.cq_completion); @@ -1405,8 +1294,12 @@ static void receiving_stream_ready(void* bctlp, grpc_error* error) { grpc_call* call = bctl->call; if (error != GRPC_ERROR_NONE) { call->receiving_stream.reset(); - add_batch_error(bctl, GRPC_ERROR_REF(error), true); - cancel_with_error(call, STATUS_FROM_SURFACE, GRPC_ERROR_REF(error)); + if (reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&bctl->batch_error)) == + GRPC_ERROR_NONE) { + gpr_atm_rel_store(&bctl->batch_error, + reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error))); + } + cancel_with_error(call, GRPC_ERROR_REF(error)); } /* If recv_state is RECV_NONE, we will save the batch_control * object with rel_cas, and will not use it after the cas. Its corresponding @@ -1442,8 +1335,7 @@ static void validate_filtered_metadata(batch_control* bctl) { call->incoming_stream_compression_algorithm, call->incoming_message_compression_algorithm); gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_INTERNAL, - error_msg); + cancel_with_status(call, GRPC_STATUS_INTERNAL, error_msg); gpr_free(error_msg); } else if ( grpc_compression_algorithm_from_message_stream_compression_algorithm( @@ -1455,8 +1347,7 @@ static void validate_filtered_metadata(batch_control* bctl) { "compression (%d).", call->incoming_stream_compression_algorithm, call->incoming_message_compression_algorithm); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_INTERNAL, - error_msg); + cancel_with_status(call, GRPC_STATUS_INTERNAL, error_msg); gpr_free(error_msg); } else { char* error_msg = nullptr; @@ -1466,8 +1357,7 @@ static void validate_filtered_metadata(batch_control* bctl) { gpr_asprintf(&error_msg, "Invalid compression algorithm value '%d'.", compression_algorithm); gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, - error_msg); + cancel_with_status(call, GRPC_STATUS_UNIMPLEMENTED, error_msg); } else if (grpc_compression_options_is_algorithm_enabled( &compression_options, compression_algorithm) == 0) { /* check if algorithm is supported by current channel config */ @@ -1476,8 +1366,7 @@ static void validate_filtered_metadata(batch_control* bctl) { gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); gpr_log(GPR_ERROR, "%s", error_msg); - cancel_with_status(call, STATUS_FROM_SURFACE, GRPC_STATUS_UNIMPLEMENTED, - error_msg); + cancel_with_status(call, GRPC_STATUS_UNIMPLEMENTED, error_msg); } gpr_free(error_msg); @@ -1495,23 +1384,12 @@ static void validate_filtered_metadata(batch_control* bctl) { } } -static void add_batch_error(batch_control* bctl, grpc_error* error, - bool has_cancelled) { - if (error == GRPC_ERROR_NONE) return; - int idx = static_cast<int>(gpr_atm_full_fetch_add(&bctl->num_errors, 1)); - if (idx == 0 && !has_cancelled) { - cancel_with_error(bctl->call, STATUS_FROM_CORE, GRPC_ERROR_REF(error)); - } - bctl->errors[idx] = error; -} - static void receiving_initial_metadata_ready(void* bctlp, grpc_error* error) { batch_control* bctl = static_cast<batch_control*>(bctlp); grpc_call* call = bctl->call; GRPC_CALL_COMBINER_STOP(&call->call_combiner, "recv_initial_metadata_ready"); - add_batch_error(bctl, GRPC_ERROR_REF(error), false); if (error == GRPC_ERROR_NONE) { grpc_metadata_batch* md = &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; @@ -1524,6 +1402,13 @@ static void receiving_initial_metadata_ready(void* bctlp, grpc_error* error) { if (md->deadline != GRPC_MILLIS_INF_FUTURE && !call->is_client) { call->send_deadline = md->deadline; } + } else { + if (reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&bctl->batch_error)) == + GRPC_ERROR_NONE) { + gpr_atm_rel_store(&bctl->batch_error, + reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error))); + } + cancel_with_error(call, GRPC_ERROR_REF(error)); } grpc_closure* saved_rsr_closure = nullptr; @@ -1561,10 +1446,9 @@ static void receiving_trailing_metadata_ready(void* bctlp, grpc_error* error) { batch_control* bctl = static_cast<batch_control*>(bctlp); grpc_call* call = bctl->call; GRPC_CALL_COMBINER_STOP(&call->call_combiner, "recv_trailing_metadata_ready"); - add_batch_error(bctl, GRPC_ERROR_REF(error), false); grpc_metadata_batch* md = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; - recv_trailing_filter(call, md); + recv_trailing_filter(call, md, GRPC_ERROR_REF(error)); finish_batch_step(bctl); } @@ -1572,7 +1456,14 @@ static void finish_batch(void* bctlp, grpc_error* error) { batch_control* bctl = static_cast<batch_control*>(bctlp); grpc_call* call = bctl->call; GRPC_CALL_COMBINER_STOP(&call->call_combiner, "on_complete"); - add_batch_error(bctl, GRPC_ERROR_REF(error), false); + if (reinterpret_cast<grpc_error*>(gpr_atm_acq_load(&bctl->batch_error)) == + GRPC_ERROR_NONE) { + gpr_atm_rel_store(&bctl->batch_error, + reinterpret_cast<gpr_atm>(GRPC_ERROR_REF(error))); + } + if (error != GRPC_ERROR_NONE) { + cancel_with_error(call, GRPC_ERROR_REF(error)); + } finish_batch_step(bctl); } @@ -1774,28 +1665,33 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops, call->send_extra_metadata_count = 1; call->send_extra_metadata[0].md = grpc_channel_get_reffed_status_elem( call->channel, op->data.send_status_from_server.status); - { - grpc_error* override_error = GRPC_ERROR_NONE; - if (op->data.send_status_from_server.status != GRPC_STATUS_OK) { - override_error = - error_from_status(op->data.send_status_from_server.status, - "Returned non-ok status"); - } - if (op->data.send_status_from_server.status_details != nullptr) { - call->send_extra_metadata[1].md = grpc_mdelem_from_slices( - GRPC_MDSTR_GRPC_MESSAGE, - grpc_slice_ref_internal( - *op->data.send_status_from_server.status_details)); - call->send_extra_metadata_count++; + grpc_error* status_error = + op->data.send_status_from_server.status == GRPC_STATUS_OK + ? GRPC_ERROR_NONE + : grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Server returned error"), + GRPC_ERROR_INT_GRPC_STATUS, + static_cast<intptr_t>( + op->data.send_status_from_server.status)); + if (op->data.send_status_from_server.status_details != nullptr) { + call->send_extra_metadata[1].md = grpc_mdelem_from_slices( + GRPC_MDSTR_GRPC_MESSAGE, + grpc_slice_ref_internal( + *op->data.send_status_from_server.status_details)); + call->send_extra_metadata_count++; + if (status_error != GRPC_ERROR_NONE) { char* msg = grpc_slice_to_c_string( GRPC_MDVALUE(call->send_extra_metadata[1].md)); - override_error = - grpc_error_set_str(override_error, GRPC_ERROR_STR_GRPC_MESSAGE, + status_error = + grpc_error_set_str(status_error, GRPC_ERROR_STR_GRPC_MESSAGE, grpc_slice_from_copied_string(msg)); gpr_free(msg); } - set_status_from_error(call, STATUS_FROM_API_OVERRIDE, override_error); } + + gpr_atm_rel_store(&call->status_error, + reinterpret_cast<gpr_atm>(status_error)); if (!prepare_application_metadata( call, static_cast<int>( diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index b3b06059d4..b34260505a 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -33,6 +33,7 @@ typedef void (*grpc_ioreq_completion_func)(grpc_call* call, int success, typedef struct grpc_call_create_args { grpc_channel* channel; + grpc_server* server; grpc_call* parent; uint32_t propagation_mask; diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 82635d3c21..054fe105c3 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -100,7 +100,6 @@ grpc_channel* grpc_channel_create_with_builder( return channel; } - memset(channel, 0, sizeof(*channel)); channel->target = target; channel->is_client = grpc_channel_stack_type_is_client(channel_stack_type); size_t channel_tracer_max_nodes = 0; // default to off @@ -166,11 +165,12 @@ grpc_channel* grpc_channel_create_with_builder( } grpc_channel_args_destroy(args); - if (channelz_enabled) { - bool is_top_level_channel = channel->is_client && !internal_channel; + // we only need to do the channelz bookkeeping for clients here. The channelz + // bookkeeping for server channels occurs in src/core/lib/surface/server.cc + if (channelz_enabled && channel->is_client) { channel->channelz_channel = channel_node_create_func( - channel, channel_tracer_max_nodes, is_top_level_channel); - channel->channelz_channel->trace()->AddTraceEvent( + channel, channel_tracer_max_nodes, !internal_channel); + channel->channelz_channel->AddTraceEvent( grpc_core::channelz::ChannelTrace::Severity::Info, grpc_slice_from_static_string("Channel created")); } @@ -428,6 +428,9 @@ void grpc_channel_internal_unref(grpc_channel* c REF_ARG) { static void destroy_channel(void* arg, grpc_error* error) { grpc_channel* channel = static_cast<grpc_channel*>(arg); if (channel->channelz_channel != nullptr) { + channel->channelz_channel->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Channel destroyed")); channel->channelz_channel->MarkChannelDestroyed(); channel->channelz_channel.reset(); } diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc index 0769d9e4f6..5dc9991f70 100644 --- a/src/core/lib/surface/completion_queue.cc +++ b/src/core/lib/surface/completion_queue.cc @@ -184,7 +184,8 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { typedef struct cq_vtable { grpc_cq_completion_type cq_completion_type; size_t data_size; - void (*init)(void* data, grpc_core::CQCallbackInterface* shutdown_callback); + void (*init)(void* data, + grpc_experimental_completion_queue_functor* shutdown_callback); void (*shutdown)(grpc_completion_queue* cq); void (*destroy)(void* data); bool (*begin_op)(grpc_completion_queue* cq, void* tag); @@ -267,7 +268,7 @@ typedef struct cq_callback_data { bool shutdown_called; /** A callback that gets invoked when the CQ completes shutdown */ - grpc_core::CQCallbackInterface* shutdown_callback; + grpc_experimental_completion_queue_functor* shutdown_callback; } cq_callback_data; /* Completion queue structure */ @@ -333,12 +334,12 @@ static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag, gpr_timespec deadline, void* reserved); // Note that cq_init_next and cq_init_pluck do not use the shutdown_callback -static void cq_init_next(void* data, - grpc_core::CQCallbackInterface* shutdown_callback); -static void cq_init_pluck(void* data, - grpc_core::CQCallbackInterface* shutdown_callback); -static void cq_init_callback(void* data, - grpc_core::CQCallbackInterface* shutdown_callback); +static void cq_init_next( + void* data, grpc_experimental_completion_queue_functor* shutdown_callback); +static void cq_init_pluck( + void* data, grpc_experimental_completion_queue_functor* shutdown_callback); +static void cq_init_callback( + void* data, grpc_experimental_completion_queue_functor* shutdown_callback); static void cq_destroy_next(void* data); static void cq_destroy_pluck(void* data); static void cq_destroy_callback(void* data); @@ -462,7 +463,7 @@ static long cq_event_queue_num_items(grpc_cq_event_queue* q) { grpc_completion_queue* grpc_completion_queue_create_internal( grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type, - grpc_core::CQCallbackInterface* shutdown_callback) { + grpc_experimental_completion_queue_functor* shutdown_callback) { GPR_TIMER_SCOPE("grpc_completion_queue_create_internal", 0); grpc_completion_queue* cq; @@ -497,8 +498,8 @@ grpc_completion_queue* grpc_completion_queue_create_internal( return cq; } -static void cq_init_next(void* data, - grpc_core::CQCallbackInterface* shutdown_callback) { +static void cq_init_next( + void* data, grpc_experimental_completion_queue_functor* shutdown_callback) { cq_next_data* cqd = static_cast<cq_next_data*>(data); /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); @@ -513,8 +514,8 @@ static void cq_destroy_next(void* data) { cq_event_queue_destroy(&cqd->queue); } -static void cq_init_pluck(void* data, - grpc_core::CQCallbackInterface* shutdown_callback) { +static void cq_init_pluck( + void* data, grpc_experimental_completion_queue_functor* shutdown_callback) { cq_pluck_data* cqd = static_cast<cq_pluck_data*>(data); /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); @@ -532,7 +533,7 @@ static void cq_destroy_pluck(void* data) { } static void cq_init_callback( - void* data, grpc_core::CQCallbackInterface* shutdown_callback) { + void* data, grpc_experimental_completion_queue_functor* shutdown_callback) { cq_callback_data* cqd = static_cast<cq_callback_data*>(data); /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); @@ -859,7 +860,8 @@ static void cq_end_op_for_callback( GRPC_ERROR_UNREF(error); - (static_cast<grpc_core::CQCallbackInterface*>(tag))->Run(is_success); + auto* functor = static_cast<grpc_experimental_completion_queue_functor*>(tag); + (*functor->functor_run)(functor, is_success); } void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error, @@ -1343,7 +1345,7 @@ static void cq_finish_shutdown_callback(grpc_completion_queue* cq) { GPR_ASSERT(cqd->shutdown_called); cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); - callback->Run(true); + (*callback->functor_run)(callback, true); } static void cq_shutdown_callback(grpc_completion_queue* cq) { @@ -1364,9 +1366,11 @@ static void cq_shutdown_callback(grpc_completion_queue* cq) { } cqd->shutdown_called = true; if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { + gpr_mu_unlock(cq->mu); cq_finish_shutdown_callback(cq); + } else { + gpr_mu_unlock(cq->mu); } - gpr_mu_unlock(cq->mu); GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)"); } diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index a7c524d8e8..d60fe6d6ef 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -48,23 +48,6 @@ typedef struct grpc_cq_completion { uintptr_t next; } grpc_cq_completion; -/// For callback CQs, the tag that is passed in for an operation must -/// actually be a pointer to an implementation of the following class. -/// When the operation completes, the tag will be typecasted from void* -/// to grpc_core::CQCallbackInterface* and then the Run method will be -/// invoked on it. In practice, the language binding (e.g., C++ API -/// implementation) is responsible for providing and using an implementation -/// of this abstract base class. -namespace grpc_core { -class CQCallbackInterface { - public: - virtual ~CQCallbackInterface() {} - virtual void Run(bool) GRPC_ABSTRACT; - - GRPC_ABSTRACT_BASE_CLASS -}; -} // namespace grpc_core - #ifndef NDEBUG void grpc_cq_internal_ref(grpc_completion_queue* cc, const char* reason, const char* file, int line); @@ -106,6 +89,6 @@ int grpc_get_cq_poll_num(grpc_completion_queue* cc); grpc_completion_queue* grpc_completion_queue_create_internal( grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type, - grpc_core::CQCallbackInterface* shutdown_callback); + grpc_experimental_completion_queue_functor* shutdown_callback); #endif /* GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H */ diff --git a/src/core/lib/surface/completion_queue_factory.cc b/src/core/lib/surface/completion_queue_factory.cc index ed92dd7eba..2616c156e4 100644 --- a/src/core/lib/surface/completion_queue_factory.cc +++ b/src/core/lib/surface/completion_queue_factory.cc @@ -31,8 +31,7 @@ static grpc_completion_queue* default_create( const grpc_completion_queue_factory* factory, const grpc_completion_queue_attributes* attr) { return grpc_completion_queue_create_internal( - attr->cq_completion_type, attr->cq_polling_type, - static_cast<grpc_core::CQCallbackInterface*>(attr->cq_shutdown_cb)); + attr->cq_completion_type, attr->cq_polling_type, attr->cq_shutdown_cb); } static grpc_completion_queue_factory_vtable default_vtable = {default_create}; @@ -73,7 +72,8 @@ grpc_completion_queue* grpc_completion_queue_create_for_pluck(void* reserved) { } grpc_completion_queue* grpc_completion_queue_create_for_callback( - void* shutdown_callback, void* reserved) { + grpc_experimental_completion_queue_functor* shutdown_callback, + void* reserved) { GPR_ASSERT(!reserved); grpc_completion_queue_attributes attr = { 2, GRPC_CQ_CALLBACK, GRPC_CQ_DEFAULT_POLLING, shutdown_callback}; diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index cb34def740..72ddc2648d 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -149,10 +149,16 @@ struct call_data { grpc_closure server_on_recv_initial_metadata; grpc_closure kill_zombie_closure; grpc_closure* on_done_recv_initial_metadata; + grpc_closure recv_trailing_metadata_ready; + grpc_error* recv_initial_metadata_error; + grpc_closure* original_recv_trailing_metadata_ready; + grpc_error* recv_trailing_metadata_error; + bool seen_recv_trailing_metadata_ready; grpc_closure publish; call_data* pending_next; + grpc_call_combiner* call_combiner; }; struct request_matcher { @@ -219,6 +225,8 @@ struct grpc_server { /** when did we print the last shutdown progress message */ gpr_timespec last_shutdown_message_time; + + grpc_core::RefCountedPtr<grpc_core::channelz::ServerNode> channelz_server; }; #define SERVER_FROM_CALL_ELEM(elem) \ @@ -364,6 +372,7 @@ static void server_ref(grpc_server* server) { static void server_delete(grpc_server* server) { registered_method* rm; size_t i; + server->channelz_server.reset(); grpc_channel_args_destroy(server->channel_args); gpr_mu_destroy(&server->mu_global); gpr_mu_destroy(&server->mu_call); @@ -721,13 +730,43 @@ static void server_on_recv_initial_metadata(void* ptr, grpc_error* error) { if (calld->host_set && calld->path_set) { /* do nothing */ } else { + /* Pass the error reference to calld->recv_initial_metadata_error */ grpc_error* src_error = error; error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Missing :authority or :path", &error, 1); + "Missing :authority or :path", &src_error, 1); GRPC_ERROR_UNREF(src_error); + calld->recv_initial_metadata_error = GRPC_ERROR_REF(error); + } + grpc_closure* closure = calld->on_done_recv_initial_metadata; + calld->on_done_recv_initial_metadata = nullptr; + if (calld->seen_recv_trailing_metadata_ready) { + GRPC_CALL_COMBINER_START(calld->call_combiner, + &calld->recv_trailing_metadata_ready, + calld->recv_trailing_metadata_error, + "continue server_recv_trailing_metadata_ready"); } + GRPC_CLOSURE_RUN(closure, error); +} - GRPC_CLOSURE_RUN(calld->on_done_recv_initial_metadata, error); +static void server_recv_trailing_metadata_ready(void* user_data, + grpc_error* error) { + grpc_call_element* elem = static_cast<grpc_call_element*>(user_data); + call_data* calld = static_cast<call_data*>(elem->call_data); + if (calld->on_done_recv_initial_metadata != nullptr) { + calld->recv_trailing_metadata_error = GRPC_ERROR_REF(error); + calld->seen_recv_trailing_metadata_ready = true; + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready, + server_recv_trailing_metadata_ready, elem, + grpc_schedule_on_exec_ctx); + GRPC_CALL_COMBINER_STOP(calld->call_combiner, + "deferring server_recv_trailing_metadata_ready " + "until after server_on_recv_initial_metadata"); + return; + } + error = + grpc_error_add_child(GRPC_ERROR_REF(error), + GRPC_ERROR_REF(calld->recv_initial_metadata_error)); + GRPC_CLOSURE_RUN(calld->original_recv_trailing_metadata_ready, error); } static void server_mutate_op(grpc_call_element* elem, @@ -745,6 +784,12 @@ static void server_mutate_op(grpc_call_element* elem, op->payload->recv_initial_metadata.recv_flags = &calld->recv_initial_metadata_flags; } + if (op->recv_trailing_metadata) { + calld->original_recv_trailing_metadata_ready = + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready; + op->payload->recv_trailing_metadata.recv_trailing_metadata_ready = + &calld->recv_trailing_metadata_ready; + } } static void server_start_transport_stream_op_batch( @@ -779,6 +824,7 @@ static void accept_stream(void* cd, grpc_transport* transport, args.channel = chand->channel; args.server_transport_data = transport_server_data; args.send_deadline = GRPC_MILLIS_INF_FUTURE; + args.server = chand->server; grpc_call* call; grpc_error* error = grpc_call_create(&args, &call); grpc_call_element* elem = @@ -824,11 +870,14 @@ static grpc_error* init_call_elem(grpc_call_element* elem, memset(calld, 0, sizeof(call_data)); calld->deadline = GRPC_MILLIS_INF_FUTURE; calld->call = grpc_call_from_top_element(elem); + calld->call_combiner = args->call_combiner; GRPC_CLOSURE_INIT(&calld->server_on_recv_initial_metadata, server_on_recv_initial_metadata, elem, grpc_schedule_on_exec_ctx); - + GRPC_CLOSURE_INIT(&calld->recv_trailing_metadata_ready, + server_recv_trailing_metadata_ready, elem, + grpc_schedule_on_exec_ctx); server_ref(chand->server); return GRPC_ERROR_NONE; } @@ -840,7 +889,7 @@ static void destroy_call_elem(grpc_call_element* elem, call_data* calld = static_cast<call_data*>(elem->call_data); GPR_ASSERT(calld->state != PENDING); - + GRPC_ERROR_UNREF(calld->recv_initial_metadata_error); if (calld->host_set) { grpc_slice_unref_internal(calld->host); } @@ -941,6 +990,7 @@ void grpc_server_register_completion_queue(grpc_server* server, } grpc_server* grpc_server_create(const grpc_channel_args* args, void* reserved) { + grpc_core::ExecCtx exec_ctx; GRPC_API_TRACE("grpc_server_create(%p, %p)", 2, (args, reserved)); grpc_server* server = @@ -957,6 +1007,20 @@ grpc_server* grpc_server_create(const grpc_channel_args* args, void* reserved) { server->channel_args = grpc_channel_args_copy(args); + const grpc_arg* arg = grpc_channel_args_find(args, GRPC_ARG_ENABLE_CHANNELZ); + if (grpc_channel_arg_get_bool(arg, false)) { + arg = grpc_channel_args_find(args, + GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE); + size_t trace_events_per_node = + grpc_channel_arg_get_integer(arg, {0, 0, INT_MAX}); + server->channelz_server = + grpc_core::MakeRefCounted<grpc_core::channelz::ServerNode>( + trace_events_per_node); + server->channelz_server->AddTraceEvent( + grpc_core::channelz::ChannelTrace::Severity::Info, + grpc_slice_from_static_string("Server created")); + } + return server; } @@ -1459,3 +1523,11 @@ int grpc_server_has_open_connections(grpc_server* server) { gpr_mu_unlock(&server->mu_global); return r; } + +grpc_core::channelz::ServerNode* grpc_server_get_channelz_node( + grpc_server* server) { + if (server == nullptr) { + return nullptr; + } + return server->channelz_server.get(); +} diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index c617cc223e..0196743ff9 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -23,6 +23,7 @@ #include <grpc/grpc.h> #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/channelz.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/transport.h" @@ -46,6 +47,9 @@ void grpc_server_setup_transport(grpc_server* server, grpc_transport* transport, grpc_pollset* accepting_pollset, const grpc_channel_args* args); +grpc_core::channelz::ServerNode* grpc_server_get_channelz_node( + grpc_server* server); + const grpc_channel_args* grpc_server_get_channel_args(grpc_server* server); int grpc_server_has_open_connections(grpc_server* server); diff --git a/src/core/lib/transport/metadata.cc b/src/core/lib/transport/metadata.cc index d10194a2fe..d164502280 100644 --- a/src/core/lib/transport/metadata.cc +++ b/src/core/lib/transport/metadata.cc @@ -342,24 +342,6 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata) { changed ? nullptr : reinterpret_cast<grpc_mdelem_data*>(metadata)); } -static size_t get_base64_encoded_size(size_t raw_length) { - static const uint8_t tail_xtra[3] = {0, 2, 3}; - return raw_length / 3 * 4 + tail_xtra[raw_length % 3]; -} - -size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem, - bool use_true_binary_metadata) { - size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); - size_t value_len = GRPC_SLICE_LENGTH(GRPC_MDVALUE(elem)); - if (grpc_is_binary_header(GRPC_MDKEY(elem))) { - return overhead_and_key + (use_true_binary_metadata - ? value_len + 1 - : get_base64_encoded_size(value_len)); - } else { - return overhead_and_key + value_len; - } -} - grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd DEBUG_ARGS) { switch (GRPC_MDELEM_STORAGE(gmd)) { case GRPC_MDELEM_STORAGE_EXTERNAL: diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 78df4bc3a3..338082276c 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -125,9 +125,6 @@ grpc_mdelem grpc_mdelem_create( bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b); -size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem, - bool use_true_binary_metadata); - /* Mutator and accessor for grpc_mdelem user data. The destructor function is used as a type tag and is checked during user_data fetch. */ void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*if_destroy_func)(void*)); diff --git a/src/core/lib/transport/metadata_batch.cc b/src/core/lib/transport/metadata_batch.cc index 49740fcd1e..761aad19c4 100644 --- a/src/core/lib/transport/metadata_batch.cc +++ b/src/core/lib/transport/metadata_batch.cc @@ -105,7 +105,7 @@ static grpc_error* maybe_link_callout(grpc_metadata_batch* batch, return GRPC_ERROR_NONE; } if (batch->idx.array[idx] == nullptr) { - if (grpc_static_callout_is_default[idx]) ++batch->list.default_count; + ++batch->list.default_count; batch->idx.array[idx] = storage; return GRPC_ERROR_NONE; } @@ -121,7 +121,7 @@ static void maybe_unlink_callout(grpc_metadata_batch* batch, if (idx == GRPC_BATCH_CALLOUTS_COUNT) { return; } - if (grpc_static_callout_is_default[idx]) --batch->list.default_count; + --batch->list.default_count; GPR_ASSERT(batch->idx.array[idx] != nullptr); batch->idx.array[idx] = nullptr; } diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h index 7068750b6f..0bcbb32d1f 100644 --- a/src/core/lib/transport/metadata_batch.h +++ b/src/core/lib/transport/metadata_batch.h @@ -82,6 +82,7 @@ void grpc_metadata_batch_set_value(grpc_linked_mdelem* storage, grpc_error* grpc_metadata_batch_link_head(grpc_metadata_batch* batch, grpc_linked_mdelem* storage) GRPC_MUST_USE_RESULT; + /** Add \a storage to the end of \a batch. storage->md is assumed to be valid. \a storage is owned by the caller and must survive for the @@ -100,6 +101,7 @@ grpc_error* grpc_metadata_batch_link_tail(grpc_metadata_batch* batch, grpc_error* grpc_metadata_batch_add_head( grpc_metadata_batch* batch, grpc_linked_mdelem* storage, grpc_mdelem elem_to_add) GRPC_MUST_USE_RESULT; + /** Add \a elem_to_add as the last element in \a batch, using \a storage as backing storage for the linked list element. \a storage is owned by the caller and must survive for the diff --git a/src/core/lib/transport/static_metadata.cc b/src/core/lib/transport/static_metadata.cc index 6a5144f21a..ed88aa3f28 100644 --- a/src/core/lib/transport/static_metadata.cc +++ b/src/core/lib/transport/static_metadata.cc @@ -568,33 +568,6 @@ grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, {&grpc_static_metadata_refcounts[102], {{g_bytes + 1101, 13}}}}, }; -bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = { - true, // :path - true, // :method - true, // :status - true, // :authority - true, // :scheme - true, // te - true, // grpc-message - true, // grpc-status - true, // grpc-payload-bin - true, // grpc-encoding - true, // grpc-accept-encoding - true, // grpc-server-stats-bin - true, // grpc-tags-bin - true, // grpc-trace-bin - true, // content-type - true, // content-encoding - true, // accept-encoding - true, // grpc-internal-encoding-request - true, // grpc-internal-stream-encoding-request - true, // user-agent - true, // host - true, // lb-token - true, // grpc-previous-rpc-attempts - true, // grpc-retry-pushback-ms -}; - const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 76, 77, 78, 79, 80, 81, 82}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index b3a10f5873..1e95f4e5bb 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -587,8 +587,6 @@ typedef union { GRPC_BATCH_CALLOUTS_COUNT) \ : GRPC_BATCH_CALLOUTS_COUNT) -extern bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT]; - extern const uint8_t grpc_static_accept_encoding_metadata[8]; #define GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS(algs) \ (GRPC_MAKE_MDELEM( \ diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc index b5268add0d..17e8026096 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc @@ -24,6 +24,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/alts/handshaker/alts_handshaker_service_api.h" const int kHandshakerClientOpNum = 4; @@ -109,7 +110,7 @@ static grpc_byte_buffer* get_serialized_start_client(alts_tsi_event* event) { if (ok) { buffer = grpc_raw_byte_buffer_create(&slice, 1 /* number of slices */); } - grpc_slice_unref(slice); + grpc_slice_unref_internal(slice); gpr_free(target_name); grpc_gcp_handshaker_req_destroy(req); return buffer; @@ -157,7 +158,7 @@ static grpc_byte_buffer* get_serialized_start_server( if (ok) { buffer = grpc_raw_byte_buffer_create(&req_slice, 1 /* number of slices */); } - grpc_slice_unref(req_slice); + grpc_slice_unref_internal(req_slice); grpc_gcp_handshaker_req_destroy(req); return buffer; } @@ -195,7 +196,7 @@ static grpc_byte_buffer* get_serialized_next(grpc_slice* bytes_received) { if (ok) { buffer = grpc_raw_byte_buffer_create(&req_slice, 1 /* number of slices */); } - grpc_slice_unref(req_slice); + grpc_slice_unref_internal(req_slice); grpc_gcp_handshaker_req_destroy(req); return buffer; } @@ -258,7 +259,7 @@ alts_handshaker_client* alts_grpc_handshaker_client_create( grpc_slice_from_static_string(ALTS_SERVICE_METHOD), &slice, gpr_inf_future(GPR_CLOCK_REALTIME), nullptr); client->base.vtable = &vtable; - grpc_slice_unref(slice); + grpc_slice_unref_internal(slice); return &client->base; } diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.cc b/src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.cc index e0e4184686..d63d3538c5 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.cc +++ b/src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.cc @@ -20,6 +20,8 @@ #include "src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.h" +#include "src/core/lib/slice/slice_internal.h" + void add_repeated_field(repeated_field** head, const void* data) { repeated_field* field = static_cast<repeated_field*>(gpr_zalloc(sizeof(*field))); @@ -67,7 +69,7 @@ grpc_slice* create_slice(const char* data, size_t size) { void destroy_slice(grpc_slice* slice) { if (slice != nullptr) { - grpc_slice_unref(*slice); + grpc_slice_unref_internal(*slice); gpr_free(slice); } } diff --git a/src/core/tsi/alts/handshaker/alts_tsi_event.cc b/src/core/tsi/alts/handshaker/alts_tsi_event.cc index ec0bf12b95..cb36d5ebd1 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_event.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_event.cc @@ -24,6 +24,8 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/lib/slice/slice_internal.h" + tsi_result alts_tsi_event_create(alts_tsi_handshaker* handshaker, tsi_handshaker_on_next_done_cb cb, void* user_data, @@ -66,8 +68,8 @@ void alts_tsi_event_destroy(alts_tsi_event* event) { grpc_byte_buffer_destroy(event->recv_buffer); grpc_metadata_array_destroy(&event->initial_metadata); grpc_metadata_array_destroy(&event->trailing_metadata); - grpc_slice_unref(event->details); - grpc_slice_unref(event->target_name); + grpc_slice_unref_internal(event->details); + grpc_slice_unref_internal(event->target_name); grpc_alts_credentials_options_destroy(event->options); gpr_free(event); } diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 1df1021bb1..34608a3de1 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -31,6 +31,7 @@ #include "src/core/lib/gpr/host_port.h" #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/alts/frame_protector/alts_frame_protector.h" #include "src/core/tsi/alts/handshaker/alts_handshaker_client.h" #include "src/core/tsi/alts/handshaker/alts_tsi_utils.h" @@ -182,7 +183,7 @@ static void handshaker_result_destroy(tsi_handshaker_result* self) { gpr_free(result->peer_identity); gpr_free(result->key_data); gpr_free(result->unused_bytes); - grpc_slice_unref(result->rpc_versions); + grpc_slice_unref_internal(result->rpc_versions); gpr_free(result); } @@ -269,12 +270,12 @@ static tsi_result handshaker_next( handshaker->has_sent_start_message = true; } else { if (!GRPC_SLICE_IS_EMPTY(handshaker->recv_bytes)) { - grpc_slice_unref(handshaker->recv_bytes); + grpc_slice_unref_internal(handshaker->recv_bytes); } handshaker->recv_bytes = grpc_slice_ref(slice); ok = alts_handshaker_client_next(handshaker->client, event, &slice); } - grpc_slice_unref(slice); + grpc_slice_unref_internal(slice); if (ok != TSI_OK) { gpr_log(GPR_ERROR, "Failed to schedule ALTS handshaker requests"); return ok; @@ -299,8 +300,8 @@ static void handshaker_destroy(tsi_handshaker* self) { alts_tsi_handshaker* handshaker = reinterpret_cast<alts_tsi_handshaker*>(self); alts_handshaker_client_destroy(handshaker->client); - grpc_slice_unref(handshaker->recv_bytes); - grpc_slice_unref(handshaker->target_name); + grpc_slice_unref_internal(handshaker->recv_bytes); + grpc_slice_unref_internal(handshaker->target_name); grpc_alts_credentials_options_destroy(handshaker->options); gpr_free(handshaker->buffer); gpr_free(handshaker); diff --git a/src/core/tsi/alts/handshaker/alts_tsi_utils.cc b/src/core/tsi/alts/handshaker/alts_tsi_utils.cc index d9b5e6c945..1747f1ad04 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_utils.cc @@ -22,6 +22,8 @@ #include <grpc/byte_buffer_reader.h> +#include "src/core/lib/slice/slice_internal.h" + tsi_result alts_tsi_utils_convert_to_tsi_result(grpc_status_code code) { switch (code) { case GRPC_STATUS_OK: @@ -47,7 +49,7 @@ grpc_gcp_handshaker_resp* alts_tsi_utils_deserialize_response( grpc_slice slice = grpc_byte_buffer_reader_readall(&bbr); grpc_gcp_handshaker_resp* resp = grpc_gcp_handshaker_resp_create(); bool ok = grpc_gcp_handshaker_resp_decode(slice, resp); - grpc_slice_unref(slice); + grpc_slice_unref_internal(slice); grpc_byte_buffer_reader_destroy(&bbr); if (!ok) { grpc_gcp_handshaker_resp_destroy(resp); diff --git a/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc b/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc index d4fd88d1e2..e7890903d5 100644 --- a/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +++ b/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc @@ -61,7 +61,7 @@ static tsi_result alts_grpc_privacy_integrity_protect( if (status != GRPC_STATUS_OK) { gpr_log(GPR_ERROR, "Failed to protect, %s", error_details); gpr_free(error_details); - grpc_slice_unref(protected_slice); + grpc_slice_unref_internal(protected_slice); return TSI_INTERNAL_ERROR; } grpc_slice_buffer_add(protected_slices, protected_slice); @@ -106,7 +106,7 @@ static tsi_result alts_grpc_privacy_integrity_unprotect( if (status != GRPC_STATUS_OK) { gpr_log(GPR_ERROR, "Failed to unprotect, %s", error_details); gpr_free(error_details); - grpc_slice_unref(unprotected_slice); + grpc_slice_unref_internal(unprotected_slice); return TSI_INTERNAL_ERROR; } grpc_slice_buffer_reset_and_unref_internal(&rp->header_sb); diff --git a/src/core/tsi/alts_transport_security.cc b/src/core/tsi/alts_transport_security.cc index 2fd408103b..dac23bbf7a 100644 --- a/src/core/tsi/alts_transport_security.cc +++ b/src/core/tsi/alts_transport_security.cc @@ -45,7 +45,9 @@ void grpc_tsi_alts_signal_for_cq_destroy() { } void grpc_tsi_alts_init() { - memset(&g_alts_resource, 0, sizeof(alts_shared_resource)); + g_alts_resource.channel = nullptr; + g_alts_resource.cq = nullptr; + g_alts_resource.is_cq_drained = false; gpr_mu_init(&g_alts_resource.mu); gpr_cv_init(&g_alts_resource.cv); } diff --git a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc index ce74fde343..f9184bcc34 100644 --- a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +++ b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc @@ -19,6 +19,7 @@ #include <grpc/support/port_platform.h> #include "src/core/lib/gprpp/mutex_lock.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/ssl/session_cache/ssl_session.h" #include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" @@ -53,7 +54,7 @@ class SslSessionLRUCache::Node { SetSession(std::move(session)); } - ~Node() { grpc_slice_unref(key_); } + ~Node() { grpc_slice_unref_internal(key_); } // Not copyable nor movable. Node(const Node&) = delete; diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index 39b891c2e1..c59059f045 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -42,8 +42,10 @@ #include <grpcpp/support/time.h> #include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/memory.h" #include "src/core/lib/gprpp/thd.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/surface/completion_queue.h" namespace grpc { @@ -53,7 +55,12 @@ Channel::Channel(const grpc::string& host, grpc_channel* channel) g_gli_initializer.summon(); } -Channel::~Channel() { grpc_channel_destroy(c_channel_); } +Channel::~Channel() { + grpc_channel_destroy(c_channel_); + if (callback_cq_ != nullptr) { + callback_cq_->Shutdown(); + } +} namespace { @@ -135,8 +142,8 @@ void Channel::PerformOpsOnCall(internal::CallOpSetInterface* ops, size_t nops = 0; grpc_op cops[MAX_OPS]; ops->FillOps(call->call(), cops, &nops); - GPR_ASSERT(GRPC_CALL_OK == - grpc_call_start_batch(call->call(), cops, nops, ops, nullptr)); + GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_batch(call->call(), cops, nops, + ops->cq_tag(), nullptr)); } void* Channel::RegisterMethod(const char* method) { @@ -185,4 +192,41 @@ bool Channel::WaitForStateChangeImpl(grpc_connectivity_state last_observed, return ok; } +namespace { +class ShutdownCallback : public grpc_experimental_completion_queue_functor { + public: + ShutdownCallback() { functor_run = &ShutdownCallback::Run; } + // TakeCQ takes ownership of the cq into the shutdown callback + // so that the shutdown callback will be responsible for destroying it + void TakeCQ(CompletionQueue* cq) { cq_ = cq; } + + // The Run function will get invoked by the completion queue library + // when the shutdown is actually complete + static void Run(grpc_experimental_completion_queue_functor* cb, int) { + auto* callback = static_cast<ShutdownCallback*>(cb); + delete callback->cq_; + grpc_core::Delete(callback); + } + + private: + CompletionQueue* cq_ = nullptr; +}; +} // namespace + +CompletionQueue* Channel::CallbackCQ() { + // TODO(vjpai): Consider using a single global CQ for the default CQ + // if there is no explicit per-channel CQ registered + std::lock_guard<std::mutex> l(mu_); + if (callback_cq_ == nullptr) { + auto* shutdown_callback = grpc_core::New<ShutdownCallback>(); + callback_cq_ = new CompletionQueue(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_CALLBACK, GRPC_CQ_DEFAULT_POLLING, + shutdown_callback}); + + // Transfer ownership of the new cq to its own shutdown callback + shutdown_callback->TakeCQ(callback_cq_); + } + return callback_cq_; +} + } // namespace grpc diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index 67ef46bebe..87902b26f0 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -16,9 +16,11 @@ * */ -#include <grpcpp/generic/generic_stub.h> +#include <functional> +#include <grpcpp/generic/generic_stub.h> #include <grpcpp/impl/rpc_method.h> +#include <grpcpp/support/client_callback.h> namespace grpc { @@ -60,4 +62,14 @@ std::unique_ptr<GenericClientAsyncResponseReader> GenericStub::PrepareUnaryCall( context, request, false)); } +void GenericStub::experimental_type::UnaryCall( + ClientContext* context, const grpc::string& method, + const ByteBuffer* request, ByteBuffer* response, + std::function<void(Status)> on_completion) { + internal::CallbackUnaryCall( + stub_->channel_.get(), + internal::RpcMethod(method.c_str(), internal::RpcMethod::NORMAL_RPC), + context, request, response, std::move(on_completion)); +} + } // namespace grpc diff --git a/src/cpp/common/alarm.cc b/src/cpp/common/alarm.cc index 15a373d8a5..5819a4210b 100644 --- a/src/cpp/common/alarm.cc +++ b/src/cpp/common/alarm.cc @@ -39,17 +39,6 @@ class AlarmImpl : public CompletionQueueTag { AlarmImpl() : cq_(nullptr), tag_(nullptr) { gpr_ref_init(&refs_, 1); grpc_timer_init_unset(&timer_); - GRPC_CLOSURE_INIT(&on_alarm_, - [](void* arg, grpc_error* error) { - // queue the op on the completion queue - AlarmImpl* alarm = static_cast<AlarmImpl*>(arg); - alarm->Ref(); - grpc_cq_end_op( - alarm->cq_, alarm, error, - [](void* arg, grpc_cq_completion* completion) {}, - arg, &alarm->completion_); - }, - this, grpc_schedule_on_exec_ctx); } ~AlarmImpl() { grpc_core::ExecCtx exec_ctx; @@ -68,6 +57,32 @@ class AlarmImpl : public CompletionQueueTag { cq_ = cq->cq(); tag_ = tag; GPR_ASSERT(grpc_cq_begin_op(cq_, this)); + GRPC_CLOSURE_INIT(&on_alarm_, + [](void* arg, grpc_error* error) { + // queue the op on the completion queue + AlarmImpl* alarm = static_cast<AlarmImpl*>(arg); + alarm->Ref(); + grpc_cq_end_op( + alarm->cq_, alarm, error, + [](void* arg, grpc_cq_completion* completion) {}, + arg, &alarm->completion_); + }, + this, grpc_schedule_on_exec_ctx); + grpc_timer_init(&timer_, grpc_timespec_to_millis_round_up(deadline), + &on_alarm_); + } + void Set(gpr_timespec deadline, std::function<void(bool)> f) { + grpc_core::ExecCtx exec_ctx; + // Don't use any CQ at all. Instead just use the timer to fire the function + callback_ = std::move(f); + Ref(); + GRPC_CLOSURE_INIT(&on_alarm_, + [](void* arg, grpc_error* error) { + AlarmImpl* alarm = static_cast<AlarmImpl*>(arg); + alarm->callback_(error == GRPC_ERROR_NONE); + alarm->Unref(); + }, + this, grpc_schedule_on_exec_ctx); grpc_timer_init(&timer_, grpc_timespec_to_millis_round_up(deadline), &on_alarm_); } @@ -95,6 +110,7 @@ class AlarmImpl : public CompletionQueueTag { // completion queue where events about this alarm will be posted grpc_completion_queue* cq_; void* tag_; + std::function<void(bool)> callback_; }; } // namespace internal @@ -113,6 +129,15 @@ void Alarm::SetInternal(CompletionQueue* cq, gpr_timespec deadline, void* tag) { static_cast<internal::AlarmImpl*>(alarm_)->Set(cq, deadline, tag); } +void Alarm::SetInternal(gpr_timespec deadline, std::function<void(bool)> f) { + // Note that we know that alarm_ is actually an internal::AlarmImpl + // but we declared it as the base pointer to avoid a forward declaration + // or exposing core data structures in the C++ public headers. + // Thus it is safe to use a static_cast to the subclass here, and the + // C++ style guide allows us to do so in this case + static_cast<internal::AlarmImpl*>(alarm_)->Set(deadline, std::move(f)); +} + Alarm::~Alarm() { if (alarm_ != nullptr) { static_cast<internal::AlarmImpl*>(alarm_)->Destroy(); diff --git a/src/cpp/ext/filters/census/server_filter.cc b/src/cpp/ext/filters/census/server_filter.cc index c7c62eefe5..b5f3d5a13a 100644 --- a/src/cpp/ext/filters/census/server_filter.cc +++ b/src/cpp/ext/filters/census/server_filter.cc @@ -93,7 +93,7 @@ void CensusServerCallData::OnDoneRecvInitialMetadataCb(void* user_data, FilterInitialMetadata(initial_metadata, &sml); calld->path_ = grpc_slice_ref_internal(sml.path); calld->method_ = GetMethod(&calld->path_); - calld->qualified_method_ = StrCat("Recv.", calld->method_); + calld->qualified_method_ = absl::StrCat("Recv.", calld->method_); const char* tracing_str = GRPC_SLICE_IS_EMPTY(sml.tracing_slice) ? "" diff --git a/src/cpp/server/channelz/channelz_service.cc b/src/cpp/server/channelz/channelz_service.cc index 77c175e5b8..e096c1f421 100644 --- a/src/cpp/server/channelz/channelz_service.cc +++ b/src/cpp/server/channelz/channelz_service.cc @@ -32,6 +32,25 @@ Status ChannelzService::GetTopChannels( ServerContext* unused, const channelz::v1::GetTopChannelsRequest* request, channelz::v1::GetTopChannelsResponse* response) { char* json_str = grpc_channelz_get_top_channels(request->start_channel_id()); + if (json_str == nullptr) { + return Status(INTERNAL, "grpc_channelz_get_top_channels returned null"); + } + google::protobuf::util::Status s = + google::protobuf::util::JsonStringToMessage(json_str, response); + gpr_free(json_str); + if (s != google::protobuf::util::Status::OK) { + return Status(INTERNAL, s.ToString()); + } + return Status::OK; +} + +Status ChannelzService::GetServers( + ServerContext* unused, const channelz::v1::GetServersRequest* request, + channelz::v1::GetServersResponse* response) { + char* json_str = grpc_channelz_get_servers(request->start_server_id()); + if (json_str == nullptr) { + return Status(INTERNAL, "grpc_channelz_get_servers returned null"); + } google::protobuf::util::Status s = google::protobuf::util::JsonStringToMessage(json_str, response); gpr_free(json_str); @@ -45,6 +64,25 @@ Status ChannelzService::GetChannel( ServerContext* unused, const channelz::v1::GetChannelRequest* request, channelz::v1::GetChannelResponse* response) { char* json_str = grpc_channelz_get_channel(request->channel_id()); + if (json_str == nullptr) { + return Status(NOT_FOUND, "No object found for that ChannelId"); + } + google::protobuf::util::Status s = + google::protobuf::util::JsonStringToMessage(json_str, response); + gpr_free(json_str); + if (s != google::protobuf::util::Status::OK) { + return Status(INTERNAL, s.ToString()); + } + return Status::OK; +} + +Status ChannelzService::GetSubchannel( + ServerContext* unused, const channelz::v1::GetSubchannelRequest* request, + channelz::v1::GetSubchannelResponse* response) { + char* json_str = grpc_channelz_get_subchannel(request->subchannel_id()); + if (json_str == nullptr) { + return Status(NOT_FOUND, "No object found for that SubchannelId"); + } google::protobuf::util::Status s = google::protobuf::util::JsonStringToMessage(json_str, response); gpr_free(json_str); diff --git a/src/cpp/server/channelz/channelz_service.h b/src/cpp/server/channelz/channelz_service.h index f619ea49e0..9e0b5b6ead 100644 --- a/src/cpp/server/channelz/channelz_service.h +++ b/src/cpp/server/channelz/channelz_service.h @@ -32,10 +32,18 @@ class ChannelzService final : public channelz::v1::Channelz::Service { Status GetTopChannels( ServerContext* unused, const channelz::v1::GetTopChannelsRequest* request, channelz::v1::GetTopChannelsResponse* response) override; + // implementation of GetServers rpc + Status GetServers(ServerContext* unused, + const channelz::v1::GetServersRequest* request, + channelz::v1::GetServersResponse* response) override; // implementation of GetChannel rpc Status GetChannel(ServerContext* unused, const channelz::v1::GetChannelRequest* request, channelz::v1::GetChannelResponse* response) override; + // implementation of GetSubchannel rpc + Status GetSubchannel(ServerContext* unused, + const channelz::v1::GetSubchannelRequest* request, + channelz::v1::GetSubchannelResponse* response) override; }; } // namespace grpc diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 36c709eb45..36f7eb81f9 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -657,6 +657,7 @@ void Server::PerformOpsOnCall(internal::CallOpSetInterface* ops, size_t nops = 0; grpc_op cops[MAX_OPS]; ops->FillOps(call->call(), cops, &nops); + // TODO(vjpai): Use ops->cq_tag once this case supports callbacks auto result = grpc_call_start_batch(call->call(), cops, nops, ops, nullptr); if (result != GRPC_CALL_OK) { gpr_log(GPR_ERROR, "Fatal: grpc_call_start_batch returned %d", result); @@ -686,9 +687,6 @@ ServerInterface::BaseAsyncRequest::~BaseAsyncRequest() { bool ServerInterface::BaseAsyncRequest::FinalizeResult(void** tag, bool* status) { - if (*status) { - context_->client_metadata_.FillMap(); - } context_->set_call(call_); context_->cq_ = call_cq_; internal::Call call(call_, server_, call_cq_, diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 6f5bde0d9f..b7254b6bb9 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -61,6 +61,9 @@ class ServerContext::CompletionOp final : public internal::CallOpSetInterface { tag_ = tag; } + /// TODO(vjpai): Allow override of cq_tag if appropriate for callback API + void* cq_tag() override { return this; } + void Unref(); private: @@ -134,7 +137,6 @@ ServerContext::ServerContext(gpr_timespec deadline, grpc_metadata_array* arr) compression_level_set_(false), has_pending_ops_(false) { std::swap(*client_metadata_.arr(), *arr); - client_metadata_.FillMap(); } ServerContext::~ServerContext() { diff --git a/src/csharp/Grpc.Core.Tests/ChannelConnectivityTest.cs b/src/csharp/Grpc.Core.Tests/ChannelConnectivityTest.cs index a43040f01a..0834ddadda 100644 --- a/src/csharp/Grpc.Core.Tests/ChannelConnectivityTest.cs +++ b/src/csharp/Grpc.Core.Tests/ChannelConnectivityTest.cs @@ -57,19 +57,23 @@ namespace Grpc.Core.Tests [Test] public async Task Channel_WaitForStateChangedAsync() { - helper.UnaryHandler = new UnaryServerMethod<string, string>((request, context) => - { - return Task.FromResult(request); - }); - Assert.ThrowsAsync(typeof(TaskCanceledException), - async () => await channel.WaitForStateChangedAsync(channel.State, DateTime.UtcNow.AddMilliseconds(10))); + async () => await channel.WaitForStateChangedAsync(channel.State, DateTime.UtcNow.AddMilliseconds(0))); var stateChangedTask = channel.WaitForStateChangedAsync(channel.State); + await channel.ConnectAsync(DateTime.UtcNow.AddMilliseconds(5000)); + await stateChangedTask; + Assert.AreEqual(ChannelState.Ready, channel.State); + } - await Calls.AsyncUnaryCall(helper.CreateUnaryCall(), "abc"); + [Test] + public async Task Channel_TryWaitForStateChangedAsync() + { + Assert.IsFalse(await channel.TryWaitForStateChangedAsync(channel.State, DateTime.UtcNow.AddMilliseconds(0))); - await stateChangedTask; + var stateChangedTask = channel.TryWaitForStateChangedAsync(channel.State); + await channel.ConnectAsync(DateTime.UtcNow.AddMilliseconds(5000)); + Assert.IsTrue(await stateChangedTask); Assert.AreEqual(ChannelState.Ready, channel.State); } diff --git a/src/csharp/Grpc.Core.Tests/ContextualMarshallerTest.cs b/src/csharp/Grpc.Core.Tests/ContextualMarshallerTest.cs new file mode 100644 index 0000000000..c3aee726f2 --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/ContextualMarshallerTest.cs @@ -0,0 +1,119 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Grpc.Core; +using Grpc.Core.Internal; +using Grpc.Core.Utils; +using NUnit.Framework; + +namespace Grpc.Core.Tests +{ + public class ContextualMarshallerTest + { + const string Host = "127.0.0.1"; + + MockServiceHelper helper; + Server server; + Channel channel; + + [SetUp] + public void Init() + { + var contextualMarshaller = new Marshaller<string>( + (str, serializationContext) => + { + if (str == "UNSERIALIZABLE_VALUE") + { + // Google.Protobuf throws exception inherited from IOException + throw new IOException("Error serializing the message."); + } + if (str == "SERIALIZE_TO_NULL") + { + return; + } + var bytes = System.Text.Encoding.UTF8.GetBytes(str); + serializationContext.Complete(bytes); + }, + (deserializationContext) => + { + var buffer = deserializationContext.PayloadAsNewBuffer(); + Assert.AreEqual(buffer.Length, deserializationContext.PayloadLength); + var s = System.Text.Encoding.UTF8.GetString(buffer); + if (s == "UNPARSEABLE_VALUE") + { + // Google.Protobuf throws exception inherited from IOException + throw new IOException("Error parsing the message."); + } + return s; + }); + helper = new MockServiceHelper(Host, contextualMarshaller); + server = helper.GetServer(); + server.Start(); + channel = helper.GetChannel(); + } + + [TearDown] + public void Cleanup() + { + channel.ShutdownAsync().Wait(); + server.ShutdownAsync().Wait(); + } + + [Test] + public void UnaryCall() + { + helper.UnaryHandler = new UnaryServerMethod<string, string>((request, context) => + { + return Task.FromResult(request); + }); + Assert.AreEqual("ABC", Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "ABC")); + } + + [Test] + public void ResponseParsingError_UnaryResponse() + { + helper.UnaryHandler = new UnaryServerMethod<string, string>((request, context) => + { + return Task.FromResult("UNPARSEABLE_VALUE"); + }); + + var ex = Assert.Throws<RpcException>(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "REQUEST")); + Assert.AreEqual(StatusCode.Internal, ex.Status.StatusCode); + } + + [Test] + public void RequestSerializationError_BlockingUnary() + { + Assert.Throws<IOException>(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "UNSERIALIZABLE_VALUE")); + } + + [Test] + public void SerializationResultIsNull_BlockingUnary() + { + Assert.Throws<NullReferenceException>(() => Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "SERIALIZE_TO_NULL")); + } + } +} diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs index 9aab54d2d0..775849d89b 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs @@ -107,6 +107,42 @@ namespace Grpc.Core.Internal.Tests } [Test] + public void AsyncUnary_RequestSerializationExceptionDoesntLeakResources() + { + string nullRequest = null; // will throw when serializing + Assert.Throws(typeof(ArgumentNullException), () => asyncCall.UnaryCallAsync(nullRequest)); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] + public void AsyncUnary_StartCallFailureDoesntLeakResources() + { + fakeCall.MakeStartCallFail(); + Assert.Throws(typeof(InvalidOperationException), () => asyncCall.UnaryCallAsync("request1")); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] + public void SyncUnary_RequestSerializationExceptionDoesntLeakResources() + { + string nullRequest = null; // will throw when serializing + Assert.Throws(typeof(ArgumentNullException), () => asyncCall.UnaryCall(nullRequest)); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] + public void SyncUnary_StartCallFailureDoesntLeakResources() + { + fakeCall.MakeStartCallFail(); + Assert.Throws(typeof(InvalidOperationException), () => asyncCall.UnaryCall("request1")); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] public void ClientStreaming_StreamingReadNotAllowed() { asyncCall.ClientStreamingCallAsync(); @@ -328,6 +364,15 @@ namespace Grpc.Core.Internal.Tests } [Test] + public void ClientStreaming_StartCallFailureDoesntLeakResources() + { + fakeCall.MakeStartCallFail(); + Assert.Throws(typeof(InvalidOperationException), () => asyncCall.ClientStreamingCallAsync()); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] public void ServerStreaming_StreamingSendNotAllowed() { asyncCall.StartServerStreamingCall("request1"); @@ -402,6 +447,27 @@ namespace Grpc.Core.Internal.Tests } [Test] + public void ServerStreaming_RequestSerializationExceptionDoesntLeakResources() + { + string nullRequest = null; // will throw when serializing + Assert.Throws(typeof(ArgumentNullException), () => asyncCall.StartServerStreamingCall(nullRequest)); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + + var responseStream = new ClientResponseStream<string, string>(asyncCall); + var readTask = responseStream.MoveNext(); + } + + [Test] + public void ServerStreaming_StartCallFailureDoesntLeakResources() + { + fakeCall.MakeStartCallFail(); + Assert.Throws(typeof(InvalidOperationException), () => asyncCall.StartServerStreamingCall("request1")); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + + [Test] public void DuplexStreaming_NoRequestNoResponse_Success() { asyncCall.StartDuplexStreamingCall(); @@ -558,6 +624,15 @@ namespace Grpc.Core.Internal.Tests AssertStreamingResponseError(asyncCall, fakeCall, readTask2, StatusCode.Cancelled); } + [Test] + public void DuplexStreaming_StartCallFailureDoesntLeakResources() + { + fakeCall.MakeStartCallFail(); + Assert.Throws(typeof(InvalidOperationException), () => asyncCall.StartDuplexStreamingCall()); + Assert.AreEqual(0, channel.GetCallReferenceCount()); + Assert.IsTrue(fakeCall.IsDisposed); + } + ClientSideStatus CreateClientSideStatus(StatusCode statusCode) { return new ClientSideStatus(new Status(statusCode, ""), new Metadata()); diff --git a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs index 581ac3384b..ef67918dab 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs @@ -31,6 +31,7 @@ namespace Grpc.Core.Internal.Tests /// </summary> internal class FakeNativeCall : INativeCall { + private bool shouldStartCallFail; public IUnaryResponseClientCallback UnaryResponseClientCallback { get; @@ -102,26 +103,31 @@ namespace Grpc.Core.Internal.Tests public void StartUnary(IUnaryResponseClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags) { + StartCallMaybeFail(); UnaryResponseClientCallback = callback; } public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags) { + StartCallMaybeFail(); throw new NotImplementedException(); } public void StartClientStreaming(IUnaryResponseClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags) { + StartCallMaybeFail(); UnaryResponseClientCallback = callback; } public void StartServerStreaming(IReceivedStatusOnClientCallback callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags) { + StartCallMaybeFail(); ReceivedStatusOnClientCallback = callback; } public void StartDuplexStreaming(IReceivedStatusOnClientCallback callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags) { + StartCallMaybeFail(); ReceivedStatusOnClientCallback = callback; } @@ -165,5 +171,22 @@ namespace Grpc.Core.Internal.Tests { IsDisposed = true; } + + /// <summary> + /// Emulate CallSafeHandle.CheckOk() failure for all future attempts + /// to start a call. + /// </summary> + public void MakeStartCallFail() + { + shouldStartCallFail = true; + } + + private void StartCallMaybeFail() + { + if (shouldStartCallFail) + { + throw new InvalidOperationException("Start call has failed."); + } + } } } diff --git a/src/csharp/Grpc.Core.Tests/MarshallerTest.cs b/src/csharp/Grpc.Core.Tests/MarshallerTest.cs new file mode 100644 index 0000000000..97f64a0575 --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/MarshallerTest.cs @@ -0,0 +1,105 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Grpc.Core; +using Grpc.Core.Internal; +using Grpc.Core.Utils; +using NUnit.Framework; + +namespace Grpc.Core.Tests +{ + public class MarshallerTest + { + [Test] + public void ContextualSerializerEmulation() + { + Func<string, byte[]> simpleSerializer = System.Text.Encoding.UTF8.GetBytes; + Func<byte[], string> simpleDeserializer = System.Text.Encoding.UTF8.GetString; + var marshaller = new Marshaller<string>(simpleSerializer, + simpleDeserializer); + + Assert.AreSame(simpleSerializer, marshaller.Serializer); + Assert.AreSame(simpleDeserializer, marshaller.Deserializer); + + // test that emulated contextual serializer and deserializer work + string origMsg = "abc"; + var serializationContext = new FakeSerializationContext(); + marshaller.ContextualSerializer(origMsg, serializationContext); + + var deserializationContext = new FakeDeserializationContext(serializationContext.Payload); + Assert.AreEqual(origMsg, marshaller.ContextualDeserializer(deserializationContext)); + } + + [Test] + public void SimpleSerializerEmulation() + { + Action<string, SerializationContext> contextualSerializer = (str, context) => + { + var bytes = System.Text.Encoding.UTF8.GetBytes(str); + context.Complete(bytes); + }; + Func<DeserializationContext, string> contextualDeserializer = (context) => + { + return System.Text.Encoding.UTF8.GetString(context.PayloadAsNewBuffer()); + }; + var marshaller = new Marshaller<string>(contextualSerializer, contextualDeserializer); + + Assert.AreSame(contextualSerializer, marshaller.ContextualSerializer); + Assert.AreSame(contextualDeserializer, marshaller.ContextualDeserializer); + + // test that emulated serializer and deserializer work + var origMsg = "abc"; + var serialized = marshaller.Serializer(origMsg); + Assert.AreEqual(origMsg, marshaller.Deserializer(serialized)); + } + + class FakeSerializationContext : SerializationContext + { + public byte[] Payload; + public override void Complete(byte[] payload) + { + this.Payload = payload; + } + } + + class FakeDeserializationContext : DeserializationContext + { + public byte[] payload; + + public FakeDeserializationContext(byte[] payload) + { + this.payload = payload; + } + + public override int PayloadLength => payload.Length; + + public override byte[] PayloadAsNewBuffer() + { + return payload; + } + } + } +} diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs index 7912b06b71..7ce929dfa3 100644 --- a/src/csharp/Grpc.Core/Channel.cs +++ b/src/csharp/Grpc.Core/Channel.cs @@ -136,7 +136,7 @@ namespace Grpc.Core /// </summary> public async Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null) { - var result = await WaitForStateChangedInternalAsync(lastObservedState, deadline).ConfigureAwait(false); + var result = await TryWaitForStateChangedAsync(lastObservedState, deadline).ConfigureAwait(false); if (!result) { throw new TaskCanceledException("Reached deadline."); @@ -147,7 +147,7 @@ namespace Grpc.Core /// Returned tasks completes once channel state has become different from /// given lastObservedState (<c>true</c> is returned) or if the wait has timed out (<c>false</c> is returned). /// </summary> - internal Task<bool> WaitForStateChangedInternalAsync(ChannelState lastObservedState, DateTime? deadline = null) + public Task<bool> TryWaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null) { GrpcPreconditions.CheckArgument(lastObservedState != ChannelState.Shutdown, "Shutdown is a terminal state. No further state changes can occur."); @@ -297,6 +297,12 @@ namespace Grpc.Core activeCallCounter.Decrement(); } + // for testing only + internal long GetCallReferenceCount() + { + return activeCallCounter.Count; + } + private ChannelState GetConnectivityState(bool tryToConnect) { try diff --git a/src/csharp/Grpc.Core/DeserializationContext.cs b/src/csharp/Grpc.Core/DeserializationContext.cs new file mode 100644 index 0000000000..5b6372ef85 --- /dev/null +++ b/src/csharp/Grpc.Core/DeserializationContext.cs @@ -0,0 +1,46 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +namespace Grpc.Core +{ + /// <summary> + /// Provides access to the payload being deserialized when deserializing messages. + /// </summary> + public abstract class DeserializationContext + { + /// <summary> + /// Get the total length of the payload in bytes. + /// </summary> + public abstract int PayloadLength { get; } + + /// <summary> + /// Gets the entire payload as a newly allocated byte array. + /// Once the byte array is returned, the byte array becomes owned by the caller and won't be ever accessed or reused by gRPC again. + /// NOTE: Obtaining the buffer as a newly allocated byte array is the simplest way of accessing the payload, + /// but it can have important consequences in high-performance scenarios. + /// In particular, using this method usually requires copying of the entire buffer one extra time. + /// Also, allocating a new buffer each time can put excessive pressure on GC, especially if + /// the payload is more than 86700 bytes large (which means the newly allocated buffer will be placed in LOH, + /// and LOH object can only be garbage collected via a full ("stop the world") GC run). + /// NOTE: Deserializers are expected not to call this method more than once per received message + /// (as there is no practical reason for doing so) and <c>DeserializationContext</c> implementations are free to assume so. + /// </summary> + /// <returns>byte array containing the entire payload.</returns> + public abstract byte[] PayloadAsNewBuffer(); + } +} diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index 66902f3caa..4cdf0ee6a7 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -17,6 +17,7 @@ #endregion using System; +using System.Threading; using System.Threading.Tasks; using Grpc.Core.Logging; using Grpc.Core.Profiling; @@ -34,6 +35,8 @@ namespace Grpc.Core.Internal readonly CallInvocationDetails<TRequest, TResponse> details; readonly INativeCall injectedNativeCall; // for testing + bool registeredWithChannel; + // Dispose of to de-register cancellation token registration IDisposable cancellationTokenRegistration; @@ -77,43 +80,59 @@ namespace Grpc.Core.Internal using (profiler.NewScope("AsyncCall.UnaryCall")) using (CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.CreateSync()) { - byte[] payload = UnsafeSerialize(msg); + bool callStartedOk = false; + try + { + unaryResponseTcs = new TaskCompletionSource<TResponse>(); - unaryResponseTcs = new TaskCompletionSource<TResponse>(); + lock (myLock) + { + GrpcPreconditions.CheckState(!started); + started = true; + Initialize(cq); - lock (myLock) - { - GrpcPreconditions.CheckState(!started); - started = true; - Initialize(cq); + halfcloseRequested = true; + readingDone = true; + } - halfcloseRequested = true; - readingDone = true; - } + byte[] payload = UnsafeSerialize(msg); - using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) - { - var ctx = details.Channel.Environment.BatchContextPool.Lease(); - try + using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) { - call.StartUnary(ctx, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); - var ev = cq.Pluck(ctx.Handle); - bool success = (ev.success != 0); + var ctx = details.Channel.Environment.BatchContextPool.Lease(); try { - using (profiler.NewScope("AsyncCall.UnaryCall.HandleBatch")) + call.StartUnary(ctx, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); + callStartedOk = true; + + var ev = cq.Pluck(ctx.Handle); + bool success = (ev.success != 0); + try + { + using (profiler.NewScope("AsyncCall.UnaryCall.HandleBatch")) + { + HandleUnaryResponse(success, ctx.GetReceivedStatusOnClient(), ctx.GetReceivedMessage(), ctx.GetReceivedInitialMetadata()); + } + } + catch (Exception e) { - HandleUnaryResponse(success, ctx.GetReceivedStatusOnClient(), ctx.GetReceivedMessage(), ctx.GetReceivedInitialMetadata()); + Logger.Error(e, "Exception occurred while invoking completion delegate."); } } - catch (Exception e) + finally { - Logger.Error(e, "Exception occurred while invoking completion delegate."); + ctx.Recycle(); } } - finally + } + finally + { + if (!callStartedOk) { - ctx.Recycle(); + lock (myLock) + { + OnFailedToStartCallLocked(); + } } } @@ -130,22 +149,35 @@ namespace Grpc.Core.Internal { lock (myLock) { - GrpcPreconditions.CheckState(!started); - started = true; + bool callStartedOk = false; + try + { + GrpcPreconditions.CheckState(!started); + started = true; - Initialize(details.Channel.CompletionQueue); + Initialize(details.Channel.CompletionQueue); - halfcloseRequested = true; - readingDone = true; + halfcloseRequested = true; + readingDone = true; + + byte[] payload = UnsafeSerialize(msg); - byte[] payload = UnsafeSerialize(msg); + unaryResponseTcs = new TaskCompletionSource<TResponse>(); + using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + { + call.StartUnary(UnaryResponseClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); + callStartedOk = true; + } - unaryResponseTcs = new TaskCompletionSource<TResponse>(); - using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + return unaryResponseTcs.Task; + } + finally { - call.StartUnary(UnaryResponseClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); + if (!callStartedOk) + { + OnFailedToStartCallLocked(); + } } - return unaryResponseTcs.Task; } } @@ -157,20 +189,32 @@ namespace Grpc.Core.Internal { lock (myLock) { - GrpcPreconditions.CheckState(!started); - started = true; + bool callStartedOk = false; + try + { + GrpcPreconditions.CheckState(!started); + started = true; - Initialize(details.Channel.CompletionQueue); + Initialize(details.Channel.CompletionQueue); - readingDone = true; + readingDone = true; + + unaryResponseTcs = new TaskCompletionSource<TResponse>(); + using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + { + call.StartClientStreaming(UnaryResponseClientCallback, metadataArray, details.Options.Flags); + callStartedOk = true; + } - unaryResponseTcs = new TaskCompletionSource<TResponse>(); - using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + return unaryResponseTcs.Task; + } + finally { - call.StartClientStreaming(UnaryResponseClientCallback, metadataArray, details.Options.Flags); + if (!callStartedOk) + { + OnFailedToStartCallLocked(); + } } - - return unaryResponseTcs.Task; } } @@ -181,21 +225,33 @@ namespace Grpc.Core.Internal { lock (myLock) { - GrpcPreconditions.CheckState(!started); - started = true; + bool callStartedOk = false; + try + { + GrpcPreconditions.CheckState(!started); + started = true; - Initialize(details.Channel.CompletionQueue); + Initialize(details.Channel.CompletionQueue); - halfcloseRequested = true; + halfcloseRequested = true; - byte[] payload = UnsafeSerialize(msg); + byte[] payload = UnsafeSerialize(msg); - streamingResponseCallFinishedTcs = new TaskCompletionSource<object>(); - using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + streamingResponseCallFinishedTcs = new TaskCompletionSource<object>(); + using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + { + call.StartServerStreaming(ReceivedStatusOnClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); + callStartedOk = true; + } + call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback); + } + finally { - call.StartServerStreaming(ReceivedStatusOnClientCallback, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags); + if (!callStartedOk) + { + OnFailedToStartCallLocked(); + } } - call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback); } } @@ -207,17 +263,29 @@ namespace Grpc.Core.Internal { lock (myLock) { - GrpcPreconditions.CheckState(!started); - started = true; + bool callStartedOk = false; + try + { + GrpcPreconditions.CheckState(!started); + started = true; - Initialize(details.Channel.CompletionQueue); + Initialize(details.Channel.CompletionQueue); - streamingResponseCallFinishedTcs = new TaskCompletionSource<object>(); - using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + streamingResponseCallFinishedTcs = new TaskCompletionSource<object>(); + using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + { + call.StartDuplexStreaming(ReceivedStatusOnClientCallback, metadataArray, details.Options.Flags); + callStartedOk = true; + } + call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback); + } + finally { - call.StartDuplexStreaming(ReceivedStatusOnClientCallback, metadataArray, details.Options.Flags); + if (!callStartedOk) + { + OnFailedToStartCallLocked(); + } } - call.StartReceiveInitialMetadata(ReceivedResponseHeadersCallback); } } @@ -327,7 +395,11 @@ namespace Grpc.Core.Internal protected override void OnAfterReleaseResourcesLocked() { - details.Channel.RemoveCallReference(this); + if (registeredWithChannel) + { + details.Channel.RemoveCallReference(this); + registeredWithChannel = false; + } } protected override void OnAfterReleaseResourcesUnlocked() @@ -394,10 +466,27 @@ namespace Grpc.Core.Internal var call = CreateNativeCall(cq); details.Channel.AddCallReference(this); + registeredWithChannel = true; InitializeInternal(call); + RegisterCancellationCallback(); } + private void OnFailedToStartCallLocked() + { + ReleaseResources(); + + // We need to execute the hook that disposes the cancellation token + // registration, but it cannot be done from under a lock. + // To make things simple, we just schedule the unregistering + // on a threadpool. + // - Once the native call is disposed, the Cancel() calls are ignored anyway + // - We don't care about the overhead as OnFailedToStartCallLocked() only happens + // when something goes very bad when initializing a call and that should + // never happen when gRPC is used correctly. + ThreadPool.QueueUserWorkItem((state) => OnAfterReleaseResourcesUnlocked()); + } + private INativeCall CreateNativeCall(CompletionQueueSafeHandle cq) { if (injectedNativeCall != null) diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index 5a53049e4b..a93dc34620 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -189,7 +189,7 @@ namespace Grpc.Core.Internal /// </summary> protected abstract Exception GetRpcExceptionClientOnly(); - private void ReleaseResources() + protected void ReleaseResources() { if (call != null) { diff --git a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs index 4d695e8850..faeb51e6f7 100644 --- a/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs +++ b/src/csharp/Grpc.Core/Internal/NativeMetadataCredentialsPlugin.cs @@ -68,8 +68,8 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionStatusMsg); - Logger.Error(e, GetMetadataExceptionLogMsg); + // eat the exception, we must not throw when inside callback from native code. + Logger.Error(e, "Exception occurred while invoking native metadata interceptor handler."); } } @@ -87,7 +87,8 @@ namespace Grpc.Core.Internal } catch (Exception e) { - Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, GetMetadataExceptionStatusMsg); + string detail = GetMetadataExceptionStatusMsg + " " + e.ToString(); + Native.grpcsharp_metadata_credentials_notify_from_plugin(callbackPtr, userDataPtr, MetadataArraySafeHandle.Create(Metadata.Empty), StatusCode.Unknown, detail); Logger.Error(e, GetMetadataExceptionLogMsg); } } diff --git a/src/csharp/Grpc.Core/Marshaller.cs b/src/csharp/Grpc.Core/Marshaller.cs index 1d758ca935..0af9aa586b 100644 --- a/src/csharp/Grpc.Core/Marshaller.cs +++ b/src/csharp/Grpc.Core/Marshaller.cs @@ -29,36 +29,129 @@ namespace Grpc.Core readonly Func<T, byte[]> serializer; readonly Func<byte[], T> deserializer; + readonly Action<T, SerializationContext> contextualSerializer; + readonly Func<DeserializationContext, T> contextualDeserializer; + /// <summary> - /// Initializes a new marshaller. + /// Initializes a new marshaller from simple serialize/deserialize functions. /// </summary> /// <param name="serializer">Function that will be used to serialize messages.</param> /// <param name="deserializer">Function that will be used to deserialize messages.</param> public Marshaller(Func<T, byte[]> serializer, Func<byte[], T> deserializer) { - this.serializer = GrpcPreconditions.CheckNotNull(serializer, "serializer"); - this.deserializer = GrpcPreconditions.CheckNotNull(deserializer, "deserializer"); + this.serializer = GrpcPreconditions.CheckNotNull(serializer, nameof(serializer)); + this.deserializer = GrpcPreconditions.CheckNotNull(deserializer, nameof(deserializer)); + this.contextualSerializer = EmulateContextualSerializer; + this.contextualDeserializer = EmulateContextualDeserializer; } /// <summary> - /// Gets the serializer function. + /// Initializes a new marshaller from serialize/deserialize fuctions that can access serialization and deserialization + /// context. Compared to the simple serializer/deserializer functions, using the contextual version provides more + /// flexibility and can lead to increased efficiency (and better performance). + /// Note: This constructor is part of an experimental API that can change or be removed without any prior notice. /// </summary> - public Func<T, byte[]> Serializer + /// <param name="serializer">Function that will be used to serialize messages.</param> + /// <param name="deserializer">Function that will be used to deserialize messages.</param> + public Marshaller(Action<T, SerializationContext> serializer, Func<DeserializationContext, T> deserializer) { - get - { - return this.serializer; - } + this.contextualSerializer = GrpcPreconditions.CheckNotNull(serializer, nameof(serializer)); + this.contextualDeserializer = GrpcPreconditions.CheckNotNull(deserializer, nameof(deserializer)); + // TODO(jtattermusch): once gRPC C# library switches to using contextual (de)serializer, + // emulating the simple (de)serializer will become unnecessary. + this.serializer = EmulateSimpleSerializer; + this.deserializer = EmulateSimpleDeserializer; } /// <summary> + /// Gets the serializer function. + /// </summary> + public Func<T, byte[]> Serializer => this.serializer; + + /// <summary> /// Gets the deserializer function. /// </summary> - public Func<byte[], T> Deserializer + public Func<byte[], T> Deserializer => this.deserializer; + + /// <summary> + /// Gets the serializer function. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public Action<T, SerializationContext> ContextualSerializer => this.contextualSerializer; + + /// <summary> + /// Gets the serializer function. + /// Note: experimental API that can change or be removed without any prior notice. + /// </summary> + public Func<DeserializationContext, T> ContextualDeserializer => this.contextualDeserializer; + + // for backward compatibility, emulate the simple serializer using the contextual one + private byte[] EmulateSimpleSerializer(T msg) { - get + // TODO(jtattermusch): avoid the allocation by passing a thread-local instance + // This code will become unnecessary once gRPC C# library switches to using contextual (de)serializer. + var context = new EmulatedSerializationContext(); + this.contextualSerializer(msg, context); + return context.GetPayload(); + } + + // for backward compatibility, emulate the simple deserializer using the contextual one + private T EmulateSimpleDeserializer(byte[] payload) + { + // TODO(jtattermusch): avoid the allocation by passing a thread-local instance + // This code will become unnecessary once gRPC C# library switches to using contextual (de)serializer. + var context = new EmulatedDeserializationContext(payload); + return this.contextualDeserializer(context); + } + + // for backward compatibility, emulate the contextual serializer using the simple one + private void EmulateContextualSerializer(T message, SerializationContext context) + { + var payload = this.serializer(message); + context.Complete(payload); + } + + // for backward compatibility, emulate the contextual deserializer using the simple one + private T EmulateContextualDeserializer(DeserializationContext context) + { + return this.deserializer(context.PayloadAsNewBuffer()); + } + + internal class EmulatedSerializationContext : SerializationContext + { + bool isComplete; + byte[] payload; + + public override void Complete(byte[] payload) + { + GrpcPreconditions.CheckState(!isComplete); + this.isComplete = true; + this.payload = payload; + } + + internal byte[] GetPayload() + { + return this.payload; + } + } + + internal class EmulatedDeserializationContext : DeserializationContext + { + readonly byte[] payload; + bool alreadyCalledPayloadAsNewBuffer; + + public EmulatedDeserializationContext(byte[] payload) + { + this.payload = GrpcPreconditions.CheckNotNull(payload); + } + + public override int PayloadLength => payload.Length; + + public override byte[] PayloadAsNewBuffer() { - return this.deserializer; + GrpcPreconditions.CheckState(!alreadyCalledPayloadAsNewBuffer); + alreadyCalledPayloadAsNewBuffer = true; + return payload; } } } @@ -77,6 +170,15 @@ namespace Grpc.Core } /// <summary> + /// Creates a marshaller from specified contextual serializer and deserializer. + /// Note: This method is part of an experimental API that can change or be removed without any prior notice. + /// </summary> + public static Marshaller<T> Create<T>(Action<T, SerializationContext> serializer, Func<DeserializationContext, T> deserializer) + { + return new Marshaller<T>(serializer, deserializer); + } + + /// <summary> /// Returns a marshaller for <c>string</c> type. This is useful for testing. /// </summary> public static Marshaller<string> StringMarshaller diff --git a/src/csharp/Grpc.Core/RpcException.cs b/src/csharp/Grpc.Core/RpcException.cs index 94429d74ce..ff89897565 100644 --- a/src/csharp/Grpc.Core/RpcException.cs +++ b/src/csharp/Grpc.Core/RpcException.cs @@ -33,10 +33,8 @@ namespace Grpc.Core /// Creates a new <c>RpcException</c> associated with given status. /// </summary> /// <param name="status">Resulting status of a call.</param> - public RpcException(Status status) : base(status.ToString()) + public RpcException(Status status) : this(status, Metadata.Empty, status.ToString()) { - this.status = status; - this.trailers = Metadata.Empty; } /// <summary> @@ -44,10 +42,8 @@ namespace Grpc.Core /// </summary> /// <param name="status">Resulting status of a call.</param> /// <param name="message">The exception message.</param> - public RpcException(Status status, string message) : base(message) + public RpcException(Status status, string message) : this(status, Metadata.Empty, message) { - this.status = status; - this.trailers = Metadata.Empty; } /// <summary> @@ -55,7 +51,17 @@ namespace Grpc.Core /// </summary> /// <param name="status">Resulting status of a call.</param> /// <param name="trailers">Response trailing metadata.</param> - public RpcException(Status status, Metadata trailers) : base(status.ToString()) + public RpcException(Status status, Metadata trailers) : this(status, trailers, status.ToString()) + { + } + + /// <summary> + /// Creates a new <c>RpcException</c> associated with given status, message and trailing response metadata. + /// </summary> + /// <param name="status">Resulting status of a call.</param> + /// <param name="trailers">Response trailing metadata.</param> + /// <param name="message">The exception message.</param> + public RpcException(Status status, Metadata trailers, string message) : base(message) { this.status = status; this.trailers = GrpcPreconditions.CheckNotNull(trailers); diff --git a/src/csharp/Grpc.Core/SerializationContext.cs b/src/csharp/Grpc.Core/SerializationContext.cs new file mode 100644 index 0000000000..cf4d1595da --- /dev/null +++ b/src/csharp/Grpc.Core/SerializationContext.cs @@ -0,0 +1,34 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +namespace Grpc.Core +{ + /// <summary> + /// Provides storage for payload when serializing a message. + /// </summary> + public abstract class SerializationContext + { + /// <summary> + /// Use the byte array as serialized form of current message and mark serialization process as complete. + /// Complete() can only be called once. By calling this method the caller gives up the ownership of the + /// payload which must not be accessed afterwards. + /// </summary> + /// <param name="payload">the serialized form of current message</param> + public abstract void Complete(byte[] payload); + } +} diff --git a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs index c83ccd2612..40447854f4 100644 --- a/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/MetadataCredentialsTest.cs @@ -153,9 +153,10 @@ namespace Grpc.IntegrationTesting [Test] public void MetadataCredentials_InterceptorThrows() { + var authInterceptorExceptionMessage = "Auth interceptor throws"; var callCredentials = CallCredentials.FromInterceptor(new AsyncAuthInterceptor((context, metadata) => { - throw new Exception("Auth interceptor throws"); + throw new Exception(authInterceptorExceptionMessage); })); var channelCredentials = ChannelCredentials.Create(TestCredentials.CreateSslCredentials(), callCredentials); channel = new Channel(Host, server.Ports.Single().BoundPort, channelCredentials, options); @@ -163,6 +164,7 @@ namespace Grpc.IntegrationTesting var ex = Assert.Throws<RpcException>(() => client.UnaryCall(new SimpleRequest { })); Assert.AreEqual(StatusCode.Unavailable, ex.Status.StatusCode); + StringAssert.Contains(authInterceptorExceptionMessage, ex.Status.Detail); } private class FakeTestService : TestService.TestServiceBase diff --git a/src/csharp/tests.json b/src/csharp/tests.json index c2f243fe0a..5683d164c6 100644 --- a/src/csharp/tests.json +++ b/src/csharp/tests.json @@ -23,8 +23,10 @@ "Grpc.Core.Tests.ClientServerTest", "Grpc.Core.Tests.CompressionTest", "Grpc.Core.Tests.ContextPropagationTest", + "Grpc.Core.Tests.ContextualMarshallerTest", "Grpc.Core.Tests.GrpcEnvironmentTest", "Grpc.Core.Tests.HalfcloseTest", + "Grpc.Core.Tests.MarshallerTest", "Grpc.Core.Tests.MarshallingErrorsTest", "Grpc.Core.Tests.MetadataTest", "Grpc.Core.Tests.PerformanceTest", diff --git a/src/objective-c/BoringSSL-GRPC.podspec b/src/objective-c/BoringSSL-GRPC.podspec index 704b35a29e..04e4d5768f 100644 --- a/src/objective-c/BoringSSL-GRPC.podspec +++ b/src/objective-c/BoringSSL-GRPC.podspec @@ -38,7 +38,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL-GRPC' - version = '0.0.1' + version = '0.0.2' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google\'s needs.' # Adapted from the homepage: @@ -81,7 +81,7 @@ Pod::Spec.new do |s| s.ios.deployment_target = '5.0' s.osx.deployment_target = '10.7' - name = 'openssl' + name = 'openssl_grpc' # When creating a dynamic framework, name it openssl.framework instead of BoringSSL.framework. # This lets users write their includes like `#include <openssl/ssl.h>` as opposed to `#include @@ -1546,6 +1546,7 @@ Pod::Spec.new do |s| sed -i'.back' '/^#define \\([A-Za-z0-9_]*\\) \\1/d' include/openssl/ssl.h sed -i'.back' 'N;/^#define \\([A-Za-z0-9_]*\\) *\\\\\\n *\\1/d' include/openssl/ssl.h sed -i'.back' 's/#ifndef md5_block_data_order/#ifndef GRPC_SHADOW_md5_block_data_order/g' crypto/fipsmodule/md5/md5.c + find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g' END_OF_COMMAND # Redefine symbols to avoid conflict when the same app also depends on OpenSSL. The list of diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.h b/src/objective-c/GRPCClient/private/NSError+GRPC.h index a63e76ee4d..fb05638b78 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.h +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.h @@ -25,6 +25,6 @@ * and whose domain is |kGRPCErrorDomain|. */ + (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode - details:(char *)details + details:(const char *)details errorString:(const char *)errorString; @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.m b/src/objective-c/GRPCClient/private/NSError+GRPC.m index 199b2ebb6c..3eefed88d6 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.m +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.m @@ -24,18 +24,20 @@ NSString *const kGRPCErrorDomain = @"io.grpc"; @implementation NSError (GRPC) + (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode - details:(char *)details + details:(const char *)details errorString:(const char *)errorString { if (statusCode == GRPC_STATUS_OK) { return nil; } - NSString *message = [NSString stringWithCString:details encoding:NSUTF8StringEncoding]; - NSString *debugMessage = [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding]; - return [NSError errorWithDomain:kGRPCErrorDomain - code:statusCode - userInfo:@{ - NSLocalizedDescriptionKey : message, - NSDebugDescriptionErrorKey : debugMessage - }]; + NSMutableDictionary *userInfo = [NSMutableDictionary dictionary]; + if (details) { + userInfo[NSLocalizedDescriptionKey] = + [NSString stringWithCString:details encoding:NSUTF8StringEncoding]; + } + if (errorString) { + userInfo[NSDebugDescriptionErrorKey] = + [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding]; + } + return [NSError errorWithDomain:kGRPCErrorDomain code:statusCode userInfo:userInfo]; } @end diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 75a669da4d..84893b92c1 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -39,7 +39,7 @@ #import "src/core/tsi/grpc_shadow_boringssl.h" -#import <openssl/ssl.h> +#import <openssl_grpc/ssl.h> static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile index 507d251b48..5d2f1340da 100644 --- a/src/objective-c/tests/Podfile +++ b/src/objective-c/tests/Podfile @@ -14,6 +14,7 @@ GRPC_LOCAL_SRC = '../../..' InteropTestsLocalSSL InteropTestsLocalCleartext InteropTestsRemoteWithCronet + UnitTests ).each do |target_name| target target_name do pod 'Protobuf', :path => "#{GRPC_LOCAL_SRC}/third_party/protobuf", :inhibit_warnings => true diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index 0e0e8babc0..f0d8123263 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -13,6 +13,8 @@ 16A9E77B6E336B3C0B9BA6E0 /* libPods-InteropTestsLocalSSL.a in Frameworks */ = {isa = PBXBuildFile; fileRef = DBEDE45BDA60DF1E1C8950C0 /* libPods-InteropTestsLocalSSL.a */; }; 20DFDF829DD993A4A00D5662 /* libPods-RxLibraryUnitTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = A58BE6DF1C62D1739EBB2C78 /* libPods-RxLibraryUnitTests.a */; }; 333E8FC01C8285B7C547D799 /* libPods-InteropTestsLocalCleartext.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FD346DB2C23F676C4842F3FF /* libPods-InteropTestsLocalCleartext.a */; }; + 5E0282E9215AA697007AC99D /* UnitTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 5E0282E8215AA697007AC99D /* UnitTests.m */; }; + 5E0282EB215AA697007AC99D /* libTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 635697C71B14FC11007A7283 /* libTests.a */; }; 5E8A5DA71D3840B4000F8BC4 /* CoreCronetEnd2EndTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5E8A5DA61D3840B4000F8BC4 /* CoreCronetEnd2EndTests.mm */; }; 5E8A5DA91D3840B4000F8BC4 /* libTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 635697C71B14FC11007A7283 /* libTests.a */; }; 5EAD6D271E27047400002378 /* CronetUnitTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 5EAD6D261E27047400002378 /* CronetUnitTests.m */; }; @@ -54,10 +56,18 @@ 91D4B3C85B6D8562F409CB48 /* libPods-InteropTestsLocalSSLCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F3AB031E0E26AC8EF30A2A2A /* libPods-InteropTestsLocalSSLCFStream.a */; }; BC111C80CBF7068B62869352 /* libPods-InteropTestsRemoteCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F44AC3F44E3491A8C0D890FE /* libPods-InteropTestsRemoteCFStream.a */; }; C3D6F4270A2FFF634D8849ED /* libPods-InteropTestsLocalCleartextCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 0BDA4BA011779D5D25B5618C /* libPods-InteropTestsLocalCleartextCFStream.a */; }; + CCF5C0719EF608276AE16374 /* libPods-UnitTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 22A3EBB488699C8CEA19707B /* libPods-UnitTests.a */; }; F15EF7852DC70770EFDB1D2C /* libPods-AllTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = CAE086D5B470DA367D415AB0 /* libPods-AllTests.a */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ + 5E0282EC215AA697007AC99D /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 635697BF1B14FC11007A7283 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 635697C61B14FC11007A7283; + remoteInfo = Tests; + }; 5E8A5DAA1D3840B4000F8BC4 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 635697BF1B14FC11007A7283 /* Project object */; @@ -138,6 +148,7 @@ 1588C85DEAF7FC0ACDEA4C02 /* Pods-InteropTestsLocalCleartext.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartext.test.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext.test.xcconfig"; sourceTree = "<group>"; }; 17F60BF2871F6AF85FB3FA12 /* Pods-InteropTestsRemoteWithCronet.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemoteWithCronet.debug.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemoteWithCronet/Pods-InteropTestsRemoteWithCronet.debug.xcconfig"; sourceTree = "<group>"; }; 20DFF2F3C97EF098FE5A3171 /* libPods-Tests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-Tests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 22A3EBB488699C8CEA19707B /* libPods-UnitTests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-UnitTests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; 2B89F3037963E6EDDD48D8C3 /* Pods-InteropTestsRemoteWithCronet.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemoteWithCronet.test.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemoteWithCronet/Pods-InteropTestsRemoteWithCronet.test.xcconfig"; sourceTree = "<group>"; }; 303F4A17EB1650FC44603D17 /* Pods-InteropTestsRemoteCFStream.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemoteCFStream.release.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemoteCFStream/Pods-InteropTestsRemoteCFStream.release.xcconfig"; sourceTree = "<group>"; }; 32748C4078AEB05F8F954361 /* Pods-InteropTestsRemoteCFStream.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemoteCFStream.debug.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemoteCFStream/Pods-InteropTestsRemoteCFStream.debug.xcconfig"; sourceTree = "<group>"; }; @@ -155,6 +166,9 @@ 55B630C1FF8C36D1EFC4E0A4 /* Pods-InteropTestsLocalSSLCFStream.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalSSLCFStream.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalSSLCFStream/Pods-InteropTestsLocalSSLCFStream.cronet.xcconfig"; sourceTree = "<group>"; }; 573450F334B331D0BED8B961 /* Pods-CoreCronetEnd2EndTests.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-CoreCronetEnd2EndTests.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-CoreCronetEnd2EndTests/Pods-CoreCronetEnd2EndTests.cronet.xcconfig"; sourceTree = "<group>"; }; 5761E98978DDDF136A58CB7E /* Pods-AllTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AllTests.release.xcconfig"; path = "Pods/Target Support Files/Pods-AllTests/Pods-AllTests.release.xcconfig"; sourceTree = "<group>"; }; + 5E0282E6215AA697007AC99D /* UnitTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = UnitTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 5E0282E8215AA697007AC99D /* UnitTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = UnitTests.m; sourceTree = "<group>"; }; + 5E0282EA215AA697007AC99D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; }; 5E8A5DA41D3840B4000F8BC4 /* CoreCronetEnd2EndTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CoreCronetEnd2EndTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 5E8A5DA61D3840B4000F8BC4 /* CoreCronetEnd2EndTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = CoreCronetEnd2EndTests.mm; sourceTree = "<group>"; }; 5EA908CF4CDA4CE218352A06 /* Pods-InteropTestsLocalSSLCFStream.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalSSLCFStream.release.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalSSLCFStream/Pods-InteropTestsLocalSSLCFStream.release.xcconfig"; sourceTree = "<group>"; }; @@ -192,7 +206,9 @@ 7BA53C6D224288D5870FE6F3 /* Pods-InteropTestsLocalCleartextCFStream.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartextCFStream.release.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartextCFStream/Pods-InteropTestsLocalCleartextCFStream.release.xcconfig"; sourceTree = "<group>"; }; 8B498B05C6DA0818B2FA91D4 /* Pods-InteropTestsLocalCleartextCFStream.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartextCFStream.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartextCFStream/Pods-InteropTestsLocalCleartextCFStream.cronet.xcconfig"; sourceTree = "<group>"; }; 943138072A9605B5B8DC1FC0 /* Pods-InteropTestsLocalCleartextCFStream.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartextCFStream.debug.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartextCFStream/Pods-InteropTestsLocalCleartextCFStream.debug.xcconfig"; sourceTree = "<group>"; }; + 94D7A5FAA13480E9A5166D7A /* Pods-UnitTests.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-UnitTests.test.xcconfig"; path = "Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests.test.xcconfig"; sourceTree = "<group>"; }; 9E9444C764F0FFF64A7EB58E /* libPods-InteropTestsRemoteWithCronet.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-InteropTestsRemoteWithCronet.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + A2DCF2570BE515B62CB924CA /* Pods-UnitTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-UnitTests.debug.xcconfig"; path = "Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests.debug.xcconfig"; sourceTree = "<group>"; }; A58BE6DF1C62D1739EBB2C78 /* libPods-RxLibraryUnitTests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-RxLibraryUnitTests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; A6F832FCEFA6F6881E620F12 /* Pods-InteropTestsRemote.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemote.test.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemote/Pods-InteropTestsRemote.test.xcconfig"; sourceTree = "<group>"; }; AA7CB64B4DD9915AE7C03163 /* Pods-InteropTestsLocalCleartext.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartext.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext.cronet.xcconfig"; sourceTree = "<group>"; }; @@ -208,9 +224,11 @@ DBEDE45BDA60DF1E1C8950C0 /* libPods-InteropTestsLocalSSL.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-InteropTestsLocalSSL.a"; sourceTree = BUILT_PRODUCTS_DIR; }; DC3CA1D948F068E76957A861 /* Pods-InteropTestsRemote.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemote.debug.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemote/Pods-InteropTestsRemote.debug.xcconfig"; sourceTree = "<group>"; }; E1486220285AF123EB124008 /* Pods-InteropTestsLocalCleartext.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartext.debug.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext.debug.xcconfig"; sourceTree = "<group>"; }; + E1E7660656D902104F728892 /* Pods-UnitTests.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-UnitTests.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests.cronet.xcconfig"; sourceTree = "<group>"; }; E4275A759BDBDF143B9B438F /* Pods-InteropTestsRemote.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemote.release.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemote/Pods-InteropTestsRemote.release.xcconfig"; sourceTree = "<group>"; }; E4FD4606D4AB8D5A314D72F0 /* Pods-InteropTestsLocalCleartextCFStream.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartextCFStream.test.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartextCFStream/Pods-InteropTestsLocalCleartextCFStream.test.xcconfig"; sourceTree = "<group>"; }; E7E4D3FD76E3B745D992AF5F /* Pods-AllTests.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AllTests.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-AllTests/Pods-AllTests.cronet.xcconfig"; sourceTree = "<group>"; }; + EBFFEC04B514CB0D4922DC40 /* Pods-UnitTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-UnitTests.release.xcconfig"; path = "Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests.release.xcconfig"; sourceTree = "<group>"; }; F3AB031E0E26AC8EF30A2A2A /* libPods-InteropTestsLocalSSLCFStream.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-InteropTestsLocalSSLCFStream.a"; sourceTree = BUILT_PRODUCTS_DIR; }; F44AC3F44E3491A8C0D890FE /* libPods-InteropTestsRemoteCFStream.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-InteropTestsRemoteCFStream.a"; sourceTree = BUILT_PRODUCTS_DIR; }; FBD98AC417B9882D32B19F28 /* libPods-CoreCronetEnd2EndTests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-CoreCronetEnd2EndTests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -219,6 +237,15 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + 5E0282E3215AA697007AC99D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 5E0282EB215AA697007AC99D /* libTests.a in Frameworks */, + CCF5C0719EF608276AE16374 /* libPods-UnitTests.a in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5E8A5DA11D3840B4000F8BC4 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -341,6 +368,7 @@ F44AC3F44E3491A8C0D890FE /* libPods-InteropTestsRemoteCFStream.a */, 0BDA4BA011779D5D25B5618C /* libPods-InteropTestsLocalCleartextCFStream.a */, F3AB031E0E26AC8EF30A2A2A /* libPods-InteropTestsLocalSSLCFStream.a */, + 22A3EBB488699C8CEA19707B /* libPods-UnitTests.a */, ); name = Frameworks; sourceTree = "<group>"; @@ -394,10 +422,23 @@ 41AA59529240A6BBBD3DB904 /* Pods-InteropTestsLocalSSLCFStream.test.xcconfig */, 55B630C1FF8C36D1EFC4E0A4 /* Pods-InteropTestsLocalSSLCFStream.cronet.xcconfig */, 5EA908CF4CDA4CE218352A06 /* Pods-InteropTestsLocalSSLCFStream.release.xcconfig */, + A2DCF2570BE515B62CB924CA /* Pods-UnitTests.debug.xcconfig */, + 94D7A5FAA13480E9A5166D7A /* Pods-UnitTests.test.xcconfig */, + E1E7660656D902104F728892 /* Pods-UnitTests.cronet.xcconfig */, + EBFFEC04B514CB0D4922DC40 /* Pods-UnitTests.release.xcconfig */, ); name = Pods; sourceTree = "<group>"; }; + 5E0282E7215AA697007AC99D /* UnitTests */ = { + isa = PBXGroup; + children = ( + 5E0282E8215AA697007AC99D /* UnitTests.m */, + 5E0282EA215AA697007AC99D /* Info.plist */, + ); + path = UnitTests; + sourceTree = "<group>"; + }; 5E8A5DA51D3840B4000F8BC4 /* CoreCronetEnd2EndTests */ = { isa = PBXGroup; children = ( @@ -432,6 +473,7 @@ 5E8A5DA51D3840B4000F8BC4 /* CoreCronetEnd2EndTests */, 5EE84BF21D4717E40050C6CC /* InteropTestsRemoteWithCronet */, 5EAD6D251E27047400002378 /* CronetUnitTests */, + 5E0282E7215AA697007AC99D /* UnitTests */, 635697C81B14FC11007A7283 /* Products */, 51E4650F34F854F41FF053B3 /* Pods */, 136D535E19727099B941D7B1 /* Frameworks */, @@ -453,6 +495,7 @@ 5EC5E421208177CC000EF4AD /* InteropTestsRemoteCFStream.xctest */, 5EC5E4312081856B000EF4AD /* InteropTestsLocalCleartextCFStream.xctest */, 5EC5E442208185CE000EF4AD /* InteropTestsLocalSSLCFStream.xctest */, + 5E0282E6215AA697007AC99D /* UnitTests.xctest */, ); name = Products; sourceTree = "<group>"; @@ -485,6 +528,26 @@ /* End PBXGroup section */ /* Begin PBXNativeTarget section */ + 5E0282E5215AA697007AC99D /* UnitTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 5E0282F2215AA697007AC99D /* Build configuration list for PBXNativeTarget "UnitTests" */; + buildPhases = ( + F07941C0BAF6A7C67AA60C48 /* [CP] Check Pods Manifest.lock */, + 5E0282E2215AA697007AC99D /* Sources */, + 5E0282E3215AA697007AC99D /* Frameworks */, + 5E0282E4215AA697007AC99D /* Resources */, + 9AD0B5E94F2AA5962EA6AA36 /* [CP] Copy Pods Resources */, + ); + buildRules = ( + ); + dependencies = ( + 5E0282ED215AA697007AC99D /* PBXTargetDependency */, + ); + name = UnitTests; + productName = UnitTests; + productReference = 5E0282E6215AA697007AC99D /* UnitTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; 5E8A5DA31D3840B4000F8BC4 /* CoreCronetEnd2EndTests */ = { isa = PBXNativeTarget; buildConfigurationList = 5E8A5DAE1D3840B4000F8BC4 /* Build configuration list for PBXNativeTarget "CoreCronetEnd2EndTests" */; @@ -729,6 +792,10 @@ LastUpgradeCheck = 0630; ORGANIZATIONNAME = gRPC; TargetAttributes = { + 5E0282E5215AA697007AC99D = { + CreatedOnToolsVersion = 9.2; + ProvisioningStyle = Automatic; + }; 5E8A5DA31D3840B4000F8BC4 = { CreatedOnToolsVersion = 7.3.1; }; @@ -794,11 +861,19 @@ 5EC5E420208177CC000EF4AD /* InteropTestsRemoteCFStream */, 5EC5E4302081856B000EF4AD /* InteropTestsLocalCleartextCFStream */, 5EC5E441208185CE000EF4AD /* InteropTestsLocalSSLCFStream */, + 5E0282E5215AA697007AC99D /* UnitTests */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ + 5E0282E4215AA697007AC99D /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5E8A5DA21D3840B4000F8BC4 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -1098,6 +1173,24 @@ shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-InteropTestsRemoteCFStream/Pods-InteropTestsRemoteCFStream-resources.sh\"\n"; showEnvVarsInLog = 0; }; + 9AD0B5E94F2AA5962EA6AA36 /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + "${SRCROOT}/Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests-resources.sh", + "${PODS_CONFIGURATION_BUILD_DIR}/gRPC/gRPCCertificates.bundle", + ); + name = "[CP] Copy Pods Resources"; + outputPaths = ( + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/gRPCCertificates.bundle", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-UnitTests/Pods-UnitTests-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; A023FB55205A7EA37D413549 /* [CP] Copy Pods Resources */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -1260,6 +1353,24 @@ shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-CoreCronetEnd2EndTests/Pods-CoreCronetEnd2EndTests-frameworks.sh\"\n"; showEnvVarsInLog = 0; }; + F07941C0BAF6A7C67AA60C48 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-UnitTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; F3D5B2CDA172580341682830 /* [CP] Check Pods Manifest.lock */ = { isa = PBXShellScriptBuildPhase; buildActionMask = 2147483647; @@ -1299,6 +1410,14 @@ /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ + 5E0282E2215AA697007AC99D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 5E0282E9215AA697007AC99D /* UnitTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5E8A5DA01D3840B4000F8BC4 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1412,6 +1531,11 @@ /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ + 5E0282ED215AA697007AC99D /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 635697C61B14FC11007A7283 /* Tests */; + targetProxy = 5E0282EC215AA697007AC99D /* PBXContainerItemProxy */; + }; 5E8A5DAB1D3840B4000F8BC4 /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 635697C61B14FC11007A7283 /* Tests */; @@ -1455,6 +1579,128 @@ /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ + 5E0282EE215AA697007AC99D /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = A2DCF2570BE515B62CB924CA /* Pods-UnitTests.debug.xcconfig */; + buildSettings = { + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + INFOPLIST_FILE = UnitTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 11.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = io.grpc.UnitTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + USER_HEADER_SEARCH_PATHS = "\"$(PODS_ROOT)/../../../..\""; + }; + name = Debug; + }; + 5E0282EF215AA697007AC99D /* Test */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 94D7A5FAA13480E9A5166D7A /* Pods-UnitTests.test.xcconfig */; + buildSettings = { + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu11; + INFOPLIST_FILE = UnitTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 11.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = io.grpc.UnitTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + USER_HEADER_SEARCH_PATHS = "\"$(PODS_ROOT)/../../../..\""; + }; + name = Test; + }; + 5E0282F0215AA697007AC99D /* Cronet */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = E1E7660656D902104F728892 /* Pods-UnitTests.cronet.xcconfig */; + buildSettings = { + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu11; + INFOPLIST_FILE = UnitTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 11.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = io.grpc.UnitTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + USER_HEADER_SEARCH_PATHS = "\"$(PODS_ROOT)/../../../..\""; + }; + name = Cronet; + }; + 5E0282F1215AA697007AC99D /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EBFFEC04B514CB0D4922DC40 /* Pods-UnitTests.release.xcconfig */; + buildSettings = { + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + GCC_C_LANGUAGE_STANDARD = gnu11; + INFOPLIST_FILE = UnitTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 11.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = io.grpc.UnitTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + TARGETED_DEVICE_FAMILY = "1,2"; + USER_HEADER_SEARCH_PATHS = "\"$(PODS_ROOT)/../../../..\""; + }; + name = Release; + }; 5E1228981E4D400F00E8504F /* Test */ = { isa = XCBuildConfiguration; buildSettings = { @@ -2602,6 +2848,17 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + 5E0282F2215AA697007AC99D /* Build configuration list for PBXNativeTarget "UnitTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 5E0282EE215AA697007AC99D /* Debug */, + 5E0282EF215AA697007AC99D /* Test */, + 5E0282F0215AA697007AC99D /* Cronet */, + 5E0282F1215AA697007AC99D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; 5E8A5DAE1D3840B4000F8BC4 /* Build configuration list for PBXNativeTarget "CoreCronetEnd2EndTests" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/UnitTests.xcscheme b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/UnitTests.xcscheme new file mode 100644 index 0000000000..3af3555f48 --- /dev/null +++ b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/UnitTests.xcscheme @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Scheme + LastUpgradeVersion = "0920" + version = "1.3"> + <BuildAction + parallelizeBuildables = "YES" + buildImplicitDependencies = "YES"> + <BuildActionEntries> + <BuildActionEntry + buildForTesting = "YES" + buildForRunning = "YES" + buildForProfiling = "NO" + buildForArchiving = "NO" + buildForAnalyzing = "NO"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "5E0282E5215AA697007AC99D" + BuildableName = "UnitTests.xctest" + BlueprintName = "UnitTests" + ReferencedContainer = "container:Tests.xcodeproj"> + </BuildableReference> + </BuildActionEntry> + </BuildActionEntries> + </BuildAction> + <TestAction + buildConfiguration = "Debug" + selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" + selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" + language = "" + shouldUseLaunchSchemeArgsEnv = "YES"> + <Testables> + <TestableReference + skipped = "NO"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "5E0282E5215AA697007AC99D" + BuildableName = "UnitTests.xctest" + BlueprintName = "UnitTests" + ReferencedContainer = "container:Tests.xcodeproj"> + </BuildableReference> + </TestableReference> + </Testables> + <AdditionalOptions> + </AdditionalOptions> + </TestAction> + <LaunchAction + buildConfiguration = "Debug" + selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" + selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" + language = "" + launchStyle = "0" + useCustomWorkingDirectory = "NO" + ignoresPersistentStateOnLaunch = "NO" + debugDocumentVersioning = "YES" + debugServiceExtension = "internal" + allowLocationSimulation = "YES"> + <MacroExpansion> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "5E0282E5215AA697007AC99D" + BuildableName = "UnitTests.xctest" + BlueprintName = "UnitTests" + ReferencedContainer = "container:Tests.xcodeproj"> + </BuildableReference> + </MacroExpansion> + <AdditionalOptions> + </AdditionalOptions> + </LaunchAction> + <ProfileAction + buildConfiguration = "Release" + shouldUseLaunchSchemeArgsEnv = "YES" + savedToolIdentifier = "" + useCustomWorkingDirectory = "NO" + debugDocumentVersioning = "YES"> + <MacroExpansion> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "5E0282E5215AA697007AC99D" + BuildableName = "UnitTests.xctest" + BlueprintName = "UnitTests" + ReferencedContainer = "container:Tests.xcodeproj"> + </BuildableReference> + </MacroExpansion> + </ProfileAction> + <AnalyzeAction + buildConfiguration = "Debug"> + </AnalyzeAction> + <ArchiveAction + buildConfiguration = "Release" + revealArchiveInOrganizer = "YES"> + </ArchiveAction> +</Scheme> diff --git a/src/objective-c/tests/UnitTests/Info.plist b/src/objective-c/tests/UnitTests/Info.plist new file mode 100644 index 0000000000..6c40a6cd0c --- /dev/null +++ b/src/objective-c/tests/UnitTests/Info.plist @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>$(DEVELOPMENT_LANGUAGE)</string> + <key>CFBundleExecutable</key> + <string>$(EXECUTABLE_NAME)</string> + <key>CFBundleIdentifier</key> + <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleName</key> + <string>$(PRODUCT_NAME)</string> + <key>CFBundlePackageType</key> + <string>BNDL</string> + <key>CFBundleShortVersionString</key> + <string>1.0</string> + <key>CFBundleVersion</key> + <string>1</string> +</dict> +</plist> diff --git a/src/objective-c/tests/UnitTests/UnitTests.m b/src/objective-c/tests/UnitTests/UnitTests.m new file mode 100644 index 0000000000..57e686d1b6 --- /dev/null +++ b/src/objective-c/tests/UnitTests/UnitTests.m @@ -0,0 +1,62 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#import <XCTest/XCTest.h> + +#import <GRPCClient/GRPCCall.h> + +#import "src/objective-c/GRPCClient/private/NSError+GRPC.h" + +@interface UnitTests : XCTestCase + +@end + +@implementation UnitTests + +- (void)testNSError { + const char *kDetails = "test details"; + const char *kErrorString = "test errorString"; + NSError *error1 = [NSError grpc_errorFromStatusCode:GRPC_STATUS_OK details:nil errorString:nil]; + NSError *error2 = [NSError grpc_errorFromStatusCode:GRPC_STATUS_CANCELLED + details:kDetails + errorString:kErrorString]; + NSError *error3 = [NSError grpc_errorFromStatusCode:GRPC_STATUS_UNAUTHENTICATED + details:kDetails + errorString:nil]; + NSError *error4 = + [NSError grpc_errorFromStatusCode:GRPC_STATUS_UNAVAILABLE details:nil errorString:nil]; + + XCTAssertNil(error1); + XCTAssertEqual(error2.code, 1); + XCTAssertEqualObjects(error2.domain, @"io.grpc"); + XCTAssertEqualObjects(error2.userInfo[NSLocalizedDescriptionKey], + [NSString stringWithUTF8String:kDetails]); + XCTAssertEqualObjects(error2.userInfo[NSDebugDescriptionErrorKey], + [NSString stringWithUTF8String:kErrorString]); + XCTAssertEqual(error3.code, 16); + XCTAssertEqualObjects(error3.domain, @"io.grpc"); + XCTAssertEqualObjects(error3.userInfo[NSLocalizedDescriptionKey], + [NSString stringWithUTF8String:kDetails]); + XCTAssertNil(error3.userInfo[NSDebugDescriptionErrorKey]); + XCTAssertEqual(error4.code, 14); + XCTAssertEqualObjects(error4.domain, @"io.grpc"); + XCTAssertNil(error4.userInfo[NSLocalizedDescriptionKey]); + XCTAssertNil(error4.userInfo[NSDebugDescriptionErrorKey]); +} + +@end diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 9dde07d55c..f37c6cf9f6 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -163,4 +163,14 @@ xcodebuild \ | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - +echo "TIME: $(date)" +xcodebuild \ + -workspace Tests.xcworkspace \ + -scheme UnitTests \ + -destination name="iPhone 8" \ + test \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - + exit 0 diff --git a/src/php/tests/unit_tests/ChannelTest.php b/src/php/tests/unit_tests/ChannelTest.php index 49b0e350f7..583c618d32 100644 --- a/src/php/tests/unit_tests/ChannelTest.php +++ b/src/php/tests/unit_tests/ChannelTest.php @@ -599,10 +599,10 @@ class ChannelTest extends PHPUnit_Framework_TestCase public function testPersistentChannelForceNewOldChannelIdle2() { - $this->channel1 = new Grpc\Channel('localhost:50029', [ + $this->channel1 = new Grpc\Channel('localhost:50032', [ "grpc_target_persist_bound" => 2, ]); - $this->channel2 = new Grpc\Channel('localhost:50029', []); + $this->channel2 = new Grpc\Channel('localhost:50032', []); // try to connect on channel2 $state = $this->channel1->getConnectivityState(true); diff --git a/src/proto/grpc/testing/package_options.proto b/src/proto/grpc/testing/package_options.proto new file mode 100644 index 0000000000..e7ecf8c196 --- /dev/null +++ b/src/proto/grpc/testing/package_options.proto @@ -0,0 +1,28 @@ +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package grpc.testing; + +// For sanity checking package definitions +option ruby_package = "Grpc.Testing.Package.Options"; + +message TestRequest { } + +message TestResponse { } + +service TestService { + rpc GetTest(TestRequest) returns (TestResponse) { } +} diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel index 3f214bf3b0..2e6839ef2d 100644 --- a/src/python/grpcio/grpc/BUILD.bazel +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -2,7 +2,7 @@ load("@grpc_python_dependencies//:requirements.bzl", "requirement") package(default_visibility = ["//visibility:public"]) -py_binary( +py_library( name = "grpcio", srcs = ["__init__.py"], deps = [ @@ -22,7 +22,6 @@ py_binary( data = [ "//:grpc", ], - main = "__init__.py", imports = ["../",], ) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 6876601785..3494c9b15a 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -24,6 +24,7 @@ from grpc import _grpcio_metadata from grpc._cython import cygrpc from grpc.framework.foundation import callable_util +logging.basicConfig() _LOGGER = logging.getLogger(__name__) _USER_AGENT = 'grpc-python/{}'.format(_grpcio_metadata.__version__) diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py index 8358cbec5b..3805c7e82a 100644 --- a/src/python/grpcio/grpc/_common.py +++ b/src/python/grpcio/grpc/_common.py @@ -20,6 +20,7 @@ import six import grpc from grpc._cython import cygrpc +logging.basicConfig() _LOGGER = logging.getLogger(__name__) CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY = { diff --git a/src/python/grpcio/grpc/_cython/BUILD.bazel b/src/python/grpcio/grpc/_cython/BUILD.bazel index 7124e83dee..cfd3a51d9b 100644 --- a/src/python/grpcio/grpc/_cython/BUILD.bazel +++ b/src/python/grpcio/grpc/_cython/BUILD.bazel @@ -8,6 +8,7 @@ pyx_library( "__init__.py", "cygrpc.pxd", "cygrpc.pyx", + "_cygrpc/_hooks.pyx.pxi", "_cygrpc/grpc_string.pyx.pxi", "_cygrpc/arguments.pyx.pxi", "_cygrpc/call.pyx.pxi", @@ -15,6 +16,7 @@ pyx_library( "_cygrpc/credentials.pyx.pxi", "_cygrpc/completion_queue.pyx.pxi", "_cygrpc/event.pyx.pxi", + "_cygrpc/fork_posix.pyx.pxi", "_cygrpc/metadata.pyx.pxi", "_cygrpc/operation.pyx.pxi", "_cygrpc/records.pyx.pxi", @@ -24,12 +26,14 @@ pyx_library( "_cygrpc/time.pyx.pxi", "_cygrpc/grpc_gevent.pyx.pxi", "_cygrpc/grpc.pxi", + "_cygrpc/_hooks.pxd.pxi", "_cygrpc/arguments.pxd.pxi", "_cygrpc/call.pxd.pxi", "_cygrpc/channel.pxd.pxi", "_cygrpc/credentials.pxd.pxi", "_cygrpc/completion_queue.pxd.pxi", "_cygrpc/event.pxd.pxi", + "_cygrpc/fork_posix.pxd.pxi", "_cygrpc/metadata.pxd.pxi", "_cygrpc/operation.pxd.pxi", "_cygrpc/records.pxd.pxi", diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 38fd9e78b2..63048e8da0 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -144,8 +144,14 @@ cdef class SSLChannelCredentials(ChannelCredentials): return grpc_ssl_credentials_create( c_pem_root_certificates, NULL, NULL, NULL) else: - c_pem_key_certificate_pair.private_key = self._private_key - c_pem_key_certificate_pair.certificate_chain = self._certificate_chain + if self._private_key: + c_pem_key_certificate_pair.private_key = self._private_key + else: + c_pem_key_certificate_pair.private_key = NULL + if self._certificate_chain: + c_pem_key_certificate_pair.certificate_chain = self._certificate_chain + else: + c_pem_key_certificate_pair.certificate_chain = NULL return grpc_ssl_credentials_create( c_pem_root_certificates, &c_pem_key_certificate_pair, NULL, NULL) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi index 00a1b23a67..334e561baa 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi @@ -14,6 +14,7 @@ import logging +logging.basicConfig() _LOGGER = logging.getLogger(__name__) # This function will ascii encode unicode string inputs if neccesary. diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index ce701724fd..5779437b92 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -18,6 +18,7 @@ import logging import time import grpc +logging.basicConfig() _LOGGER = logging.getLogger(__name__) cdef class Server: diff --git a/src/python/grpcio/grpc/_plugin_wrapping.py b/src/python/grpcio/grpc/_plugin_wrapping.py index 916ee080b6..88ab4d8371 100644 --- a/src/python/grpcio/grpc/_plugin_wrapping.py +++ b/src/python/grpcio/grpc/_plugin_wrapping.py @@ -20,6 +20,7 @@ import grpc from grpc import _common from grpc._cython import cygrpc +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 7276a7fd90..daa000a6e1 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -27,6 +27,7 @@ from grpc import _interceptor from grpc._cython import cygrpc from grpc.framework.foundation import callable_util +logging.basicConfig() _LOGGER = logging.getLogger(__name__) _SHUTDOWN_TAG = 'shutdown' diff --git a/src/python/grpcio/grpc/framework/foundation/callable_util.py b/src/python/grpcio/grpc/framework/foundation/callable_util.py index 24daf3406f..fb8d5f7c1e 100644 --- a/src/python/grpcio/grpc/framework/foundation/callable_util.py +++ b/src/python/grpcio/grpc/framework/foundation/callable_util.py @@ -21,6 +21,7 @@ import logging import six +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio/grpc/framework/foundation/logging_pool.py b/src/python/grpcio/grpc/framework/foundation/logging_pool.py index 216e3990db..7702d1785f 100644 --- a/src/python/grpcio/grpc/framework/foundation/logging_pool.py +++ b/src/python/grpcio/grpc/framework/foundation/logging_pool.py @@ -17,6 +17,7 @@ import logging from concurrent import futures +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio/grpc/framework/foundation/stream_util.py b/src/python/grpcio/grpc/framework/foundation/stream_util.py index 1faaf29bd7..9184f95873 100644 --- a/src/python/grpcio/grpc/framework/foundation/stream_util.py +++ b/src/python/grpcio/grpc/framework/foundation/stream_util.py @@ -19,6 +19,7 @@ import threading from grpc.framework.foundation import stream _NO_VALUE = object() +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 0f68e823d7..3ad82e9cad 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -92,7 +92,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/error.cc', 'src/core/lib/iomgr/ev_epoll1_linux.cc', 'src/core/lib/iomgr/ev_epollex_linux.cc', - 'src/core/lib/iomgr/ev_epollsig_linux.cc', 'src/core/lib/iomgr/ev_poll_posix.cc', 'src/core/lib/iomgr/ev_posix.cc', 'src/core/lib/iomgr/ev_windows.cc', @@ -223,6 +222,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/transport/frame_settings.cc', 'src/core/ext/transport/chttp2/transport/frame_window_update.cc', 'src/core/ext/transport/chttp2/transport/hpack_encoder.cc', + 'src/core/ext/transport/chttp2/transport/hpack_mapping.cc', 'src/core/ext/transport/chttp2/transport/hpack_parser.cc', 'src/core/ext/transport/chttp2/transport/hpack_table.cc', 'src/core/ext/transport/chttp2/transport/http2_settings.cc', diff --git a/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py b/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py index 191b1c1726..d7205ca579 100644 --- a/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py +++ b/src/python/grpcio_testing/grpc_testing/_channel/_invocation.py @@ -18,6 +18,7 @@ import threading import grpc _NOT_YET_OBSERVED = object() +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio_testing/grpc_testing/_server/_rpc.py b/src/python/grpcio_testing/grpc_testing/_server/_rpc.py index b856da100f..736b714dc6 100644 --- a/src/python/grpcio_testing/grpc_testing/_server/_rpc.py +++ b/src/python/grpcio_testing/grpc_testing/_server/_rpc.py @@ -18,6 +18,7 @@ import threading import grpc from grpc_testing import _common +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio_testing/grpc_testing/_time.py b/src/python/grpcio_testing/grpc_testing/_time.py index 75e6db3458..9692c34e6f 100644 --- a/src/python/grpcio_testing/grpc_testing/_time.py +++ b/src/python/grpcio_testing/grpc_testing/_time.py @@ -21,6 +21,7 @@ import time as _time import grpc import grpc_testing +logging.basicConfig() _LOGGER = logging.getLogger(__name__) diff --git a/src/python/grpcio_tests/tests/interop/server.py b/src/python/grpcio_tests/tests/interop/server.py index fd28d498a1..768cdaf468 100644 --- a/src/python/grpcio_tests/tests/interop/server.py +++ b/src/python/grpcio_tests/tests/interop/server.py @@ -25,6 +25,7 @@ from tests.interop import methods from tests.interop import resources from tests.unit import test_common +logging.basicConfig() _ONE_DAY_IN_SECONDS = 60 * 60 * 24 _LOGGER = logging.getLogger(__name__) diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index b6c0791469..fc641dae80 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -819,6 +819,7 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { unsigned write_flag = 0; void* tag = (void*)&st; + grpc_ruby_fork_guard(); if (RTYPEDDATA_DATA(self) == NULL) { rb_raise(grpc_rb_eCallError, "Cannot run batch on closed call"); return Qnil; diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c index 3f0dc530cf..6d4b2293a2 100644 --- a/src/ruby/ext/grpc/rb_channel.c +++ b/src/ruby/ext/grpc/rb_channel.c @@ -217,6 +217,7 @@ static VALUE grpc_rb_channel_init(int argc, VALUE* argv, VALUE self) { MEMZERO(&args, grpc_channel_args, 1); grpc_ruby_once_init(); + grpc_ruby_fork_guard(); rb_thread_call_without_gvl( wait_until_channel_polling_thread_started_no_gil, &stop_waiting_for_thread_start, @@ -374,6 +375,7 @@ static VALUE grpc_rb_channel_watch_connectivity_state(VALUE self, watch_state_stack stack; void* op_success = 0; + grpc_ruby_fork_guard(); TypedData_Get_Struct(self, grpc_rb_channel, &grpc_channel_data_type, wrapper); if (wrapper->bg_wrapped == NULL) { @@ -415,6 +417,7 @@ static VALUE grpc_rb_channel_create_call(VALUE self, VALUE parent, VALUE mask, grpc_slice* host_slice_ptr = NULL; char* tmp_str = NULL; + grpc_ruby_fork_guard(); if (host != Qnil) { host_slice = grpc_slice_from_copied_buffer(RSTRING_PTR(host), RSTRING_LEN(host)); diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index f065a857db..872aed0cfc 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -23,9 +23,13 @@ #include <math.h> #include <ruby/vm.h> +#include <stdbool.h> #include <sys/time.h> +#include <sys/types.h> +#include <unistd.h> #include <grpc/grpc.h> +#include <grpc/support/log.h> #include <grpc/support/time.h> #include "rb_call.h" #include "rb_call_credentials.h" @@ -255,7 +259,26 @@ static void Init_grpc_time_consts() { id_tv_nsec = rb_intern("tv_nsec"); } -static void grpc_rb_shutdown(void) { grpc_shutdown(); } +#if GPR_WINDOWS +static void grpc_ruby_set_init_pid(void) {} +static bool grpc_ruby_forked_after_init(void) { return false; } +#else +static pid_t grpc_init_pid; + +static void grpc_ruby_set_init_pid(void) { + GPR_ASSERT(grpc_init_pid == 0); + grpc_init_pid = getpid(); +} + +static bool grpc_ruby_forked_after_init(void) { + GPR_ASSERT(grpc_init_pid != 0); + return grpc_init_pid != getpid(); +} +#endif + +static void grpc_rb_shutdown(void) { + if (!grpc_ruby_forked_after_init()) grpc_shutdown(); +} /* Initialize the GRPC module structs */ @@ -276,10 +299,17 @@ VALUE sym_metadata = Qundef; static gpr_once g_once_init = GPR_ONCE_INIT; static void grpc_ruby_once_init_internal() { + grpc_ruby_set_init_pid(); grpc_init(); atexit(grpc_rb_shutdown); } +void grpc_ruby_fork_guard() { + if (grpc_ruby_forked_after_init()) { + rb_raise(rb_eRuntimeError, "grpc cannot be used before and after forking"); + } +} + static VALUE bg_thread_init_rb_mu = Qundef; static int bg_thread_init_done = 0; diff --git a/src/ruby/ext/grpc/rb_grpc.h b/src/ruby/ext/grpc/rb_grpc.h index 577902319e..4118435ecf 100644 --- a/src/ruby/ext/grpc/rb_grpc.h +++ b/src/ruby/ext/grpc/rb_grpc.h @@ -69,4 +69,6 @@ gpr_timespec grpc_rb_time_timeval(VALUE time, int interval); void grpc_ruby_once_init(); +void grpc_ruby_fork_guard(); + #endif /* GRPC_RB_H_ */ diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index f44f89dfa0..1e7d7f687f 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -97,7 +97,10 @@ grpc_resource_quota_resize_type grpc_resource_quota_resize_import; grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_import; grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import; +grpc_channelz_get_servers_type grpc_channelz_get_servers_import; grpc_channelz_get_channel_type grpc_channelz_get_channel_import; +grpc_channelz_get_subchannel_type grpc_channelz_get_subchannel_import; +grpc_channelz_get_socket_type grpc_channelz_get_socket_import; grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import; grpc_server_add_insecure_channel_from_fd_type grpc_server_add_insecure_channel_from_fd_import; grpc_use_signal_type grpc_use_signal_import; @@ -351,7 +354,10 @@ void grpc_rb_load_imports(HMODULE library) { grpc_resource_quota_set_max_threads_import = (grpc_resource_quota_set_max_threads_type) GetProcAddress(library, "grpc_resource_quota_set_max_threads"); grpc_resource_quota_arg_vtable_import = (grpc_resource_quota_arg_vtable_type) GetProcAddress(library, "grpc_resource_quota_arg_vtable"); grpc_channelz_get_top_channels_import = (grpc_channelz_get_top_channels_type) GetProcAddress(library, "grpc_channelz_get_top_channels"); + grpc_channelz_get_servers_import = (grpc_channelz_get_servers_type) GetProcAddress(library, "grpc_channelz_get_servers"); grpc_channelz_get_channel_import = (grpc_channelz_get_channel_type) GetProcAddress(library, "grpc_channelz_get_channel"); + grpc_channelz_get_subchannel_import = (grpc_channelz_get_subchannel_type) GetProcAddress(library, "grpc_channelz_get_subchannel"); + grpc_channelz_get_socket_import = (grpc_channelz_get_socket_type) GetProcAddress(library, "grpc_channelz_get_socket"); grpc_insecure_channel_create_from_fd_import = (grpc_insecure_channel_create_from_fd_type) GetProcAddress(library, "grpc_insecure_channel_create_from_fd"); grpc_server_add_insecure_channel_from_fd_import = (grpc_server_add_insecure_channel_from_fd_type) GetProcAddress(library, "grpc_server_add_insecure_channel_from_fd"); grpc_use_signal_import = (grpc_use_signal_type) GetProcAddress(library, "grpc_use_signal"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 99433f0e40..ed4b6264b0 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -107,7 +107,7 @@ extern grpc_completion_queue_create_for_next_type grpc_completion_queue_create_f typedef grpc_completion_queue*(*grpc_completion_queue_create_for_pluck_type)(void* reserved); extern grpc_completion_queue_create_for_pluck_type grpc_completion_queue_create_for_pluck_import; #define grpc_completion_queue_create_for_pluck grpc_completion_queue_create_for_pluck_import -typedef grpc_completion_queue*(*grpc_completion_queue_create_for_callback_type)(void* shutdown_callback, void* reserved); +typedef grpc_completion_queue*(*grpc_completion_queue_create_for_callback_type)(grpc_experimental_completion_queue_functor* shutdown_callback, void* reserved); extern grpc_completion_queue_create_for_callback_type grpc_completion_queue_create_for_callback_import; #define grpc_completion_queue_create_for_callback grpc_completion_queue_create_for_callback_import typedef grpc_completion_queue*(*grpc_completion_queue_create_type)(const grpc_completion_queue_factory* factory, const grpc_completion_queue_attributes* attributes, void* reserved); @@ -266,9 +266,18 @@ extern grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import typedef char*(*grpc_channelz_get_top_channels_type)(intptr_t start_channel_id); extern grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import; #define grpc_channelz_get_top_channels grpc_channelz_get_top_channels_import +typedef char*(*grpc_channelz_get_servers_type)(intptr_t start_server_id); +extern grpc_channelz_get_servers_type grpc_channelz_get_servers_import; +#define grpc_channelz_get_servers grpc_channelz_get_servers_import typedef char*(*grpc_channelz_get_channel_type)(intptr_t channel_id); extern grpc_channelz_get_channel_type grpc_channelz_get_channel_import; #define grpc_channelz_get_channel grpc_channelz_get_channel_import +typedef char*(*grpc_channelz_get_subchannel_type)(intptr_t subchannel_id); +extern grpc_channelz_get_subchannel_type grpc_channelz_get_subchannel_import; +#define grpc_channelz_get_subchannel grpc_channelz_get_subchannel_import +typedef char*(*grpc_channelz_get_socket_type)(intptr_t socket_id); +extern grpc_channelz_get_socket_type grpc_channelz_get_socket_import; +#define grpc_channelz_get_socket grpc_channelz_get_socket_import typedef grpc_channel*(*grpc_insecure_channel_create_from_fd_type)(const char* target, int fd, const grpc_channel_args* args); extern grpc_insecure_channel_create_from_fd_type grpc_insecure_channel_create_from_fd_import; #define grpc_insecure_channel_create_from_fd grpc_insecure_channel_create_from_fd_import @@ -401,10 +410,10 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials* creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import -typedef grpc_alts_credentials_options*(*grpc_alts_credentials_client_options_create_type)(); +typedef grpc_alts_credentials_options*(*grpc_alts_credentials_client_options_create_type)(void); extern grpc_alts_credentials_client_options_create_type grpc_alts_credentials_client_options_create_import; #define grpc_alts_credentials_client_options_create grpc_alts_credentials_client_options_create_import -typedef grpc_alts_credentials_options*(*grpc_alts_credentials_server_options_create_type)(); +typedef grpc_alts_credentials_options*(*grpc_alts_credentials_server_options_create_type)(void); extern grpc_alts_credentials_server_options_create_type grpc_alts_credentials_server_options_create_import; #define grpc_alts_credentials_server_options_create grpc_alts_credentials_server_options_create_import typedef void(*grpc_alts_credentials_client_options_add_target_service_account_type)(grpc_alts_credentials_options* options, const char* service_account); diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c index 88e6a0cfd5..2931f34409 100644 --- a/src/ruby/ext/grpc/rb_server.c +++ b/src/ruby/ext/grpc/rb_server.c @@ -243,6 +243,8 @@ static VALUE grpc_rb_server_request_call(VALUE self) { static VALUE grpc_rb_server_start(VALUE self) { grpc_rb_server* s = NULL; TypedData_Get_Struct(self, grpc_rb_server, &grpc_rb_server_data_type, s); + + grpc_ruby_fork_guard(); if (s->wrapped == NULL) { rb_raise(rb_eRuntimeError, "destroyed!"); } else { diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb index 5fd1805aab..efb0e4233d 100644 --- a/src/ruby/lib/grpc/generic/rpc_desc.rb +++ b/src/ruby/lib/grpc/generic/rpc_desc.rb @@ -32,7 +32,7 @@ module GRPC # @return [Proc] { |instance| marshalled(instance) } def marshal_proc - proc { |o| o.class.method(marshal_method).call(o).to_s } + proc { |o| o.class.send(marshal_method, o).to_s } end # @param [:input, :output] target determines whether to produce the an @@ -42,9 +42,9 @@ module GRPC # @return [Proc] An unmarshal proc { |marshalled(instance)| instance } def unmarshal_proc(target) fail ArgumentError unless [:input, :output].include?(target) - unmarshal_class = method(target).call + unmarshal_class = send(target) unmarshal_class = unmarshal_class.type if unmarshal_class.is_a? Stream - proc { |o| unmarshal_class.method(unmarshal_method).call(o) } + proc { |o| unmarshal_class.send(unmarshal_method, o) } end def handle_request_response(active_call, mth, inter_ctx) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 838ac45927..3b5a0ce27f 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -136,7 +136,7 @@ module GRPC begin blk, args = worker_queue.pop blk.call(*args) - rescue StandardError => e + rescue StandardError, GRPC::Core::CallError => e GRPC.logger.warn('Error in worker thread') GRPC.logger.warn(e) end diff --git a/src/ruby/pb/test/client.rb b/src/ruby/pb/test/client.rb index cfed7ca12a..7710d34657 100755 --- a/src/ruby/pb/test/client.rb +++ b/src/ruby/pb/test/client.rb @@ -95,7 +95,7 @@ end # creates a test stub that accesses host:port securely. def create_stub(opts) - address = "#{opts.host}:#{opts.port}" + address = "#{opts.server_host}:#{opts.server_port}" # Provide channel args that request compression by default # for compression interop tests @@ -703,8 +703,8 @@ class NamedTests end # Args is used to hold the command line info. -Args = Struct.new(:default_service_account, :host, :host_override, - :oauth_scope, :port, :secure, :test_case, +Args = Struct.new(:default_service_account, :server_host, :host_override, + :oauth_scope, :server_port, :secure, :test_case, :use_test_ca) # validates the command line options, returning them as a Hash. @@ -715,7 +715,7 @@ def parse_args opts.on('--oauth_scope scope', 'Scope for OAuth tokens') { |v| args['oauth_scope'] = v } opts.on('--server_host SERVER_HOST', 'server hostname') do |v| - args['host'] = v + args['server_host'] = v end opts.on('--default_service_account email_address', 'email address of the default service account') do |v| @@ -725,7 +725,9 @@ def parse_args 'override host via a HTTP header') do |v| args['host_override'] = v end - opts.on('--server_port SERVER_PORT', 'server port') { |v| args['port'] = v } + opts.on('--server_port SERVER_PORT', 'server port') do |v| + args['server_port'] = v + end # instance_methods(false) gives only the methods defined in that class test_cases = NamedTests.instance_methods(false).map(&:to_s) test_case_list = test_cases.join(',') @@ -744,7 +746,7 @@ def parse_args end def _check_args(args) - %w(host port test_case).each do |a| + %w(server_host server_port test_case).each do |a| if args[a].nil? fail(OptionParser::MissingArgument, "please specify --#{a}") end diff --git a/src/ruby/spec/channel_spec.rb b/src/ruby/spec/channel_spec.rb index 3c9eca47ec..adba6db99c 100644 --- a/src/ruby/spec/channel_spec.rb +++ b/src/ruby/spec/channel_spec.rb @@ -13,6 +13,7 @@ # limitations under the License. require 'spec_helper' +require 'English' def load_test_certs test_root = File.join(File.dirname(__FILE__), 'testdata') @@ -27,6 +28,28 @@ describe GRPC::Core::Channel do GRPC::Core::ChannelCredentials.new(load_test_certs[0]) end + def fork_with_propagated_error_message + pipe_read, pipe_write = IO.pipe + pid = fork do + pipe_read.close + begin + yield + rescue => exc + pipe_write.syswrite(exc.message) + end + pipe_write.close + end + pipe_write.close + + exc_message = pipe_read.read + Process.wait(pid) + + unless $CHILD_STATUS.success? + raise "forked process failed with #{$CHILD_STATUS}" + end + raise exc_message unless exc_message.empty? + end + shared_examples '#new' do it 'take a host name without channel args' do blk = proc do @@ -79,6 +102,14 @@ describe GRPC::Core::Channel do blk = construct_with_args(args) expect(&blk).to_not raise_error end + + it 'raises if grpc was initialized in another process' do + blk = construct_with_args({}) + expect(&blk).not_to raise_error + expect do + fork_with_propagated_error_message(&blk) + end.to raise_error(RuntimeError, 'grpc cannot be used before and after forking') + end end describe '#new for secure channels' do @@ -121,6 +152,19 @@ describe GRPC::Core::Channel do end expect(&blk).to raise_error(RuntimeError) end + + it 'raises if grpc was initialized in another process' do + ch = GRPC::Core::Channel.new(fake_host, nil, :this_channel_is_insecure) + + deadline = Time.now + 5 + + blk = proc do + fork_with_propagated_error_message do + ch.create_call(nil, nil, 'dummy_method', nil, deadline) + end + end + expect(&blk).to raise_error(RuntimeError, 'grpc cannot be used before and after forking') + end end describe '#destroy' do diff --git a/src/ruby/spec/client_auth_spec.rb b/src/ruby/spec/client_auth_spec.rb index c563ed3227..5d2e6d24ec 100644 --- a/src/ruby/spec/client_auth_spec.rb +++ b/src/ruby/spec/client_auth_spec.rb @@ -30,7 +30,7 @@ end def create_server_creds test_root = File.join(File.dirname(__FILE__), 'testdata') - p "test root: #{test_root}" + GRPC.logger.info("test root: #{test_root}") files = ['ca.pem', 'server1.key', 'server1.pem'] creds = files.map { |f| File.open(File.join(test_root, f)).read } GRPC::Core::ServerCredentials.new( @@ -59,7 +59,7 @@ class SslTestService def a_client_streaming_rpc(call) check_peer_cert(call) - call.each_remote_read.each { |r| p r } + call.each_remote_read.each { |r| GRPC.logger.info(r) } EchoMsg.new end @@ -70,7 +70,7 @@ class SslTestService def a_bidi_rpc(requests, call) check_peer_cert(call) - requests.each { |r| p r } + requests.each { |r| GRPC.logger.info(r) } [EchoMsg.new, EchoMsg.new] end end @@ -116,11 +116,11 @@ describe 'client-server auth' do it 'client-server auth with server streaming RPCs' do responses = @stub.a_server_streaming_rpc(EchoMsg.new) - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end it 'client-server auth with bidi RPCs' do responses = @stub.a_bidi_rpc([EchoMsg.new, EchoMsg.new]) - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end end diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb index fc50362ca5..200139d849 100644 --- a/src/ruby/spec/generic/client_stub_spec.rb +++ b/src/ruby/spec/generic/client_stub_spec.rb @@ -293,7 +293,7 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength describe 'without a call operation' do def get_response(stub, credentials: nil) - puts credentials.inspect + GRPC.logger.info(credentials.inspect) stub.request_response(@method, @sent_msg, noop, noop, metadata: @metadata, credentials: credentials) @@ -342,13 +342,15 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength it 'sends metadata to the server ok when running start_call first' do run_op_view_metadata_test(true) check_op_view_of_finished_client_call( - @op, @server_initial_md, @server_trailing_md) { |r| p r } + @op, @server_initial_md, @server_trailing_md + ) { |r| GRPC.logger.info(r) } end it 'does not crash when used after the call has been finished' do run_op_view_metadata_test(false) check_op_view_of_finished_client_call( - @op, @server_initial_md, @server_trailing_md) { |r| p r } + @op, @server_initial_md, @server_trailing_md + ) { |r| GRPC.logger.info(r) } end end end @@ -435,13 +437,15 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength it 'sends metadata to the server ok when running start_call first' do run_op_view_metadata_test(true) check_op_view_of_finished_client_call( - @op, @server_initial_md, @server_trailing_md) { |r| p r } + @op, @server_initial_md, @server_trailing_md + ) { |r| GRPC.logger.info(r) } end it 'does not crash when used after the call has been finished' do run_op_view_metadata_test(false) check_op_view_of_finished_client_call( - @op, @server_initial_md, @server_trailing_md) { |r| p r } + @op, @server_initial_md, @server_trailing_md + ) { |r| GRPC.logger.info(r) } end end end @@ -578,7 +582,7 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength run_op_view_metadata_test(true) check_op_view_of_finished_client_call( @op, @server_initial_md, @server_trailing_md) do |responses| - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end end @@ -586,7 +590,7 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength run_op_view_metadata_test(false) check_op_view_of_finished_client_call( @op, @server_initial_md, @server_trailing_md) do |responses| - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end end @@ -895,7 +899,7 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength run_op_view_metadata_test(true) check_op_view_of_finished_client_call( @op, @server_initial_md, @server_trailing_md) do |responses| - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end end @@ -903,7 +907,7 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength run_op_view_metadata_test(false) check_op_view_of_finished_client_call( @op, @server_initial_md, @server_trailing_md) do |responses| - responses.each { |r| p r } + responses.each { |r| GRPC.logger.info(r) } end end diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e072d0c45f..44a6134086 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -125,7 +125,7 @@ class CheckCallAfterFinishedService fail 'shouldnt reuse service' unless @server_side_call.nil? @server_side_call = call # iterate through requests so call can complete - call.each_remote_read.each { |r| p r } + call.each_remote_read.each { |r| GRPC.logger.info(r) } EchoMsg.new end @@ -138,7 +138,7 @@ class CheckCallAfterFinishedService def a_bidi_rpc(requests, call) fail 'shouldnt reuse service' unless @server_side_call.nil? @server_side_call = call - requests.each { |r| p r } + requests.each { |r| GRPC.logger.info(r) } [EchoMsg.new, EchoMsg.new] end end @@ -560,7 +560,7 @@ describe GRPC::RpcServer do 'connect_k1' => 'connect_v1' } wanted_md.each do |key, value| - puts "key: #{key}" + GRPC.logger.info("key: #{key}") expect(op.metadata[key]).to eq(value) end @srv.stop diff --git a/src/ruby/spec/pb/codegen/package_option_spec.rb b/src/ruby/spec/pb/codegen/package_option_spec.rb new file mode 100644 index 0000000000..46d23cd651 --- /dev/null +++ b/src/ruby/spec/pb/codegen/package_option_spec.rb @@ -0,0 +1,53 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'spec_helper' +require 'open3' +require 'tmpdir' + +describe 'Code Generation Options' do + it 'should generate and respect package options' do + fail 'CONFIG env variable unexpectedly unset' unless ENV['CONFIG'] + bins_sub_dir = ENV['CONFIG'] + + src_dir = File.join(File.dirname(__FILE__), '..', '..', '..', '..') + pb_dir = File.join(src_dir, 'proto') + bins_dir = File.join(src_dir, '..', 'bins', bins_sub_dir) + + plugin = File.join(bins_dir, 'grpc_ruby_plugin') + protoc = File.join(bins_dir, 'protobuf', 'protoc') + + # Generate the service from the proto + Dir.mktmpdir(nil, File.dirname(__FILE__)) do |tmp_dir| + gen_file = system(protoc, + '-I.', + 'grpc/testing/package_options.proto', + "--grpc_out=#{tmp_dir}", # generate the service + "--ruby_out=#{tmp_dir}", # generate the definitions + "--plugin=protoc-gen-grpc=#{plugin}", + chdir: pb_dir, + out: File::NULL) + + expect(gen_file).to be_truthy + begin + $LOAD_PATH.push(tmp_dir) + expect { Grpc::Testing::Package::Options::TestService::Service }.to raise_error(NameError) + expect(require('grpc/testing/package_options_services_pb')).to be_truthy + expect { Grpc::Testing::Package::Options::TestService::Service }.to_not raise_error + ensure + $LOAD_PATH.delete(tmp_dir) + end + end + end +end diff --git a/src/ruby/spec/support/services.rb b/src/ruby/spec/support/services.rb index 27239cd66c..6e693f1cde 100644 --- a/src/ruby/spec/support/services.rb +++ b/src/ruby/spec/support/services.rb @@ -50,7 +50,9 @@ class EchoService def a_client_streaming_rpc(call) # iterate through requests so call can complete call.output_metadata.update(@trailing_metadata) - call.each_remote_read.each { |r| p r } + call.each_remote_read.each do |r| + GRPC.logger.info(r) + end EchoMsg.new end @@ -61,7 +63,9 @@ class EchoService def a_bidi_rpc(requests, call) call.output_metadata.update(@trailing_metadata) - requests.each { |r| p r } + requests.each do |r| + GRPC.logger.info(r) + end [EchoMsg.new, EchoMsg.new] end end @@ -71,35 +75,37 @@ EchoStub = EchoService.rpc_stub_class # For testing server interceptors class TestServerInterceptor < GRPC::ServerInterceptor def request_response(request:, call:, method:) - p "Received request/response call at method #{method}" \ - " with request #{request} for call #{call}" + GRPC.logger.info("Received request/response call at method #{method}" \ + " with request #{request} for call #{call}") call.output_metadata[:interc] = 'from_request_response' - p "[GRPC::Ok] (#{method.owner.name}.#{method.name})" + GRPC.logger.info("[GRPC::Ok] (#{method.owner.name}.#{method.name})") yield end def client_streamer(call:, method:) call.output_metadata[:interc] = 'from_client_streamer' call.each_remote_read.each do |r| - p "In interceptor: #{r}" + GRPC.logger.info("In interceptor: #{r}") end - p "Received client streamer call at method #{method} for call #{call}" + GRPC.logger.info( + "Received client streamer call at method #{method} for call #{call}" + ) yield end def server_streamer(request:, call:, method:) - p "Received server streamer call at method #{method} with request" \ - " #{request} for call #{call}" + GRPC.logger.info("Received server streamer call at method #{method} with request" \ + " #{request} for call #{call}") call.output_metadata[:interc] = 'from_server_streamer' yield end def bidi_streamer(requests:, call:, method:) requests.each do |r| - p "Bidi request: #{r}" + GRPC.logger.info("Bidi request: #{r}") end - p "Received bidi streamer call at method #{method} with requests" \ - " #{requests} for call #{call}" + GRPC.logger.info("Received bidi streamer call at method #{method} with requests" \ + " #{requests} for call #{call}") call.output_metadata[:interc] = 'from_bidi_streamer' yield end @@ -108,38 +114,38 @@ end # For testing client interceptors class TestClientInterceptor < GRPC::ClientInterceptor def request_response(request:, call:, method:, metadata: {}) - p "Intercepted request/response call at method #{method}" \ + GRPC.logger.info("Intercepted request/response call at method #{method}" \ " with request #{request} for call #{call}" \ - " and metadata: #{metadata}" + " and metadata: #{metadata}") metadata['foo'] = 'bar_from_request_response' yield end def client_streamer(requests:, call:, method:, metadata: {}) - p "Received client streamer call at method #{method}" \ + GRPC.logger.info("Received client streamer call at method #{method}" \ " with requests #{requests} for call #{call}" \ - " and metadata: #{metadata}" + " and metadata: #{metadata}") requests.each do |r| - p "In client interceptor: #{r}" + GRPC.logger.info("In client interceptor: #{r}") end metadata['foo'] = 'bar_from_client_streamer' yield end def server_streamer(request:, call:, method:, metadata: {}) - p "Received server streamer call at method #{method}" \ + GRPC.logger.info("Received server streamer call at method #{method}" \ " with request #{request} for call #{call}" \ - " and metadata: #{metadata}" + " and metadata: #{metadata}") metadata['foo'] = 'bar_from_server_streamer' yield end def bidi_streamer(requests:, call:, method:, metadata: {}) - p "Received bidi streamer call at method #{method}" \ + GRPC.logger.info("Received bidi streamer call at method #{method}" \ "with requests #{requests} for call #{call}" \ - " and metadata: #{metadata}" + " and metadata: #{metadata}") requests.each do |r| - p "In client interceptor: #{r}" + GRPC.logger.info("In client interceptor: #{r}") end metadata['foo'] = 'bar_from_bidi_streamer' yield diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index f912154301..98b6344a4b 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -174,7 +174,7 @@ ss.header_mappings_dir = '.' ss.libraries = 'z' ss.dependency "#{s.name}/Interface", version - ss.dependency 'BoringSSL-GRPC', '0.0.1' + ss.dependency 'BoringSSL-GRPC', '0.0.2' ss.dependency 'nanopb', '~> 0.3' ss.compiler_flags = '-DGRPC_SHADOW_BORINGSSL_SYMBOLS' @@ -223,5 +223,6 @@ s.prepare_command = <<-END_OF_COMMAND find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include <nanopb/\\1>;g' find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g' END_OF_COMMAND end diff --git a/templates/src/objective-c/BoringSSL-GRPC.podspec.template b/templates/src/objective-c/BoringSSL-GRPC.podspec.template index 986216fdbf..2b3bb8d97a 100644 --- a/templates/src/objective-c/BoringSSL-GRPC.podspec.template +++ b/templates/src/objective-c/BoringSSL-GRPC.podspec.template @@ -43,7 +43,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL-GRPC' - version = '0.0.1' + version = '0.0.2' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google\'s needs.' # Adapted from the homepage: @@ -86,7 +86,7 @@ s.ios.deployment_target = '5.0' s.osx.deployment_target = '10.7' - name = 'openssl' + name = 'openssl_grpc' # When creating a dynamic framework, name it openssl.framework instead of BoringSSL.framework. # This lets users write their includes like `#include <openssl/ssl.h>` as opposed to `#include @@ -1551,6 +1551,7 @@ sed -i'.back' '/^#define \\([A-Za-z0-9_]*\\) \\1/d' include/openssl/ssl.h sed -i'.back' 'N;/^#define \\([A-Za-z0-9_]*\\) *\\\\\\n *\\1/d' include/openssl/ssl.h sed -i'.back' 's/#ifndef md5_block_data_order/#ifndef GRPC_SHADOW_md5_block_data_order/g' crypto/fipsmodule/md5/md5.c + find . -type f \\( -path '*.h' -or -path '*.cc' -or -path '*.c' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include <openssl/;#include <openssl_grpc/;g' END_OF_COMMAND # Redefine symbols to avoid conflict when the same app also depends on OpenSSL. The list of diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template index 4fb7b4642d..34ea645cfa 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM google/dart:latest + FROM google/dart:2.0 # Upgrade Dart to version 2. RUN apt-get update && apt-get upgrade -y dart diff --git a/test/cpp/util/.clang-tidy b/test/.clang-tidy index f951ee5fea..f951ee5fea 100644 --- a/test/cpp/util/.clang-tidy +++ b/test/.clang-tidy diff --git a/test/core/channel/channel_stack_test.cc b/test/core/channel/channel_stack_test.cc index 2f5329a96d..6f0bfa06d2 100644 --- a/test/core/channel/channel_stack_test.cc +++ b/test/core/channel/channel_stack_test.cc @@ -124,7 +124,7 @@ static void test_create_channel_stack(void) { gpr_now(GPR_CLOCK_MONOTONIC), /* start_time */ GRPC_MILLIS_INF_FUTURE, /* deadline */ nullptr, /* arena */ - nullptr /* call_combiner */ + nullptr, /* call_combiner */ }; grpc_error* error = grpc_call_stack_init(channel_stack, 1, free_call, call_stack, &args); diff --git a/test/core/channel/channel_trace_test.cc b/test/core/channel/channel_trace_test.cc index f224457a55..d594445bfb 100644 --- a/test/core/channel/channel_trace_test.cc +++ b/test/core/channel/channel_trace_test.cc @@ -40,6 +40,17 @@ namespace grpc_core { namespace channelz { namespace testing { + +// testing peer to access channel internals +class ChannelNodePeer { + public: + explicit ChannelNodePeer(ChannelNode* node) : node_(node) {} + ChannelTrace* trace() const { return &node_->trace_; } + + private: + ChannelNode* node_; +}; + namespace { grpc_json* GetJsonChild(grpc_json* parent, const char* key) { @@ -156,28 +167,29 @@ TEST_P(ChannelTracerTest, ComplexTest) { ChannelFixture channel1(GetParam()); RefCountedPtr<ChannelNode> sc1 = MakeRefCounted<ChannelNode>(channel1.channel(), GetParam(), true); - tracer.AddTraceEventReferencingSubchannel( + ChannelNodePeer sc1_peer(sc1.get()); + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Info, grpc_slice_from_static_string("subchannel one created"), sc1); ValidateChannelTrace(&tracer, 3, GetParam()); - AddSimpleTrace(sc1->trace()); - AddSimpleTrace(sc1->trace()); - AddSimpleTrace(sc1->trace()); - ValidateChannelTrace(sc1->trace(), 3, GetParam()); - AddSimpleTrace(sc1->trace()); - AddSimpleTrace(sc1->trace()); - AddSimpleTrace(sc1->trace()); - ValidateChannelTrace(sc1->trace(), 6, GetParam()); + AddSimpleTrace(sc1_peer.trace()); + AddSimpleTrace(sc1_peer.trace()); + AddSimpleTrace(sc1_peer.trace()); + ValidateChannelTrace(sc1_peer.trace(), 3, GetParam()); + AddSimpleTrace(sc1_peer.trace()); + AddSimpleTrace(sc1_peer.trace()); + AddSimpleTrace(sc1_peer.trace()); + ValidateChannelTrace(sc1_peer.trace(), 6, GetParam()); AddSimpleTrace(&tracer); AddSimpleTrace(&tracer); ValidateChannelTrace(&tracer, 5, GetParam()); ChannelFixture channel2(GetParam()); RefCountedPtr<ChannelNode> sc2 = MakeRefCounted<ChannelNode>(channel2.channel(), GetParam(), true); - tracer.AddTraceEventReferencingChannel( + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Info, grpc_slice_from_static_string("LB channel two created"), sc2); - tracer.AddTraceEventReferencingSubchannel( + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Warning, grpc_slice_from_static_string("subchannel one inactive"), sc1); ValidateChannelTrace(&tracer, 7, GetParam()); @@ -203,33 +215,35 @@ TEST_P(ChannelTracerTest, TestNesting) { ChannelFixture channel1(GetParam()); RefCountedPtr<ChannelNode> sc1 = MakeRefCounted<ChannelNode>(channel1.channel(), GetParam(), true); - tracer.AddTraceEventReferencingChannel( + ChannelNodePeer sc1_peer(sc1.get()); + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Info, grpc_slice_from_static_string("subchannel one created"), sc1); ValidateChannelTrace(&tracer, 3, GetParam()); - AddSimpleTrace(sc1->trace()); + AddSimpleTrace(sc1_peer.trace()); ChannelFixture channel2(GetParam()); RefCountedPtr<ChannelNode> conn1 = MakeRefCounted<ChannelNode>(channel2.channel(), GetParam(), true); + ChannelNodePeer conn1_peer(conn1.get()); // nesting one level deeper. - sc1->trace()->AddTraceEventReferencingSubchannel( + sc1_peer.trace()->AddTraceEventWithReference( ChannelTrace::Severity::Info, grpc_slice_from_static_string("connection one created"), conn1); ValidateChannelTrace(&tracer, 3, GetParam()); - AddSimpleTrace(conn1->trace()); + AddSimpleTrace(conn1_peer.trace()); AddSimpleTrace(&tracer); AddSimpleTrace(&tracer); ValidateChannelTrace(&tracer, 5, GetParam()); - ValidateChannelTrace(conn1->trace(), 1, GetParam()); + ValidateChannelTrace(conn1_peer.trace(), 1, GetParam()); ChannelFixture channel3(GetParam()); RefCountedPtr<ChannelNode> sc2 = MakeRefCounted<ChannelNode>(channel3.channel(), GetParam(), true); - tracer.AddTraceEventReferencingSubchannel( + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Info, grpc_slice_from_static_string("subchannel two created"), sc2); // this trace should not get added to the parents children since it is already // present in the tracer. - tracer.AddTraceEventReferencingChannel( + tracer.AddTraceEventWithReference( ChannelTrace::Severity::Warning, grpc_slice_from_static_string("subchannel one inactive"), sc1); AddSimpleTrace(&tracer); diff --git a/test/core/channel/channelz_registry_test.cc b/test/core/channel/channelz_registry_test.cc index 581e867584..c02d525c81 100644 --- a/test/core/channel/channelz_registry_test.cc +++ b/test/core/channel/channelz_registry_test.cc @@ -44,22 +44,22 @@ namespace channelz { namespace testing { TEST(ChannelzRegistryTest, UuidStartsAboveZeroTest) { - ChannelNode* channelz_channel = nullptr; - intptr_t uuid = ChannelzRegistry::RegisterChannelNode(channelz_channel); + BaseNode* channelz_channel = nullptr; + intptr_t uuid = ChannelzRegistry::Register(channelz_channel); EXPECT_GT(uuid, 0) << "First uuid chose must be greater than zero. Zero if " "reserved according to " "https://github.com/grpc/proposal/blob/master/" "A14-channelz.md"; - ChannelzRegistry::UnregisterChannelNode(uuid); + ChannelzRegistry::Unregister(uuid); } TEST(ChannelzRegistryTest, UuidsAreIncreasing) { - ChannelNode* channelz_channel = nullptr; + BaseNode* channelz_channel = nullptr; std::vector<intptr_t> uuids; uuids.reserve(10); for (int i = 0; i < 10; ++i) { // reregister the same object. It's ok since we are just testing uuids - uuids.push_back(ChannelzRegistry::RegisterChannelNode(channelz_channel)); + uuids.push_back(ChannelzRegistry::Register(channelz_channel)); } for (size_t i = 1; i < uuids.size(); ++i) { EXPECT_LT(uuids[i - 1], uuids[i]) << "Uuids must always be increasing"; @@ -68,30 +68,30 @@ TEST(ChannelzRegistryTest, UuidsAreIncreasing) { TEST(ChannelzRegistryTest, RegisterGetTest) { // we hackily jam an intptr_t into this pointer to check for equality later - ChannelNode* channelz_channel = (ChannelNode*)42; - intptr_t uuid = ChannelzRegistry::RegisterChannelNode(channelz_channel); - ChannelNode* retrieved = ChannelzRegistry::GetChannelNode(uuid); + BaseNode* channelz_channel = (BaseNode*)42; + intptr_t uuid = ChannelzRegistry::Register(channelz_channel); + BaseNode* retrieved = ChannelzRegistry::Get(uuid); EXPECT_EQ(channelz_channel, retrieved); } TEST(ChannelzRegistryTest, RegisterManyItems) { // we hackily jam an intptr_t into this pointer to check for equality later - ChannelNode* channelz_channel = (ChannelNode*)42; + BaseNode* channelz_channel = (BaseNode*)42; for (int i = 0; i < 100; i++) { - intptr_t uuid = ChannelzRegistry::RegisterChannelNode(channelz_channel); - ChannelNode* retrieved = ChannelzRegistry::GetChannelNode(uuid); + intptr_t uuid = ChannelzRegistry::Register(channelz_channel); + BaseNode* retrieved = ChannelzRegistry::Get(uuid); EXPECT_EQ(channelz_channel, retrieved); } } TEST(ChannelzRegistryTest, NullIfNotPresentTest) { // we hackily jam an intptr_t into this pointer to check for equality later - ChannelNode* channelz_channel = (ChannelNode*)42; - intptr_t uuid = ChannelzRegistry::RegisterChannelNode(channelz_channel); + BaseNode* channelz_channel = (BaseNode*)42; + intptr_t uuid = ChannelzRegistry::Register(channelz_channel); // try to pull out a uuid that does not exist. - ChannelNode* nonexistant = ChannelzRegistry::GetChannelNode(uuid + 1); + BaseNode* nonexistant = ChannelzRegistry::Get(uuid + 1); EXPECT_EQ(nonexistant, nullptr); - ChannelNode* retrieved = ChannelzRegistry::GetChannelNode(uuid); + BaseNode* retrieved = ChannelzRegistry::Get(uuid); EXPECT_EQ(channelz_channel, retrieved); } diff --git a/test/core/channel/channelz_test.cc b/test/core/channel/channelz_test.cc index ad5f86d934..bcda30d9f2 100644 --- a/test/core/channel/channelz_test.cc +++ b/test/core/channel/channelz_test.cc @@ -31,6 +31,7 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/json/json.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/server.h" #include "test/core/util/test_config.h" #include "test/cpp/util/channel_trace_proto_helper.h" @@ -44,16 +45,16 @@ namespace channelz { namespace testing { // testing peer to access channel internals -class ChannelNodePeer { +class CallCountingHelperPeer { public: - ChannelNodePeer(ChannelNode* channel) : channel_(channel) {} - grpc_millis last_call_started_millis() { + explicit CallCountingHelperPeer(CallCountingHelper* node) : node_(node) {} + grpc_millis last_call_started_millis() const { return (grpc_millis)gpr_atm_no_barrier_load( - &channel_->last_call_started_millis_); + &node_->last_call_started_millis_); } private: - ChannelNode* channel_; + CallCountingHelper* node_; }; namespace { @@ -102,6 +103,25 @@ void ValidateGetTopChannels(size_t expected_channels) { gpr_free(core_api_json_str); } +void ValidateGetServers(size_t expected_servers) { + char* json_str = ChannelzRegistry::GetServers(0); + grpc::testing::ValidateGetServersResponseProtoJsonTranslation(json_str); + grpc_json* parsed_json = grpc_json_parse_string(json_str); + // This check will naturally have to change when we support pagination. + // tracked: https://github.com/grpc/grpc/issues/16019. + ValidateJsonArraySize(parsed_json, "server", expected_servers); + grpc_json* end = GetJsonChild(parsed_json, "end"); + ASSERT_NE(end, nullptr); + EXPECT_EQ(end->type, GRPC_JSON_TRUE); + grpc_json_destroy(parsed_json); + gpr_free(json_str); + // also check that the core API formats this correctly + char* core_api_json_str = grpc_channelz_get_servers(0); + grpc::testing::ValidateGetServersResponseProtoJsonTranslation( + core_api_json_str); + gpr_free(core_api_json_str); +} + class ChannelFixture { public: ChannelFixture(int max_trace_nodes = 0) { @@ -124,6 +144,28 @@ class ChannelFixture { grpc_channel* channel_; }; +class ServerFixture { + public: + explicit ServerFixture(int max_trace_nodes = 0) { + grpc_arg server_a[] = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE), + max_trace_nodes), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ), true), + }; + grpc_channel_args server_args = {GPR_ARRAY_SIZE(server_a), server_a}; + server_ = grpc_server_create(&server_args, nullptr); + } + + ~ServerFixture() { grpc_server_destroy(server_); } + + grpc_server* server() const { return server_; } + + private: + grpc_server* server_; +}; + struct validate_channel_data_args { int64_t calls_started; int64_t calls_failed; @@ -157,14 +199,21 @@ void ValidateChannel(ChannelNode* channel, validate_channel_data_args args) { ValidateCounters(json_str, args); gpr_free(json_str); // also check that the core API formats this the correct way - char* core_api_json_str = grpc_channelz_get_channel(channel->channel_uuid()); + char* core_api_json_str = grpc_channelz_get_channel(channel->uuid()); grpc::testing::ValidateGetChannelResponseProtoJsonTranslation( core_api_json_str); gpr_free(core_api_json_str); } -grpc_millis GetLastCallStartedMillis(ChannelNode* channel) { - ChannelNodePeer peer(channel); +void ValidateServer(ServerNode* server, validate_channel_data_args args) { + char* json_str = server->RenderJsonString(); + grpc::testing::ValidateServerProtoJsonTranslation(json_str); + ValidateCounters(json_str, args); + gpr_free(json_str); +} + +grpc_millis GetLastCallStartedMillis(CallCountingHelper* channel) { + CallCountingHelperPeer peer(channel); return peer.last_call_started_millis(); } @@ -215,31 +264,29 @@ TEST_P(ChannelzChannelTest, BasicChannelAPIFunctionality) { TEST_P(ChannelzChannelTest, LastCallStartedMillis) { grpc_core::ExecCtx exec_ctx; - ChannelFixture channel(GetParam()); - ChannelNode* channelz_channel = - grpc_channel_get_channelz_node(channel.channel()); + CallCountingHelper counter; // start a call to set the last call started timestamp - channelz_channel->RecordCallStarted(); - grpc_millis millis1 = GetLastCallStartedMillis(channelz_channel); + counter.RecordCallStarted(); + grpc_millis millis1 = GetLastCallStartedMillis(&counter); // time gone by should not affect the timestamp ChannelzSleep(100); - grpc_millis millis2 = GetLastCallStartedMillis(channelz_channel); + grpc_millis millis2 = GetLastCallStartedMillis(&counter); EXPECT_EQ(millis1, millis2); // calls succeeded or failed should not affect the timestamp ChannelzSleep(100); - channelz_channel->RecordCallFailed(); - channelz_channel->RecordCallSucceeded(); - grpc_millis millis3 = GetLastCallStartedMillis(channelz_channel); + counter.RecordCallFailed(); + counter.RecordCallSucceeded(); + grpc_millis millis3 = GetLastCallStartedMillis(&counter); EXPECT_EQ(millis1, millis3); // another call started should affect the timestamp // sleep for extra long to avoid flakes (since we cache Now()) ChannelzSleep(5000); - channelz_channel->RecordCallStarted(); - grpc_millis millis4 = GetLastCallStartedMillis(channelz_channel); + counter.RecordCallStarted(); + grpc_millis millis4 = GetLastCallStartedMillis(&counter); EXPECT_NE(millis1, millis4); } -TEST(ChannelzGetTopChannelsTest, BasicTest) { +TEST(ChannelzGetTopChannelsTest, BasicGetTopChannelsTest) { grpc_core::ExecCtx exec_ctx; ChannelFixture channel; ValidateGetTopChannels(1); @@ -275,9 +322,49 @@ TEST(ChannelzGetTopChannelsTest, InternalChannelTest) { grpc_channel_destroy(internal_channel); } +class ChannelzServerTest : public ::testing::TestWithParam<size_t> {}; + +TEST_P(ChannelzServerTest, BasicServerAPIFunctionality) { + grpc_core::ExecCtx exec_ctx; + ServerFixture server(10); + ServerNode* channelz_server = grpc_server_get_channelz_node(server.server()); + channelz_server->RecordCallStarted(); + channelz_server->RecordCallFailed(); + channelz_server->RecordCallSucceeded(); + ValidateServer(channelz_server, {1, 1, 1}); + channelz_server->RecordCallStarted(); + channelz_server->RecordCallFailed(); + channelz_server->RecordCallSucceeded(); + channelz_server->RecordCallStarted(); + channelz_server->RecordCallFailed(); + channelz_server->RecordCallSucceeded(); + ValidateServer(channelz_server, {3, 3, 3}); +} + +TEST(ChannelzGetServersTest, BasicGetServersTest) { + grpc_core::ExecCtx exec_ctx; + ServerFixture server; + ValidateGetServers(1); +} + +TEST(ChannelzGetServersTest, NoServersTest) { + grpc_core::ExecCtx exec_ctx; + ValidateGetServers(0); +} + +TEST(ChannelzGetServersTest, ManyServersTest) { + grpc_core::ExecCtx exec_ctx; + ServerFixture servers[10]; + (void)servers; // suppress unused variable error + ValidateGetServers(10); +} + INSTANTIATE_TEST_CASE_P(ChannelzChannelTestSweep, ChannelzChannelTest, ::testing::Values(0, 1, 2, 6, 10, 15)); +INSTANTIATE_TEST_CASE_P(ChannelzServerTestSweep, ChannelzServerTest, + ::testing::Values(0, 1, 2, 6, 10, 15)); + } // namespace testing } // namespace channelz } // namespace grpc_core diff --git a/test/core/client_channel/parse_address_test.cc b/test/core/client_channel/parse_address_test.cc index ae157fbb8b..004549fa62 100644 --- a/test/core/client_channel/parse_address_test.cc +++ b/test/core/client_channel/parse_address_test.cc @@ -91,6 +91,15 @@ static void test_grpc_parse_ipv6(const char* uri_text, const char* host, grpc_uri_destroy(uri); } +/* Test parsing invalid ipv6 addresses (valid uri_text but invalid ipv6 addr) */ +static void test_grpc_parse_ipv6_invalid(const char* uri_text) { + grpc_core::ExecCtx exec_ctx; + grpc_uri* uri = grpc_uri_parse(uri_text, 0); + grpc_resolved_address addr; + GPR_ASSERT(!grpc_parse_ipv6(uri, &addr)); + grpc_uri_destroy(uri); +} + int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); @@ -100,5 +109,10 @@ int main(int argc, char** argv) { test_grpc_parse_ipv6("ipv6:[2001:db8::1]:12345", "2001:db8::1", 12345, 0); test_grpc_parse_ipv6("ipv6:[2001:db8::1%252]:12345", "2001:db8::1", 12345, 2); + /* Address length greater than GRPC_INET6_ADDRSTRLEN */ + test_grpc_parse_ipv6_invalid( + "ipv6:WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW45%" + "v6:45%x$1*"); + grpc_shutdown(); } diff --git a/test/core/end2end/fixtures/http_proxy_fixture.cc b/test/core/end2end/fixtures/http_proxy_fixture.cc index ea9c000efb..b235c10165 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.cc +++ b/test/core/end2end/fixtures/http_proxy_fixture.cc @@ -52,6 +52,16 @@ #include "test/core/util/port.h" struct grpc_end2end_http_proxy { + grpc_end2end_http_proxy() + : proxy_name(nullptr), + server(nullptr), + channel_args(nullptr), + mu(nullptr), + pollset(nullptr), + combiner(nullptr) { + gpr_ref_init(&users, 1); + combiner = grpc_combiner_create(); + } char* proxy_name; grpc_core::Thread thd; grpc_tcp_server* server; @@ -519,11 +529,7 @@ static void thread_main(void* arg) { grpc_end2end_http_proxy* grpc_end2end_http_proxy_create( grpc_channel_args* args) { grpc_core::ExecCtx exec_ctx; - grpc_end2end_http_proxy* proxy = - static_cast<grpc_end2end_http_proxy*>(gpr_malloc(sizeof(*proxy))); - memset(proxy, 0, sizeof(*proxy)); - proxy->combiner = grpc_combiner_create(); - gpr_ref_init(&proxy->users, 1); + grpc_end2end_http_proxy* proxy = grpc_core::New<grpc_end2end_http_proxy>(); // Construct proxy address. const int proxy_port = grpc_pick_unused_port_or_die(); gpr_join_host_port(&proxy->proxy_name, "localhost", proxy_port); @@ -573,7 +579,7 @@ void grpc_end2end_http_proxy_destroy(grpc_end2end_http_proxy* proxy) { GRPC_CLOSURE_CREATE(destroy_pollset, proxy->pollset, grpc_schedule_on_exec_ctx)); GRPC_COMBINER_UNREF(proxy->combiner, "test"); - gpr_free(proxy); + grpc_core::Delete(proxy); } const char* grpc_end2end_http_proxy_get_proxy_name( diff --git a/test/core/end2end/fixtures/proxy.cc b/test/core/end2end/fixtures/proxy.cc index 042c858b4c..869b6e846d 100644 --- a/test/core/end2end/fixtures/proxy.cc +++ b/test/core/end2end/fixtures/proxy.cc @@ -30,6 +30,17 @@ #include "test/core/util/port.h" struct grpc_end2end_proxy { + grpc_end2end_proxy() + : proxy_port(nullptr), + server_port(nullptr), + cq(nullptr), + server(nullptr), + client(nullptr), + shutdown(false), + new_call(nullptr) { + memset(&new_call_details, 0, sizeof(new_call_details)); + memset(&new_call_metadata, 0, sizeof(new_call_metadata)); + } grpc_core::Thread thd; char* proxy_port; char* server_port; @@ -79,9 +90,7 @@ grpc_end2end_proxy* grpc_end2end_proxy_create(const grpc_end2end_proxy_def* def, int proxy_port = grpc_pick_unused_port_or_die(); int server_port = grpc_pick_unused_port_or_die(); - grpc_end2end_proxy* proxy = - static_cast<grpc_end2end_proxy*>(gpr_malloc(sizeof(*proxy))); - memset(proxy, 0, sizeof(*proxy)); + grpc_end2end_proxy* proxy = grpc_core::New<grpc_end2end_proxy>(); gpr_join_host_port(&proxy->proxy_port, "localhost", proxy_port); gpr_join_host_port(&proxy->server_port, "localhost", server_port); @@ -128,7 +137,7 @@ void grpc_end2end_proxy_destroy(grpc_end2end_proxy* proxy) { grpc_channel_destroy(proxy->client); grpc_completion_queue_destroy(proxy->cq); grpc_call_details_destroy(&proxy->new_call_details); - gpr_free(proxy); + grpc_core::Delete(proxy); } static void unrefpc(proxy_call* pc, const char* reason) { diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index db2886d8e3..6d9ffcfb91 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -13,277 +13,347 @@ # See the License for the specific language governing permissions and # limitations under the License. -POLLERS = ['epollex', 'epollsig', 'epoll1', 'poll', 'poll-cv'] - -load("//bazel:grpc_build_system.bzl", "grpc_sh_test", "grpc_cc_binary", "grpc_cc_library") - """Generates the appropriate build.json data for all the end2end tests.""" +load("//bazel:grpc_build_system.bzl", "grpc_cc_binary", "grpc_cc_library") -def fixture_options(fullstack=True, includes_proxy=False, dns_resolver=True, - name_resolution=True, secure=True, tracing=False, - platforms=['windows', 'linux', 'mac', 'posix'], - is_inproc=False, is_http2=True, supports_proxy_auth=False, - supports_write_buffering=True, client_channel=True): - return struct( - fullstack=fullstack, - includes_proxy=includes_proxy, - dns_resolver=dns_resolver, - name_resolution=name_resolution, - secure=secure, - tracing=tracing, - is_inproc=is_inproc, - is_http2=is_http2, - supports_proxy_auth=supports_proxy_auth, - supports_write_buffering=supports_write_buffering, - client_channel=client_channel, - #platforms=platforms, - ) +POLLERS = ["epollex", "epoll1", "poll", "poll-cv"] +def _fixture_options( + fullstack = True, + includes_proxy = False, + dns_resolver = True, + name_resolution = True, + secure = True, + tracing = False, + _platforms = ["windows", "linux", "mac", "posix"], + is_inproc = False, + is_http2 = True, + supports_proxy_auth = False, + supports_write_buffering = True, + client_channel = True): + return struct( + fullstack = fullstack, + includes_proxy = includes_proxy, + dns_resolver = dns_resolver, + name_resolution = name_resolution, + secure = secure, + tracing = tracing, + is_inproc = is_inproc, + is_http2 = is_http2, + supports_proxy_auth = supports_proxy_auth, + supports_write_buffering = supports_write_buffering, + client_channel = client_channel, + #_platforms=_platforms, + ) # maps fixture name to whether it requires the security library END2END_FIXTURES = { - 'h2_compress': fixture_options(), - 'h2_census': fixture_options(), + "h2_compress": _fixture_options(), + "h2_census": _fixture_options(), # TODO(juanlishen): This is disabled for now, but should be considered to re-enable once we have # decided how the load reporting service should be enabled. - #'h2_load_reporting': fixture_options(), - 'h2_fakesec': fixture_options(), - 'h2_fd': fixture_options(dns_resolver=False, fullstack=False, - client_channel=False, - platforms=['linux', 'mac', 'posix']), - 'h2_full': fixture_options(), - 'h2_full+pipe': fixture_options(platforms=['linux']), - 'h2_full+trace': fixture_options(tracing=True), - 'h2_full+workarounds': fixture_options(), - 'h2_http_proxy': fixture_options(supports_proxy_auth=True), - 'h2_oauth2': fixture_options(), - 'h2_proxy': fixture_options(includes_proxy=True), - 'h2_sockpair_1byte': fixture_options(fullstack=False, dns_resolver=False, - client_channel=False), - 'h2_sockpair': fixture_options(fullstack=False, dns_resolver=False, - client_channel=False), - 'h2_sockpair+trace': fixture_options(fullstack=False, dns_resolver=False, - tracing=True, client_channel=False), - 'h2_ssl': fixture_options(secure=True), - 'h2_local': fixture_options(secure=True, dns_resolver=False, platforms=['linux', 'mac', 'posix']), - 'h2_ssl_proxy': fixture_options(includes_proxy=True, secure=True), - 'h2_uds': fixture_options(dns_resolver=False, - platforms=['linux', 'mac', 'posix']), - 'inproc': fixture_options(fullstack=False, dns_resolver=False, - name_resolution=False, is_inproc=True, - is_http2=False, supports_write_buffering=False, - client_channel=False), + #'h2_load_reporting': _fixture_options(), + "h2_fakesec": _fixture_options(), + "h2_fd": _fixture_options( + dns_resolver = False, + fullstack = False, + client_channel = False, + _platforms = ["linux", "mac", "posix"], + ), + "h2_full": _fixture_options(), + "h2_full+pipe": _fixture_options(_platforms = ["linux"]), + "h2_full+trace": _fixture_options(tracing = True), + "h2_full+workarounds": _fixture_options(), + "h2_http_proxy": _fixture_options(supports_proxy_auth = True), + "h2_oauth2": _fixture_options(), + "h2_proxy": _fixture_options(includes_proxy = True), + "h2_sockpair_1byte": _fixture_options( + fullstack = False, + dns_resolver = False, + client_channel = False, + ), + "h2_sockpair": _fixture_options( + fullstack = False, + dns_resolver = False, + client_channel = False, + ), + "h2_sockpair+trace": _fixture_options( + fullstack = False, + dns_resolver = False, + tracing = True, + client_channel = False, + ), + "h2_ssl": _fixture_options(secure = True), + "h2_local": _fixture_options(secure = True, dns_resolver = False, _platforms = ["linux", "mac", "posix"]), + "h2_ssl_proxy": _fixture_options(includes_proxy = True, secure = True), + "h2_uds": _fixture_options( + dns_resolver = False, + _platforms = ["linux", "mac", "posix"], + ), + "inproc": _fixture_options( + fullstack = False, + dns_resolver = False, + name_resolution = False, + is_inproc = True, + is_http2 = False, + supports_write_buffering = False, + client_channel = False, + ), } - -def test_options(needs_fullstack=False, needs_dns=False, needs_names=False, - proxyable=True, secure=False, traceable=False, - exclude_inproc=False, needs_http2=False, - needs_proxy_auth=False, needs_write_buffering=False, - needs_client_channel=False): - return struct( - needs_fullstack=needs_fullstack, - needs_dns=needs_dns, - needs_names=needs_names, - proxyable=proxyable, - secure=secure, - traceable=traceable, - exclude_inproc=exclude_inproc, - needs_http2=needs_http2, - needs_proxy_auth=needs_proxy_auth, - needs_write_buffering=needs_write_buffering, - needs_client_channel=needs_client_channel, - ) - +def _test_options( + needs_fullstack = False, + needs_dns = False, + needs_names = False, + proxyable = True, + secure = False, + traceable = False, + exclude_inproc = False, + needs_http2 = False, + needs_proxy_auth = False, + needs_write_buffering = False, + needs_client_channel = False): + return struct( + needs_fullstack = needs_fullstack, + needs_dns = needs_dns, + needs_names = needs_names, + proxyable = proxyable, + secure = secure, + traceable = traceable, + exclude_inproc = exclude_inproc, + needs_http2 = needs_http2, + needs_proxy_auth = needs_proxy_auth, + needs_write_buffering = needs_write_buffering, + needs_client_channel = needs_client_channel, + ) # maps test names to options END2END_TESTS = { - 'bad_hostname': test_options(needs_names=True), - 'bad_ping': test_options(needs_fullstack=True,proxyable=False), - 'binary_metadata': test_options(), - 'resource_quota_server': test_options(proxyable=False), - 'call_creds': test_options(secure=True), - 'call_host_override': test_options(needs_fullstack=True, needs_dns=True, - needs_names=True), - 'cancel_after_accept': test_options(), - 'cancel_after_client_done': test_options(), - 'cancel_after_invoke': test_options(), - 'cancel_after_round_trip': test_options(), - 'cancel_before_invoke': test_options(), - 'cancel_in_a_vacuum': test_options(), - 'cancel_with_status': test_options(), - 'compressed_payload': test_options(proxyable=False, exclude_inproc=True), - 'connectivity': test_options(needs_fullstack=True, needs_names=True, - proxyable=False), - 'channelz': test_options(), - 'default_host': test_options(needs_fullstack=True, needs_dns=True, - needs_names=True), - 'disappearing_server': test_options(needs_fullstack=True,needs_names=True), - 'empty_batch': test_options(), - 'filter_causes_close': test_options(), - 'filter_call_init_fails': test_options(), - 'graceful_server_shutdown': test_options(exclude_inproc=True), - 'hpack_size': test_options(proxyable=False, traceable=False, - exclude_inproc=True), - 'high_initial_seqno': test_options(), - 'idempotent_request': test_options(), - 'invoke_large_request': test_options(), - 'keepalive_timeout': test_options(proxyable=False, needs_http2=True), - 'large_metadata': test_options(), - 'max_concurrent_streams': test_options(proxyable=False, - exclude_inproc=True), - 'max_connection_age': test_options(exclude_inproc=True), - 'max_connection_idle': test_options(needs_fullstack=True, proxyable=False), - 'max_message_length': test_options(), - 'negative_deadline': test_options(), - 'network_status_change': test_options(), - 'no_error_on_hotpath': test_options(proxyable=False), - 'no_logging': test_options(traceable=False), - 'no_op': test_options(), - 'payload': test_options(), + "bad_hostname": _test_options(needs_names = True), + "bad_ping": _test_options(needs_fullstack = True, proxyable = False), + "binary_metadata": _test_options(), + "resource_quota_server": _test_options(proxyable = False), + "call_creds": _test_options(secure = True), + "call_host_override": _test_options( + needs_fullstack = True, + needs_dns = True, + needs_names = True, + ), + "cancel_after_accept": _test_options(), + "cancel_after_client_done": _test_options(), + "cancel_after_invoke": _test_options(), + "cancel_after_round_trip": _test_options(), + "cancel_before_invoke": _test_options(), + "cancel_in_a_vacuum": _test_options(), + "cancel_with_status": _test_options(), + "compressed_payload": _test_options(proxyable = False, exclude_inproc = True), + "connectivity": _test_options( + needs_fullstack = True, + needs_names = True, + proxyable = False, + ), + "channelz": _test_options(), + "default_host": _test_options( + needs_fullstack = True, + needs_dns = True, + needs_names = True, + ), + "disappearing_server": _test_options(needs_fullstack = True, needs_names = True), + "empty_batch": _test_options(), + "filter_causes_close": _test_options(), + "filter_call_init_fails": _test_options(), + "graceful_server_shutdown": _test_options(exclude_inproc = True), + "hpack_size": _test_options( + proxyable = False, + traceable = False, + exclude_inproc = True, + ), + "high_initial_seqno": _test_options(), + "idempotent_request": _test_options(), + "invoke_large_request": _test_options(), + "keepalive_timeout": _test_options(proxyable = False, needs_http2 = True), + "large_metadata": _test_options(), + "max_concurrent_streams": _test_options( + proxyable = False, + exclude_inproc = True, + ), + "max_connection_age": _test_options(exclude_inproc = True), + "max_connection_idle": _test_options(needs_fullstack = True, proxyable = False), + "max_message_length": _test_options(), + "negative_deadline": _test_options(), + "network_status_change": _test_options(), + "no_error_on_hotpath": _test_options(proxyable = False), + "no_logging": _test_options(traceable = False), + "no_op": _test_options(), + "payload": _test_options(), # TODO(juanlishen): This is disabled for now because it depends on some generated functions in # end2end_tests.cc, which are not generated because they would depend on OpenCensus while # OpenCensus can only be built via Bazel so far. - # 'load_reporting_hook': test_options(), - 'ping_pong_streaming': test_options(), - 'ping': test_options(needs_fullstack=True, proxyable=False), - 'proxy_auth': test_options(needs_proxy_auth=True), - 'registered_call': test_options(), - 'request_with_flags': test_options(proxyable=False), - 'request_with_payload': test_options(), + # 'load_reporting_hook': _test_options(), + "ping_pong_streaming": _test_options(), + "ping": _test_options(needs_fullstack = True, proxyable = False), + "proxy_auth": _test_options(needs_proxy_auth = True), + "registered_call": _test_options(), + "request_with_flags": _test_options(proxyable = False), + "request_with_payload": _test_options(), # TODO(roth): Remove proxyable=False for all retry tests once we # have a way for the proxy to propagate the fact that trailing # metadata is available when initial metadata is returned. # See https://github.com/grpc/grpc/issues/14467 for context. - 'retry': test_options(needs_client_channel=True, proxyable=False), - 'retry_cancellation': test_options(needs_client_channel=True, - proxyable=False), - 'retry_disabled': test_options(needs_client_channel=True, proxyable=False), - 'retry_exceeds_buffer_size_in_initial_batch': test_options( - needs_client_channel=True, proxyable=False), - 'retry_exceeds_buffer_size_in_subsequent_batch': test_options( - needs_client_channel=True, proxyable=False), - 'retry_non_retriable_status': test_options(needs_client_channel=True, - proxyable=False), - 'retry_non_retriable_status_before_recv_trailing_metadata_started': - test_options(needs_client_channel=True, proxyable=False), - 'retry_recv_initial_metadata': test_options(needs_client_channel=True, - proxyable=False), - 'retry_recv_message': test_options(needs_client_channel=True, - proxyable=False), - 'retry_server_pushback_delay': test_options(needs_client_channel=True, - proxyable=False), - 'retry_server_pushback_disabled': test_options(needs_client_channel=True, - proxyable=False), - 'retry_streaming': test_options(needs_client_channel=True, proxyable=False), - 'retry_streaming_after_commit': test_options(needs_client_channel=True, - proxyable=False), - 'retry_streaming_succeeds_before_replay_finished': test_options( - needs_client_channel=True, proxyable=False), - 'retry_throttled': test_options(needs_client_channel=True, - proxyable=False), - 'retry_too_many_attempts': test_options(needs_client_channel=True, - proxyable=False), - 'server_finishes_request': test_options(), - 'shutdown_finishes_calls': test_options(), - 'shutdown_finishes_tags': test_options(), - 'simple_cacheable_request': test_options(), - 'simple_delayed_request': test_options(needs_fullstack=True), - 'simple_metadata': test_options(), - 'simple_request': test_options(), - 'streaming_error_response': test_options(), - 'stream_compression_compressed_payload': test_options(proxyable=False, - exclude_inproc=True), - 'stream_compression_payload': test_options(exclude_inproc=True), - 'stream_compression_ping_pong_streaming': test_options(exclude_inproc=True), - 'trailing_metadata': test_options(), - 'authority_not_supported': test_options(), - 'filter_latency': test_options(), - 'filter_status_code': test_options(), - 'workaround_cronet_compression': test_options(), - 'write_buffering': test_options(needs_write_buffering=True), - 'write_buffering_at_end': test_options(needs_write_buffering=True), + "retry": _test_options(needs_client_channel = True, proxyable = False), + "retry_cancellation": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_disabled": _test_options(needs_client_channel = True, proxyable = False), + "retry_exceeds_buffer_size_in_initial_batch": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_exceeds_buffer_size_in_subsequent_batch": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_non_retriable_status": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_non_retriable_status_before_recv_trailing_metadata_started": _test_options(needs_client_channel = True, proxyable = False), + "retry_recv_initial_metadata": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_recv_message": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_server_pushback_delay": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_server_pushback_disabled": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_streaming": _test_options(needs_client_channel = True, proxyable = False), + "retry_streaming_after_commit": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_streaming_succeeds_before_replay_finished": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_throttled": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "retry_too_many_attempts": _test_options( + needs_client_channel = True, + proxyable = False, + ), + "server_finishes_request": _test_options(), + "shutdown_finishes_calls": _test_options(), + "shutdown_finishes_tags": _test_options(), + "simple_cacheable_request": _test_options(), + "simple_delayed_request": _test_options(needs_fullstack = True), + "simple_metadata": _test_options(), + "simple_request": _test_options(), + "streaming_error_response": _test_options(), + "stream_compression_compressed_payload": _test_options( + proxyable = False, + exclude_inproc = True, + ), + "stream_compression_payload": _test_options(exclude_inproc = True), + "stream_compression_ping_pong_streaming": _test_options(exclude_inproc = True), + "trailing_metadata": _test_options(), + "authority_not_supported": _test_options(), + "filter_latency": _test_options(), + "filter_status_code": _test_options(), + "workaround_cronet_compression": _test_options(), + "write_buffering": _test_options(needs_write_buffering = True), + "write_buffering_at_end": _test_options(needs_write_buffering = True), } - -def compatible(fopt, topt): - if topt.needs_fullstack: - if not fopt.fullstack: - return False - if topt.needs_dns: - if not fopt.dns_resolver: - return False - if topt.needs_names: - if not fopt.name_resolution: - return False - if not topt.proxyable: - if fopt.includes_proxy: - return False - if not topt.traceable: - if fopt.tracing: - return False - if topt.exclude_inproc: - if fopt.is_inproc: - return False - if topt.needs_http2: - if not fopt.is_http2: - return False - if topt.needs_proxy_auth: - if not fopt.supports_proxy_auth: - return False - if topt.needs_write_buffering: - if not fopt.supports_write_buffering: - return False - if topt.needs_client_channel: - if not fopt.client_channel: - return False - return True - +def _compatible(fopt, topt): + if topt.needs_fullstack: + if not fopt.fullstack: + return False + if topt.needs_dns: + if not fopt.dns_resolver: + return False + if topt.needs_names: + if not fopt.name_resolution: + return False + if not topt.proxyable: + if fopt.includes_proxy: + return False + if not topt.traceable: + if fopt.tracing: + return False + if topt.exclude_inproc: + if fopt.is_inproc: + return False + if topt.needs_http2: + if not fopt.is_http2: + return False + if topt.needs_proxy_auth: + if not fopt.supports_proxy_auth: + return False + if topt.needs_write_buffering: + if not fopt.supports_write_buffering: + return False + if topt.needs_client_channel: + if not fopt.client_channel: + return False + return True def grpc_end2end_tests(): - grpc_cc_library( - name = 'end2end_tests', - srcs = ['end2end_tests.cc', 'end2end_test_utils.cc'] + [ - 'tests/%s.cc' % t - for t in sorted(END2END_TESTS.keys())], - hdrs = [ - 'tests/cancel_test_helpers.h', - 'end2end_tests.h' - ], - language = "C++", - deps = [ - ':cq_verifier', - ':ssl_test_data', - ':http_proxy', - ':proxy', - ] - ) - - for f, fopt in END2END_FIXTURES.items(): - grpc_cc_binary( - name = '%s_test' % f, - srcs = ['fixtures/%s.cc' % f], - language = "C++", - deps = [ - ':end2end_tests', - '//test/core/util:grpc_test_util', - '//:grpc', - '//test/core/util:gpr_test_util', - '//:gpr', - ], + grpc_cc_library( + name = "end2end_tests", + srcs = ["end2end_tests.cc", "end2end_test_utils.cc"] + [ + "tests/%s.cc" % t + for t in sorted(END2END_TESTS.keys()) + ], + hdrs = [ + "tests/cancel_test_helpers.h", + "end2end_tests.h", + ], + language = "C++", + deps = [ + ":cq_verifier", + ":ssl_test_data", + ":http_proxy", + ":proxy", + ], ) - for t, topt in END2END_TESTS.items(): - #print(compatible(fopt, topt), f, t, fopt, topt) - if not compatible(fopt, topt): continue - for poller in POLLERS: - native.sh_test( - name = '%s_test@%s@poller=%s' % (f, t, poller), - data = [':%s_test' % f], - srcs = ['end2end_test.sh'], - args = [ - '$(location %s_test)' % f, - t, - poller, - ], + + for f, fopt in END2END_FIXTURES.items(): + grpc_cc_binary( + name = "%s_test" % f, + srcs = ["fixtures/%s.cc" % f], + language = "C++", + deps = [ + ":end2end_tests", + "//test/core/util:grpc_test_util", + "//:grpc", + "//test/core/util:gpr_test_util", + "//:gpr", + ], ) + for t, topt in END2END_TESTS.items(): + #print(_compatible(fopt, topt), f, t, fopt, topt) + if not _compatible(fopt, topt): + continue + for poller in POLLERS: + native.sh_test( + name = "%s_test@%s@poller=%s" % (f, t, poller), + data = [":%s_test" % f], + srcs = ["end2end_test.sh"], + args = [ + "$(location %s_test)" % f, + t, + poller, + ], + ) diff --git a/test/core/end2end/inproc_callback_test.cc b/test/core/end2end/inproc_callback_test.cc index 0d6c7c75a8..310030046a 100644 --- a/test/core/end2end/inproc_callback_test.cc +++ b/test/core/end2end/inproc_callback_test.cc @@ -37,13 +37,16 @@ typedef struct inproc_fixture_data { namespace { template <typename F> -class CQDeletingCallback : public grpc_core::CQCallbackInterface { +class CQDeletingCallback : public grpc_experimental_completion_queue_functor { public: - explicit CQDeletingCallback(F f) : func_(f) {} - ~CQDeletingCallback() override {} - void Run(bool ok) override { - func_(ok); - grpc_core::Delete(this); + explicit CQDeletingCallback(F f) : func_(f) { + functor_run = &CQDeletingCallback::Run; + } + ~CQDeletingCallback() {} + static void Run(grpc_experimental_completion_queue_functor* cb, int ok) { + auto* callback = static_cast<CQDeletingCallback*>(cb); + callback->func_(static_cast<bool>(ok)); + grpc_core::Delete(callback); } private: @@ -51,18 +54,24 @@ class CQDeletingCallback : public grpc_core::CQCallbackInterface { }; template <typename F> -grpc_core::CQCallbackInterface* NewDeletingCallback(F f) { +grpc_experimental_completion_queue_functor* NewDeletingCallback(F f) { return grpc_core::New<CQDeletingCallback<F>>(f); } -class ShutdownCallback : public grpc_core::CQCallbackInterface { +class ShutdownCallback : public grpc_experimental_completion_queue_functor { public: ShutdownCallback() : done_(false) { + functor_run = &ShutdownCallback::StaticRun; gpr_mu_init(&mu_); gpr_cv_init(&cv_); } - ~ShutdownCallback() override {} - void Run(bool ok) override { + ~ShutdownCallback() {} + static void StaticRun(grpc_experimental_completion_queue_functor* cb, + int ok) { + auto* callback = static_cast<ShutdownCallback*>(cb); + callback->Run(static_cast<bool>(ok)); + } + void Run(bool ok) { gpr_log(GPR_DEBUG, "CQ shutdown notification invoked"); gpr_mu_lock(&mu_); done_ = true; @@ -170,7 +179,7 @@ static void verify_tags(gpr_timespec deadline) { // This function creates a callback functor that emits the // desired tag into the global tag set -static grpc_core::CQCallbackInterface* tag(intptr_t t) { +static grpc_experimental_completion_queue_functor* tag(intptr_t t) { auto func = [t](bool ok) { gpr_mu_lock(&tags_mu); gpr_log(GPR_DEBUG, "Completing operation %" PRIdPTR, t); diff --git a/test/core/end2end/tests/channelz.cc b/test/core/end2end/tests/channelz.cc index 533703a2be..40a0370f0e 100644 --- a/test/core/end2end/tests/channelz.cc +++ b/test/core/end2end/tests/channelz.cc @@ -22,6 +22,7 @@ #include <string.h> #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/server.h" #include <grpc/byte_buffer.h> #include <grpc/grpc.h> @@ -198,17 +199,21 @@ static void run_one_request(grpc_end2end_test_config config, static void test_channelz(grpc_end2end_test_config config) { grpc_end2end_test_fixture f; - grpc_arg client_a; - client_a.type = GRPC_ARG_INTEGER; - client_a.key = const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ); - client_a.value.integer = true; - grpc_channel_args client_args = {1, &client_a}; + grpc_arg arg; + arg.type = GRPC_ARG_INTEGER; + arg.key = const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ); + arg.value.integer = true; + grpc_channel_args args = {1, &arg}; - f = begin_test(config, "test_channelz", &client_args, nullptr); + f = begin_test(config, "test_channelz", &args, &args); grpc_core::channelz::ChannelNode* channelz_channel = grpc_channel_get_channelz_node(f.client); - GPR_ASSERT(channelz_channel != nullptr); + + grpc_core::channelz::ServerNode* channelz_server = + grpc_server_get_channelz_node(f.server); + GPR_ASSERT(channelz_server != nullptr); + char* json = channelz_channel->RenderJsonString(); GPR_ASSERT(json != nullptr); // nothing is present yet @@ -235,7 +240,19 @@ static void test_channelz(grpc_end2end_test_config config) { GPR_ASSERT(nullptr != strstr(json, "\"callsStarted\":\"2\"")); GPR_ASSERT(nullptr != strstr(json, "\"callsFailed\":\"1\"")); GPR_ASSERT(nullptr != strstr(json, "\"callsSucceeded\":\"1\"")); - // channel tracing is not enables, so these should not be preset. + // channel tracing is not enabled, so these should not be preset. + GPR_ASSERT(nullptr == strstr(json, "\"trace\"")); + GPR_ASSERT(nullptr == strstr(json, "\"description\":\"Channel created\"")); + GPR_ASSERT(nullptr == strstr(json, "\"severity\":\"CT_INFO\"")); + gpr_free(json); + + json = channelz_server->RenderJsonString(); + GPR_ASSERT(json != nullptr); + gpr_log(GPR_INFO, "%s", json); + GPR_ASSERT(nullptr != strstr(json, "\"callsStarted\":\"2\"")); + GPR_ASSERT(nullptr != strstr(json, "\"callsFailed\":\"1\"")); + GPR_ASSERT(nullptr != strstr(json, "\"callsSucceeded\":\"1\"")); + // channel tracing is not enabled, so these should not be preset. GPR_ASSERT(nullptr == strstr(json, "\"trace\"")); GPR_ASSERT(nullptr == strstr(json, "\"description\":\"Channel created\"")); GPR_ASSERT(nullptr == strstr(json, "\"severity\":\"CT_INFO\"")); @@ -248,22 +265,24 @@ static void test_channelz(grpc_end2end_test_config config) { static void test_channelz_with_channel_trace(grpc_end2end_test_config config) { grpc_end2end_test_fixture f; - grpc_arg client_a[2]; - client_a[0].type = GRPC_ARG_INTEGER; - client_a[0].key = - const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE); - client_a[0].value.integer = 5; - client_a[1].type = GRPC_ARG_INTEGER; - client_a[1].key = const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ); - client_a[1].value.integer = true; - grpc_channel_args client_args = {GPR_ARRAY_SIZE(client_a), client_a}; - - f = begin_test(config, "test_channelz_with_channel_trace", &client_args, - nullptr); + grpc_arg arg[2]; + arg[0].type = GRPC_ARG_INTEGER; + arg[0].key = const_cast<char*>(GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE); + arg[0].value.integer = 5; + arg[1].type = GRPC_ARG_INTEGER; + arg[1].key = const_cast<char*>(GRPC_ARG_ENABLE_CHANNELZ); + arg[1].value.integer = true; + grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg}; + + f = begin_test(config, "test_channelz_with_channel_trace", &args, &args); grpc_core::channelz::ChannelNode* channelz_channel = grpc_channel_get_channelz_node(f.client); - GPR_ASSERT(channelz_channel != nullptr); + + grpc_core::channelz::ServerNode* channelz_server = + grpc_server_get_channelz_node(f.server); + GPR_ASSERT(channelz_server != nullptr); + char* json = channelz_channel->RenderJsonString(); GPR_ASSERT(json != nullptr); gpr_log(GPR_INFO, "%s", json); @@ -272,6 +291,14 @@ static void test_channelz_with_channel_trace(grpc_end2end_test_config config) { GPR_ASSERT(nullptr != strstr(json, "\"severity\":\"CT_INFO\"")); gpr_free(json); + json = channelz_server->RenderJsonString(); + GPR_ASSERT(json != nullptr); + gpr_log(GPR_INFO, "%s", json); + GPR_ASSERT(nullptr != strstr(json, "\"trace\"")); + GPR_ASSERT(nullptr != strstr(json, "\"description\":\"Server created\"")); + GPR_ASSERT(nullptr != strstr(json, "\"severity\":\"CT_INFO\"")); + gpr_free(json); + end_test(&f); config.tear_down_data(&f); } diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc index ba3cbfa6d1..5ffc3d00a3 100644 --- a/test/core/end2end/tests/filter_status_code.cc +++ b/test/core/end2end/tests/filter_status_code.cc @@ -16,6 +16,14 @@ * */ +/* This test verifies - + * 1) grpc_call_final_info passed to the filters on destroying a call contains + * the proper status. + * 2) If the response has both an HTTP status code and a gRPC status code, then + * we should prefer the gRPC status code as mentioned in + * https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md + */ + #include "test/core/end2end/end2end_tests.h" #include <limits.h> @@ -249,6 +257,22 @@ typedef struct final_status_data { grpc_call_stack* call; } final_status_data; +static void server_start_transport_stream_op_batch( + grpc_call_element* elem, grpc_transport_stream_op_batch* op) { + auto* data = static_cast<final_status_data*>(elem->call_data); + if (data->call == g_server_call_stack) { + if (op->send_initial_metadata) { + auto* batch = op->payload->send_initial_metadata.send_initial_metadata; + if (batch->idx.named.status != nullptr) { + /* Replace the HTTP status with 404 */ + grpc_metadata_batch_substitute(batch, batch->idx.named.status, + GRPC_MDELEM_STATUS_404); + } + } + } + grpc_call_next_op(elem, op); +} + static grpc_error* init_call_elem(grpc_call_element* elem, const grpc_call_element_args* args) { final_status_data* data = static_cast<final_status_data*>(elem->call_data); @@ -307,7 +331,7 @@ static const grpc_channel_filter test_client_filter = { "client_filter_status_code"}; static const grpc_channel_filter test_server_filter = { - grpc_call_next_op, + server_start_transport_stream_op_batch, grpc_channel_next_op, sizeof(final_status_data), init_call_elem, diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index 675d9e6278..70ee83acd2 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -40,6 +40,7 @@ grpc_cc_library( grpc_cc_test( name = "combiner_test", srcs = ["combiner_test.cc"], + data = ["//third_party/toolchains:RBE_USE_MACHINE_TYPE_LARGE"], language = "C++", deps = [ "//:gpr", @@ -47,7 +48,6 @@ grpc_cc_test( "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", ], - data = ["//third_party/toolchains:RBE_USE_MACHINE_TYPE_LARGE"], ) grpc_cc_test( @@ -89,18 +89,6 @@ grpc_cc_test( ) grpc_cc_test( - name = "ev_epollsig_linux_test", - srcs = ["ev_epollsig_linux_test.cc"], - language = "C++", - deps = [ - "//:gpr", - "//:grpc", - "//test/core/util:gpr_test_util", - "//test/core/util:grpc_test_util", - ], -) - -grpc_cc_test( name = "fd_conservation_posix_test", srcs = ["fd_conservation_posix_test.cc"], language = "C++", @@ -136,7 +124,6 @@ grpc_cc_test( ], ) - grpc_cc_test( name = "load_file_test", srcs = ["load_file_test.cc"], @@ -150,8 +137,8 @@ grpc_cc_test( ) grpc_cc_test( - name = "pollset_set_test", - srcs = ["pollset_set_test.cc"], + name = "resolve_address_posix_test", + srcs = ["resolve_address_posix_test.cc"], language = "C++", deps = [ "//:gpr", @@ -162,8 +149,11 @@ grpc_cc_test( ) grpc_cc_test( - name = "resolve_address_posix_test", - srcs = ["resolve_address_posix_test.cc"], + name = "resolve_address_using_ares_resolver_test", + srcs = ["resolve_address_test.cc"], + args = [ + "--resolver=ares", + ], language = "C++", deps = [ "//:gpr", @@ -174,8 +164,11 @@ grpc_cc_test( ) grpc_cc_test( - name = "resolve_address_test", + name = "resolve_address_using_native_resolver_test", srcs = ["resolve_address_test.cc"], + args = [ + "--resolver=native", + ], language = "C++", deps = [ "//:gpr", @@ -258,7 +251,6 @@ grpc_cc_test( ], ) - grpc_cc_test( name = "tcp_server_posix_test", srcs = ["tcp_server_posix_test.cc"], diff --git a/test/core/iomgr/error_test.cc b/test/core/iomgr/error_test.cc index a1628a1f71..d78a8c2af3 100644 --- a/test/core/iomgr/error_test.cc +++ b/test/core/iomgr/error_test.cc @@ -187,16 +187,6 @@ static void test_os_error() { GRPC_ERROR_UNREF(error); } -static void test_special() { - grpc_error* error = GRPC_ERROR_NONE; - error = grpc_error_add_child( - error, GRPC_ERROR_CREATE_FROM_STATIC_STRING("test child")); - intptr_t i; - GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &i)); - GPR_ASSERT(i == GRPC_STATUS_OK); - GRPC_ERROR_UNREF(error); -} - static void test_overflow() { grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Overflow"); @@ -235,7 +225,6 @@ int main(int argc, char** argv) { test_os_error(); test_create_referencing(); test_create_referencing_many(); - test_special(); test_overflow(); grpc_shutdown(); diff --git a/test/core/iomgr/ev_epollsig_linux_test.cc b/test/core/iomgr/ev_epollsig_linux_test.cc deleted file mode 100644 index 28c9dd408c..0000000000 --- a/test/core/iomgr/ev_epollsig_linux_test.cc +++ /dev/null @@ -1,321 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -#include "src/core/lib/iomgr/port.h" - -/* This test only relevant on linux systems where epoll() is available */ -#ifdef GRPC_LINUX_EPOLL_CREATE1 -#include "src/core/lib/iomgr/ev_epollsig_linux.h" -#include "src/core/lib/iomgr/ev_posix.h" - -#include <errno.h> -#include <string.h> -#include <unistd.h> - -#include <grpc/grpc.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> - -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/gprpp/thd.h" -#include "src/core/lib/iomgr/iomgr.h" -#include "test/core/util/test_config.h" - -typedef struct test_pollset { - grpc_pollset* pollset; - gpr_mu* mu; -} test_pollset; - -typedef struct test_fd { - int inner_fd; - grpc_fd* fd; -} test_fd; - -/* num_fds should be an even number */ -static void test_fd_init(test_fd* tfds, int* fds, int num_fds) { - int i; - int r; - - /* Create some dummy file descriptors. Currently using pipe file descriptors - * for this test but we could use any other type of file descriptors. Also, - * since pipe() used in this test creates two fds in each call, num_fds should - * be an even number */ - GPR_ASSERT((num_fds % 2) == 0); - for (i = 0; i < num_fds; i = i + 2) { - r = pipe(fds + i); - if (r != 0) { - gpr_log(GPR_ERROR, "Error in creating pipe. %d (%s)", errno, - strerror(errno)); - return; - } - } - - for (i = 0; i < num_fds; i++) { - tfds[i].inner_fd = fds[i]; - tfds[i].fd = grpc_fd_create(fds[i], "test_fd", false); - } -} - -static void test_fd_cleanup(test_fd* tfds, int num_fds) { - int release_fd; - int i; - - for (i = 0; i < num_fds; i++) { - grpc_fd_shutdown(tfds[i].fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("test_fd_cleanup")); - grpc_core::ExecCtx::Get()->Flush(); - - grpc_fd_orphan(tfds[i].fd, nullptr, &release_fd, "test_fd_cleanup"); - grpc_core::ExecCtx::Get()->Flush(); - - GPR_ASSERT(release_fd == tfds[i].inner_fd); - close(tfds[i].inner_fd); - } -} - -static void test_pollset_init(test_pollset* pollsets, int num_pollsets) { - int i; - for (i = 0; i < num_pollsets; i++) { - pollsets[i].pollset = - static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); - grpc_pollset_init(pollsets[i].pollset, &pollsets[i].mu); - } -} - -static void destroy_pollset(void* p, grpc_error* error) { - grpc_pollset_destroy(static_cast<grpc_pollset*>(p)); -} - -static void test_pollset_cleanup(test_pollset* pollsets, int num_pollsets) { - grpc_closure destroyed; - int i; - - for (i = 0; i < num_pollsets; i++) { - GRPC_CLOSURE_INIT(&destroyed, destroy_pollset, pollsets[i].pollset, - grpc_schedule_on_exec_ctx); - grpc_pollset_shutdown(pollsets[i].pollset, &destroyed); - - grpc_core::ExecCtx::Get()->Flush(); - gpr_free(pollsets[i].pollset); - } -} - - /* - * Cases to test: - * case 1) Polling islands of both fd and pollset are NULL - * case 2) Polling island of fd is NULL but that of pollset is not-NULL - * case 3) Polling island of fd is not-NULL but that of pollset is NULL - * case 4) Polling islands of both fd and pollset are not-NULL and: - * case 4.1) Polling islands of fd and pollset are equal - * case 4.2) Polling islands of fd and pollset are NOT-equal (This results - * in a merge) - * */ - -#define NUM_FDS 8 -#define NUM_POLLSETS 4 - -static void test_add_fd_to_pollset() { - grpc_core::ExecCtx exec_ctx; - test_fd tfds[NUM_FDS]; - int fds[NUM_FDS]; - test_pollset pollsets[NUM_POLLSETS]; - void* expected_pi = nullptr; - int i; - - test_fd_init(tfds, fds, NUM_FDS); - test_pollset_init(pollsets, NUM_POLLSETS); - - /*Step 1. - * Create three polling islands (This will exercise test case 1 and 2) with - * the following configuration: - * polling island 0 = { fds:0,1,2, pollsets:0} - * polling island 1 = { fds:3,4, pollsets:1} - * polling island 2 = { fds:5,6,7 pollsets:2} - * - *Step 2. - * Add pollset 3 to polling island 0 (by adding fds 0 and 1 to pollset 3) - * (This will exercise test cases 3 and 4.1). The configuration becomes: - * polling island 0 = { fds:0,1,2, pollsets:0,3} <<< pollset 3 added here - * polling island 1 = { fds:3,4, pollsets:1} - * polling island 2 = { fds:5,6,7 pollsets:2} - * - *Step 3. - * Merge polling islands 0 and 1 by adding fd 0 to pollset 1 (This will - * exercise test case 4.2). The configuration becomes: - * polling island (merged) = {fds: 0,1,2,3,4, pollsets: 0,1,3} - * polling island 2 = {fds: 5,6,7 pollsets: 2} - * - *Step 4. - * Finally do one more merge by adding fd 3 to pollset 2. - * polling island (merged) = {fds: 0,1,2,3,4,5,6,7, pollsets: 0,1,2,3} - */ - - /* == Step 1 == */ - for (i = 0; i <= 2; i++) { - grpc_pollset_add_fd(pollsets[0].pollset, tfds[i].fd); - grpc_core::ExecCtx::Get()->Flush(); - } - - for (i = 3; i <= 4; i++) { - grpc_pollset_add_fd(pollsets[1].pollset, tfds[i].fd); - grpc_core::ExecCtx::Get()->Flush(); - } - - for (i = 5; i <= 7; i++) { - grpc_pollset_add_fd(pollsets[2].pollset, tfds[i].fd); - grpc_core::ExecCtx::Get()->Flush(); - } - - /* == Step 2 == */ - for (i = 0; i <= 1; i++) { - grpc_pollset_add_fd(pollsets[3].pollset, tfds[i].fd); - grpc_core::ExecCtx::Get()->Flush(); - } - - /* == Step 3 == */ - grpc_pollset_add_fd(pollsets[1].pollset, tfds[0].fd); - grpc_core::ExecCtx::Get()->Flush(); - - /* == Step 4 == */ - grpc_pollset_add_fd(pollsets[2].pollset, tfds[3].fd); - grpc_core::ExecCtx::Get()->Flush(); - - /* All polling islands are merged at this point */ - - /* Compare Fd:0's polling island with that of all other Fds */ - expected_pi = grpc_fd_get_polling_island(tfds[0].fd); - for (i = 1; i < NUM_FDS; i++) { - GPR_ASSERT(grpc_are_polling_islands_equal( - expected_pi, grpc_fd_get_polling_island(tfds[i].fd))); - } - - /* Compare Fd:0's polling island with that of all other pollsets */ - for (i = 0; i < NUM_POLLSETS; i++) { - GPR_ASSERT(grpc_are_polling_islands_equal( - expected_pi, grpc_pollset_get_polling_island(pollsets[i].pollset))); - } - - test_fd_cleanup(tfds, NUM_FDS); - test_pollset_cleanup(pollsets, NUM_POLLSETS); -} - -#undef NUM_FDS -#undef NUM_POLLSETS - -typedef struct threading_shared { - gpr_mu* mu; - grpc_pollset* pollset; - grpc_wakeup_fd* wakeup_fd; - grpc_fd* wakeup_desc; - grpc_closure on_wakeup; - int wakeups; -} threading_shared; - -static __thread int thread_wakeups = 0; - -static void test_threading_loop(void* arg) { - threading_shared* shared = static_cast<threading_shared*>(arg); - while (thread_wakeups < 1000000) { - grpc_core::ExecCtx exec_ctx; - grpc_pollset_worker* worker; - gpr_mu_lock(shared->mu); - GPR_ASSERT(GRPC_LOG_IF_ERROR( - "pollset_work", - grpc_pollset_work(shared->pollset, &worker, GRPC_MILLIS_INF_FUTURE))); - gpr_mu_unlock(shared->mu); - } -} - -static void test_threading_wakeup(void* arg, grpc_error* error) { - threading_shared* shared = static_cast<threading_shared*>(arg); - ++shared->wakeups; - ++thread_wakeups; - if (error == GRPC_ERROR_NONE) { - GPR_ASSERT(GRPC_LOG_IF_ERROR( - "consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd))); - grpc_fd_notify_on_read(shared->wakeup_desc, &shared->on_wakeup); - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next", - grpc_wakeup_fd_wakeup(shared->wakeup_fd))); - } -} - -static void test_threading(void) { - threading_shared shared; - shared.pollset = static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); - grpc_pollset_init(shared.pollset, &shared.mu); - - grpc_core::Thread thds[10]; - for (auto& th : thds) { - th = grpc_core::Thread("test_thread", test_threading_loop, &shared); - th.Start(); - } - grpc_wakeup_fd fd; - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&fd))); - shared.wakeup_fd = &fd; - shared.wakeup_desc = grpc_fd_create(fd.read_fd, "wakeup", false); - shared.wakeups = 0; - { - grpc_core::ExecCtx exec_ctx; - grpc_pollset_add_fd(shared.pollset, shared.wakeup_desc); - grpc_fd_notify_on_read( - shared.wakeup_desc, - GRPC_CLOSURE_INIT(&shared.on_wakeup, test_threading_wakeup, &shared, - grpc_schedule_on_exec_ctx)); - } - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_first", - grpc_wakeup_fd_wakeup(shared.wakeup_fd))); - for (auto& th : thds) { - th.Join(); - } - fd.read_fd = 0; - grpc_wakeup_fd_destroy(&fd); - { - grpc_core::ExecCtx exec_ctx; - grpc_fd_shutdown(shared.wakeup_desc, GRPC_ERROR_CANCELLED); - grpc_fd_orphan(shared.wakeup_desc, nullptr, nullptr, "done"); - grpc_pollset_shutdown(shared.pollset, - GRPC_CLOSURE_CREATE(destroy_pollset, shared.pollset, - grpc_schedule_on_exec_ctx)); - } - gpr_free(shared.pollset); -} - -int main(int argc, char** argv) { - const char* poll_strategy = nullptr; - grpc_test_init(argc, argv); - grpc_init(); - { - grpc_core::ExecCtx exec_ctx; - - poll_strategy = grpc_get_poll_strategy_name(); - if (poll_strategy != nullptr && strcmp(poll_strategy, "epollsig") == 0) { - test_add_fd_to_pollset(); - test_threading(); - } else { - gpr_log(GPR_INFO, - "Skipping the test. The test is only relevant for 'epollsig' " - "strategy. and the current strategy is: '%s'", - poll_strategy); - } - } - - grpc_shutdown(); - return 0; -} -#else /* defined(GRPC_LINUX_EPOLL_CREATE1) */ -int main(int argc, char** argv) { return 0; } -#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */ diff --git a/test/core/iomgr/pollset_set_test.cc b/test/core/iomgr/pollset_set_test.cc deleted file mode 100644 index 1aae1daa02..0000000000 --- a/test/core/iomgr/pollset_set_test.cc +++ /dev/null @@ -1,447 +0,0 @@ -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -#include "src/core/lib/iomgr/port.h" - -/* This test only relevant on linux systems where epoll is available */ -#ifdef GRPC_LINUX_EPOLL_CREATE1 - -#include <errno.h> -#include <string.h> -#include <unistd.h> - -#include <grpc/grpc.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> - -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/iomgr/ev_posix.h" -#include "src/core/lib/iomgr/iomgr.h" -#include "test/core/util/test_config.h" - -/******************************************************************************* - * test_pollset_set - */ - -typedef struct test_pollset_set { - grpc_pollset_set* pss; -} test_pollset_set; - -void init_test_pollset_sets(test_pollset_set* pollset_sets, const int num_pss) { - for (int i = 0; i < num_pss; i++) { - pollset_sets[i].pss = grpc_pollset_set_create(); - } -} - -void cleanup_test_pollset_sets(test_pollset_set* pollset_sets, - const int num_pss) { - for (int i = 0; i < num_pss; i++) { - grpc_pollset_set_destroy(pollset_sets[i].pss); - pollset_sets[i].pss = nullptr; - } -} - -/******************************************************************************* - * test_pollset - */ - -typedef struct test_pollset { - grpc_pollset* ps; - gpr_mu* mu; -} test_pollset; - -static void init_test_pollsets(test_pollset* pollsets, const int num_pollsets) { - for (int i = 0; i < num_pollsets; i++) { - pollsets[i].ps = - static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); - grpc_pollset_init(pollsets[i].ps, &pollsets[i].mu); - } -} - -static void destroy_pollset(void* p, grpc_error* error) { - grpc_pollset_destroy(static_cast<grpc_pollset*>(p)); -} - -static void cleanup_test_pollsets(test_pollset* pollsets, - const int num_pollsets) { - grpc_closure destroyed; - for (int i = 0; i < num_pollsets; i++) { - GRPC_CLOSURE_INIT(&destroyed, destroy_pollset, pollsets[i].ps, - grpc_schedule_on_exec_ctx); - grpc_pollset_shutdown(pollsets[i].ps, &destroyed); - - grpc_core::ExecCtx::Get()->Flush(); - gpr_free(pollsets[i].ps); - pollsets[i].ps = nullptr; - } -} - -/******************************************************************************* - * test_fd - */ - -typedef struct test_fd { - grpc_fd* fd; - grpc_wakeup_fd wakeup_fd; - - bool is_on_readable_called; /* Is on_readable closure is called ? */ - grpc_closure on_readable; /* Closure to call when this fd is readable */ -} test_fd; - -void on_readable(void* tfd, grpc_error* error) { - (static_cast<test_fd*>(tfd))->is_on_readable_called = true; -} - -static void reset_test_fd(test_fd* tfd) { - tfd->is_on_readable_called = false; - - GRPC_CLOSURE_INIT(&tfd->on_readable, on_readable, tfd, - grpc_schedule_on_exec_ctx); - grpc_fd_notify_on_read(tfd->fd, &tfd->on_readable); -} - -static void init_test_fds(test_fd* tfds, const int num_fds) { - for (int i = 0; i < num_fds; i++) { - GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_init(&tfds[i].wakeup_fd)); - tfds[i].fd = grpc_fd_create(GRPC_WAKEUP_FD_GET_READ_FD(&tfds[i].wakeup_fd), - "test_fd", false); - reset_test_fd(&tfds[i]); - } -} - -static void cleanup_test_fds(test_fd* tfds, const int num_fds) { - int release_fd; - - for (int i = 0; i < num_fds; i++) { - grpc_fd_shutdown(tfds[i].fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("fd cleanup")); - grpc_core::ExecCtx::Get()->Flush(); - - /* grpc_fd_orphan frees the memory allocated for grpc_fd. Normally it also - * calls close() on the underlying fd. In our case, we are using - * grpc_wakeup_fd and we would like to destroy it ourselves (by calling - * grpc_wakeup_fd_destroy). To prevent grpc_fd from calling close() on the - * underlying fd, call it with a non-NULL 'release_fd' parameter */ - grpc_fd_orphan(tfds[i].fd, nullptr, &release_fd, "test_fd_cleanup"); - grpc_core::ExecCtx::Get()->Flush(); - - grpc_wakeup_fd_destroy(&tfds[i].wakeup_fd); - } -} - -static void make_test_fds_readable(test_fd* tfds, const int num_fds) { - for (int i = 0; i < num_fds; i++) { - GPR_ASSERT(GRPC_ERROR_NONE == grpc_wakeup_fd_wakeup(&tfds[i].wakeup_fd)); - } -} - -static void verify_readable_and_reset(test_fd* tfds, const int num_fds) { - for (int i = 0; i < num_fds; i++) { - /* Verify that the on_readable callback was called */ - GPR_ASSERT(tfds[i].is_on_readable_called); - - /* Reset the tfd[i] structure */ - GPR_ASSERT(GRPC_ERROR_NONE == - grpc_wakeup_fd_consume_wakeup(&tfds[i].wakeup_fd)); - reset_test_fd(&tfds[i]); - } -} - -/******************************************************************************* - * Main tests - */ - -/* Test some typical scenarios in pollset_set */ -static void pollset_set_test_basic() { - /* We construct the following structure for this test: - * - * +---> FD0 (Added before PSS1, PS1 and PS2 are added to PSS0) - * | - * +---> FD5 (Added after PSS1, PS1 and PS2 are added to PSS0) - * | - * | - * | +---> FD1 (Added before PSS1 is added to PSS0) - * | | - * | +---> FD6 (Added after PSS1 is added to PSS0) - * | | - * +---> PSS1--+ +--> FD2 (Added before PS0 is added to PSS1) - * | | | - * | +---> PS0---+ - * | | - * PSS0---+ +--> FD7 (Added after PS0 is added to PSS1) - * | - * | - * | +---> FD3 (Added before PS1 is added to PSS0) - * | | - * +---> PS1---+ - * | | - * | +---> FD8 (Added after PS1 added to PSS0) - * | - * | - * | +---> FD4 (Added before PS2 is added to PSS0) - * | | - * +---> PS2---+ - * | - * +---> FD9 (Added after PS2 is added to PSS0) - */ - grpc_core::ExecCtx exec_ctx; - grpc_pollset_worker* worker; - grpc_millis deadline; - - test_fd tfds[10]; - test_pollset pollsets[3]; - test_pollset_set pollset_sets[2]; - const int num_fds = GPR_ARRAY_SIZE(tfds); - const int num_ps = GPR_ARRAY_SIZE(pollsets); - const int num_pss = GPR_ARRAY_SIZE(pollset_sets); - - init_test_fds(tfds, num_fds); - init_test_pollsets(pollsets, num_ps); - init_test_pollset_sets(pollset_sets, num_pss); - - /* Construct the pollset_set/pollset/fd tree (see diagram above) */ - - grpc_pollset_set_add_fd(pollset_sets[0].pss, tfds[0].fd); - grpc_pollset_set_add_fd(pollset_sets[1].pss, tfds[1].fd); - - grpc_pollset_add_fd(pollsets[0].ps, tfds[2].fd); - grpc_pollset_add_fd(pollsets[1].ps, tfds[3].fd); - grpc_pollset_add_fd(pollsets[2].ps, tfds[4].fd); - - grpc_pollset_set_add_pollset_set(pollset_sets[0].pss, pollset_sets[1].pss); - - grpc_pollset_set_add_pollset(pollset_sets[1].pss, pollsets[0].ps); - grpc_pollset_set_add_pollset(pollset_sets[0].pss, pollsets[1].ps); - grpc_pollset_set_add_pollset(pollset_sets[0].pss, pollsets[2].ps); - - grpc_pollset_set_add_fd(pollset_sets[0].pss, tfds[5].fd); - grpc_pollset_set_add_fd(pollset_sets[1].pss, tfds[6].fd); - - grpc_pollset_add_fd(pollsets[0].ps, tfds[7].fd); - grpc_pollset_add_fd(pollsets[1].ps, tfds[8].fd); - grpc_pollset_add_fd(pollsets[2].ps, tfds[9].fd); - - grpc_core::ExecCtx::Get()->Flush(); - - /* Test that if any FD in the above structure is readable, it is observable by - * doing grpc_pollset_work on any pollset - * - * For every pollset, do the following: - * - (Ensure that all FDs are in reset state) - * - Make all FDs readable - * - Call grpc_pollset_work() on the pollset - * - Flush the exec_ctx - * - Verify that on_readable call back was called for all FDs (and - * reset the FDs) - * */ - for (int i = 0; i < num_ps; i++) { - make_test_fds_readable(tfds, num_fds); - - gpr_mu_lock(pollsets[i].mu); - deadline = grpc_timespec_to_millis_round_up( - grpc_timeout_milliseconds_to_deadline(2)); - GPR_ASSERT(GRPC_ERROR_NONE == - grpc_pollset_work(pollsets[i].ps, &worker, deadline)); - gpr_mu_unlock(pollsets[i].mu); - - grpc_core::ExecCtx::Get()->Flush(); - - verify_readable_and_reset(tfds, num_fds); - grpc_core::ExecCtx::Get()->Flush(); - } - - /* Test tear down */ - grpc_pollset_set_del_fd(pollset_sets[0].pss, tfds[0].fd); - grpc_pollset_set_del_fd(pollset_sets[0].pss, tfds[5].fd); - grpc_pollset_set_del_fd(pollset_sets[1].pss, tfds[1].fd); - grpc_pollset_set_del_fd(pollset_sets[1].pss, tfds[6].fd); - grpc_core::ExecCtx::Get()->Flush(); - - grpc_pollset_set_del_pollset(pollset_sets[1].pss, pollsets[0].ps); - grpc_pollset_set_del_pollset(pollset_sets[0].pss, pollsets[1].ps); - grpc_pollset_set_del_pollset(pollset_sets[0].pss, pollsets[2].ps); - - grpc_pollset_set_del_pollset_set(pollset_sets[0].pss, pollset_sets[1].pss); - grpc_core::ExecCtx::Get()->Flush(); - - cleanup_test_fds(tfds, num_fds); - cleanup_test_pollsets(pollsets, num_ps); - cleanup_test_pollset_sets(pollset_sets, num_pss); -} - -/* Same FD added multiple times to the pollset_set tree */ -void pollset_set_test_dup_fds() { - /* We construct the following structure for this test: - * - * +---> FD0 - * | - * | - * PSS0---+ - * | +---> FD0 (also under PSS0) - * | | - * +---> PSS1--+ +--> FD1 (also under PSS1) - * | | - * +---> PS ---+ - * | | - * | +--> FD2 - * +---> FD1 - */ - grpc_core::ExecCtx exec_ctx; - grpc_pollset_worker* worker; - grpc_millis deadline; - - test_fd tfds[3]; - test_pollset pollset; - test_pollset_set pollset_sets[2]; - const int num_fds = GPR_ARRAY_SIZE(tfds); - const int num_ps = 1; - const int num_pss = GPR_ARRAY_SIZE(pollset_sets); - - init_test_fds(tfds, num_fds); - init_test_pollsets(&pollset, num_ps); - init_test_pollset_sets(pollset_sets, num_pss); - - /* Construct the structure */ - grpc_pollset_set_add_fd(pollset_sets[0].pss, tfds[0].fd); - grpc_pollset_set_add_fd(pollset_sets[1].pss, tfds[0].fd); - grpc_pollset_set_add_fd(pollset_sets[1].pss, tfds[1].fd); - - grpc_pollset_add_fd(pollset.ps, tfds[1].fd); - grpc_pollset_add_fd(pollset.ps, tfds[2].fd); - - grpc_pollset_set_add_pollset(pollset_sets[1].pss, pollset.ps); - grpc_pollset_set_add_pollset_set(pollset_sets[0].pss, pollset_sets[1].pss); - - /* Test. Make all FDs readable and make sure that can be observed by doing a - * grpc_pollset_work on the pollset 'PS' */ - make_test_fds_readable(tfds, num_fds); - - gpr_mu_lock(pollset.mu); - deadline = grpc_timespec_to_millis_round_up( - grpc_timeout_milliseconds_to_deadline(2)); - GPR_ASSERT(GRPC_ERROR_NONE == - grpc_pollset_work(pollset.ps, &worker, deadline)); - gpr_mu_unlock(pollset.mu); - grpc_core::ExecCtx::Get()->Flush(); - - verify_readable_and_reset(tfds, num_fds); - grpc_core::ExecCtx::Get()->Flush(); - - /* Tear down */ - grpc_pollset_set_del_fd(pollset_sets[0].pss, tfds[0].fd); - grpc_pollset_set_del_fd(pollset_sets[1].pss, tfds[0].fd); - grpc_pollset_set_del_fd(pollset_sets[1].pss, tfds[1].fd); - - grpc_pollset_set_del_pollset(pollset_sets[1].pss, pollset.ps); - grpc_pollset_set_del_pollset_set(pollset_sets[0].pss, pollset_sets[1].pss); - grpc_core::ExecCtx::Get()->Flush(); - - cleanup_test_fds(tfds, num_fds); - cleanup_test_pollsets(&pollset, num_ps); - cleanup_test_pollset_sets(pollset_sets, num_pss); -} - -/* Pollset_set with an empty pollset */ -void pollset_set_test_empty_pollset() { - /* We construct the following structure for this test: - * - * +---> PS0 (EMPTY) - * | - * +---> FD0 - * | - * PSS0---+ - * | +---> FD1 - * | | - * +---> PS1--+ - * | - * +---> FD2 - */ - grpc_core::ExecCtx exec_ctx; - grpc_pollset_worker* worker; - grpc_millis deadline; - - test_fd tfds[3]; - test_pollset pollsets[2]; - test_pollset_set pollset_set; - const int num_fds = GPR_ARRAY_SIZE(tfds); - const int num_ps = GPR_ARRAY_SIZE(pollsets); - const int num_pss = 1; - - init_test_fds(tfds, num_fds); - init_test_pollsets(pollsets, num_ps); - init_test_pollset_sets(&pollset_set, num_pss); - - /* Construct the structure */ - grpc_pollset_set_add_fd(pollset_set.pss, tfds[0].fd); - grpc_pollset_add_fd(pollsets[1].ps, tfds[1].fd); - grpc_pollset_add_fd(pollsets[1].ps, tfds[2].fd); - - grpc_pollset_set_add_pollset(pollset_set.pss, pollsets[0].ps); - grpc_pollset_set_add_pollset(pollset_set.pss, pollsets[1].ps); - - /* Test. Make all FDs readable and make sure that can be observed by doing - * grpc_pollset_work on the empty pollset 'PS0' */ - make_test_fds_readable(tfds, num_fds); - - gpr_mu_lock(pollsets[0].mu); - deadline = grpc_timespec_to_millis_round_up( - grpc_timeout_milliseconds_to_deadline(2)); - GPR_ASSERT(GRPC_ERROR_NONE == - grpc_pollset_work(pollsets[0].ps, &worker, deadline)); - gpr_mu_unlock(pollsets[0].mu); - grpc_core::ExecCtx::Get()->Flush(); - - verify_readable_and_reset(tfds, num_fds); - grpc_core::ExecCtx::Get()->Flush(); - - /* Tear down */ - grpc_pollset_set_del_fd(pollset_set.pss, tfds[0].fd); - grpc_pollset_set_del_pollset(pollset_set.pss, pollsets[0].ps); - grpc_pollset_set_del_pollset(pollset_set.pss, pollsets[1].ps); - grpc_core::ExecCtx::Get()->Flush(); - - cleanup_test_fds(tfds, num_fds); - cleanup_test_pollsets(pollsets, num_ps); - cleanup_test_pollset_sets(&pollset_set, num_pss); -} - -int main(int argc, char** argv) { - grpc_test_init(argc, argv); - grpc_init(); - { - grpc_core::ExecCtx exec_ctx; - const char* poll_strategy = grpc_get_poll_strategy_name(); - - if (poll_strategy != nullptr && - (strcmp(poll_strategy, "epollsig") == 0 || - strcmp(poll_strategy, "epoll-threadpool") == 0)) { - pollset_set_test_basic(); - pollset_set_test_dup_fds(); - pollset_set_test_empty_pollset(); - } else { - gpr_log(GPR_INFO, - "Skipping the test. The test is only relevant for 'epoll' " - "strategy. and the current strategy is: '%s'", - poll_strategy); - } - } - grpc_shutdown(); - return 0; -} -#else /* defined(GRPC_LINUX_EPOLL_CREATE1) */ -int main(int argc, char** argv) { return 0; } -#endif /* !defined(GRPC_LINUX_EPOLL_CREATE1) */ diff --git a/test/core/iomgr/resolve_address_test.cc b/test/core/iomgr/resolve_address_test.cc index 2fb831a6a4..52e4840c7c 100644 --- a/test/core/iomgr/resolve_address_test.cc +++ b/test/core/iomgr/resolve_address_test.cc @@ -22,8 +22,14 @@ #include <grpc/support/log.h> #include <grpc/support/sync.h> #include <grpc/support/time.h> + +#include <string.h> + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" +#include "test/core/util/cmdline.h" #include "test/core/util/test_config.h" static gpr_timespec test_deadline(void) { @@ -240,6 +246,28 @@ static void test_unparseable_hostports(void) { } int main(int argc, char** argv) { + // First set the resolver type based off of --resolver + const char* resolver_type = nullptr; + gpr_cmdline* cl = gpr_cmdline_create("resolve address test"); + gpr_cmdline_add_string(cl, "resolver", "Resolver type (ares or native)", + &resolver_type); + gpr_cmdline_parse(cl, argc, argv); + const char* cur_resolver = gpr_getenv("GRPC_DNS_RESOLVER"); + if (cur_resolver != nullptr && strlen(cur_resolver) != 0) { + gpr_log(GPR_INFO, "Warning: overriding resolver setting of %s", + cur_resolver); + } + if (gpr_stricmp(resolver_type, "native") == 0) { + gpr_setenv("GRPC_DNS_RESOLVER", "native"); + } else if (gpr_stricmp(resolver_type, "ares") == 0) { +#ifndef GRPC_UV + gpr_setenv("GRPC_DNS_RESOLVER", "ares"); +#endif + } else { + gpr_log(GPR_ERROR, "--resolver_type was not set to ares or native"); + abort(); + } + // Run the test. grpc_test_init(argc, argv); grpc_init(); { @@ -250,10 +278,18 @@ int main(int argc, char** argv) { test_missing_default_port(); test_ipv6_with_port(); test_ipv6_without_port(); - test_invalid_ip_addresses(); - test_unparseable_hostports(); + if (gpr_stricmp(resolver_type, "ares") != 0) { + // These tests can trigger DNS queries to the nearby nameserver + // that need to come back in order for the test to succeed. + // c-ares is prone to not using the local system caches that the + // native getaddrinfo implementations take advantage of, so running + // these unit tests under c-ares risks flakiness. + test_invalid_ip_addresses(); + test_unparseable_hostports(); + } grpc_executor_shutdown(); } + gpr_cmdline_destroy(cl); grpc_shutdown(); return 0; diff --git a/test/core/security/linux_system_roots_test.cc b/test/core/security/linux_system_roots_test.cc index fce9c8dcc5..24d446de35 100644 --- a/test/core/security/linux_system_roots_test.cc +++ b/test/core/security/linux_system_roots_test.cc @@ -41,10 +41,6 @@ #include "gtest/gtest.h" -#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR -#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" -#endif - namespace grpc { namespace { @@ -68,7 +64,6 @@ TEST(CreateRootCertsBundleTest, ReturnsEmpty) { } TEST(CreateRootCertsBundleTest, BundlesCorrectly) { - gpr_setenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR, "true"); // Test that CreateRootCertsBundle returns a correct slice. grpc_slice roots_bundle = grpc_empty_slice(); GRPC_LOG_IF_ERROR( @@ -81,7 +76,6 @@ TEST(CreateRootCertsBundleTest, BundlesCorrectly) { char* bundle_str = grpc_slice_to_c_string(roots_bundle); EXPECT_STREQ(result_str, bundle_str); // Clean up. - unsetenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); gpr_free(result_str); gpr_free(bundle_str); grpc_slice_unref(roots_bundle); diff --git a/test/core/security/security_connector_test.cc b/test/core/security/security_connector_test.cc index 82d77eef8b..9dd37b975b 100644 --- a/test/core/security/security_connector_test.cc +++ b/test/core/security/security_connector_test.cc @@ -415,6 +415,7 @@ static void test_default_ssl_roots(void) { /* Now setup a permanent failure for the overridden roots and we should get an empty slice. */ + gpr_setenv("GRPC_NOT_USE_SYSTEM_SSL_ROOTS", "true"); grpc_set_ssl_roots_override_callback(override_roots_permanent_failure); roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); GPR_ASSERT(GRPC_SLICE_IS_EMPTY(roots)); diff --git a/test/core/surface/completion_queue_test.cc b/test/core/surface/completion_queue_test.cc index b889fd0fc6..f7ce8a7042 100644 --- a/test/core/surface/completion_queue_test.cc +++ b/test/core/surface/completion_queue_test.cc @@ -369,11 +369,15 @@ static void test_callback(void) { LOG_TEST("test_callback"); bool got_shutdown = false; - class ShutdownCallback : public grpc_core::CQCallbackInterface { + class ShutdownCallback : public grpc_experimental_completion_queue_functor { public: - ShutdownCallback(bool* done) : done_(done) {} + ShutdownCallback(bool* done) : done_(done) { + functor_run = &ShutdownCallback::Run; + } ~ShutdownCallback() {} - void Run(bool ok) override { *done_ = ok; } + static void Run(grpc_experimental_completion_queue_functor* cb, int ok) { + *static_cast<ShutdownCallback*>(cb)->done_ = static_cast<bool>(ok); + } private: bool* done_; @@ -391,14 +395,17 @@ static void test_callback(void) { grpc_completion_queue_factory_lookup(&attr), &attr, nullptr); int counter = 0; - class TagCallback : public grpc_core::CQCallbackInterface { + class TagCallback : public grpc_experimental_completion_queue_functor { public: - TagCallback(int* counter, int tag) : counter_(counter), tag_(tag) {} + TagCallback(int* counter, int tag) : counter_(counter), tag_(tag) { + functor_run = &TagCallback::Run; + } ~TagCallback() {} - void Run(bool ok) override { - GPR_ASSERT(ok); - *counter_ += tag_; - grpc_core::Delete(this); + static void Run(grpc_experimental_completion_queue_functor* cb, int ok) { + GPR_ASSERT(static_cast<bool>(ok)); + auto* callback = static_cast<TagCallback*>(cb); + *callback->counter_ += callback->tag_; + grpc_core::Delete(callback); }; private: diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index b832a1661b..3ebdb88a08 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -136,7 +136,10 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_resource_quota_set_max_threads); printf("%lx", (unsigned long) grpc_resource_quota_arg_vtable); printf("%lx", (unsigned long) grpc_channelz_get_top_channels); + printf("%lx", (unsigned long) grpc_channelz_get_servers); printf("%lx", (unsigned long) grpc_channelz_get_channel); + printf("%lx", (unsigned long) grpc_channelz_get_subchannel); + printf("%lx", (unsigned long) grpc_channelz_get_socket); printf("%lx", (unsigned long) grpc_auth_property_iterator_next); printf("%lx", (unsigned long) grpc_auth_context_property_iterator); printf("%lx", (unsigned long) grpc_auth_context_peer_identity); diff --git a/test/core/transport/chttp2/hpack_encoder_test.cc b/test/core/transport/chttp2/hpack_encoder_test.cc index 2a57198ab6..ab819f9092 100644 --- a/test/core/transport/chttp2/hpack_encoder_test.cc +++ b/test/core/transport/chttp2/hpack_encoder_test.cc @@ -202,7 +202,7 @@ static void verify_table_size_change_match_elem_size(const char* key, grpc_mdelem elem = grpc_mdelem_from_slices( grpc_slice_intern(grpc_slice_from_static_string(key)), grpc_slice_intern(grpc_slice_from_static_string(value))); - size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, use_true_binary); + size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, use_true_binary); size_t initial_table_size = g_compressor.table_size; grpc_linked_mdelem* e = static_cast<grpc_linked_mdelem*>(gpr_malloc(sizeof(*e))); diff --git a/test/core/transport/metadata_test.cc b/test/core/transport/metadata_test.cc index 4be34f72d9..8ab9639dfa 100644 --- a/test/core/transport/metadata_test.cc +++ b/test/core/transport/metadata_test.cc @@ -27,6 +27,7 @@ #include <grpc/support/string_util.h> #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" +#include "src/core/ext/transport/chttp2/transport/hpack_table.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" @@ -293,7 +294,7 @@ static void verify_ascii_header_size(const char* key, const char* value, grpc_mdelem elem = grpc_mdelem_from_slices( maybe_intern(grpc_slice_from_static_string(key), intern_key), maybe_intern(grpc_slice_from_static_string(value), intern_value)); - size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, false); + size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, false); size_t expected_size = 32 + strlen(key) + strlen(value); GPR_ASSERT(expected_size == elem_size); GRPC_MDELEM_UNREF(elem); @@ -307,7 +308,7 @@ static void verify_binary_header_size(const char* key, const uint8_t* value, maybe_intern(grpc_slice_from_static_buffer(value, value_len), intern_value)); GPR_ASSERT(grpc_is_binary_header(GRPC_MDKEY(elem))); - size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, false); + size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, false); grpc_slice value_slice = grpc_slice_from_copied_buffer( reinterpret_cast<const char*>(value), value_len); grpc_slice base64_encoded = grpc_chttp2_base64_encode(value_slice); diff --git a/test/core/tsi/alts/fake_handshaker/BUILD b/test/core/tsi/alts/fake_handshaker/BUILD index a09a046d27..98cd628a7d 100644 --- a/test/core/tsi/alts/fake_handshaker/BUILD +++ b/test/core/tsi/alts/fake_handshaker/BUILD @@ -37,21 +37,22 @@ grpc_cc_library( name = "fake_handshaker_lib", testonly = True, srcs = ["fake_handshaker_server.cc"], + hdrs = ["fake_handshaker_server.h"], language = "C++", deps = [ "handshaker_proto", "transport_security_common_proto", "//:grpc++", - "//test/cpp/util:test_config", ], ) grpc_cc_binary( name = "fake_handshaker_server", testonly = True, - srcs = ["fake_handshaker_server.cc"], + srcs = ["fake_handshaker_server_main.cc"], language = "C++", deps = [ + "//test/cpp/util:test_config", "fake_handshaker_lib", ], ) diff --git a/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.cc b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.cc index f6a4791b49..ba246b07eb 100644 --- a/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.cc +++ b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.cc @@ -15,12 +15,12 @@ * limitations under the License. * */ +#include "test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h" #include <memory> #include <sstream> #include <string> -#include <gflags/gflags.h> #include <grpc/grpc.h> #include <grpc/support/log.h> #include <grpcpp/impl/codegen/async_stream.h> @@ -32,10 +32,6 @@ #include "test/core/tsi/alts/fake_handshaker/handshaker.grpc.pb.h" #include "test/core/tsi/alts/fake_handshaker/handshaker.pb.h" #include "test/core/tsi/alts/fake_handshaker/transport_security_common.pb.h" -#include "test/cpp/util/test_config.h" - -DEFINE_int32(handshaker_port, 55056, - "TCP port on which the fake handshaker server listens to."); // Fake handshake messages. constexpr char kClientInitFrame[] = "ClientInit"; @@ -243,26 +239,9 @@ class FakeHandshakerService : public HandshakerService::Service { } }; -} // namespace gcp -} // namespace grpc - -void RunServer() { - GPR_ASSERT(FLAGS_handshaker_port != 0); - std::ostringstream server_address; - server_address << "[::1]:" << FLAGS_handshaker_port; - grpc::gcp::FakeHandshakerService service; - grpc::ServerBuilder builder; - builder.AddListeningPort(server_address.str(), - grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - std::unique_ptr<grpc::Server> server(builder.BuildAndStart()); - gpr_log(GPR_INFO, "Fake handshaker server listening on %s", - server_address.str().c_str()); - server->Wait(); +std::unique_ptr<grpc::Service> CreateFakeHandshakerService() { + return std::unique_ptr<grpc::Service>{new grpc::gcp::FakeHandshakerService}; } -int main(int argc, char** argv) { - grpc::testing::InitTest(&argc, &argv, true); - RunServer(); - return 0; -} +} // namespace gcp +} // namespace grpc diff --git a/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h new file mode 100644 index 0000000000..eb4bfdffa1 --- /dev/null +++ b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h @@ -0,0 +1,29 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include <memory> +#include <string> + +#include <grpcpp/grpcpp.h> + +namespace grpc { +namespace gcp { + +std::unique_ptr<grpc::Service> CreateFakeHandshakerService(); + +} // namespace gcp +} // namespace grpc diff --git a/test/core/tsi/alts/fake_handshaker/fake_handshaker_server_main.cc b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server_main.cc new file mode 100644 index 0000000000..60351533d9 --- /dev/null +++ b/test/core/tsi/alts/fake_handshaker/fake_handshaker_server_main.cc @@ -0,0 +1,53 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include "test/core/tsi/alts/fake_handshaker/fake_handshaker_server.h" + +#include <sstream> + +#include <gflags/gflags.h> +#include <grpc/support/log.h> +#include <grpcpp/impl/codegen/service_type.h> +#include <grpcpp/server_builder.h> + +#include "test/cpp/util/test_config.h" + +DEFINE_int32(handshaker_port, 55056, + "TCP port on which the fake handshaker server listens to."); + +static void RunFakeHandshakerServer(const std::string& server_address) { + std::unique_ptr<grpc::Service> service = + grpc::gcp::CreateFakeHandshakerService(); + grpc::ServerBuilder builder; + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + builder.RegisterService(service.get()); + gpr_log(GPR_INFO, "Fake handshaker server listening on %s", + server_address.c_str()); + std::unique_ptr<grpc::Server> server = builder.BuildAndStart(); + server->Wait(); +} + +int main(int argc, char** argv) { + grpc::testing::InitTest(&argc, &argv, true); + + GPR_ASSERT(FLAGS_handshaker_port != 0); + std::ostringstream server_address; + server_address << "[::1]:" << FLAGS_handshaker_port; + + RunFakeHandshakerServer(server_address.str()); + return 0; +} diff --git a/test/core/util/port_isolated_runtime_environment.cc b/test/core/util/port_isolated_runtime_environment.cc index ff8342ff4a..1f678c08e5 100644 --- a/test/core/util/port_isolated_runtime_environment.cc +++ b/test/core/util/port_isolated_runtime_environment.cc @@ -16,9 +16,12 @@ * */ -/* When running tests on remote machines, the framework takes a round-robin pick - * of a port within certain range. There is no need to recycle ports. +/* When individual tests run in an isolated runtime environment (e.g. each test + * runs in a separate container) the framework takes a round-robin pick of a + * port within certain range. There is no need to recycle ports. */ +#include <grpc/support/atm.h> +#include <grpc/support/log.h> #include <grpc/support/time.h> #include <stdlib.h> #include "src/core/lib/iomgr/port.h" @@ -27,23 +30,25 @@ #include "test/core/util/port.h" -#define MIN_PORT 49152 -#define MAX_PORT 65536 +#define MIN_PORT 1025 +#define MAX_PORT 32766 -int get_random_starting_port() { +static int get_random_port_offset() { srand(gpr_now(GPR_CLOCK_REALTIME).tv_nsec); - return rand() % (MAX_PORT - MIN_PORT + 1) + MIN_PORT; + double rnd = static_cast<double>(rand()) / + (static_cast<double>(RAND_MAX) + 1.0); // values from [0,1) + return static_cast<int>(rnd * (MAX_PORT - MIN_PORT + 1)); } -static int s_allocated_port = get_random_starting_port(); +static int s_initial_offset = get_random_port_offset(); +static gpr_atm s_pick_counter = 0; int grpc_pick_unused_port_or_die(void) { - int allocated_port = s_allocated_port++; - if (s_allocated_port == MAX_PORT) { - s_allocated_port = MIN_PORT; - } - - return allocated_port; + int orig_counter_val = + static_cast<int>(gpr_atm_full_fetch_add(&s_pick_counter, 1)); + GPR_ASSERT(orig_counter_val < (MAX_PORT - MIN_PORT + 1)); + return MIN_PORT + + (s_initial_offset + orig_counter_val) % (MAX_PORT - MIN_PORT + 1); } void grpc_recycle_unused_port(int port) { (void)port; } diff --git a/test/cpp/codegen/compiler_test_golden b/test/cpp/codegen/compiler_test_golden index 756f6a0224..93e1e68654 100644 --- a/test/cpp/codegen/compiler_test_golden +++ b/test/cpp/codegen/compiler_test_golden @@ -26,6 +26,7 @@ #include "src/proto/grpc/testing/compiler_test.pb.h" +#include <functional> #include <grpcpp/impl/codegen/async_generic_service.h> #include <grpcpp/impl/codegen/async_stream.h> #include <grpcpp/impl/codegen/async_unary_call.h> @@ -105,6 +106,23 @@ class ServiceA final { return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); } // Method A4 trailing comment 1 + class experimental_async_interface { + public: + virtual ~experimental_async_interface() {} + // MethodA1 leading comment 1 + virtual void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0; + // MethodA1 trailing comment 1 + // MethodA2 detached leading comment 1 + // + // Method A2 leading comment 1 + // Method A2 leading comment 2 + // MethodA2 trailing comment 1 + // Method A3 leading comment 1 + // Method A3 trailing comment 1 + // Method A4 leading comment 1 + // Method A4 trailing comment 1 + }; + virtual class experimental_async_interface* experimental_async() { return nullptr; } private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; @@ -155,9 +173,21 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); } + class experimental_async final : + public StubInterface::experimental_async_interface { + public: + void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override; + private: + friend class Stub; + explicit experimental_async(Stub* stub): stub_(stub) { } + Stub* stub() { return stub_; } + Stub* stub_; + }; + class experimental_async_interface* experimental_async() override { return &async_stub_; } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; + class experimental_async async_stub_{this}; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientWriter< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) override; @@ -488,6 +518,14 @@ class ServiceB final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); } // MethodB1 trailing comment 1 + class experimental_async_interface { + public: + virtual ~experimental_async_interface() {} + // MethodB1 leading comment 1 + virtual void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0; + // MethodB1 trailing comment 1 + }; + virtual class experimental_async_interface* experimental_async() { return nullptr; } private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; @@ -502,9 +540,21 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); } + class experimental_async final : + public StubInterface::experimental_async_interface { + public: + void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override; + private: + friend class Stub; + explicit experimental_async(Stub* stub): stub_(stub) { } + Stub* stub() { return stub_; } + Stub* stub_; + }; + class experimental_async_interface* experimental_async() override { return &async_stub_; } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; + class experimental_async async_stub_{this}; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; const ::grpc::internal::RpcMethod rpcmethod_MethodB1_; diff --git a/test/cpp/common/alarm_test.cc b/test/cpp/common/alarm_test.cc index 57d958349e..e909d03658 100644 --- a/test/cpp/common/alarm_test.cc +++ b/test/cpp/common/alarm_test.cc @@ -16,9 +16,13 @@ * */ +#include <condition_variable> +#include <memory> +#include <mutex> +#include <thread> + #include <grpcpp/alarm.h> #include <grpcpp/completion_queue.h> -#include <thread> #include <gtest/gtest.h> @@ -43,6 +47,66 @@ TEST(AlarmTest, RegularExpiry) { EXPECT_EQ(junk, output_tag); } +struct Completion { + bool completed = false; + std::mutex mu; + std::condition_variable cv; +}; + +TEST(AlarmTest, CallbackRegularExpiry) { + Alarm alarm; + + auto c = std::make_shared<Completion>(); + alarm.experimental().Set( + std::chrono::system_clock::now() + std::chrono::seconds(1), [c](bool ok) { + EXPECT_TRUE(ok); + std::lock_guard<std::mutex> l(c->mu); + c->completed = true; + c->cv.notify_one(); + }); + + std::unique_lock<std::mutex> l(c->mu); + EXPECT_TRUE(c->cv.wait_until( + l, std::chrono::system_clock::now() + std::chrono::seconds(10), + [c] { return c->completed; })); +} + +TEST(AlarmTest, CallbackZeroExpiry) { + Alarm alarm; + + auto c = std::make_shared<Completion>(); + alarm.experimental().Set(grpc_timeout_seconds_to_deadline(0), [c](bool ok) { + EXPECT_TRUE(ok); + std::lock_guard<std::mutex> l(c->mu); + c->completed = true; + c->cv.notify_one(); + }); + + std::unique_lock<std::mutex> l(c->mu); + EXPECT_TRUE(c->cv.wait_until( + l, std::chrono::system_clock::now() + std::chrono::seconds(10), + [c] { return c->completed; })); +} + +TEST(AlarmTest, CallbackNegativeExpiry) { + Alarm alarm; + + auto c = std::make_shared<Completion>(); + alarm.experimental().Set( + std::chrono::system_clock::now() + std::chrono::seconds(-1), + [c](bool ok) { + EXPECT_TRUE(ok); + std::lock_guard<std::mutex> l(c->mu); + c->completed = true; + c->cv.notify_one(); + }); + + std::unique_lock<std::mutex> l(c->mu); + EXPECT_TRUE(c->cv.wait_until( + l, std::chrono::system_clock::now() + std::chrono::seconds(10), + [c] { return c->completed; })); +} + TEST(AlarmTest, MultithreadedRegularExpiry) { CompletionQueue cq; void* junk = reinterpret_cast<void*>(1618033); @@ -182,6 +246,26 @@ TEST(AlarmTest, Cancellation) { EXPECT_EQ(junk, output_tag); } +TEST(AlarmTest, CallbackCancellation) { + Alarm alarm; + + auto c = std::make_shared<Completion>(); + alarm.experimental().Set( + std::chrono::system_clock::now() + std::chrono::seconds(10), + [c](bool ok) { + EXPECT_FALSE(ok); + std::lock_guard<std::mutex> l(c->mu); + c->completed = true; + c->cv.notify_one(); + }); + alarm.Cancel(); + + std::unique_lock<std::mutex> l(c->mu); + EXPECT_TRUE(c->cv.wait_until( + l, std::chrono::system_clock::now() + std::chrono::seconds(1), + [c] { return c->completed; })); +} + TEST(AlarmTest, SetDestruction) { CompletionQueue cq; void* junk = reinterpret_cast<void*>(1618033); @@ -200,6 +284,26 @@ TEST(AlarmTest, SetDestruction) { EXPECT_EQ(junk, output_tag); } +TEST(AlarmTest, CallbackSetDestruction) { + auto c = std::make_shared<Completion>(); + { + Alarm alarm; + alarm.experimental().Set( + std::chrono::system_clock::now() + std::chrono::seconds(10), + [c](bool ok) { + EXPECT_FALSE(ok); + std::lock_guard<std::mutex> l(c->mu); + c->completed = true; + c->cv.notify_one(); + }); + } + + std::unique_lock<std::mutex> l(c->mu); + EXPECT_TRUE(c->cv.wait_until( + l, std::chrono::system_clock::now() + std::chrono::seconds(1), + [c] { return c->completed; })); +} + TEST(AlarmTest, UnsetDestruction) { CompletionQueue cq; Alarm alarm; diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index 75dec56a60..0415efc1ef 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -98,6 +98,25 @@ grpc_cc_binary( ], ) +grpc_cc_test( + name = "client_callback_end2end_test", + srcs = ["client_callback_end2end_test.cc"], + external_deps = [ + "gtest", + ], + deps = [ + ":test_service_impl", + "//:gpr", + "//:grpc", + "//:grpc++", + "//src/proto/grpc/testing:echo_messages_proto", + "//src/proto/grpc/testing:echo_proto", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + "//test/cpp/util:test_util", + ], +) + grpc_cc_library( name = "end2end_test_lib", testonly = True, diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index c9246f0806..d97ea071d1 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -178,7 +178,7 @@ class Verifier { EXPECT_EQ(it2->second.ok, ok); } } else { - gpr_log(GPR_ERROR, "Unexpected tag: %p", tag); + gpr_log(GPR_ERROR, "Unexpected tag: %p", got_tag); abort(); } } diff --git a/test/cpp/end2end/channelz_service_test.cc b/test/cpp/end2end/channelz_service_test.cc index 933e4a1ff6..e96d68f93b 100644 --- a/test/cpp/end2end/channelz_service_test.cc +++ b/test/cpp/end2end/channelz_service_test.cc @@ -35,10 +35,16 @@ #include "test/core/util/test_config.h" #include "test/cpp/end2end/test_service_impl.h" +#include <google/protobuf/text_format.h> + #include <gtest/gtest.h> using grpc::channelz::v1::GetChannelRequest; using grpc::channelz::v1::GetChannelResponse; +using grpc::channelz::v1::GetServersRequest; +using grpc::channelz::v1::GetServersResponse; +using grpc::channelz::v1::GetSubchannelRequest; +using grpc::channelz::v1::GetSubchannelResponse; using grpc::channelz::v1::GetTopChannelsRequest; using grpc::channelz::v1::GetTopChannelsResponse; @@ -140,7 +146,7 @@ class ChannelzServerTest : public ::testing::Test { ClientContext context; Status s = echo_stub_->Echo(&context, request, &response); EXPECT_EQ(response.message(), request.message()); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); } void SendFailedEcho(int channel_idx) { @@ -156,6 +162,19 @@ class ChannelzServerTest : public ::testing::Test { EXPECT_FALSE(s.ok()); } + // Uses GetTopChannels to return the channel_id of a particular channel, + // so that the unit tests may test GetChannel call. + intptr_t GetChannelId(int channel_idx) { + GetTopChannelsRequest request; + GetTopChannelsResponse response; + request.set_start_channel_id(0); + ClientContext context; + Status s = channelz_stub_->GetTopChannels(&context, request, &response); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); + EXPECT_GT(response.channel_size(), channel_idx); + return response.channel(channel_idx).ref().channel_id(); + } + static string to_string(const int number) { std::stringstream strs; strs << number; @@ -190,7 +209,7 @@ TEST_F(ChannelzServerTest, BasicTest) { request.set_start_channel_id(0); ClientContext context; Status s = channelz_stub_->GetTopChannels(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel_size(), 1); } @@ -202,7 +221,7 @@ TEST_F(ChannelzServerTest, HighStartId) { request.set_start_channel_id(10000); ClientContext context; Status s = channelz_stub_->GetTopChannels(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel_size(), 0); } @@ -212,10 +231,10 @@ TEST_F(ChannelzServerTest, SuccessfulRequestTest) { SendSuccessfulEcho(0); GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(1); + request.set_channel_id(GetChannelId(0)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), 1); EXPECT_EQ(response.channel().data().calls_succeeded(), 1); EXPECT_EQ(response.channel().data().calls_failed(), 0); @@ -227,10 +246,10 @@ TEST_F(ChannelzServerTest, FailedRequestTest) { SendFailedEcho(0); GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(1); + request.set_channel_id(GetChannelId(0)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), 1); EXPECT_EQ(response.channel().data().calls_succeeded(), 0); EXPECT_EQ(response.channel().data().calls_failed(), 1); @@ -250,10 +269,10 @@ TEST_F(ChannelzServerTest, ManyRequestsTest) { } GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(1); + request.set_channel_id(GetChannelId(0)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), kNumSuccess + kNumFailed); EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); @@ -269,7 +288,7 @@ TEST_F(ChannelzServerTest, ManyChannels) { request.set_start_channel_id(0); ClientContext context; Status s = channelz_stub_->GetTopChannels(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel_size(), kNumChannels); } @@ -292,10 +311,10 @@ TEST_F(ChannelzServerTest, ManyRequestsManyChannels) { { GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(1); + request.set_channel_id(GetChannelId(0)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), kNumSuccess); EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); EXPECT_EQ(response.channel().data().calls_failed(), 0); @@ -305,10 +324,10 @@ TEST_F(ChannelzServerTest, ManyRequestsManyChannels) { { GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(2); + request.set_channel_id(GetChannelId(1)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), kNumFailed); EXPECT_EQ(response.channel().data().calls_succeeded(), 0); EXPECT_EQ(response.channel().data().calls_failed(), kNumFailed); @@ -318,10 +337,10 @@ TEST_F(ChannelzServerTest, ManyRequestsManyChannels) { { GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(3); + request.set_channel_id(GetChannelId(2)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), kNumSuccess + kNumFailed); EXPECT_EQ(response.channel().data().calls_succeeded(), kNumSuccess); @@ -332,16 +351,103 @@ TEST_F(ChannelzServerTest, ManyRequestsManyChannels) { { GetChannelRequest request; GetChannelResponse response; - request.set_channel_id(4); + request.set_channel_id(GetChannelId(3)); ClientContext context; Status s = channelz_stub_->GetChannel(&context, request, &response); - EXPECT_TRUE(s.ok()); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); EXPECT_EQ(response.channel().data().calls_started(), 0); EXPECT_EQ(response.channel().data().calls_succeeded(), 0); EXPECT_EQ(response.channel().data().calls_failed(), 0); } } +TEST_F(ChannelzServerTest, ManySubchannels) { + ResetStubs(); + const int kNumChannels = 4; + ConfigureProxy(kNumChannels); + const int kNumSuccess = 10; + const int kNumFailed = 11; + for (int i = 0; i < kNumSuccess; ++i) { + SendSuccessfulEcho(0); + SendSuccessfulEcho(2); + } + for (int i = 0; i < kNumFailed; ++i) { + SendFailedEcho(1); + SendFailedEcho(2); + } + GetTopChannelsRequest gtc_request; + GetTopChannelsResponse gtc_response; + gtc_request.set_start_channel_id(0); + ClientContext context; + Status s = + channelz_stub_->GetTopChannels(&context, gtc_request, >c_response); + EXPECT_TRUE(s.ok()) << s.error_message(); + EXPECT_EQ(gtc_response.channel_size(), kNumChannels); + for (int i = 0; i < gtc_response.channel_size(); ++i) { + // if the channel sent no RPCs, then expect no subchannels to have been + // created. + if (gtc_response.channel(i).data().calls_started() == 0) { + EXPECT_EQ(gtc_response.channel(i).subchannel_ref_size(), 0); + continue; + } + // The resolver must return at least one address. + ASSERT_GT(gtc_response.channel(i).subchannel_ref_size(), 0); + GetSubchannelRequest gsc_request; + GetSubchannelResponse gsc_response; + gsc_request.set_subchannel_id( + gtc_response.channel(i).subchannel_ref(0).subchannel_id()); + ClientContext context; + Status s = + channelz_stub_->GetSubchannel(&context, gsc_request, &gsc_response); + EXPECT_TRUE(s.ok()) << s.error_message(); + EXPECT_EQ(gtc_response.channel(i).data().calls_started(), + gsc_response.subchannel().data().calls_started()); + EXPECT_EQ(gtc_response.channel(i).data().calls_succeeded(), + gsc_response.subchannel().data().calls_succeeded()); + EXPECT_EQ(gtc_response.channel(i).data().calls_failed(), + gsc_response.subchannel().data().calls_failed()); + } +} + +TEST_F(ChannelzServerTest, BasicServerTest) { + ResetStubs(); + ConfigureProxy(1); + GetServersRequest request; + GetServersResponse response; + request.set_start_server_id(0); + ClientContext context; + Status s = channelz_stub_->GetServers(&context, request, &response); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); + EXPECT_EQ(response.server_size(), 1); +} + +TEST_F(ChannelzServerTest, ServerCallTest) { + ResetStubs(); + ConfigureProxy(1); + const int kNumSuccess = 10; + const int kNumFailed = 11; + for (int i = 0; i < kNumSuccess; ++i) { + SendSuccessfulEcho(0); + } + for (int i = 0; i < kNumFailed; ++i) { + SendFailedEcho(0); + } + GetServersRequest request; + GetServersResponse response; + request.set_start_server_id(0); + ClientContext context; + Status s = channelz_stub_->GetServers(&context, request, &response); + EXPECT_TRUE(s.ok()) << "s.error_message() = " << s.error_message(); + EXPECT_EQ(response.server_size(), 1); + EXPECT_EQ(response.server(0).data().calls_succeeded(), kNumSuccess); + EXPECT_EQ(response.server(0).data().calls_failed(), kNumFailed); + // This is success+failure+1 because the call that retrieved this information + // will be counted as started. It will not track success/failure until after + // it has returned, so that is not included in the response. + EXPECT_EQ(response.server(0).data().calls_started(), + kNumSuccess + kNumFailed + 1); +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/end2end/client_callback_end2end_test.cc b/test/cpp/end2end/client_callback_end2end_test.cc new file mode 100644 index 0000000000..62a85641c7 --- /dev/null +++ b/test/cpp/end2end/client_callback_end2end_test.cc @@ -0,0 +1,241 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <functional> +#include <mutex> +#include <thread> + +#include <grpcpp/channel.h> +#include <grpcpp/client_context.h> +#include <grpcpp/create_channel.h> +#include <grpcpp/generic/generic_stub.h> +#include <grpcpp/impl/codegen/proto_utils.h> +#include <grpcpp/server.h> +#include <grpcpp/server_builder.h> +#include <grpcpp/server_context.h> +#include <grpcpp/support/client_callback.h> + +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/core/util/test_config.h" +#include "test/cpp/end2end/test_service_impl.h" +#include "test/cpp/util/byte_buffer_proto_helper.h" + +#include <gtest/gtest.h> + +namespace grpc { +namespace testing { +namespace { + +class ClientCallbackEnd2endTest : public ::testing::Test { + protected: + ClientCallbackEnd2endTest() {} + + void SetUp() override { + ServerBuilder builder; + builder.RegisterService(&service_); + + server_ = builder.BuildAndStart(); + is_server_started_ = true; + } + + void ResetStub() { + ChannelArguments args; + channel_ = server_->InProcessChannel(args); + stub_ = grpc::testing::EchoTestService::NewStub(channel_); + generic_stub_.reset(new GenericStub(channel_)); + } + + void TearDown() override { + if (is_server_started_) { + server_->Shutdown(); + } + } + + void SendRpcs(int num_rpcs, bool with_binary_metadata) { + grpc::string test_string(""); + for (int i = 0; i < num_rpcs; i++) { + EchoRequest request; + EchoResponse response; + ClientContext cli_ctx; + + test_string += "Hello world. "; + request.set_message(test_string); + + if (with_binary_metadata) { + char bytes[8] = {'\0', '\1', '\2', '\3', + '\4', '\5', '\6', static_cast<char>(i)}; + cli_ctx.AddMetadata("custom-bin", grpc::string(bytes, 8)); + } + + cli_ctx.set_compression_algorithm(GRPC_COMPRESS_GZIP); + + std::mutex mu; + std::condition_variable cv; + bool done = false; + stub_->experimental_async()->Echo( + &cli_ctx, &request, &response, + [&request, &response, &done, &mu, &cv](Status s) { + GPR_ASSERT(s.ok()); + + EXPECT_EQ(request.message(), response.message()); + std::lock_guard<std::mutex> l(mu); + done = true; + cv.notify_one(); + }); + std::unique_lock<std::mutex> l(mu); + while (!done) { + cv.wait(l); + } + } + } + + void SendRpcsGeneric(int num_rpcs, bool maybe_except) { + const grpc::string kMethodName("/grpc.testing.EchoTestService/Echo"); + grpc::string test_string(""); + for (int i = 0; i < num_rpcs; i++) { + EchoRequest request; + std::unique_ptr<ByteBuffer> send_buf; + ByteBuffer recv_buf; + ClientContext cli_ctx; + + test_string += "Hello world. "; + request.set_message(test_string); + send_buf = SerializeToByteBuffer(&request); + + std::mutex mu; + std::condition_variable cv; + bool done = false; + generic_stub_->experimental().UnaryCall( + &cli_ctx, kMethodName, send_buf.get(), &recv_buf, + [&request, &recv_buf, &done, &mu, &cv, maybe_except](Status s) { + GPR_ASSERT(s.ok()); + + EchoResponse response; + EXPECT_TRUE(ParseFromByteBuffer(&recv_buf, &response)); + EXPECT_EQ(request.message(), response.message()); + std::lock_guard<std::mutex> l(mu); + done = true; + cv.notify_one(); +#if GRPC_ALLOW_EXCEPTIONS + if (maybe_except) { + throw - 1; + } +#else + GPR_ASSERT(!maybe_except); +#endif + }); + std::unique_lock<std::mutex> l(mu); + while (!done) { + cv.wait(l); + } + } + } + + bool is_server_started_; + std::shared_ptr<Channel> channel_; + std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_; + std::unique_ptr<grpc::GenericStub> generic_stub_; + TestServiceImpl service_; + std::unique_ptr<Server> server_; +}; + +TEST_F(ClientCallbackEnd2endTest, SimpleRpc) { + ResetStub(); + SendRpcs(1, false); +} + +TEST_F(ClientCallbackEnd2endTest, SequentialRpcs) { + ResetStub(); + SendRpcs(10, false); +} + +TEST_F(ClientCallbackEnd2endTest, SequentialRpcsWithVariedBinaryMetadataValue) { + ResetStub(); + SendRpcs(10, true); +} + +TEST_F(ClientCallbackEnd2endTest, SequentialGenericRpcs) { + ResetStub(); + SendRpcsGeneric(10, false); +} + +#if GRPC_ALLOW_EXCEPTIONS +TEST_F(ClientCallbackEnd2endTest, ExceptingRpc) { + ResetStub(); + SendRpcsGeneric(10, true); +} +#endif + +TEST_F(ClientCallbackEnd2endTest, MultipleRpcsWithVariedBinaryMetadataValue) { + ResetStub(); + std::vector<std::thread> threads; + threads.reserve(10); + for (int i = 0; i < 10; ++i) { + threads.emplace_back([this] { SendRpcs(10, true); }); + } + for (int i = 0; i < 10; ++i) { + threads[i].join(); + } +} + +TEST_F(ClientCallbackEnd2endTest, MultipleRpcs) { + ResetStub(); + std::vector<std::thread> threads; + threads.reserve(10); + for (int i = 0; i < 10; ++i) { + threads.emplace_back([this] { SendRpcs(10, false); }); + } + for (int i = 0; i < 10; ++i) { + threads[i].join(); + } +} + +TEST_F(ClientCallbackEnd2endTest, CancelRpcBeforeStart) { + ResetStub(); + EchoRequest request; + EchoResponse response; + ClientContext context; + request.set_message("hello"); + context.TryCancel(); + + std::mutex mu; + std::condition_variable cv; + bool done = false; + stub_->experimental_async()->Echo( + &context, &request, &response, [&response, &done, &mu, &cv](Status s) { + EXPECT_EQ("", response.message()); + EXPECT_EQ(grpc::StatusCode::CANCELLED, s.error_code()); + std::lock_guard<std::mutex> l(mu); + done = true; + cv.notify_one(); + }); + std::unique_lock<std::mutex> l(mu); + while (!done) { + cv.wait(l); + } +} + +} // namespace +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index e5d6132012..a9d68ab058 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -119,6 +119,7 @@ class ClientLbEnd2endTest : public ::testing::Test { } void SetUp() override { + grpc_init(); response_generator_ = grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>(); } @@ -127,6 +128,7 @@ class ClientLbEnd2endTest : public ::testing::Test { for (size_t i = 0; i < servers_.size(); ++i) { servers_[i]->Shutdown(); } + grpc_shutdown(); } void CreateServers(size_t num_servers, @@ -560,7 +562,23 @@ TEST_F(ClientLbEnd2endTest, PickFirstUpdateSuperset) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } -TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { +class ClientLbEnd2endWithParamTest + : public ClientLbEnd2endTest, + public ::testing::WithParamInterface<bool> { + protected: + void SetUp() override { + grpc_subchannel_index_test_only_set_force_creation(GetParam()); + ClientLbEnd2endTest::SetUp(); + } + + void TearDown() override { + ClientLbEnd2endTest::TearDown(); + grpc_subchannel_index_test_only_set_force_creation(false); + } +}; + +TEST_P(ClientLbEnd2endWithParamTest, PickFirstManyUpdates) { + gpr_log(GPR_INFO, "subchannel force creation: %d", GetParam()); // Start servers and send one RPC per server. const int kNumServers = 3; StartServers(kNumServers); @@ -570,20 +588,21 @@ TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { for (size_t i = 0; i < servers_.size(); ++i) { ports.emplace_back(servers_[i]->port_); } - for (const bool force_creation : {true, false}) { - grpc_subchannel_index_test_only_set_force_creation(force_creation); - gpr_log(GPR_INFO, "Force subchannel creation: %d", force_creation); - for (size_t i = 0; i < 1000; ++i) { - std::shuffle(ports.begin(), ports.end(), - std::mt19937(std::random_device()())); - SetNextResolution(ports); - if (i % 10 == 0) CheckRpcSendOk(stub, DEBUG_LOCATION); - } + for (size_t i = 0; i < 1000; ++i) { + std::shuffle(ports.begin(), ports.end(), + std::mt19937(std::random_device()())); + SetNextResolution(ports); + // We should re-enter core at the end of the loop to give the resolution + // setting closure a chance to run. + if ((i + 1) % 10 == 0) CheckRpcSendOk(stub, DEBUG_LOCATION); } // Check LB policy name for the channel. EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +INSTANTIATE_TEST_CASE_P(SubchannelForceCreation, ClientLbEnd2endWithParamTest, + ::testing::Bool()); + TEST_F(ClientLbEnd2endTest, PickFirstReresolutionNoSelected) { // Prepare the ports for up servers and down servers. const int kNumServers = 3; @@ -984,8 +1003,6 @@ TEST_F(ClientLbEnd2endTest, RoundRobinSingleReconnect) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); grpc_test_init(argc, argv); - grpc_init(); const auto result = RUN_ALL_TESTS(); - grpc_shutdown(); return result; } diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index 7bcf23c0eb..a4b1a85f85 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -57,6 +57,7 @@ DEFINE_string( "half_duplex : half-duplex streaming;\n" "jwt_token_creds: large_unary with JWT token auth;\n" "large_unary : single request and (large) response;\n" + "long_lived_channel: sends large_unary rpcs over a long-lived channel;\n" "oauth2_auth_token: raw oauth2 access token auth;\n" "per_rpc_creds: raw oauth2 access token on a single rpc;\n" "ping_pong : full-duplex streaming;\n" @@ -84,10 +85,12 @@ DEFINE_bool(do_not_abort_on_transient_failures, false, "whether abort() is called or not. It does not control whether the " "test is retried in case of transient failures (and currently the " "interop tests are not retried even if this flag is set to true)"); - DEFINE_int32(soak_iterations, 1000, "number of iterations to use for the two soak tests; rpc_soak and " "channel_soak"); +DEFINE_int32(iteration_interval, 10, + "The interval in seconds between rpcs. This is used by " + "long_connection test"); using grpc::testing::CreateChannelForTestCase; using grpc::testing::GetServiceAccountJsonKey; @@ -163,6 +166,9 @@ int main(int argc, char** argv) { FLAGS_soak_iterations); actions["rpc_soak"] = std::bind(&grpc::testing::InteropClient::DoRpcSoakTest, &client, FLAGS_soak_iterations); + actions["long_lived_channel"] = + std::bind(&grpc::testing::InteropClient::DoLongLivedChannelTest, &client, + FLAGS_soak_iterations, FLAGS_iteration_interval); UpdateActions(&actions); diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index b7ce90803b..a99cf8122f 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -1052,6 +1052,34 @@ bool InteropClient::DoChannelSoakTest(int32_t soak_iterations) { return true; } +bool InteropClient::DoLongLivedChannelTest(int32_t soak_iterations, + int32_t iteration_interval) { + gpr_log(GPR_DEBUG, "Sending %d RPCs...", soak_iterations); + GPR_ASSERT(soak_iterations > 0); + GPR_ASSERT(iteration_interval > 0); + SimpleRequest request; + SimpleResponse response; + int num_failures = 0; + for (int i = 0; i < soak_iterations; ++i) { + gpr_log(GPR_DEBUG, "Sending RPC number %d...", i); + if (!PerformLargeUnary(&request, &response)) { + gpr_log(GPR_ERROR, "Iteration %d failed.", i); + num_failures++; + } + gpr_sleep_until( + gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(iteration_interval, GPR_TIMESPAN))); + } + if (num_failures == 0) { + gpr_log(GPR_DEBUG, "long_lived_channel test done."); + return true; + } else { + gpr_log(GPR_DEBUG, "long_lived_channel test failed with %d rpc failures.", + num_failures); + return false; + } +} + bool InteropClient::DoUnimplementedService() { gpr_log(GPR_DEBUG, "Sending a request for an unimplemented service..."); diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index e5be44d1d4..0ceff55c5c 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -76,6 +76,8 @@ class InteropClient { // languages bool DoChannelSoakTest(int32_t soak_iterations); bool DoRpcSoakTest(int32_t soak_iterations); + bool DoLongLivedChannelTest(int32_t soak_iterations, + int32_t iteration_interval); // Auth tests. // username is a string containing the user email diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 6fcf048bf3..ba4b57ad22 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -27,6 +27,7 @@ #include "src/core/ext/transport/chttp2/transport/hpack_encoder.h" #include "src/core/ext/transport/chttp2/transport/hpack_parser.h" +#include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/static_metadata.h" @@ -456,8 +457,9 @@ static void BM_HpackParserParseHeader(benchmark::State& state) { std::vector<grpc_slice> benchmark_slices = Fixture::GetBenchmarkSlices(); grpc_chttp2_hpack_parser p; grpc_chttp2_hpack_parser_init(&p); + const int kArenaSize = 4096 * 4096; + p.on_header_user_data = gpr_arena_create(kArenaSize); p.on_header = OnHeader; - p.on_header_user_data = nullptr; for (auto slice : init_slices) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_chttp2_hpack_parser_parse(&p, slice)); } @@ -466,7 +468,14 @@ static void BM_HpackParserParseHeader(benchmark::State& state) { GPR_ASSERT(GRPC_ERROR_NONE == grpc_chttp2_hpack_parser_parse(&p, slice)); } grpc_core::ExecCtx::Get()->Flush(); + // Recreate arena every 4k iterations to avoid oom + if (0 == (state.iterations() & 0xfff)) { + gpr_arena_destroy((gpr_arena*)p.on_header_user_data); + p.on_header_user_data = gpr_arena_create(kArenaSize); + } } + // Clean up + gpr_arena_destroy((gpr_arena*)p.on_header_user_data); for (auto slice : init_slices) grpc_slice_unref(slice); for (auto slice : benchmark_slices) grpc_slice_unref(slice); grpc_chttp2_hpack_parser_destroy(&p); @@ -766,8 +775,58 @@ class RepresentativeServerTrailingMetadata { static void free_timeout(void* p) { gpr_free(p); } -// New implementation. -static void OnHeaderNew(void* user_data, grpc_mdelem md) { +// Benchmark the current on_initial_header implementation +static void OnInitialHeader(void* user_data, grpc_mdelem md) { + // Setup for benchmark. This will bloat the absolute values of this benchmark + grpc_chttp2_incoming_metadata_buffer buffer; + grpc_chttp2_incoming_metadata_buffer_init(&buffer, (gpr_arena*)user_data); + bool seen_error = false; + + // Below here is the code we actually care about benchmarking + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && + !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { + seen_error = true; + } + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) { + grpc_millis* cached_timeout = + static_cast<grpc_millis*>(grpc_mdelem_get_user_data(md, free_timeout)); + grpc_millis timeout; + if (cached_timeout != nullptr) { + timeout = *cached_timeout; + } else { + if (GPR_UNLIKELY( + !grpc_http2_decode_timeout(GRPC_MDVALUE(md), &timeout))) { + char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md)); + gpr_log(GPR_ERROR, "Ignoring bad timeout value '%s'", val); + gpr_free(val); + timeout = GRPC_MILLIS_INF_FUTURE; + } + if (GRPC_MDELEM_IS_INTERNED(md)) { + /* not already parsed: parse it now, and store the + * result away */ + cached_timeout = + static_cast<grpc_millis*>(gpr_malloc(sizeof(grpc_millis))); + *cached_timeout = timeout; + grpc_mdelem_set_user_data(md, free_timeout, cached_timeout); + } + } + benchmark::DoNotOptimize(timeout); + GRPC_MDELEM_UNREF(md); + } else { + const size_t new_size = buffer.size + GRPC_MDELEM_LENGTH(md); + if (!seen_error) { + buffer.size = new_size; + } + grpc_error* error = grpc_chttp2_incoming_metadata_buffer_add(&buffer, md); + if (error != GRPC_ERROR_NONE) { + GPR_ASSERT(0); + } + } + grpc_chttp2_incoming_metadata_buffer_destroy(&buffer); +} + +// Benchmark timeout handling +static void OnHeaderTimeout(void* user_data, grpc_mdelem md) { if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) { grpc_millis* cached_timeout = static_cast<grpc_millis*>(grpc_mdelem_get_user_data(md, free_timeout)); @@ -853,8 +912,13 @@ BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, RepresentativeServerInitialMetadata, UnrefHeader); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, RepresentativeServerTrailingMetadata, UnrefHeader); - -BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, SameDeadline, OnHeaderNew); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, + RepresentativeClientInitialMetadata, OnInitialHeader); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, + MoreRepresentativeClientInitialMetadata, OnInitialHeader); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, + RepresentativeServerInitialMetadata, OnInitialHeader); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, SameDeadline, OnHeaderTimeout); } // namespace hpack_parser_fixtures diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index 9d70277fbc..d390ae08f6 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -48,6 +48,7 @@ namespace testing { class FixtureConfiguration { public: + virtual ~FixtureConfiguration() {} virtual void ApplyCommonChannelArguments(ChannelArguments* c) const { c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX); c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX); diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc index e4ba37e2d6..bce72985dc 100644 --- a/test/cpp/microbenchmarks/helpers.cc +++ b/test/cpp/microbenchmarks/helpers.cc @@ -38,6 +38,7 @@ void TrackCounters::AddLabel(const grpc::string& label) { } void TrackCounters::AddToLabel(std::ostream& out, benchmark::State& state) { +#ifdef GRPC_COLLECT_STATS grpc_stats_data stats_end; grpc_stats_collect(&stats_end); grpc_stats_data stats; @@ -53,6 +54,7 @@ void TrackCounters::AddToLabel(std::ostream& out, benchmark::State& state) { << " " << grpc_stats_histogram_name[i] << "-99p:" << grpc_stats_histo_percentile(&stats, (grpc_stats_histograms)i, 99.0); } +#endif #ifdef GPR_LOW_LEVEL_COUNTERS grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot(); out << " locks/iter:" diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h index 4aabd4094a..25d34b5f87 100644 --- a/test/cpp/microbenchmarks/helpers.h +++ b/test/cpp/microbenchmarks/helpers.h @@ -63,6 +63,7 @@ extern gpr_atm gpr_now_call_count; class TrackCounters { public: TrackCounters() { grpc_stats_collect(&stats_begin_); } + virtual ~TrackCounters() {} virtual void Finish(benchmark::State& state); virtual void AddLabel(const grpc::string& label); virtual void AddToLabel(std::ostream& out, benchmark::State& state); diff --git a/test/cpp/naming/address_sorting_test.cc b/test/cpp/naming/address_sorting_test.cc index 04c300876c..fc6721d0ba 100644 --- a/test/cpp/naming/address_sorting_test.cc +++ b/test/cpp/naming/address_sorting_test.cc @@ -216,7 +216,7 @@ TEST_F(AddressSortingTest, TestDepriotizesUnreachableAddresses) { {"1.2.3.4:443", AF_INET}, {"5.6.7.8:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "1.2.3.4:443", "5.6.7.8:443", @@ -235,7 +235,7 @@ TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { {"[2607:f8b0:400a:801::1002]:443", AF_INET6}, {"1.2.3.4:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "1.2.3.4:443", "[2607:f8b0:400a:801::1002]:443", @@ -255,7 +255,7 @@ TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { {"[2607:f8b0:400a:801::1002]:443", AF_INET6}, {"1.2.3.4:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[2607:f8b0:400a:801::1002]:443", "1.2.3.4:443", @@ -279,7 +279,7 @@ TEST_F(AddressSortingTest, TestDepriotizesNonMatchingScope) { {"[2000:f8b0:400a:801::1002]:443", AF_INET6}, {"[fec0::5000]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[fec0::5000]:443", "[2000:f8b0:400a:801::1002]:443", @@ -302,7 +302,7 @@ TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTable) { {"[2002::5001]:443", AF_INET6}, {"[2001::5001]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[2001::5001]:443", "[2002::5001]:443", @@ -325,7 +325,7 @@ TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { {"[2001::5001]:443", AF_INET6}, {"[2002::5001]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[2001::5001]:443", "[2002::5001]:443", @@ -348,7 +348,7 @@ TEST_F(AddressSortingTest, {"[3ffe::5001]:443", AF_INET6}, {"1.2.3.4:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs( lb_addrs, { // The AF_INET address should be IPv4-mapped by the sort, @@ -381,7 +381,7 @@ TEST_F(AddressSortingTest, {v4_compat_dest, AF_INET6}, {"[::1]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[::1]:443", v4_compat_dest, @@ -404,7 +404,7 @@ TEST_F(AddressSortingTest, {"[1234::2]:443", AF_INET6}, {"[::1]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs( lb_addrs, { @@ -428,7 +428,7 @@ TEST_F(AddressSortingTest, {"[2001::1234]:443", AF_INET6}, {"[2000::5001]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs( lb_addrs, { // The 2000::/16 address should match the ::/0 prefix rule @@ -452,7 +452,7 @@ TEST_F( {"[2001::1231]:443", AF_INET6}, {"[2000::5001]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[2000::5001]:443", "[2001::1231]:443", @@ -473,7 +473,7 @@ TEST_F(AddressSortingTest, {"[fec0::1234]:443", AF_INET6}, {"[fc00::5001]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[fc00::5001]:443", "[fec0::1234]:443", @@ -498,7 +498,7 @@ TEST_F( {"[::ffff:1.1.1.2]:443", AF_INET6}, {"[1234::2]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { // ::ffff:0:2 should match the v4-mapped // precedence entry and be deprioritized. @@ -525,7 +525,7 @@ TEST_F(AddressSortingTest, TestPrefersSmallerScope) { {"[3ffe::5001]:443", AF_INET6}, {"[fec0::1234]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[fec0::1234]:443", "[3ffe::5001]:443", @@ -550,7 +550,7 @@ TEST_F(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { {"[3ffe:5001::]:443", AF_INET6}, {"[3ffe:1234::]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe:1234::]:443", "[3ffe:5001::]:443", @@ -571,7 +571,7 @@ TEST_F(AddressSortingTest, {"[3ffe::5001]:443", AF_INET6}, {"[3ffe::1234]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe::1234]:443", "[3ffe::5001]:443", @@ -591,7 +591,7 @@ TEST_F(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { {"[3ffe:8000::]:443", AF_INET6}, {"[3ffe:2000::]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe:2000::]:443", "[3ffe:8000::]:443", @@ -611,7 +611,7 @@ TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { {"[3ffe:6::]:443", AF_INET6}, {"[3ffe:c::]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe:c::]:443", "[3ffe:6::]:443", @@ -633,7 +633,7 @@ TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { {"[3ffe:1111:1111:1110::]:443", AF_INET6}, {"[3ffe:1111:1111:1111::]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe:1111:1111:1111::]:443", "[3ffe:1111:1111:1110::]:443", @@ -655,7 +655,7 @@ TEST_F(AddressSortingTest, TestStableSort) { {"[3ffe::1234]:443", AF_INET6}, {"[3ffe::1235]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe::1234]:443", "[3ffe::1235]:443", @@ -681,7 +681,7 @@ TEST_F(AddressSortingTest, TestStableSortFiveElements) { {"[3ffe::1234]:443", AF_INET6}, {"[3ffe::1235]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe::1231]:443", "[3ffe::1232]:443", @@ -702,7 +702,7 @@ TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExist) { {"[3ffe::1234]:443", AF_INET6}, {"[3ffe::1235]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[3ffe::1231]:443", "[3ffe::1232]:443", @@ -720,7 +720,7 @@ TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { {"[::ffff:5.6.7.8]:443", AF_INET6}, {"1.2.3.4:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[::ffff:5.6.7.8]:443", "1.2.3.4:443", @@ -748,7 +748,7 @@ TEST_F(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { {"[fec0::2000]:443", AF_INET6}, {v4_compat_dest, AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { // The sort should be stable since @@ -769,7 +769,7 @@ TEST_F(AddressSortingTest, TestPrefersIpv6Loopback) { {"[::1]:443", AF_INET6}, {"127.0.0.1:443", AF_INET}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[::1]:443", "127.0.0.1:443", @@ -783,7 +783,7 @@ TEST_F(AddressSortingTest, TestPrefersIpv6LoopbackInputsFlipped) { {"127.0.0.1:443", AF_INET}, {"[::1]:443", AF_INET6}, }); - grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + grpc_cares_wrapper_address_sorting_sort(lb_addrs); VerifyLbAddrOutputs(lb_addrs, { "[::1]:443", "127.0.0.1:443", diff --git a/test/cpp/qps/qps_benchmark_script.bzl b/test/cpp/qps/qps_benchmark_script.bzl index 0904ceb036..b2b67d988c 100644 --- a/test/cpp/qps/qps_benchmark_script.bzl +++ b/test/cpp/qps/qps_benchmark_script.bzl @@ -49,6 +49,9 @@ def qps_json_driver_batch(): "//test/cpp/util:test_config", "//test/cpp/util:test_util", ], + tags = [ + "qps_json_driver", + ], ) def json_run_localhost_batch(): @@ -71,4 +74,7 @@ def json_run_localhost_batch(): "//test/cpp/util:test_config", "//test/cpp/util:test_util", ], + tags = [ + "json_run_localhost", + ], ) diff --git a/test/cpp/util/channel_trace_proto_helper.cc b/test/cpp/util/channel_trace_proto_helper.cc index b4704bfe6a..42a436d49b 100644 --- a/test/cpp/util/channel_trace_proto_helper.cc +++ b/test/cpp/util/channel_trace_proto_helper.cc @@ -82,5 +82,18 @@ void ValidateGetChannelResponseProtoJsonTranslation(char* json_c_str) { json_c_str); } +void ValidateSubchannelProtoJsonTranslation(char* json_c_str) { + VaidateProtoJsonTranslation<grpc::channelz::v1::Subchannel>(json_c_str); +} + +void ValidateServerProtoJsonTranslation(char* json_c_str) { + VaidateProtoJsonTranslation<grpc::channelz::v1::Server>(json_c_str); +} + +void ValidateGetServersResponseProtoJsonTranslation(char* json_c_str) { + VaidateProtoJsonTranslation<grpc::channelz::v1::GetServersResponse>( + json_c_str); +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/util/channel_trace_proto_helper.h b/test/cpp/util/channel_trace_proto_helper.h index 18e3d54b6b..67c363e89b 100644 --- a/test/cpp/util/channel_trace_proto_helper.h +++ b/test/cpp/util/channel_trace_proto_helper.h @@ -26,6 +26,9 @@ void ValidateChannelTraceProtoJsonTranslation(char* json_c_str); void ValidateChannelProtoJsonTranslation(char* json_c_str); void ValidateGetTopChannelsResponseProtoJsonTranslation(char* json_c_str); void ValidateGetChannelResponseProtoJsonTranslation(char* json_c_str); +void ValidateSubchannelProtoJsonTranslation(char* json_c_str); +void ValidateServerProtoJsonTranslation(char* json_c_str); +void ValidateGetServersResponseProtoJsonTranslation(char* json_c_str); } // namespace testing } // namespace grpc diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index acf4ef8ef1..1125b2d945 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -19,6 +19,11 @@ #include "test/cpp/util/cli_credentials.h" #include <gflags/gflags.h> +#include <grpc/slice.h> +#include <grpc/support/log.h> +#include <grpcpp/impl/codegen/slice.h> + +#include "src/core/lib/iomgr/load_file.h" DEFINE_bool( enable_ssl, false, @@ -28,19 +33,52 @@ DEFINE_bool(use_auth, false, "--channel_creds_type=gdc."); DEFINE_string( access_token, "", - "The access token that will be sent to the server to authenticate RPCs."); + "The access token that will be sent to the server to authenticate RPCs. " + "Deprecated. Use --call_creds=access_token=<token>."); DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); DEFINE_string( + ssl_client_cert, "", + "If not empty, load this PEM formated client certificate file. Requires " + "use of --ssl_client_key."); +DEFINE_string( + ssl_client_key, "", + "If not empty, load this PEM formated private key. Requires use of " + "--ssl_client_cert"); +DEFINE_string( channel_creds_type, "", "The channel creds type: insecure, ssl, gdc (Google Default Credentials) " "or alts."); +DEFINE_string( + call_creds, "", + "Call credentials to use: none (default), or access_token=<token>. If " + "provided, the call creds are composited on top of channel creds."); namespace grpc { namespace testing { +namespace { + +const char ACCESS_TOKEN_PREFIX[] = "access_token="; +constexpr int ACCESS_TOKEN_PREFIX_LEN = + sizeof(ACCESS_TOKEN_PREFIX) / sizeof(*ACCESS_TOKEN_PREFIX) - 1; + +bool IsAccessToken(const grpc::string& auth) { + return auth.length() > ACCESS_TOKEN_PREFIX_LEN && + auth.compare(0, ACCESS_TOKEN_PREFIX_LEN, ACCESS_TOKEN_PREFIX) == 0; +} + +grpc::string AccessToken(const grpc::string& auth) { + if (!IsAccessToken(auth)) { + return ""; + } + return grpc::string(auth, ACCESS_TOKEN_PREFIX_LEN); +} + +} // namespace + grpc::string CliCredentials::GetDefaultChannelCredsType() const { // Compatibility logic for --enable_ssl. if (FLAGS_enable_ssl) { @@ -59,12 +97,42 @@ grpc::string CliCredentials::GetDefaultChannelCredsType() const { return "insecure"; } +grpc::string CliCredentials::GetDefaultCallCreds() const { + if (!FLAGS_access_token.empty()) { + fprintf(stderr, + "warning: --access_token is deprecated. Use " + "--call_creds=access_token=<token>.\n"); + return grpc::string("access_token=") + FLAGS_access_token; + } + return "none"; +} + std::shared_ptr<grpc::ChannelCredentials> CliCredentials::GetChannelCredentials() const { if (FLAGS_channel_creds_type.compare("insecure") == 0) { return grpc::InsecureChannelCredentials(); } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { - return grpc::SslCredentials(grpc::SslCredentialsOptions()); + grpc::SslCredentialsOptions ssl_creds_options; + // TODO(@Capstan): This won't affect Google Default Credentials using SSL. + if (!FLAGS_ssl_client_cert.empty()) { + grpc_slice cert_slice = grpc_empty_slice(); + GRPC_LOG_IF_ERROR( + "load_file", + grpc_load_file(FLAGS_ssl_client_cert.c_str(), 1, &cert_slice)); + ssl_creds_options.pem_cert_chain = + grpc::StringFromCopiedSlice(cert_slice); + grpc_slice_unref(cert_slice); + } + if (!FLAGS_ssl_client_key.empty()) { + grpc_slice key_slice = grpc_empty_slice(); + GRPC_LOG_IF_ERROR( + "load_file", + grpc_load_file(FLAGS_ssl_client_key.c_str(), 1, &key_slice)); + ssl_creds_options.pem_private_key = + grpc::StringFromCopiedSlice(key_slice); + grpc_slice_unref(key_slice); + } + return grpc::SslCredentials(ssl_creds_options); } else if (FLAGS_channel_creds_type.compare("gdc") == 0) { return grpc::GoogleDefaultCredentials(); } else if (FLAGS_channel_creds_type.compare("alts") == 0) { @@ -80,18 +148,30 @@ CliCredentials::GetChannelCredentials() const { std::shared_ptr<grpc::CallCredentials> CliCredentials::GetCallCredentials() const { - if (!FLAGS_access_token.empty()) { - if (FLAGS_use_auth) { - fprintf(stderr, - "warning: use_auth is ignored when access_token is provided."); - } - return grpc::AccessTokenCredentials(FLAGS_access_token); + if (IsAccessToken(FLAGS_call_creds)) { + return grpc::AccessTokenCredentials(AccessToken(FLAGS_call_creds)); + } + if (FLAGS_call_creds.compare("none") != 0) { + // Nothing to do; creds, if any, are baked into the channel. + return std::shared_ptr<grpc::CallCredentials>(); } + fprintf(stderr, + "--call_creds=%s invalid; must be none " + "or access_token=<token>.\n", + FLAGS_call_creds.c_str()); return std::shared_ptr<grpc::CallCredentials>(); } std::shared_ptr<grpc::ChannelCredentials> CliCredentials::GetCredentials() const { + if (FLAGS_call_creds.empty()) { + FLAGS_call_creds = GetDefaultCallCreds(); + } else if (!FLAGS_access_token.empty() && !IsAccessToken(FLAGS_call_creds)) { + fprintf(stderr, + "warning: ignoring --access_token because --call_creds " + "already set to %s.\n", + FLAGS_call_creds.c_str()); + } if (FLAGS_channel_creds_type.empty()) { FLAGS_channel_creds_type = GetDefaultChannelCredsType(); } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { @@ -106,7 +186,7 @@ std::shared_ptr<grpc::ChannelCredentials> CliCredentials::GetCredentials() FLAGS_channel_creds_type.c_str()); } // Legacy transport upgrade logic for insecure requests. - if (!FLAGS_access_token.empty() && + if (IsAccessToken(FLAGS_call_creds) && FLAGS_channel_creds_type.compare("insecure") == 0) { fprintf(stderr, "warning: --channel_creds_type=insecure upgraded to ssl because " @@ -126,10 +206,16 @@ const grpc::string CliCredentials::GetCredentialUsage() const { return " --enable_ssl ; Set whether to use ssl (deprecated)\n" " --use_auth ; Set whether to create default google" " credentials\n" + " ; (deprecated)\n" " --access_token ; Set the access token in metadata," " overrides --use_auth\n" + " ; (deprecated)\n" " --ssl_target ; Set server host for ssl validation\n" - " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n"; + " --ssl_client_cert ; Client cert for ssl\n" + " --ssl_client_key ; Client private key for ssl\n" + " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n" + " --call_creds ; Set to none, or" + " access_token=<token>\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 4636d3ca14..472c7abef2 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -36,6 +36,9 @@ class CliCredentials { // Returns the appropriate channel_creds_type value for the set of legacy // flag arguments. virtual grpc::string GetDefaultChannelCredsType() const; + // Returns the appropriate call_creds value for the set of legacy flag + // arguments. + virtual grpc::string GetDefaultCallCreds() const; // Returns the base transport channel credentials. Child classes can override // to support additional channel_creds_types unknown to this base class. virtual std::shared_ptr<grpc::ChannelCredentials> GetChannelCredentials() diff --git a/test/distrib/csharp/run_distrib_test.sh b/test/distrib/csharp/run_distrib_test.sh index f9371410e7..61924b91df 100755 --- a/test/distrib/csharp/run_distrib_test.sh +++ b/test/distrib/csharp/run_distrib_test.sh @@ -21,7 +21,8 @@ unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_windows_dotnetcli.zip ./update_version.sh auto -nuget restore +# Retry "nuget restore" to work around https://github.com/grpc/grpc/issues/16312 +nuget restore || nuget restore || nuget restore xbuild DistribTest.sln diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 25da3fdd5f..ab2e5a671a 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -23,10 +23,14 @@ import subprocess import re import perfection -# configuration: a list of either strings or 2-tuples of strings -# a single string represents a static grpc_mdstr -# a 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will -# also be created) +# Configuration: a list of either strings or 2-tuples of strings or 3-tuples of +# strings. +# A single string represents a static grpc_mdstr. +# A 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will +# also be created). +# A 3-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will +# also be created), with the last value equivalent to the mdelem's static hpack +# table index as defined by RFC 7541 CONFIG = [ # metadata strings @@ -72,105 +76,104 @@ CONFIG = [ ('grpc-encoding', 'deflate'), ('te', 'trailers'), ('content-type', 'application/grpc'), - (':method', 'POST'), - (':status', '200'), - (':status', '404'), - (':scheme', 'http'), - (':scheme', 'https'), - (':scheme', 'grpc'), - (':authority', ''), - (':method', 'GET'), + (':method', 'POST', 3), + (':status', '200', 8), + (':status', '404', 13), + (':scheme', 'http', 6), + (':scheme', 'https', 7), + (':scheme', 'grpc', 0), + (':authority', '', 1), + (':method', 'GET', 2), (':method', 'PUT'), - (':path', '/'), - (':path', '/index.html'), - (':status', '204'), - (':status', '206'), - (':status', '304'), - (':status', '400'), - (':status', '500'), - ('accept-charset', ''), + (':path', '/', 4), + (':path', '/index.html', 5), + (':status', '204', 9), + (':status', '206', 10), + (':status', '304', 11), + (':status', '400', 12), + (':status', '500', 14), + ('accept-charset', '', 15), ('accept-encoding', ''), - ('accept-encoding', 'gzip, deflate'), - ('accept-language', ''), - ('accept-ranges', ''), - ('accept', ''), - ('access-control-allow-origin', ''), - ('age', ''), - ('allow', ''), - ('authorization', ''), - ('cache-control', ''), - ('content-disposition', ''), + ('accept-encoding', 'gzip, deflate', 16), + ('accept-language', '', 17), + ('accept-ranges', '', 18), + ('accept', '', 19), + ('access-control-allow-origin', '', 20), + ('age', '', 21), + ('allow', '', 22), + ('authorization', '', 23), + ('cache-control', '', 24), + ('content-disposition', '', 25), ('content-encoding', 'identity'), ('content-encoding', 'gzip'), - ('content-encoding', ''), - ('content-language', ''), - ('content-length', ''), - ('content-location', ''), - ('content-range', ''), - ('content-type', ''), - ('cookie', ''), - ('date', ''), - ('etag', ''), - ('expect', ''), - ('expires', ''), - ('from', ''), - ('host', ''), - ('if-match', ''), - ('if-modified-since', ''), - ('if-none-match', ''), - ('if-range', ''), - ('if-unmodified-since', ''), - ('last-modified', ''), + ('content-encoding', '', 26), + ('content-language', '', 27), + ('content-length', '', 28), + ('content-location', '', 29), + ('content-range', '', 30), + ('content-type', '', 31), + ('cookie', '', 32), + ('date', '', 33), + ('etag', '', 34), + ('expect', '', 35), + ('expires', '', 36), + ('from', '', 37), + ('host', '', 38), + ('if-match', '', 39), + ('if-modified-since', '', 40), + ('if-none-match', '', 41), + ('if-range', '', 42), + ('if-unmodified-since', '', 43), + ('last-modified', '', 44), ('lb-token', ''), ('lb-cost-bin', ''), - ('link', ''), - ('location', ''), - ('max-forwards', ''), - ('proxy-authenticate', ''), - ('proxy-authorization', ''), - ('range', ''), - ('referer', ''), - ('refresh', ''), - ('retry-after', ''), - ('server', ''), - ('set-cookie', ''), - ('strict-transport-security', ''), - ('transfer-encoding', ''), - ('user-agent', ''), - ('vary', ''), - ('via', ''), - ('www-authenticate', ''), + ('link', '', 45), + ('location', '', 46), + ('max-forwards', '', 47), + ('proxy-authenticate', '', 48), + ('proxy-authorization', '', 49), + ('range', '', 50), + ('referer', '', 51), + ('refresh', '', 52), + ('retry-after', '', 53), + ('server', '', 54), + ('set-cookie', '', 55), + ('strict-transport-security', '', 56), + ('transfer-encoding', '', 57), + ('user-agent', '', 58), + ('vary', '', 59), + ('via', '', 60), + ('www-authenticate', '', 61), ] -# Entries marked with is_default=True are ignored when counting -# non-default initial metadata that prevents the chttp2 server from -# sending a Trailers-Only response. +# All entries here are ignored when counting non-default initial metadata that +# prevents the chttp2 server from sending a Trailers-Only response. METADATA_BATCH_CALLOUTS = [ - # (name, is_default) - (':path', True), - (':method', True), - (':status', True), - (':authority', True), - (':scheme', True), - ('te', True), - ('grpc-message', True), - ('grpc-status', True), - ('grpc-payload-bin', True), - ('grpc-encoding', True), - ('grpc-accept-encoding', True), - ('grpc-server-stats-bin', True), - ('grpc-tags-bin', True), - ('grpc-trace-bin', True), - ('content-type', True), - ('content-encoding', True), - ('accept-encoding', True), - ('grpc-internal-encoding-request', True), - ('grpc-internal-stream-encoding-request', True), - ('user-agent', True), - ('host', True), - ('lb-token', True), - ('grpc-previous-rpc-attempts', True), - ('grpc-retry-pushback-ms', True), + # (name) + (':path'), + (':method'), + (':status'), + (':authority'), + (':scheme'), + ('te'), + ('grpc-message'), + ('grpc-status'), + ('grpc-payload-bin'), + ('grpc-encoding'), + ('grpc-accept-encoding'), + ('grpc-server-stats-bin'), + ('grpc-tags-bin'), + ('grpc-trace-bin'), + ('content-type'), + ('content-encoding'), + ('accept-encoding'), + ('grpc-internal-encoding-request'), + ('grpc-internal-stream-encoding-request'), + ('user-agent'), + ('host'), + ('lb-token'), + ('grpc-previous-rpc-attempts'), + ('grpc-retry-pushback-ms'), ] COMPRESSION_ALGORITHMS = [ @@ -252,7 +255,7 @@ all_elems = list() static_userdata = {} # put metadata batch callouts first, to make the check of if a static metadata # string is a callout trivial -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: if elem not in all_strs: all_strs.append(elem) for elem in CONFIG: @@ -323,6 +326,17 @@ else: os.path.dirname(sys.argv[0]), '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w') +HPACK_H = open( + os.path.join( + os.path.dirname(sys.argv[0]), + '../../../src/core/ext/transport/chttp2/transport/hpack_mapping.h'), + 'w') +HPACK_C = open( + os.path.join( + os.path.dirname(sys.argv[0]), + '../../../src/core/ext/transport/chttp2/transport/hpack_mapping.cc'), + 'w') + # copy-paste copyright notice from this file with open(sys.argv[0]) as my_source: copyright = [] @@ -337,7 +351,8 @@ with open(sys.argv[0]) as my_source: if line[0] != '#': break copyright.append(line) - put_banner([H, C], [line[2:].rstrip() for line in copyright]) + put_banner([H, C, HPACK_H, HPACK_C], + [line[2:].rstrip() for line in copyright]) hex_bytes = [ord(c) for c in 'abcdefABCDEF0123456789'] @@ -364,16 +379,44 @@ See metadata.h for an explanation of the interface here, and metadata.cc for an explanation of what's going on. """.splitlines()) +put_banner([HPACK_H, HPACK_C], """WARNING: Auto-generated code. + +To make changes to this file, change +tools/codegen/core/gen_static_metadata.py, and then re-run it. + +This file contains the mapping from the index of each metadata element in the +grpc static metadata table to the index of that element in the hpack static +metadata table. If the element is not contained in the static hpack table, then +the returned index is 0. +""".splitlines()) + print >> H, '#ifndef GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H' print >> H, '#define GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H' print >> H +print >> H, '#include <grpc/support/port_platform.h>' +print >> H print >> H, '#include "src/core/lib/transport/metadata.h"' print >> H - +print >> C, '#include <grpc/support/port_platform.h>' +print >> C print >> C, '#include "src/core/lib/transport/static_metadata.h"' print >> C print >> C, '#include "src/core/lib/slice/slice_internal.h"' print >> C +print >> HPACK_H, ('#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_' + 'MAPPING_H') +print >> HPACK_H, ('#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_' + 'MAPPING_H') +print >> HPACK_H +print >> HPACK_H, '#include <grpc/support/port_platform.h>' +print >> HPACK_H +print >> HPACK_H, '#include "src/core/lib/transport/static_metadata.h"' +print >> HPACK_H +print >> HPACK_C, '#include <grpc/support/port_platform.h>' +print >> HPACK_C +print >> HPACK_C, ('#include ' + '"src/core/ext/transport/chttp2/transport/hpack_mapping.h"') +print >> HPACK_C str_ofs = 0 id2strofs = {} @@ -388,7 +431,7 @@ def slice_def(i): # validate configuration -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: assert elem in all_strs print >> H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs) @@ -450,10 +493,28 @@ print >> H, ('extern grpc_mdelem_data ' print >> H, ('extern uintptr_t ' 'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];') for i, elem in enumerate(all_elems): - print >> H, '/* "%s": "%s" */' % elem + print >> H, '/* "%s": "%s" */' % (elem[0], elem[1]) print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], ' 'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i) print >> H + +# Print out the chttp2 mapping between static mdelem index and the hpack static +# table index +print >> HPACK_H, ('extern const uint8_t grpc_hpack_static_mdelem_indices[' + 'GRPC_STATIC_MDELEM_COUNT];') +print >> HPACK_H +print >> HPACK_C, ('const uint8_t grpc_hpack_static_mdelem_indices[' + 'GRPC_STATIC_MDELEM_COUNT] = {') +indices = '' +for elem in all_elems: + index = 0 + if len(elem) == 3: + index = elem[2] + indices += '%d,' % index +print >> HPACK_C, ' %s' % indices +print >> HPACK_C, '};' +print >> HPACK_C + print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] ' '= {') print >> C, ' %s' % ','.join( @@ -546,12 +607,13 @@ print >> C, '}' print >> C print >> C, 'grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {' -for a, b in all_elems: - print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b))) +for elem in all_elems: + print >> C, '{%s,%s},' % (slice_def(str_idx(elem[0])), + slice_def(str_idx(elem[1]))) print >> C, '};' print >> H, 'typedef enum {' -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: print >> H, ' %s,' % mangle(elem, 'batch').upper() print >> H, ' GRPC_BATCH_CALLOUTS_COUNT' print >> H, '} grpc_metadata_batch_callouts_index;' @@ -559,7 +621,7 @@ print >> H print >> H, 'typedef union {' print >> H, ' struct grpc_linked_mdelem *array[GRPC_BATCH_CALLOUTS_COUNT];' print >> H, ' struct {' -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower() print >> H, ' } named;' print >> H, '} grpc_metadata_batch_callouts;' @@ -567,14 +629,6 @@ print >> H print >> H, '#define GRPC_BATCH_INDEX_OF(slice) \\' print >> H, ' (GRPC_IS_STATIC_METADATA_STRING((slice)) ? (grpc_metadata_batch_callouts_index)GPR_CLAMP(GRPC_STATIC_METADATA_INDEX((slice)), 0, GRPC_BATCH_CALLOUTS_COUNT) : GRPC_BATCH_CALLOUTS_COUNT)' print >> H -print >> H, ('extern bool grpc_static_callout_is_default[' - 'GRPC_BATCH_CALLOUTS_COUNT];') -print >> H -print >> C, 'bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {' -for elem, is_default in METADATA_BATCH_CALLOUTS: - print >> C, ' %s, // %s' % (str(is_default).lower(), elem) -print >> C, '};' -print >> C print >> H, 'extern const uint8_t grpc_static_accept_encoding_metadata[%d];' % ( 1 << len(COMPRESSION_ALGORITHMS)) @@ -599,5 +653,8 @@ print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GR print >> H, '#endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */' +print >> HPACK_H, ('#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_' + 'MAPPING_H */') + H.close() C.close() diff --git a/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile index bff20b5d06..897354891c 100644 --- a/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM google/dart:latest +FROM google/dart:2.0 # Upgrade Dart to version 2. RUN apt-get update && apt-get upgrade -y dart diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 9a97ee84f2..3b7fd1fa8e 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -945,7 +945,9 @@ include/grpcpp/impl/codegen/async_unary_call.h \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ +include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ +include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -997,6 +999,7 @@ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ +include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 0cd4cfd647..c1bcdfd3d0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -946,7 +946,9 @@ include/grpcpp/impl/codegen/async_unary_call.h \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ +include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ +include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ @@ -999,6 +1001,7 @@ include/grpcpp/support/async_stream.h \ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ +include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ @@ -1077,7 +1080,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.h \ src/core/lib/iomgr/ev_epollex_linux.h \ -src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.h \ src/core/lib/iomgr/exec_ctx.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 7cd1dc7bf3..9186056733 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1013,6 +1013,8 @@ src/core/ext/transport/chttp2/transport/frame_window_update.cc \ src/core/ext/transport/chttp2/transport/frame_window_update.h \ src/core/ext/transport/chttp2/transport/hpack_encoder.cc \ src/core/ext/transport/chttp2/transport/hpack_encoder.h \ +src/core/ext/transport/chttp2/transport/hpack_mapping.cc \ +src/core/ext/transport/chttp2/transport/hpack_mapping.h \ src/core/ext/transport/chttp2/transport/hpack_parser.cc \ src/core/ext/transport/chttp2/transport/hpack_parser.h \ src/core/ext/transport/chttp2/transport/hpack_table.cc \ @@ -1179,8 +1181,6 @@ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epoll1_linux.h \ src/core/lib/iomgr/ev_epollex_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.h \ -src/core/lib/iomgr/ev_epollsig_linux.cc \ -src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.cc \ diff --git a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh index c7d6ba6d44..01e9427e1c 100755 --- a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh +++ b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh @@ -24,4 +24,4 @@ cd "$(dirname "$0")/../../.." # after finishing each build. We only leave files we want to keep: # - reports and artifacts # - directory containing the kokoro scripts to prevent deleting a script while being executed. -time find . -type f -not -iname "*sponge_log.xml" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + +time find . -type f -not -iname "*sponge_log.*" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index a6367ad962..dfef004a60 100644..100755 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -15,5 +15,6 @@ export UPLOAD_TEST_RESULTS=true EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg index 4a0badf43b..07f7f0c659 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg index a2cfe021e1..8f2813febf 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/grpc_basictests_multilang.cfg index 4433d14cd7..f8a5a4aea5 100644 --- a/tools/internal_ci/linux/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 8b42779366..bb2a85138c 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -60,7 +60,7 @@ export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instan --platforms=//third_party/toolchains:rbe_ubuntu1604 \ --test_env=GRPC_VERBOSITY=debug \ --remote_instance_name=projects/grpc-testing/instances/default_instance \ - $1 \ + $@ \ -- //test/... || FAILED="true" if [ "$UPLOAD_TEST_RESULTS" != "" ] diff --git a/tools/internal_ci/linux/grpc_build_artifacts.cfg b/tools/internal_ci/linux/grpc_build_artifacts.cfg index 88fc6b7b35..1e04a1e788 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg index 619e3ea3a9..2737e2f345 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg index 619e3ea3a9..2737e2f345 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg index 11c211fd2b..9a430db0f9 100644 --- a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg index 6a4a163dfc..23a676cf72 100644 --- a/tools/internal_ci/linux/grpc_build_packages.cfg +++ b/tools/internal_ci/linux/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg index 2f08e15e63..aef6a7a1dc 100644 --- a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg index 794a51d3f1..6eb37b7dec 100644 --- a/tools/internal_ci/linux/grpc_coverage.cfg +++ b/tools/internal_ci/linux/grpc_coverage.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh" timeout_mins: 420 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg index 0f1d79355a..848d571333 100644 --- a/tools/internal_ci/linux/grpc_distribtests.cfg +++ b/tools/internal_ci/linux/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg index bc6c8e8f80..734bdfd78a 100644 --- a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_standalone.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_full_performance_master.cfg b/tools/internal_ci/linux/grpc_full_performance_master.cfg index 8852130a13..8ad93ee052 100644 --- a/tools/internal_ci/linux/grpc_full_performance_master.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_master.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_full_performance_release.cfg b/tools/internal_ci/linux/grpc_full_performance_release.cfg index e9a4bcdcaf..11a95e889f 100644 --- a/tools/internal_ci/linux/grpc_full_performance_release.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_release.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_release.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_alts.cfg b/tools/internal_ci/linux/grpc_interop_alts.cfg index bda76faf44..4684aba96b 100644 --- a/tools/internal_ci/linux/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_matrix.cfg b/tools/internal_ci/linux/grpc_interop_matrix.cfg index ae59e930f7..696a55c0df 100644 --- a/tools/internal_ci/linux/grpc_interop_matrix.cfg +++ b/tools/internal_ci/linux/grpc_interop_matrix.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh" timeout_mins: 300 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg index 81f2fe9dec..9b35adcece 100644 --- a/tools/internal_ci/linux/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg index 8dfc529947..de4db81e6b 100644 --- a/tools/internal_ci/linux/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg index 76e5028477..f417f24bb1 100644 --- a/tools/internal_ci/linux/grpc_portability.cfg +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_portability_build_only.cfg b/tools/internal_ci/linux/grpc_portability_build_only.cfg index 4acd9353fb..fab9dde3e5 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.cfg +++ b/tools/internal_ci/linux/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_publish_packages.cfg b/tools/internal_ci/linux/grpc_publish_packages.cfg index 82d571d642..dc9fe7d0a7 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.cfg +++ b/tools/internal_ci/linux/grpc_publish_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_publish_packages.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg index b20d2ffba8..704d5c6cbc 100644 --- a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg +++ b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 30 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_sanity.cfg b/tools/internal_ci/linux/grpc_sanity.cfg index 9f65918e23..341471bbb5 100644 --- a/tools/internal_ci/linux/grpc_sanity.cfg +++ b/tools/internal_ci/linux/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 2ba7d469ec..366b5cbe34 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -15,4 +15,5 @@ export UPLOAD_TEST_RESULTS=true EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh index 2aebb65552..39c991f291 100644 --- a/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh @@ -14,5 +14,6 @@ # limitations under the License. EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg index 8124f5c1b3..8a67d28ce4 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg index ecedc73e44..a681978b6e 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg index 577cb28ae5..249ecc99ce 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg index 9e0b724b2e..665d7f3d2c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg index 0fda74cf44..163274d4dc 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg index 199a8905d9..b65cd3e05c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg index f7e8d8ad33..59f38f0d1b 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg index c1253b30f7..e91a612878 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg index cb18e8e868..b1eb575605 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg index d14c79a1f6..7321effc12 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg index 9269c345f0..47301d6141 100644 --- a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg index 0f83299aab..276c34f0cf 100644 --- a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg index e86b3ab475..78358eac28 100644 --- a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh index edd8f92975..3dee115300 100644 --- a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh @@ -14,4 +14,5 @@ # limitations under the License. EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" +EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg index 06a4372ce2..d6d70677dd 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg index f875327c1b..7b22c6afef 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg index 6658a804d8..6c9dd6ef8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg index 957a91ef2b..8700c74c8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg index dbbfce90cb..02162d860b 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg index fb0cefa160..95582184ed 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index 1daf7a514e..d444596a81 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index a8503b7bcb..3891cc37e8 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index 12af4581eb..91ce1627af 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index 0d3803bf23..fcdc2de37b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 557561810b..de2a74051b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg index cb15ca34fd..51c291f7df 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index 53bda1ff0a..e6f9c7ec87 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index d359eb601a..f2a83fe95a 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_build_artifacts.cfg b/tools/internal_ci/macos/grpc_build_artifacts.cfg index 4da61faed3..c73cf4359f 100644 --- a/tools/internal_ci/macos/grpc_build_artifacts.cfg +++ b/tools/internal_ci/macos/grpc_build_artifacts.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg index ae88f39b90..156ec6fe33 100644 --- a/tools/internal_ci/macos/grpc_distribtests.cfg +++ b/tools/internal_ci/macos/grpc_distribtests.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_interop.cfg b/tools/internal_ci/macos/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/grpc_interop.cfg +++ b/tools/internal_ci/macos/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_interop_toprod.cfg b/tools/internal_ci/macos/grpc_interop_toprod.cfg index c92c397daa..2cfc8a2d6d 100644 --- a/tools/internal_ci/macos/grpc_interop_toprod.cfg +++ b/tools/internal_ci/macos/grpc_interop_toprod.cfg @@ -21,7 +21,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/interop/service_account/GrpcTes timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg index 30c01d3e2f..c759397b78 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg index b63ee713bc..4d68341405 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_interop.cfg b/tools/internal_ci/macos/pull_request/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/pull_request/grpc_interop.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg index 8942bc7ba7..fb215bdf99 100644 --- a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg @@ -20,7 +20,7 @@ timeout_mins: 60 gfile_resources: "/bigstore/grpc-testing-secrets/github_credentials/oauth_token.txt" action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests.cfg b/tools/internal_ci/windows/grpc_basictests.cfg index 8e644e4c5e..fcf5237bf3 100644 --- a/tools/internal_ci/windows/grpc_basictests.cfg +++ b/tools/internal_ci/windows/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/grpc_basictests_dbg.cfg index 28d53cdc7b..4e5e7b6545 100644 --- a/tools/internal_ci/windows/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_opt.cfg b/tools/internal_ci/windows/grpc_basictests_opt.cfg index 4b7a965977..f5db6a9897 100644 --- a/tools/internal_ci/windows/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_build_artifacts.cfg b/tools/internal_ci/windows/grpc_build_artifacts.cfg index 38b0abd519..f45cfda121 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.cfg +++ b/tools/internal_ci/windows/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg index 65a8b1eef3..b351bbb11b 100644 --- a/tools/internal_ci/windows/grpc_build_packages.cfg +++ b/tools/internal_ci/windows/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg index 1766e601e5..e12d4bf98d 100644 --- a/tools/internal_ci/windows/grpc_distribtests.cfg +++ b/tools/internal_ci/windows/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg index 33a50fdc45..7f32ba4c70 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_standalone.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_portability.cfg b/tools/internal_ci/windows/grpc_portability.cfg index 94e71753ef..aa86c06632 100644 --- a/tools/internal_ci/windows/grpc_portability.cfg +++ b/tools/internal_ci/windows/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_portability_build_only.cfg b/tools/internal_ci/windows/grpc_portability_build_only.cfg index 3bc27f1f24..3dace627eb 100644 --- a/tools/internal_ci/windows/grpc_portability_build_only.cfg +++ b/tools/internal_ci/windows/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg index 91777cd7cb..a8e6f0675b 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg index 81a1e9d3c8..ce957232fd 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg index 3bb6510ca7..7d2abbfb19 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_portability.cfg b/tools/internal_ci/windows/pull_request/grpc_portability.cfg index 2bda487629..6f332416fe 100644 --- a/tools/internal_ci/windows/pull_request/grpc_portability.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index bb9222d953..15b53d1716 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -99,6 +99,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.1': None }, + { + 'v1.15.0': None + }, ], 'go': [ { @@ -184,6 +187,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.0': None }, + { + 'v1.15.0': None + }, ], 'python': [ { @@ -228,6 +234,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.1': None }, + { + 'v1.15.0': None + }, ], 'node': [ { @@ -316,6 +325,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.1': None }, + { + 'v1.15.0': None + }, ], 'php': [ { @@ -360,6 +372,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.1': None }, + { + 'v1.15.0': None + }, ], 'csharp': [ { @@ -409,6 +424,9 @@ LANG_RELEASE_MATRIX = { { 'v1.14.1': None }, + { + 'v1.15.0': None + }, ], } diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8ea5126fde..29b72dca43 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -493,23 +493,6 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "ev_epollsig_linux_test", - "src": [ - "test/core/iomgr/ev_epollsig_linux_test.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", "name": "fake_resolver_test", "src": [ "test/core/client_channel/resolvers/fake_resolver_test.cc" @@ -1863,9 +1846,9 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "pollset_set_test", + "name": "resolve_address_posix_test", "src": [ - "test/core/iomgr/pollset_set_test.cc" + "test/core/iomgr/resolve_address_posix_test.cc" ], "third_party": false, "type": "target" @@ -1880,9 +1863,9 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "resolve_address_posix_test", + "name": "resolve_address_using_ares_resolver_test", "src": [ - "test/core/iomgr/resolve_address_posix_test.cc" + "test/core/iomgr/resolve_address_test.cc" ], "third_party": false, "type": "target" @@ -1897,7 +1880,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "resolve_address_test", + "name": "resolve_address_using_native_resolver_test", "src": [ "test/core/iomgr/resolve_address_test.cc" ], @@ -3331,6 +3314,25 @@ "grpc++_test_util", "grpc_test_util" ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "client_callback_end2end_test", + "src": [ + "test/cpp/end2end/client_callback_end2end_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], "headers": [ "src/proto/grpc/lb/v1/load_balancer.grpc.pb.h", "src/proto/grpc/lb/v1/load_balancer.pb.h", @@ -9515,7 +9517,6 @@ "src/core/lib/iomgr/error.cc", "src/core/lib/iomgr/ev_epoll1_linux.cc", "src/core/lib/iomgr/ev_epollex_linux.cc", - "src/core/lib/iomgr/ev_epollsig_linux.cc", "src/core/lib/iomgr/ev_poll_posix.cc", "src/core/lib/iomgr/ev_posix.cc", "src/core/lib/iomgr/ev_windows.cc", @@ -9696,7 +9697,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", - "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -9848,7 +9848,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", - "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -10661,6 +10660,7 @@ "src/core/ext/transport/chttp2/transport/frame_settings.h", "src/core/ext/transport/chttp2/transport/frame_window_update.h", "src/core/ext/transport/chttp2/transport/hpack_encoder.h", + "src/core/ext/transport/chttp2/transport/hpack_mapping.h", "src/core/ext/transport/chttp2/transport/hpack_parser.h", "src/core/ext/transport/chttp2/transport/hpack_table.h", "src/core/ext/transport/chttp2/transport/http2_settings.h", @@ -10698,6 +10698,8 @@ "src/core/ext/transport/chttp2/transport/frame_window_update.h", "src/core/ext/transport/chttp2/transport/hpack_encoder.cc", "src/core/ext/transport/chttp2/transport/hpack_encoder.h", + "src/core/ext/transport/chttp2/transport/hpack_mapping.cc", + "src/core/ext/transport/chttp2/transport/hpack_mapping.h", "src/core/ext/transport/chttp2/transport/hpack_parser.cc", "src/core/ext/transport/chttp2/transport/hpack_parser.h", "src/core/ext/transport/chttp2/transport/hpack_table.cc", @@ -11081,7 +11083,9 @@ "include/grpcpp/impl/codegen/byte_buffer.h", "include/grpcpp/impl/codegen/call.h", "include/grpcpp/impl/codegen/call_hook.h", + "include/grpcpp/impl/codegen/callback_common.h", "include/grpcpp/impl/codegen/channel_interface.h", + "include/grpcpp/impl/codegen/client_callback.h", "include/grpcpp/impl/codegen/client_context.h", "include/grpcpp/impl/codegen/client_unary_call.h", "include/grpcpp/impl/codegen/completion_queue.h", @@ -11147,7 +11151,9 @@ "include/grpcpp/impl/codegen/byte_buffer.h", "include/grpcpp/impl/codegen/call.h", "include/grpcpp/impl/codegen/call_hook.h", + "include/grpcpp/impl/codegen/callback_common.h", "include/grpcpp/impl/codegen/channel_interface.h", + "include/grpcpp/impl/codegen/client_callback.h", "include/grpcpp/impl/codegen/client_context.h", "include/grpcpp/impl/codegen/client_unary_call.h", "include/grpcpp/impl/codegen/completion_queue.h", @@ -11305,6 +11311,7 @@ "include/grpcpp/support/async_unary_call.h", "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", + "include/grpcpp/support/client_callback.h", "include/grpcpp/support/config.h", "include/grpcpp/support/proto_buffer_reader.h", "include/grpcpp/support/proto_buffer_writer.h", @@ -11409,6 +11416,7 @@ "include/grpcpp/support/async_unary_call.h", "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", + "include/grpcpp/support/client_callback.h", "include/grpcpp/support/config.h", "include/grpcpp/support/proto_buffer_reader.h", "include/grpcpp/support/proto_buffer_writer.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fba76d69d1..4fdd26efa4 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -605,26 +605,6 @@ "args": [], "benchmark": false, "ci_platforms": [ - "linux" - ], - "cpu_cost": 3, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "gtest": false, - "language": "c", - "name": "ev_epollsig_linux_test", - "platforms": [ - "linux" - ], - "uses_polling": true - }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ "linux", "mac", "posix", @@ -2105,7 +2085,9 @@ "args": [], "benchmark": false, "ci_platforms": [ - "linux" + "linux", + "mac", + "posix" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -2115,38 +2097,44 @@ "flaky": false, "gtest": false, "language": "c", - "name": "pollset_set_test", + "name": "resolve_address_posix_test", "platforms": [ - "linux" + "linux", + "mac", + "posix" ], "uses_polling": true }, { - "args": [], + "args": [ + "--resolver=ares" + ], "benchmark": false, "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], + "exclude_iomgrs": [], "flaky": false, "gtest": false, "language": "c", - "name": "resolve_address_posix_test", + "name": "resolve_address_using_ares_resolver_test", "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, { - "args": [], + "args": [ + "--resolver=native" + ], "benchmark": false, "ci_platforms": [ "linux", @@ -2160,7 +2148,7 @@ "flaky": false, "gtest": false, "language": "c", - "name": "resolve_address_test", + "name": "resolve_address_using_native_resolver_test", "platforms": [ "linux", "mac", @@ -3978,6 +3966,30 @@ "posix", "windows" ], + "cpu_cost": 0.5, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "client_callback_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 561f453da7..b732e1e03e 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -176,13 +176,15 @@ class JobSpec(object): timeout_retries=0, kill_handler=None, cpu_cost=1.0, - verbose_success=False): + verbose_success=False, + logfilename=None): """ Arguments: cmdline: a list of arguments to pass as the command line environ: a dictionary of environment variables to set in the child process kill_handler: a handler that will be called whenever job.kill() is invoked cpu_cost: number of cores per second this job needs + logfilename: use given file to store job's output, rather than using a temporary file """ if environ is None: environ = {} @@ -197,6 +199,11 @@ class JobSpec(object): self.kill_handler = kill_handler self.cpu_cost = cpu_cost self.verbose_success = verbose_success + self.logfilename = logfilename + if self.logfilename and self.flake_retries != 0 and self.timeout_retries != 0: + # Forbidden to avoid overwriting the test log when retrying. + raise Exception( + 'Cannot use custom logfile when retries are enabled') def identity(self): return '%r %r' % (self.cmdline, self.environ) @@ -261,7 +268,15 @@ class Job(object): return self._spec def start(self): - self._tempfile = tempfile.TemporaryFile() + if self._spec.logfilename: + # make sure the log directory exists + logfile_dir = os.path.dirname( + os.path.abspath(self._spec.logfilename)) + if not os.path.exists(logfile_dir): + os.makedirs(logfile_dir) + self._logfile = open(self._spec.logfilename, 'w+') + else: + self._logfile = tempfile.TemporaryFile() env = dict(os.environ) env.update(self._spec.environ) env.update(self._add_env) @@ -277,7 +292,7 @@ class Job(object): measure_cpu_costs = False try_start = lambda: subprocess.Popen(args=cmdline, stderr=subprocess.STDOUT, - stdout=self._tempfile, + stdout=self._logfile, cwd=self._spec.cwd, shell=self._spec.shell, env=env) @@ -300,7 +315,7 @@ class Job(object): """Poll current state of the job. Prints messages at completion.""" def stdout(self=self): - stdout = read_from_start(self._tempfile) + stdout = read_from_start(self._logfile) self.result.message = stdout[-_MAX_RESULT_SIZE:] return stdout diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index d29ebc6219..74f329048f 100644 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -40,13 +40,14 @@ _RESULTS_SCHEMA = [ ('test_case', 'STRING', 'Name of test case'), ('result', 'STRING', 'Test or build result'), ('timestamp', 'TIMESTAMP', 'Timestamp of test run'), + ('duration', 'FLOAT', 'Duration of the test run'), ] _TABLE_ID = 'rbe_test_results' def _get_api_key(): """Returns string with API key to access ResultStore. - Intended to be used in Kokoro envrionment.""" + Intended to be used in Kokoro environment.""" api_key_directory = os.getenv('KOKORO_GFILE_DIR') api_key_file = os.path.join(api_key_directory, 'resultstore_api_key') assert os.path.isfile(api_key_file), 'Must add --api_key arg if not on ' \ @@ -57,7 +58,7 @@ def _get_api_key(): def _get_invocation_id(): """Returns String of Bazel invocation ID. Intended to be used in - Kokoro envirionment.""" + Kokoro environment.""" bazel_id_directory = os.getenv('KOKORO_ARTIFACTS_DIR') bazel_id_file = os.path.join(bazel_id_directory, 'bazel_invocation_ids') assert os.path.isfile(bazel_id_file), 'bazel_invocation_ids file, written ' \ @@ -66,6 +67,16 @@ def _get_invocation_id(): return f.read().replace('\n', '') +def _parse_test_duration(duration_str): + """Parse test duration string in '123.567s' format""" + try: + if duration_str.endswith('s'): + duration_str = duration_str[:-1] + return float(duration_str) + except: + return None + + def _upload_results_to_bq(rows): """Upload test results to a BQ table. @@ -205,6 +216,8 @@ if __name__ == "__main__": result, 'timestamp': action['timing']['startTime'], + 'duration': + _parse_test_duration(action['timing']['duration']), } }) except Exception as e: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ecb5e1d899..3d73f9ec0e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -61,7 +61,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { } _POLLING_STRATEGIES = { - 'linux': ['epollex', 'epollsig', 'epoll1', 'poll', 'poll-cv'], + 'linux': ['epollex', 'epoll1', 'poll', 'poll-cv'], 'mac': ['poll'], } @@ -1430,7 +1430,7 @@ argp.add_argument( default=None, type=str, help='Only use the specified comma-delimited list of polling engines. ' - 'Example: --force_use_pollers epollsig,poll ' + 'Example: --force_use_pollers epoll1,poll ' ' (This flag has no effect if --force_default_poller flag is also used)') argp.add_argument( '--max_time', default=-1, type=int, help='Maximum test runtime in seconds') diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 0af9e0cbbe..00fc68ad17 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -43,9 +43,6 @@ _OBJC_RUNTESTS_TIMEOUT = 90 * 60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 -# report suffix is important for reports to get picked up by internal CI -_REPORT_SUFFIX = 'sponge_log.xml' - def _safe_report_name(name): """Reports with '+' in target name won't show correctly in ResultStore""" @@ -53,13 +50,16 @@ def _safe_report_name(name): def _report_filename(name): - """Generates report file name""" - return 'report_%s_%s' % (_safe_report_name(name), _REPORT_SUFFIX) + """Generates report file name with directory structure that leads to better presentation by internal CI""" + # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. + return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml') -def _report_filename_internal_ci(name): - """Generates report file name that leads to better presentation by internal CI""" - return '%s/%s' % (_safe_report_name(name), _REPORT_SUFFIX) +def _report_logfilename(name): + """Generates log file name that corresponds to name generated by _report_filename""" + # 'sponge_log.log' suffix must be there for log to get recognized as "target log" + # for the corresponding 'sponge_log.xml' report. + return '%s/%s' % (_safe_report_name(name), 'sponge_log.log') def _docker_jobspec(name, @@ -80,7 +80,8 @@ def _docker_jobspec(name, ] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, - timeout_seconds=timeout_seconds) + timeout_seconds=timeout_seconds, + logfilename=_report_logfilename(name)) return test_job @@ -107,7 +108,8 @@ def _workspace_jobspec(name, ] + runtests_args, environ=env, shortname='run_tests_%s' % name, - timeout_seconds=timeout_seconds) + timeout_seconds=timeout_seconds, + logfilename=_report_logfilename(name)) return test_job @@ -507,8 +509,9 @@ if __name__ == "__main__": default=False, action='store_const', const=True, - help='Put reports into subdirectories to improve presentation of ' - 'results by Internal CI.') + help= + '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of ' + 'results by Kokoro.') argp.add_argument( '--bq_result_table', default='', @@ -517,9 +520,6 @@ if __name__ == "__main__": help='Upload test results to a specified BQ table.') args = argp.parse_args() - if args.internal_ci: - _report_filename = _report_filename_internal_ci # override the function - extra_args = [] if args.build_only: extra_args.append('--build_only') |