diff options
113 files changed, 2064 insertions, 492 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index d0a65b4493..180f416609 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -332,6 +332,7 @@ add_dependencies(buildtests_c alarm_test) add_dependencies(buildtests_c algorithm_test) add_dependencies(buildtests_c alloc_test) add_dependencies(buildtests_c alpn_test) +add_dependencies(buildtests_c arena_test) add_dependencies(buildtests_c bad_server_response_test) add_dependencies(buildtests_c bdp_estimator_test) add_dependencies(buildtests_c bin_decoder_test) @@ -574,6 +575,9 @@ add_dependencies(buildtests_cxx alarm_cpp_test) add_dependencies(buildtests_cxx async_end2end_test) add_dependencies(buildtests_cxx auth_property_iterator_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_arena) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_call_create) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -603,6 +607,9 @@ endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_metadata) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_pollset) +endif() add_dependencies(buildtests_cxx channel_arguments_test) add_dependencies(buildtests_cxx channel_filter_test) add_dependencies(buildtests_cxx cli_call_test) @@ -690,6 +697,7 @@ add_library(gpr src/core/lib/profiling/basic_timers.c src/core/lib/profiling/stap_timers.c src/core/lib/support/alloc.c + src/core/lib/support/arena.c src/core/lib/support/avl.c src/core/lib/support/backoff.c src/core/lib/support/cmdline.c @@ -4112,6 +4120,31 @@ target_link_libraries(alpn_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(arena_test + test/core/support/arena_test.c +) + + +target_include_directories(arena_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(arena_test + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr_test_util + gpr +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(bad_server_response_test test/core/end2end/bad_server_response_test.c ) @@ -7626,6 +7659,45 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_arena + test/cpp/microbenchmarks/bm_arena.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_arena + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_arena + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_call_create test/cpp/microbenchmarks/bm_call_create.cc third_party/googletest/src/gtest-all.cc @@ -8014,6 +8086,45 @@ target_link_libraries(bm_metadata endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(bm_pollset + test/cpp/microbenchmarks/bm_pollset.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_pollset + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_pollset + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) add_executable(channel_arguments_test test/cpp/common/channel_arguments_test.cc @@ -189,7 +189,7 @@ CC_ubsan = clang CXX_ubsan = clang++ LD_ubsan = clang LDXX_ubsan = clang++ -CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined,unsigned-integer-overflow -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs +CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow DEFINES_ubsan = NDEBUG @@ -906,6 +906,7 @@ algorithm_test: $(BINDIR)/$(CONFIG)/algorithm_test alloc_test: $(BINDIR)/$(CONFIG)/alloc_test alpn_test: $(BINDIR)/$(CONFIG)/alpn_test api_fuzzer: $(BINDIR)/$(CONFIG)/api_fuzzer +arena_test: $(BINDIR)/$(CONFIG)/arena_test bad_server_response_test: $(BINDIR)/$(CONFIG)/bad_server_response_test bdp_estimator_test: $(BINDIR)/$(CONFIG)/bdp_estimator_test bin_decoder_test: $(BINDIR)/$(CONFIG)/bin_decoder_test @@ -1047,6 +1048,7 @@ wakeup_fd_cv_test: $(BINDIR)/$(CONFIG)/wakeup_fd_cv_test alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test +bm_arena: $(BINDIR)/$(CONFIG)/bm_arena bm_call_create: $(BINDIR)/$(CONFIG)/bm_call_create bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_closure: $(BINDIR)/$(CONFIG)/bm_closure @@ -1057,6 +1059,7 @@ bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata +bm_pollset: $(BINDIR)/$(CONFIG)/bm_pollset channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test cli_call_test: $(BINDIR)/$(CONFIG)/cli_call_test @@ -1285,6 +1288,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/algorithm_test \ $(BINDIR)/$(CONFIG)/alloc_test \ $(BINDIR)/$(CONFIG)/alpn_test \ + $(BINDIR)/$(CONFIG)/arena_test \ $(BINDIR)/$(CONFIG)/bad_server_response_test \ $(BINDIR)/$(CONFIG)/bdp_estimator_test \ $(BINDIR)/$(CONFIG)/bin_decoder_test \ @@ -1468,6 +1472,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_arena \ $(BINDIR)/$(CONFIG)/bm_call_create \ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack \ $(BINDIR)/$(CONFIG)/bm_closure \ @@ -1478,6 +1483,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ + $(BINDIR)/$(CONFIG)/bm_pollset \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ @@ -1583,6 +1589,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/alarm_cpp_test \ $(BINDIR)/$(CONFIG)/async_end2end_test \ $(BINDIR)/$(CONFIG)/auth_property_iterator_test \ + $(BINDIR)/$(CONFIG)/bm_arena \ $(BINDIR)/$(CONFIG)/bm_call_create \ $(BINDIR)/$(CONFIG)/bm_chttp2_hpack \ $(BINDIR)/$(CONFIG)/bm_closure \ @@ -1593,6 +1600,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_fullstack_trickle \ $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \ $(BINDIR)/$(CONFIG)/bm_metadata \ + $(BINDIR)/$(CONFIG)/bm_pollset \ $(BINDIR)/$(CONFIG)/channel_arguments_test \ $(BINDIR)/$(CONFIG)/channel_filter_test \ $(BINDIR)/$(CONFIG)/cli_call_test \ @@ -1663,6 +1671,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/alloc_test || ( echo test alloc_test failed ; exit 1 ) $(E) "[RUN] Testing alpn_test" $(Q) $(BINDIR)/$(CONFIG)/alpn_test || ( echo test alpn_test failed ; exit 1 ) + $(E) "[RUN] Testing arena_test" + $(Q) $(BINDIR)/$(CONFIG)/arena_test || ( echo test arena_test failed ; exit 1 ) $(E) "[RUN] Testing bad_server_response_test" $(Q) $(BINDIR)/$(CONFIG)/bad_server_response_test || ( echo test bad_server_response_test failed ; exit 1 ) $(E) "[RUN] Testing bdp_estimator_test" @@ -1923,6 +1933,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/async_end2end_test || ( echo test async_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing auth_property_iterator_test" $(Q) $(BINDIR)/$(CONFIG)/auth_property_iterator_test || ( echo test auth_property_iterator_test failed ; exit 1 ) + $(E) "[RUN] Testing bm_arena" + $(Q) $(BINDIR)/$(CONFIG)/bm_arena || ( echo test bm_arena failed ; exit 1 ) $(E) "[RUN] Testing bm_call_create" $(Q) $(BINDIR)/$(CONFIG)/bm_call_create || ( echo test bm_call_create failed ; exit 1 ) $(E) "[RUN] Testing bm_chttp2_hpack" @@ -1943,6 +1955,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" $(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 ) + $(E) "[RUN] Testing bm_pollset" + $(Q) $(BINDIR)/$(CONFIG)/bm_pollset || ( echo test bm_pollset failed ; exit 1 ) $(E) "[RUN] Testing channel_arguments_test" $(Q) $(BINDIR)/$(CONFIG)/channel_arguments_test || ( echo test channel_arguments_test failed ; exit 1 ) $(E) "[RUN] Testing channel_filter_test" @@ -2594,6 +2608,7 @@ LIBGPR_SRC = \ src/core/lib/profiling/basic_timers.c \ src/core/lib/profiling/stap_timers.c \ src/core/lib/support/alloc.c \ + src/core/lib/support/arena.c \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/cmdline.c \ @@ -8094,6 +8109,38 @@ endif endif +ARENA_TEST_SRC = \ + test/core/support/arena_test.c \ + +ARENA_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ARENA_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/arena_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/arena_test: $(ARENA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(ARENA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/arena_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/support/arena_test.o: $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_arena_test: $(ARENA_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(ARENA_TEST_OBJS:.o=.dep) +endif +endif + + BAD_SERVER_RESPONSE_TEST_SRC = \ test/core/end2end/bad_server_response_test.c \ @@ -12639,6 +12686,49 @@ endif endif +BM_ARENA_SRC = \ + test/cpp/microbenchmarks/bm_arena.cc \ + +BM_ARENA_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_ARENA_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_arena: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_arena: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_arena: $(PROTOBUF_DEP) $(BM_ARENA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_ARENA_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_arena + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_arena.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_arena: $(BM_ARENA_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_ARENA_OBJS:.o=.dep) +endif +endif + + BM_CALL_CREATE_SRC = \ test/cpp/microbenchmarks/bm_call_create.cc \ @@ -13069,6 +13159,49 @@ endif endif +BM_POLLSET_SRC = \ + test/cpp/microbenchmarks/bm_pollset.cc \ + +BM_POLLSET_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_POLLSET_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_pollset: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_pollset: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_pollset: $(PROTOBUF_DEP) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_pollset + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_pollset: $(BM_POLLSET_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_POLLSET_OBJS:.o=.dep) +endif +endif + + CHANNEL_ARGUMENTS_TEST_SRC = \ test/cpp/common/channel_arguments_test.cc \ diff --git a/binding.gyp b/binding.gyp index c521a27c30..59b6b600e9 100644 --- a/binding.gyp +++ b/binding.gyp @@ -544,6 +544,7 @@ 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', diff --git a/build.yaml b/build.yaml index a32a8a06d3..997f55df97 100644 --- a/build.yaml +++ b/build.yaml @@ -85,6 +85,7 @@ filegroups: - include/grpc/support/useful.h headers: - src/core/lib/profiling/timers.h + - src/core/lib/support/arena.h - src/core/lib/support/backoff.h - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h @@ -101,6 +102,7 @@ filegroups: - src/core/lib/profiling/basic_timers.c - src/core/lib/profiling/stap_timers.c - src/core/lib/support/alloc.c + - src/core/lib/support/arena.c - src/core/lib/support/avl.c - src/core/lib/support/backoff.c - src/core/lib/support/cmdline.c @@ -1483,6 +1485,14 @@ targets: - test/core/end2end/fuzzers/api_fuzzer_corpus dict: test/core/end2end/fuzzers/api_fuzzer.dictionary maxlen: 2048 +- name: arena_test + build: test + language: c + src: + - test/core/support/arena_test.c + deps: + - gpr_test_util + - gpr - name: bad_server_response_test build: test language: c @@ -2589,6 +2599,8 @@ targets: - grpc - gpr_test_util - gpr + exclude_iomgrs: + - uv platforms: - mac - linux @@ -3052,6 +3064,26 @@ targets: - grpc - gpr_test_util - gpr +- name: bm_arena + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_arena.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix - name: bm_call_create build: test language: c++ @@ -3268,6 +3300,26 @@ targets: - mac - linux - posix +- name: bm_pollset + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_pollset.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + platforms: + - mac + - linux + - posix - name: channel_arguments_test gtest: true build: test @@ -4233,8 +4285,8 @@ configs: TSAN_OPTIONS: suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 ubsan: CC: clang - CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined,unsigned-integer-overflow - -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs + CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer + -Wno-unused-command-line-argument -Wvarargs CXX: clang++ DEFINES: NDEBUG LD: clang @@ -4242,7 +4294,7 @@ configs: LDXX: clang++ compile_the_world: true test_environ: - UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt defaults: benchmark: CPPFLAGS: -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX @@ -39,6 +39,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/profiling/basic_timers.c \ src/core/lib/profiling/stap_timers.c \ src/core/lib/support/alloc.c \ + src/core/lib/support/arena.c \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/cmdline.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 027babcda4..8a5dcbb99c 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -196,6 +196,7 @@ Pod::Spec.new do |s| # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', + 'src/core/lib/support/arena.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', @@ -211,6 +212,7 @@ Pod::Spec.new do |s| 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', @@ -675,6 +677,7 @@ Pod::Spec.new do |s| 'src/core/plugin_registry/grpc_plugin_registry.c' ss.private_header_files = 'src/core/lib/profiling/timers.h', + 'src/core/lib/support/arena.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', diff --git a/grpc.gemspec b/grpc.gemspec index 8d5b7b2ab1..825c23d7a8 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -82,6 +82,7 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_posix.h ) s.files += %w( include/grpc/impl/codegen/sync_windows.h ) s.files += %w( src/core/lib/profiling/timers.h ) + s.files += %w( src/core/lib/support/arena.h ) s.files += %w( src/core/lib/support/backoff.h ) s.files += %w( src/core/lib/support/block_annotate.h ) s.files += %w( src/core/lib/support/env.h ) @@ -97,6 +98,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/profiling/basic_timers.c ) s.files += %w( src/core/lib/profiling/stap_timers.c ) s.files += %w( src/core/lib/support/alloc.c ) + s.files += %w( src/core/lib/support/arena.c ) s.files += %w( src/core/lib/support/avl.c ) s.files += %w( src/core/lib/support/backoff.c ) s.files += %w( src/core/lib/support/cmdline.c ) diff --git a/package.xml b/package.xml index 4167bef26e..67669286c3 100644 --- a/package.xml +++ b/package.xml @@ -91,6 +91,7 @@ <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_posix.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/sync_windows.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/profiling/timers.h" role="src" /> + <file baseinstalldir="/" name="src/core/lib/support/arena.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/backoff.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/block_annotate.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/env.h" role="src" /> @@ -106,6 +107,7 @@ <file baseinstalldir="/" name="src/core/lib/profiling/basic_timers.c" role="src" /> <file baseinstalldir="/" name="src/core/lib/profiling/stap_timers.c" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/alloc.c" role="src" /> + <file baseinstalldir="/" name="src/core/lib/support/arena.c" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/avl.c" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/backoff.c" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/cmdline.c" role="src" /> diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 4841da8da8..242ce06a16 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -101,18 +101,20 @@ class IndentScope { // TODO(https://github.com/google/protobuf/issues/888): // Export `ModuleName` from protobuf's // `src/google/protobuf/compiler/python/python_generator.cc` file. -grpc::string ModuleName(const grpc::string& filename) { +grpc::string ModuleName(const grpc::string& filename, + const grpc::string& import_prefix) { grpc::string basename = StripProto(filename); basename = StringReplace(basename, "-", "_"); basename = StringReplace(basename, "/", "."); - return basename + "_pb2"; + return import_prefix + basename + "_pb2"; } // TODO(https://github.com/google/protobuf/issues/888): // Export `ModuleAlias` from protobuf's // `src/google/protobuf/compiler/python/python_generator.cc` file. -grpc::string ModuleAlias(const grpc::string& filename) { - grpc::string module_name = ModuleName(filename); +grpc::string ModuleAlias(const grpc::string& filename, + const grpc::string& import_prefix) { + grpc::string module_name = ModuleName(filename, import_prefix); // We can't have dots in the module name, so we replace each with _dot_. // But that could lead to a collision between a.b and a_dot_b, so we also // duplicate each underscore. @@ -189,7 +191,7 @@ bool PrivateGenerator::GetModuleAndMessagePath(const Descriptor* type, grpc::string generator_file_name = file->name(); grpc::string module; if (generator_file_name != file_name || generate_in_pb2_grpc) { - module = ModuleAlias(file_name) + "."; + module = ModuleAlias(file_name, config.import_prefix) + "."; } else { module = ""; } @@ -666,8 +668,10 @@ bool PrivateGenerator::PrintPreamble() { for (int k = 0; k < 2; ++k) { const Descriptor* type = types[k]; grpc::string type_file_name = type->file()->name(); - grpc::string module_name = ModuleName(type_file_name); - grpc::string module_alias = ModuleAlias(type_file_name); + grpc::string module_name = + ModuleName(type_file_name, config.import_prefix); + grpc::string module_alias = + ModuleAlias(type_file_name, config.import_prefix); imports_set.insert(std::make_tuple(module_name, module_alias)); } } @@ -766,7 +770,9 @@ pair<bool, grpc::string> PrivateGenerator::GetGrpcServices() { } // namespace GeneratorConfiguration::GeneratorConfiguration() - : grpc_package_root("grpc"), beta_package_root("grpc.beta") {} + : grpc_package_root("grpc"), + beta_package_root("grpc.beta"), + import_prefix("") {} PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config) : config_(config) {} diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index 6a95255d40..b91059fad7 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -45,7 +45,10 @@ namespace grpc_python_generator { struct GeneratorConfiguration { GeneratorConfiguration(); grpc::string grpc_package_root; + // TODO(https://github.com/grpc/grpc/issues/8622): Drop this. grpc::string beta_package_root; + // TODO(https://github.com/google/protobuf/issues/888): Drop this. + grpc::string import_prefix; }; class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c index b80d831557..69d488ad18 100644 --- a/src/core/ext/census/grpc_filter.c +++ b/src/core/ext/census/grpc_filter.c @@ -78,7 +78,8 @@ static void client_mutate_op(grpc_call_element *elem, call_data *calld = elem->call_data; channel_data *chand = elem->channel_data; if (op->send_initial_metadata) { - extract_and_annotate_method_tag(op->send_initial_metadata, calld, chand); + extract_and_annotate_method_tag( + op->payload->send_initial_metadata.send_initial_metadata, calld, chand); } } @@ -107,9 +108,12 @@ static void server_mutate_op(grpc_call_element *elem, call_data *calld = elem->call_data; if (op->recv_initial_metadata) { /* substitute our callback for the op callback */ - calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->recv_initial_metadata_ready; - op->recv_initial_metadata_ready = &calld->finish_recv; + calld->recv_initial_metadata = + op->payload->recv_initial_metadata.recv_initial_metadata; + calld->on_done_recv = + op->payload->recv_initial_metadata.recv_initial_metadata_ready; + op->payload->recv_initial_metadata.recv_initial_metadata_ready = + &calld->finish_recv; } } diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index d308096212..3d9d70559a 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -71,7 +71,8 @@ */ typedef enum { - WAIT_FOR_READY_UNSET, + /* zero so it can be default initialized */ + WAIT_FOR_READY_UNSET = 0, WAIT_FOR_READY_FALSE, WAIT_FOR_READY_TRUE } wait_for_ready_value; @@ -631,7 +632,8 @@ static void cc_destroy_channel_elem(grpc_exec_ctx *exec_ctx, #define CANCELLED_CALL ((grpc_subchannel_call *)1) typedef enum { - GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING, + /* zero so that it can be default-initialized */ + GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING = 0, GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL } subchannel_creation_phase; @@ -653,7 +655,6 @@ typedef struct client_channel_call_data { gpr_timespec call_start_time; gpr_timespec deadline; method_parameters *method_params; - grpc_closure read_service_config; grpc_error *cancel_error; @@ -726,6 +727,47 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) { gpr_free(ops); } +// Sets calld->method_params. +// If the method params specify a timeout, populates +// *per_method_deadline and returns true. +static bool set_call_method_params_from_service_config_locked( + grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + gpr_timespec *per_method_deadline) { + channel_data *chand = elem->channel_data; + call_data *calld = elem->call_data; + if (chand->method_params_table != NULL) { + calld->method_params = grpc_method_config_table_get( + exec_ctx, chand->method_params_table, calld->path); + if (calld->method_params != NULL) { + method_parameters_ref(calld->method_params); + if (gpr_time_cmp(calld->method_params->timeout, + gpr_time_0(GPR_TIMESPAN)) != 0) { + *per_method_deadline = + gpr_time_add(calld->call_start_time, calld->method_params->timeout); + return true; + } + } + } + return false; +} + +static void apply_final_configuration_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + /* apply service-config level configuration to the call (now that we're + * certain it exists) */ + call_data *calld = elem->call_data; + gpr_timespec per_method_deadline; + if (set_call_method_params_from_service_config_locked(exec_ctx, elem, + &per_method_deadline)) { + // If the deadline from the service config is shorter than the one + // from the client API, reset the deadline timer. + if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { + calld->deadline = per_method_deadline; + grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); + } + } +} + static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_call_element *elem = arg; @@ -851,6 +893,7 @@ static bool pick_subchannel_locked( } GPR_ASSERT(error == GRPC_ERROR_NONE); if (chand->lb_policy != NULL) { + apply_final_configuration_locked(exec_ctx, elem); grpc_lb_policy *lb_policy = chand->lb_policy; GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel"); // If the application explicitly set wait_for_ready, use that. @@ -1063,114 +1106,18 @@ static void cc_start_transport_stream_op(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("cc_start_transport_stream_op", 0); } -// Sets calld->method_params. -// If the method params specify a timeout, populates -// *per_method_deadline and returns true. -static bool set_call_method_params_from_service_config_locked( - grpc_exec_ctx *exec_ctx, grpc_call_element *elem, - gpr_timespec *per_method_deadline) { - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - if (chand->method_params_table != NULL) { - calld->method_params = grpc_method_config_table_get( - exec_ctx, chand->method_params_table, calld->path); - if (calld->method_params != NULL) { - method_parameters_ref(calld->method_params); - if (gpr_time_cmp(calld->method_params->timeout, - gpr_time_0(GPR_TIMESPAN)) != 0) { - *per_method_deadline = - gpr_time_add(calld->call_start_time, calld->method_params->timeout); - return true; - } - } - } - return false; -} - -// Gets data from the service config. Invoked when the resolver returns -// its initial result. -static void read_service_config_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - grpc_call_element *elem = arg; - call_data *calld = elem->call_data; - // If this is an error, there's no point in looking at the service config. - if (error == GRPC_ERROR_NONE) { - gpr_timespec per_method_deadline; - if (set_call_method_params_from_service_config_locked( - exec_ctx, elem, &per_method_deadline)) { - // If the deadline from the service config is shorter than the one - // from the client API, reset the deadline timer. - if (gpr_time_cmp(per_method_deadline, calld->deadline) < 0) { - calld->deadline = per_method_deadline; - grpc_deadline_state_reset(exec_ctx, elem, calld->deadline); - } - } - } - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "read_service_config"); -} - -static void initial_read_service_config_locked(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error_ignored) { - grpc_call_element *elem = arg; - channel_data *chand = elem->channel_data; - call_data *calld = elem->call_data; - // If the resolver has already returned results, then we can access - // the service config parameters immediately. Otherwise, we need to - // defer that work until the resolver returns an initial result. - if (chand->lb_policy != NULL) { - // We already have a resolver result, so check for service config. - gpr_timespec per_method_deadline; - if (set_call_method_params_from_service_config_locked( - exec_ctx, elem, &per_method_deadline)) { - calld->deadline = gpr_time_min(calld->deadline, per_method_deadline); - } - } else { - // We don't yet have a resolver result, so register a callback to - // get the service config data once the resolver returns. - // Take a reference to the call stack to be owned by the callback. - GRPC_CALL_STACK_REF(calld->owning_call, "read_service_config"); - grpc_closure_init(&calld->read_service_config, read_service_config_locked, - elem, grpc_combiner_scheduler(chand->combiner, false)); - grpc_closure_list_append(&chand->waiting_for_config_closures, - &calld->read_service_config, GRPC_ERROR_NONE); - } - // Start the deadline timer with the current deadline value. If we - // do not yet have service config data, then the timer may be reset - // later. - grpc_deadline_state_start(exec_ctx, elem, calld->deadline); - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, - "initial_read_service_config"); -} - /* Constructor for call_data */ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { - channel_data *chand = elem->channel_data; call_data *calld = elem->call_data; // Initialize data members. grpc_deadline_state_init(exec_ctx, elem, args->call_stack); calld->path = grpc_slice_ref_internal(args->path); calld->call_start_time = args->start_time; calld->deadline = gpr_convert_clock_type(args->deadline, GPR_CLOCK_MONOTONIC); - calld->method_params = NULL; - calld->cancel_error = GRPC_ERROR_NONE; - gpr_atm_rel_store(&calld->subchannel_call, 0); - calld->connected_subchannel = NULL; - calld->waiting_ops = NULL; - calld->waiting_ops_count = 0; - calld->waiting_ops_capacity = 0; - calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING; calld->owning_call = args->call_stack; - calld->pollent = NULL; - GRPC_CALL_STACK_REF(calld->owning_call, "initial_read_service_config"); - grpc_closure_sched( - exec_ctx, - grpc_closure_init(&calld->read_service_config, - initial_read_service_config_locked, elem, - grpc_combiner_scheduler(chand->combiner, false)), - GRPC_ERROR_NONE); + grpc_deadline_state_start(exec_ctx, elem, calld->deadline); return GRPC_ERROR_NONE; } diff --git a/src/core/ext/client_channel/parse_address.c b/src/core/ext/client_channel/parse_address.c index 8ae15fc72b..cd1b2cd80c 100644 --- a/src/core/ext/client_channel/parse_address.c +++ b/src/core/ext/client_channel/parse_address.c @@ -128,6 +128,7 @@ int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr) { GPR_ASSERT(host_end >= host); char host_without_scope[INET6_ADDRSTRLEN]; size_t host_without_scope_len = (size_t)(host_end - host); + uint32_t sin6_scope_id = 0; strncpy(host_without_scope, host, host_without_scope_len); host_without_scope[host_without_scope_len] = '\0'; if (inet_pton(AF_INET6, host_without_scope, &in6->sin6_addr) == 0) { @@ -136,10 +137,12 @@ int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr) { } if (gpr_parse_bytes_to_uint32(host_end + 1, strlen(host) - host_without_scope_len - 1, - &in6->sin6_scope_id) == 0) { + &sin6_scope_id) == 0) { gpr_log(GPR_ERROR, "invalid ipv6 scope id: '%s'", host_end + 1); goto done; } + // Handle "sin6_scope_id" being type "u_long". See grpc issue ##10027. + in6->sin6_scope_id = sin6_scope_id; } else { if (inet_pton(AF_INET6, host, &in6->sin6_addr) == 0) { gpr_log(GPR_ERROR, "invalid ipv6 address: '%s'", host); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index de76ed4743..8e403086c0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -511,6 +511,10 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error) { if (!t->closed) { + if (!grpc_error_has_clear_grpc_status(error)) { + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNAVAILABLE); + } if (t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE) { if (t->close_transport_on_writes_finished == NULL) { t->close_transport_on_writes_finished = @@ -520,10 +524,6 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_error_add_child(t->close_transport_on_writes_finished, error); return; } - if (!grpc_error_has_clear_grpc_status(error)) { - error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNAVAILABLE); - } t->closed = 1; connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "close_transport"); diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index f487533c41..9b4b1a7b84 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -91,7 +91,7 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_ping_parser *p = parser; while (p->byte != 8 && cur != end) { - p->opaque_8bytes |= (((uint64_t)*cur) << (8 * p->byte)); + p->opaque_8bytes |= (((uint64_t)*cur) << (56 - 8 * p->byte)); cur++; p->byte++; } diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 01a03533da..cf5acfaac3 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -54,6 +54,7 @@ #include "third_party/objective_c/Cronet/bidirectional_stream_c.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 +#define GRPC_FLUSH_READ_SIZE 4096 #define CRONET_LOG(...) \ do { \ @@ -151,11 +152,17 @@ struct write_state { struct op_state { bool state_op_done[OP_NUM_OPS]; bool state_callback_received[OP_NUM_OPS]; + /* A non-zero gRPC status code has been seen */ bool fail_state; + /* Transport is discarding all buffered messages */ bool flush_read; bool flush_cronet_when_ready; bool pending_write_for_trailer; - bool unprocessed_send_message; + bool pending_send_message; + /* User requested RECV_TRAILING_METADATA */ + bool pending_recv_trailing_metadata; + /* Cronet has not issued a callback of a bidirectional read */ + bool pending_read_from_cronet; grpc_error *cancel_error; /* data structure for storing data coming from server */ struct read_state rs; @@ -248,11 +255,35 @@ static const char *op_id_string(enum e_op_id i) { return "UNKNOWN"; } -static void free_read_buffer(stream_obj *s) { +static void null_and_maybe_free_read_buffer(stream_obj *s) { if (s->state.rs.read_buffer && s->state.rs.read_buffer != s->state.rs.grpc_header_bytes) { gpr_free(s->state.rs.read_buffer); - s->state.rs.read_buffer = NULL; + } + s->state.rs.read_buffer = NULL; +} + +static void maybe_flush_read(stream_obj *s) { + /* To enter flush read state (discarding all the buffered messages in + * transport layer), two conditions must be satisfied: 1) non-zero grpc status + * has been received, and 2) an op requesting the status code + * (RECV_TRAILING_METADATA) is issued by the user. (See + * doc/status_ordering.md) */ + /* Whenever the evaluation of any of the two condition is changed, we check + * whether we should enter the flush read state. */ + if (s->state.pending_recv_trailing_metadata && s->state.fail_state) { + if (!s->state.flush_read) { + CRONET_LOG(GPR_DEBUG, "%p: Flush read", s); + s->state.flush_read = true; + null_and_maybe_free_read_buffer(s); + s->state.rs.read_buffer = gpr_malloc(GRPC_FLUSH_READ_SIZE); + if (!s->state.pending_read_from_cronet) { + CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, + GRPC_FLUSH_READ_SIZE); + s->state.pending_read_from_cronet = true; + } + } } } @@ -279,7 +310,11 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { storage->head = new_op; storage->num_pending_ops++; if (op->send_message) { - s->state.unprocessed_send_message = true; + s->state.pending_send_message = true; + } + if (op->recv_trailing_metadata) { + s->state.pending_recv_trailing_metadata = true; + maybe_flush_read(s); } CRONET_LOG(GPR_DEBUG, "adding new op %p. %d in the queue.", new_op, storage->num_pending_ops); @@ -367,7 +402,7 @@ static void on_failed(bidirectional_stream *stream, int net_error) { gpr_free(s->state.ws.write_buffer); s->state.ws.write_buffer = NULL; } - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -390,7 +425,7 @@ static void on_canceled(bidirectional_stream *stream) { gpr_free(s->state.ws.write_buffer); s->state.ws.write_buffer = NULL; } - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -405,7 +440,7 @@ static void on_succeeded(bidirectional_stream *stream) { bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_SUCCEEDED] = true; s->cbs = NULL; - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); gpr_mu_unlock(&s->mu); execute_from_storage(s); } @@ -473,6 +508,7 @@ static void on_response_headers_received( CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, s->state.rs.remaining_bytes); + s->state.pending_read_from_cronet = true; } gpr_mu_unlock(&s->mu); grpc_exec_ctx_finish(&exec_ctx); @@ -504,10 +540,13 @@ static void on_read_completed(bidirectional_stream *stream, char *data, CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); gpr_mu_lock(&s->mu); + s->state.pending_read_from_cronet = false; s->state.state_callback_received[OP_RECV_MESSAGE] = true; if (count > 0 && s->state.flush_read) { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); - bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, 4096); + bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, + GRPC_FLUSH_READ_SIZE); + s->state.pending_read_from_cronet = true; gpr_mu_unlock(&s->mu); } else if (count > 0) { s->state.rs.received_bytes += count; @@ -518,16 +557,14 @@ static void on_read_completed(bidirectional_stream *stream, char *data, bidirectional_stream_read( s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); + s->state.pending_read_from_cronet = true; gpr_mu_unlock(&s->mu); } else { gpr_mu_unlock(&s->mu); execute_from_storage(s); } } else { - if (s->state.flush_read) { - gpr_free(s->state.rs.read_buffer); - s->state.rs.read_buffer = NULL; - } + null_and_maybe_free_read_buffer(s); s->state.rs.read_stream_closed = true; gpr_mu_unlock(&s->mu); execute_from_storage(s); @@ -564,6 +601,7 @@ static void on_response_trailers_received( if (0 == strcmp(trailers->headers[i].key, "grpc-status") && 0 != strcmp(trailers->headers[i].value, "0")) { s->state.fail_state = true; + maybe_flush_read(s); } } s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; @@ -778,7 +816,7 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; /* we haven't sent message yet */ - else if (stream_state->unprocessed_send_message && + else if (stream_state->pending_send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; /* we haven't got on_write_completed for the send yet */ @@ -878,9 +916,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, char *url = NULL; const char *method = "POST"; s->header_array.headers = NULL; - convert_metadata_to_cronet_headers( - stream_op->send_initial_metadata->list.head, t->host, &url, - &s->header_array.headers, &s->header_array.count, &method); + convert_metadata_to_cronet_headers(stream_op->payload->send_initial_metadata + .send_initial_metadata->list.head, + t->host, &url, &s->header_array.headers, + &s->header_array.count, &method); s->header_array.capacity = s->header_array.count; CRONET_LOG(GPR_DEBUG, "bidirectional_stream_start(%p, %s)", s->cbs, url); bidirectional_stream_start(s->cbs, url, 0, method, &s->header_array, false); @@ -900,7 +939,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } else if (stream_op->send_message && op_can_be_run(stream_op, s, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); - stream_state->unprocessed_send_message = false; + stream_state->pending_send_message = false; if (stream_state->state_callback_received[OP_FAILED]) { result = NO_ACTION_POSSIBLE; CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); @@ -908,13 +947,14 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, grpc_slice_buffer write_slice_buffer; grpc_slice slice; grpc_slice_buffer_init(&write_slice_buffer); - grpc_byte_stream_next(NULL, stream_op->send_message, &slice, - stream_op->send_message->length, NULL); + grpc_byte_stream_next( + NULL, stream_op->payload->send_message.send_message, &slice, + stream_op->payload->send_message.send_message->length, NULL); /* Check that compression flag is OFF. We don't support compression yet. */ - if (stream_op->send_message->flags != 0) { + if (stream_op->payload->send_message.send_message->flags != 0) { gpr_log(GPR_ERROR, "Compression is not supported"); - GPR_ASSERT(stream_op->send_message->flags == 0); + GPR_ASSERT(stream_op->payload->send_message.send_message->flags == 0); } grpc_slice_buffer_add(&write_slice_buffer, slice); if (write_slice_buffer.count != 1) { @@ -972,17 +1012,23 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, OP_RECV_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_NONE); + grpc_closure_sched( + exec_ctx, + stream_op->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_NONE); } else if (stream_state->state_callback_received[OP_FAILED]) { - grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_NONE); + grpc_closure_sched( + exec_ctx, + stream_op->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_NONE); } else { grpc_chttp2_incoming_metadata_buffer_publish( exec_ctx, &oas->s->state.rs.initial_metadata, - stream_op->recv_initial_metadata); - grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready, - GRPC_ERROR_NONE); + stream_op->payload->recv_initial_metadata.recv_initial_metadata); + grpc_closure_sched( + exec_ctx, + stream_op->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_NONE); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; @@ -991,20 +1037,31 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); - grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, + grpc_closure_sched(exec_ctx, + stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); stream_state->state_op_done[OP_RECV_MESSAGE] = true; result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_state->state_callback_received[OP_FAILED]) { CRONET_LOG(GPR_DEBUG, "Stream failed."); - grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, + grpc_closure_sched(exec_ctx, + stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); stream_state->state_op_done[OP_RECV_MESSAGE] = true; result = ACTION_TAKEN_NO_CALLBACK; } else if (stream_state->rs.read_stream_closed == true) { /* No more data will be received */ CRONET_LOG(GPR_DEBUG, "read stream closed"); - grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, + grpc_closure_sched(exec_ctx, + stream_op->payload->recv_message.recv_message_ready, + GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; + result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_state->flush_read) { + CRONET_LOG(GPR_DEBUG, "flush read"); + grpc_closure_sched(exec_ctx, + stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -1029,6 +1086,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_WITH_CALLBACK; } else { stream_state->rs.remaining_bytes = 0; @@ -1038,8 +1096,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, - GRPC_ERROR_NONE); + grpc_closure_sched( + exec_ctx, stream_op->payload->recv_message.recv_message_ready, + GRPC_ERROR_NONE); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -1047,11 +1106,13 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.received_bytes = 0; + stream_state->rs.length_field_received = false; CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { @@ -1064,6 +1125,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, true; /* Indicates that at least one read request has been made */ bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_WITH_CALLBACK; } else { result = NO_ACTION_POSSIBLE; @@ -1075,7 +1137,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, uint8_t *dst_p = GRPC_SLICE_START_PTR(read_data_slice); memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); - free_read_buffer(s); + null_and_maybe_free_read_buffer(s); grpc_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); @@ -1083,7 +1145,8 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_closure_sched(exec_ctx, stream_op->recv_message_ready, + grpc_closure_sched(exec_ctx, + stream_op->payload->recv_message.recv_message_ready, GRPC_ERROR_NONE); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -1096,6 +1159,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs); bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); + stream_state->pending_read_from_cronet = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && @@ -1105,12 +1169,12 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( exec_ctx, &oas->s->state.rs.trailing_metadata, - stream_op->recv_trailing_metadata); + stream_op->payload->recv_trailing_metadata.recv_trailing_metadata); stream_state->rs.trailing_metadata_valid = false; } stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->cancel_error && + } else if (stream_op->cancel_stream && op_can_be_run(stream_op, s, &oas->state, OP_CANCEL_ERROR)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); CRONET_LOG(GPR_DEBUG, "W: bidirectional_stream_cancel(%p)", s->cbs); @@ -1122,7 +1186,8 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } stream_state->state_op_done[OP_CANCEL_ERROR] = true; if (!stream_state->cancel_error) { - stream_state->cancel_error = GRPC_ERROR_REF(stream_op->cancel_error); + stream_state->cancel_error = + GRPC_ERROR_REF(stream_op->payload->cancel_stream.cancel_error); } } else if (stream_op->on_complete && op_can_be_run(stream_op, s, &oas->state, OP_ON_COMPLETE)) { @@ -1153,15 +1218,6 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, make a note */ if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; - } else if (stream_state->fail_state && !stream_state->flush_read) { - CRONET_LOG(GPR_DEBUG, "running: %p flush read", oas); - if (stream_state->rs.read_buffer && - stream_state->rs.read_buffer != stream_state->rs.grpc_header_bytes) { - gpr_free(stream_state->rs.read_buffer); - stream_state->rs.read_buffer = NULL; - } - stream_state->rs.read_buffer = gpr_malloc(4096); - stream_state->flush_read = true; } else { result = NO_ACTION_POSSIBLE; } @@ -1190,7 +1246,9 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->state.fail_state = s->state.flush_read = false; s->state.cancel_error = NULL; s->state.flush_cronet_when_ready = s->state.pending_write_for_trailer = false; - s->state.unprocessed_send_message = false; + s->state.pending_send_message = false; + s->state.pending_recv_trailing_metadata = false; + s->state.pending_read_from_cronet = false; s->curr_gs = gs; s->curr_ct = (grpc_cronet_transport *)gt; @@ -1209,37 +1267,33 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_transport_stream_op *op) { CRONET_LOG(GPR_DEBUG, "perform_stream_op"); - stream_obj *s = (stream_obj *)gs; - add_to_storage(s, op); if (op->send_initial_metadata && - header_has_authority(op->send_initial_metadata->list.head)) { + header_has_authority(op->payload->send_initial_metadata + .send_initial_metadata->list.head)) { /* Cronet does not support :authority header field. We cancel the call when - this field is present in metadata */ - bidirectional_stream_header_array header_array; - bidirectional_stream_header *header; - bidirectional_stream cbs; - CRONET_LOG(GPR_DEBUG, - ":authority header is provided but not supported;" - " cancel operations"); - /* Notify application that operation is cancelled by forging trailers */ - header_array.count = 1; - header_array.capacity = 1; - header_array.headers = gpr_malloc(sizeof(bidirectional_stream_header)); - header = (bidirectional_stream_header *)header_array.headers; - header->key = "grpc-status"; - header->value = "1"; /* Return status GRPC_STATUS_CANCELLED */ - cbs.annotation = (void *)s; - s->state.state_op_done[OP_CANCEL_ERROR] = true; - on_response_trailers_received(&cbs, &header_array); - gpr_free(header_array.headers); - } else { - execute_from_storage(s); + this field is present in metadata */ + if (op->recv_initial_metadata) { + grpc_closure_sched( + exec_ctx, + op->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_CANCELLED); + } + if (op->recv_message) { + grpc_closure_sched(exec_ctx, op->payload->recv_message.recv_message_ready, + GRPC_ERROR_CANCELLED); + } + grpc_closure_sched(exec_ctx, op->on_complete, GRPC_ERROR_CANCELLED); + return; } + stream_obj *s = (stream_obj *)gs; + add_to_storage(s, op); + execute_from_storage(s); } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, void *and_free_memory) { stream_obj *s = (stream_obj *)gs; + null_and_maybe_free_read_buffer(s); GRPC_ERROR_UNREF(s->state.cancel_error); } diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 7cdbe30198..1127fff756 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -140,14 +140,16 @@ grpc_error *grpc_error_ref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return err; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->refs.count, err->refs.count + 1, file, line, func); - gpr_ref(&err->refs); + gpr_atm_no_barrier_load(&err->atomics.refs.count), + gpr_atm_no_barrier_load(&err->atomics.refs.count) + 1, file, line, + func); + gpr_ref(&err->atomics.refs); return err; } #else grpc_error *grpc_error_ref(grpc_error *err) { if (grpc_error_is_special(err)) return err; - gpr_ref(&err->refs); + gpr_ref(&err->atomics.refs); return err; } #endif @@ -182,7 +184,7 @@ static void error_destroy(grpc_error *err) { GPR_ASSERT(!grpc_error_is_special(err)); unref_errs(err); unref_strs(err); - gpr_free((void *)gpr_atm_acq_load(&err->error_string)); + gpr_free((void *)gpr_atm_acq_load(&err->atomics.error_string)); gpr_free(err); } @@ -191,15 +193,17 @@ void grpc_error_unref(grpc_error *err, const char *file, int line, const char *func) { if (grpc_error_is_special(err)) return; gpr_log(GPR_DEBUG, "%p: %" PRIdPTR " -> %" PRIdPTR " [%s:%d %s]", err, - err->refs.count, err->refs.count - 1, file, line, func); - if (gpr_unref(&err->refs)) { + gpr_atm_no_barrier_load(&err->atomics.refs.count), + gpr_atm_no_barrier_load(&err->atomics.refs.count) - 1, file, line, + func); + if (gpr_unref(&err->atomics.refs)) { error_destroy(err); } } #else void grpc_error_unref(grpc_error *err) { if (grpc_error_is_special(err)) return; - if (gpr_unref(&err->refs)) { + if (gpr_unref(&err->atomics.refs)) { error_destroy(err); } } @@ -328,8 +332,8 @@ grpc_error *grpc_error_create(const char *file, int line, const char *desc, internal_set_time(&err, GRPC_ERROR_TIME_CREATED, gpr_now(GPR_CLOCK_REALTIME)); - gpr_atm_no_barrier_store(&err->error_string, 0); - gpr_ref_init(&err->refs, 1); + gpr_atm_no_barrier_store(&err->atomics.error_string, 0); + gpr_ref_init(&err->atomics.refs, 1); GPR_TIMER_END("grpc_error_create", 0); return err; } @@ -369,7 +373,7 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { grpc_slice_from_static_string("cancelled")); internal_set_int(&out, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_CANCELLED); } - } else if (gpr_ref_is_unique(&in->refs)) { + } else if (gpr_ref_is_unique(&in->atomics.refs)) { out = in; } else { uint8_t new_arena_capacity = in->arena_capacity; @@ -382,10 +386,14 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { #ifdef GRPC_ERROR_REFCOUNT_DEBUG gpr_log(GPR_DEBUG, "%p create copying %p", out, in); #endif - memcpy(out, in, sizeof(*in) + in->arena_size * sizeof(intptr_t)); + // bulk memcpy of the rest of the struct. + size_t skip = sizeof(&out->atomics); + memcpy((void *)((uintptr_t)out + skip), (void *)((uintptr_t)in + skip), + sizeof(*in) + (in->arena_size * sizeof(intptr_t)) - skip); + // manually set the atomics and the new capacity + gpr_atm_no_barrier_store(&out->atomics.error_string, 0); + gpr_ref_init(&out->atomics.refs, 1); out->arena_capacity = new_arena_capacity; - gpr_atm_no_barrier_store(&out->error_string, 0); - gpr_ref_init(&out->refs, 1); ref_strs(out); ref_errs(out); GRPC_ERROR_UNREF(in); @@ -692,7 +700,7 @@ const char *grpc_error_string(grpc_error *err) { if (err == GRPC_ERROR_OOM) return oom_error_string; if (err == GRPC_ERROR_CANCELLED) return cancelled_error_string; - void *p = (void *)gpr_atm_acq_load(&err->error_string); + void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string); if (p != NULL) { GPR_TIMER_END("grpc_error_string", 0); return p; @@ -712,9 +720,9 @@ const char *grpc_error_string(grpc_error *err) { char *out = finish_kvs(&kvs); - if (!gpr_atm_rel_cas(&err->error_string, 0, (gpr_atm)out)) { + if (!gpr_atm_rel_cas(&err->atomics.error_string, 0, (gpr_atm)out)) { gpr_free(out); - out = (char *)gpr_atm_no_barrier_load(&err->error_string); + out = (char *)gpr_atm_no_barrier_load(&err->atomics.error_string); } GPR_TIMER_END("grpc_error_string", 0); diff --git a/src/core/lib/iomgr/error_internal.h b/src/core/lib/iomgr/error_internal.h index fb4814e41f..7f204df1b2 100644 --- a/src/core/lib/iomgr/error_internal.h +++ b/src/core/lib/iomgr/error_internal.h @@ -46,14 +46,25 @@ struct grpc_linked_error { uint8_t next; }; +// c core representation of an error. See error.h for high level description of +// this object. struct grpc_error { - gpr_refcount refs; + // All atomics in grpc_error must be stored in this nested struct. The rest of + // the object is memcpy-ed in bulk in copy_and_unref. + struct atomics { + gpr_refcount refs; + gpr_atm error_string; + } atomics; + // These arrays index into dynamic arena at the bottom of the struct. + // UINT8_MAX is used as a sentinel value. uint8_t ints[GRPC_ERROR_INT_MAX]; uint8_t strs[GRPC_ERROR_STR_MAX]; uint8_t times[GRPC_ERROR_TIME_MAX]; + // The child errors are stored in the arena, but are effectively a linked list + // structure, since they are contained withing grpc_linked_error objects. uint8_t first_err; uint8_t last_err; - gpr_atm error_string; + // The arena is dynamically reallocated with a grow factor of 1.5. uint8_t arena_size; uint8_t arena_capacity; intptr_t arena[0]; diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index af33949c69..a2f81bcd78 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -39,6 +39,7 @@ #include <string.h> +#include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <grpc/support/sync.h> @@ -61,25 +62,30 @@ gpr_mu grpc_polling_mu; immediately in the next loop iteration. Note: In the future, if there is a bug that involves missing wakeups in the future, try adding a uv_async_t to kick the loop differently */ -uv_timer_t dummy_uv_handle; +uv_timer_t *dummy_uv_handle; size_t grpc_pollset_size() { return sizeof(grpc_pollset); } void dummy_timer_cb(uv_timer_t *handle) {} +void dummy_handle_close_cb(uv_handle_t *handle) { gpr_free(handle); } + void grpc_pollset_global_init(void) { gpr_mu_init(&grpc_polling_mu); - uv_timer_init(uv_default_loop(), &dummy_uv_handle); + dummy_uv_handle = gpr_malloc(sizeof(uv_timer_t)); + uv_timer_init(uv_default_loop(), dummy_uv_handle); grpc_pollset_work_run_loop = 1; } -static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; } - void grpc_pollset_global_shutdown(void) { gpr_mu_destroy(&grpc_polling_mu); - uv_close((uv_handle_t *)&dummy_uv_handle, timer_close_cb); + uv_close((uv_handle_t *)dummy_uv_handle, dummy_handle_close_cb); } +static void timer_run_cb(uv_timer_t *timer) {} + +static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; } + void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &grpc_polling_mu; uv_timer_init(uv_default_loop(), &pollset->timer); @@ -95,7 +101,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, uv_run(uv_default_loop(), UV_RUN_NOWAIT); } else { // kick the loop once - uv_timer_start(&dummy_uv_handle, dummy_timer_cb, 0, 0); + uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); } grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } @@ -111,8 +117,6 @@ void grpc_pollset_destroy(grpc_pollset *pollset) { } } -static void timer_run_cb(uv_timer_t *timer) {} - grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { @@ -145,7 +149,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_error *grpc_pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - uv_timer_start(&dummy_uv_handle, dummy_timer_cb, 0, 0); + uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/iomgr/resolve_address_uv.c b/src/core/lib/iomgr/resolve_address_uv.c index 79ff910738..4d715be94c 100644 --- a/src/core/lib/iomgr/resolve_address_uv.c +++ b/src/core/lib/iomgr/resolve_address_uv.c @@ -40,6 +40,7 @@ #include <grpc/support/host_port.h> #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include <grpc/support/useful.h> #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" @@ -54,8 +55,36 @@ typedef struct request { grpc_closure *on_done; grpc_resolved_addresses **addresses; struct addrinfo *hints; + char *host; + char *port; } request; +static int retry_named_port_failure(int status, request *r, + uv_getaddrinfo_cb getaddrinfo_cb) { + if (status != 0) { + // This loop is copied from resolve_address_posix.c + char *svc[][2] = {{"http", "80"}, {"https", "443"}}; + for (size_t i = 0; i < GPR_ARRAY_SIZE(svc); i++) { + if (strcmp(r->port, svc[i][0]) == 0) { + int retry_status; + uv_getaddrinfo_t *req = gpr_malloc(sizeof(uv_getaddrinfo_t)); + req->data = r; + retry_status = uv_getaddrinfo(uv_default_loop(), req, getaddrinfo_cb, + r->host, svc[i][1], r->hints); + if (retry_status < 0 || getaddrinfo_cb == NULL) { + // The callback will not be called + gpr_free(req); + } + return retry_status; + } + } + } + /* If this function calls uv_getaddrinfo, it will return that function's + return value. That function only returns numbers <=0, so we can safely + return 1 to indicate that we never retried */ + return 1; +} + static grpc_error *handle_addrinfo_result(int status, struct addrinfo *result, grpc_resolved_addresses **addresses) { struct addrinfo *resp; @@ -97,13 +126,21 @@ static void getaddrinfo_callback(uv_getaddrinfo_t *req, int status, request *r = (request *)req->data; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error; + int retry_status; + + gpr_free(req); + retry_status = retry_named_port_failure(status, r, getaddrinfo_callback); + if (retry_status == 0) { + // The request is being retried. Nothing should be done here + return; + } + /* Either no retry was attempted, or the retry failed. Either way, the + original error probably has more interesting information */ error = handle_addrinfo_result(status, res, r->addresses); grpc_closure_sched(&exec_ctx, r->on_done, error); grpc_exec_ctx_finish(&exec_ctx); - gpr_free(r->hints); gpr_free(r); - gpr_free(req); uv_freeaddrinfo(res); } @@ -143,6 +180,7 @@ static grpc_error *blocking_resolve_address_impl( uv_getaddrinfo_t req; int s; grpc_error *err; + int retry_status; req.addrinfo = NULL; @@ -158,6 +196,12 @@ static grpc_error *blocking_resolve_address_impl( hints.ai_flags = AI_PASSIVE; /* for wildcard IP address */ s = uv_getaddrinfo(uv_default_loop(), &req, NULL, host, port, &hints); + request r = { + .addresses = addresses, .hints = &hints, .host = host, .port = port}; + retry_status = retry_named_port_failure(s, &r, NULL); + if (retry_status <= 0) { + s = retry_status; + } err = handle_addrinfo_result(s, req.addrinfo, addresses); done: @@ -200,6 +244,8 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name, r = gpr_malloc(sizeof(request)); r->on_done = on_done; r->addresses = addrs; + r->host = host; + r->port = port; req = gpr_malloc(sizeof(uv_getaddrinfo_t)); req->data = r; @@ -222,6 +268,8 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name, gpr_free(r); gpr_free(req); gpr_free(hints); + gpr_free(host); + gpr_free(port); } } diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index ae66577caf..618483d9cb 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -76,7 +76,6 @@ static void uv_tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", connect->addr_name, str); - grpc_error_free_string(str); } if (error == GRPC_ERROR_NONE) { /* error == NONE implies that the timer ran out, and wasn't cancelled. If diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index d1bcd89af1..71e295770a 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -109,8 +109,8 @@ struct grpc_udp_server { grpc_pollset **pollsets; /* number of pollsets in the pollsets array */ size_t pollset_count; - /* The parent grpc server */ - grpc_server *grpc_server; + /* opaque object to pass to callbacks */ + void *user_data; }; grpc_udp_server *grpc_udp_server_create(void) { @@ -178,7 +178,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { /* Call the orphan_cb to signal that the FD is about to be closed and * should no longer be used. */ GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); @@ -204,7 +204,7 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd); + sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE("Server destroyed")); } @@ -299,7 +299,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Tell the registered callback that data is available to read. */ GPR_ASSERT(sp->read_cb); - sp->read_cb(exec_ctx, sp->emfd, sp->server->grpc_server); + sp->read_cb(exec_ctx, sp->emfd, sp->server->user_data); /* Re-arm the notification event so we get another chance to read. */ grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); @@ -322,7 +322,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Tell the registered callback that the socket is writeable. */ GPR_ASSERT(sp->write_cb); - sp->write_cb(exec_ctx, sp->emfd); + sp->write_cb(exec_ctx, sp->emfd, sp->server->user_data); /* Re-arm the notification event so we get another chance to write. */ grpc_fd_notify_on_write(exec_ctx, sp->emfd, &sp->write_closure); @@ -464,13 +464,13 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index) { void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, grpc_pollset **pollsets, size_t pollset_count, - grpc_server *server) { + void *user_data) { size_t i; gpr_mu_lock(&s->mu); grpc_udp_listener *sp; GPR_ASSERT(s->active_ports == 0); s->pollsets = pollsets; - s->grpc_server = server; + s->user_data = user_data; sp = s->head; while (sp != NULL) { diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index ed63fa7d81..90842a47f0 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -47,23 +47,23 @@ typedef struct grpc_udp_server grpc_udp_server; /* Called when data is available to read from the socket. */ typedef void (*grpc_udp_server_read_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - struct grpc_server *server); + void *user_data); /* Called when the socket is writeable. */ -typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd); +typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, + void *user_data); /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd); + grpc_fd *emfd, void *user_data); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); -/* Start listening to bound ports */ +/* Start listening to bound ports. user_data is passed to callbacks. */ void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *udp_server, grpc_pollset **pollsets, size_t pollset_count, - struct grpc_server *server); + void *user_data); int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index); diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c new file mode 100644 index 0000000000..7bcb983f24 --- /dev/null +++ b/src/core/lib/support/arena.c @@ -0,0 +1,98 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/support/arena.h" +#include <grpc/support/alloc.h> +#include <grpc/support/atm.h> +#include <grpc/support/log.h> +#include <grpc/support/useful.h> + +#define ROUND_UP_TO_ALIGNMENT_SIZE(x) \ + (((x) + GPR_MAX_ALIGNMENT - 1u) & ~(GPR_MAX_ALIGNMENT - 1u)) + +typedef struct zone { + size_t size_begin; + size_t size_end; + gpr_atm next_atm; +} zone; + +struct gpr_arena { + gpr_atm size_so_far; + zone initial_zone; +}; + +gpr_arena *gpr_arena_create(size_t initial_size) { + initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size); + gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size); + a->initial_zone.size_end = initial_size; + return a; +} + +size_t gpr_arena_destroy(gpr_arena *arena) { + gpr_atm size = gpr_atm_no_barrier_load(&arena->size_so_far); + zone *z = (zone *)gpr_atm_no_barrier_load(&arena->initial_zone.next_atm); + gpr_free(arena); + while (z) { + zone *next_z = (zone *)gpr_atm_no_barrier_load(&z->next_atm); + gpr_free(z); + z = next_z; + } + return (size_t)size; +} + +void *gpr_arena_alloc(gpr_arena *arena, size_t size) { + size = ROUND_UP_TO_ALIGNMENT_SIZE(size); + size_t start = + (size_t)gpr_atm_no_barrier_fetch_add(&arena->size_so_far, size); + zone *z = &arena->initial_zone; + while (start > z->size_end) { + zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm); + if (next_z == NULL) { + size_t next_z_size = (size_t)gpr_atm_no_barrier_load(&arena->size_so_far); + next_z = gpr_zalloc(sizeof(zone) + next_z_size); + next_z->size_begin = z->size_end; + next_z->size_end = z->size_end + next_z_size; + if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) { + gpr_free(next_z); + next_z = (zone *)gpr_atm_acq_load(&z->next_atm); + } + } + z = next_z; + } + if (start + size > z->size_end) { + return gpr_arena_alloc(arena, size); + } + GPR_ASSERT(start >= z->size_begin); + GPR_ASSERT(start + size <= z->size_end); + return ((char *)(z + 1)) + start - z->size_begin; +} diff --git a/src/core/lib/support/arena.h b/src/core/lib/support/arena.h new file mode 100644 index 0000000000..c28033ffc3 --- /dev/null +++ b/src/core/lib/support/arena.h @@ -0,0 +1,54 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// \file Arena based allocator +// Allows very fast allocation of memory, but that memory cannot be freed until +// the arena as a whole is freed +// Tracks the total memory allocated against it, so that future arenas can +// pre-allocate the right amount of memory + +#ifndef GRPC_CORE_LIB_SUPPORT_ARENA_H +#define GRPC_CORE_LIB_SUPPORT_ARENA_H + +#include <stddef.h> + +typedef struct gpr_arena gpr_arena; + +// Create an arena, with \a initial_size bytes in the first allocated buffer +gpr_arena *gpr_arena_create(size_t initial_size); +// Allocate \a size bytes from the arena +void *gpr_arena_alloc(gpr_arena *arena, size_t size); +// Destroy an arena, returning the total number of bytes allocated +size_t gpr_arena_destroy(gpr_arena *arena); + +#endif /* GRPC_CORE_LIB_SUPPORT_ARENA_H */ diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index cf4410f2c0..bd34ed1b91 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -139,8 +139,8 @@ struct grpc_call { grpc_call *parent; grpc_call *first_child; gpr_timespec start_time; - /* TODO(ctiller): share with cq if possible? */ - gpr_mu mu; + /* protects first_child, and child next/prev links */ + gpr_mu child_list_mu; /* client or server call */ bool is_client; @@ -155,8 +155,8 @@ struct grpc_call { bool received_initial_metadata; bool receiving_message; bool requested_final_op; - bool received_final_op; - bool sent_any_op; + gpr_atm any_ops_sent_atm; + gpr_atm received_final_op_atm; /* have we received initial metadata */ bool has_initial_md_been_received; @@ -271,7 +271,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_TIMER_BEGIN("grpc_call_create", 0); call = gpr_zalloc(sizeof(grpc_call) + channel_stack->call_stack_size); *out_call = call; - gpr_mu_init(&call->mu); + gpr_mu_init(&call->child_list_mu); call->channel = args->channel; call->cq = args->cq; call->parent = args->parent_call; @@ -310,7 +310,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - gpr_mu_lock(&args->parent_call->mu); + gpr_mu_lock(&args->parent_call->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -338,6 +338,10 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; + if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { + cancel_with_error(exec_ctx, call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + } } if (args->parent_call->first_child == NULL) { @@ -350,7 +354,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, call; } - gpr_mu_unlock(&args->parent_call->mu); + gpr_mu_unlock(&args->parent_call->child_list_mu); } call->send_deadline = send_deadline; @@ -432,7 +436,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - gpr_mu_destroy(&c->mu); + gpr_mu_destroy(&c->child_list_mu); for (ii = 0; ii < c->send_extra_metadata_count; ii++) { GRPC_MDELEM_UNREF(exec_ctx, c->send_extra_metadata[ii].md); } @@ -453,7 +457,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { GRPC_ERROR_UNREF( - unpack_received_status(gpr_atm_no_barrier_load(&c->status[i])).error); + unpack_received_status(gpr_atm_acq_load(&c->status[i])).error); } grpc_call_stack_destroy(exec_ctx, CALL_STACK_FROM_CALL(c), &c->final_info, c); @@ -470,7 +474,7 @@ void grpc_call_destroy(grpc_call *c) { GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); if (parent) { - gpr_mu_lock(&parent->mu); + gpr_mu_lock(&parent->child_list_mu); if (c == parent->first_child) { parent->first_child = c->sibling_next; if (c == parent->first_child) { @@ -479,15 +483,14 @@ void grpc_call_destroy(grpc_call *c) { c->sibling_prev->sibling_next = c->sibling_next; c->sibling_next->sibling_prev = c->sibling_prev; } - gpr_mu_unlock(&parent->mu); + gpr_mu_unlock(&parent->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); } - gpr_mu_lock(&c->mu); GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = c->sent_any_op && !c->received_final_op; - gpr_mu_unlock(&c->mu); + cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && + !gpr_atm_acq_load(&c->received_final_op_atm); if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); @@ -551,56 +554,26 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, "c=%p, status=%d, description=%s, reserved=%p)", 4, (c, (int)status, description, reserved)); GPR_ASSERT(reserved == NULL); - gpr_mu_lock(&c->mu); cancel_with_status(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, status, description); - gpr_mu_unlock(&c->mu); grpc_exec_ctx_finish(&exec_ctx); return GRPC_CALL_OK; } -typedef struct termination_closure { - grpc_closure closure; - grpc_call *call; - grpc_transport_stream_op op; - grpc_transport_stream_op_payload payload; -} termination_closure; - -static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp, - grpc_error *error) { - termination_closure *tc = tcp; - GRPC_CALL_INTERNAL_UNREF(exec_ctx, tc->call, "termination"); - gpr_free(tc); -} - -static void send_termination(grpc_exec_ctx *exec_ctx, void *tcp, +static void done_termination(grpc_exec_ctx *exec_ctx, void *call, grpc_error *error) { - termination_closure *tc = tcp; - memset(&tc->op, 0, sizeof(tc->op)); - tc->op.payload = &tc->payload; - tc->op.cancel_stream = true; - tc->op.payload->cancel_stream.cancel_error = GRPC_ERROR_REF(error); - /* reuse closure to catch completion */ - tc->op.on_complete = grpc_closure_init(&tc->closure, done_termination, tc, - grpc_schedule_on_exec_ctx); - execute_op(exec_ctx, tc->call, &tc->op); -} - -static void terminate_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, - grpc_error *error) { - termination_closure *tc = gpr_malloc(sizeof(*tc)); - memset(tc, 0, sizeof(*tc)); - tc->call = c; - GRPC_CALL_INTERNAL_REF(tc->call, "termination"); - grpc_closure_sched(exec_ctx, grpc_closure_init(&tc->closure, send_termination, - tc, grpc_schedule_on_exec_ctx), - error); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "termination"); } static void cancel_with_error(grpc_exec_ctx *exec_ctx, grpc_call *c, status_source source, grpc_error *error) { + GRPC_CALL_INTERNAL_REF(c, "termination"); set_status_from_error(exec_ctx, c, source, GRPC_ERROR_REF(error)); - terminate_with_error(exec_ctx, c, error); + grpc_transport_stream_op *op = grpc_make_transport_stream_op( + grpc_closure_create(done_termination, c, grpc_schedule_on_exec_ctx)); + op->cancel_stream = true; + op->payload->cancel_stream.cancel_error = error; + execute_op(exec_ctx, c, op); } static grpc_error *error_from_status(grpc_status_code status, @@ -714,9 +687,7 @@ static void set_incoming_compression_algorithm( grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm( grpc_call *call) { grpc_compression_algorithm algorithm; - gpr_mu_lock(&call->mu); algorithm = call->incoming_compression_algorithm; - gpr_mu_unlock(&call->mu); return algorithm; } @@ -728,9 +699,7 @@ static grpc_compression_algorithm compression_algorithm_for_level_locked( uint32_t grpc_call_test_only_get_message_flags(grpc_call *call) { uint32_t flags; - gpr_mu_lock(&call->mu); flags = call->test_only_last_message_flags; - gpr_mu_unlock(&call->mu); return flags; } @@ -784,9 +753,7 @@ static void set_encodings_accepted_by_peer(grpc_exec_ctx *exec_ctx, uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call *call) { uint32_t encodings_accepted_by_peer; - gpr_mu_lock(&call->mu); encodings_accepted_by_peer = call->encodings_accepted_by_peer; - gpr_mu_unlock(&call->mu); return encodings_accepted_by_peer; } @@ -1033,16 +1000,13 @@ static batch_control *allocate_batch_control(grpc_call *call, const grpc_op *ops, size_t num_ops) { int slot = batch_slot_for_op(ops[0].op); - for (size_t i = 1; i < num_ops; i++) { - int op_slot = batch_slot_for_op(ops[i].op); - slot = GPR_MIN(slot, op_slot); - } batch_control *bctl = &call->active_batches[slot]; if (bctl->call != NULL) { return NULL; } memset(bctl, 0, sizeof(*bctl)); bctl->call = call; + bctl->op.payload = &call->stream_op_payload; return bctl; } @@ -1055,7 +1019,7 @@ static void finish_batch_completion(grpc_exec_ctx *exec_ctx, void *user_data, } static grpc_error *consolidate_batch_errors(batch_control *bctl) { - size_t n = (size_t)gpr_atm_no_barrier_load(&bctl->num_errors); + size_t n = (size_t)gpr_atm_acq_load(&bctl->num_errors); if (n == 0) { return GRPC_ERROR_NONE; } else if (n == 1) { @@ -1082,8 +1046,6 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, grpc_call *call = bctl->call; grpc_error *error = consolidate_batch_errors(bctl); - gpr_mu_lock(&call->mu); - if (bctl->op.send_initial_metadata) { grpc_metadata_batch_destroy( exec_ctx, @@ -1102,20 +1064,23 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; recv_trailing_filter(exec_ctx, call, md); - call->received_final_op = true; /* propagate cancellation to any interested children */ + gpr_atm_rel_store(&call->received_final_op_atm, 1); + gpr_mu_lock(&call->child_list_mu); child_call = call->first_child; if (child_call != NULL) { do { next_child_call = child_call->sibling_next; if (child_call->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - grpc_call_cancel(child_call, NULL); + cancel_with_error(exec_ctx, child_call, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); } child_call = next_child_call; } while (child_call != call->first_child); } + gpr_mu_unlock(&call->child_list_mu); if (call->is_client) { get_final_status(call, set_status_value_directly, @@ -1129,7 +1094,6 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); error = GRPC_ERROR_NONE; } - gpr_mu_unlock(&call->mu); if (bctl->completion_data.notify_tag.is_closure) { /* unrefs bctl->error */ @@ -1221,7 +1185,6 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; grpc_call *call = bctl->call; - gpr_mu_lock(&bctl->call->mu); if (error != GRPC_ERROR_NONE) { if (call->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); @@ -1233,11 +1196,9 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } if (call->has_initial_md_been_received || error != GRPC_ERROR_NONE || call->receiving_stream == NULL) { - gpr_mu_unlock(&bctl->call->mu); process_data_after_md(exec_ctx, bctlp); } else { call->saved_receiving_stream_ready_bctlp = bctlp; - gpr_mu_unlock(&bctl->call->mu); } } @@ -1296,7 +1257,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error, bool has_cancelled) { if (error == GRPC_ERROR_NONE) return; - int idx = (int)gpr_atm_no_barrier_fetch_add(&bctl->num_errors, 1); + int idx = (int)gpr_atm_full_fetch_add(&bctl->num_errors, 1); if (idx == 0 && !has_cancelled) { cancel_with_error(exec_ctx, bctl->call, STATUS_FROM_CORE, GRPC_ERROR_REF(error)); @@ -1309,8 +1270,6 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, batch_control *bctl = bctlp; grpc_call *call = bctl->call; - gpr_mu_lock(&call->mu); - add_batch_error(exec_ctx, bctl, GRPC_ERROR_REF(error), false); if (error == GRPC_ERROR_NONE) { grpc_metadata_batch *md = @@ -1336,11 +1295,9 @@ static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, receiving_stream_ready, call->saved_receiving_stream_ready_bctlp, grpc_schedule_on_exec_ctx); call->saved_receiving_stream_ready_bctlp = NULL; - grpc_closure_sched(exec_ctx, saved_rsr_closure, GRPC_ERROR_REF(error)); + grpc_closure_run(exec_ctx, saved_rsr_closure, GRPC_ERROR_REF(error)); } - gpr_mu_unlock(&call->mu); - finish_batch_step(exec_ctx, bctl); } @@ -1394,11 +1351,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->completion_data.notify_tag.is_closure = (uint8_t)(is_notify_tag_closure != 0); - gpr_mu_lock(&call->mu); grpc_transport_stream_op *stream_op = &bctl->op; grpc_transport_stream_op_payload *stream_op_payload = &call->stream_op_payload; - memset(stream_op, 0, sizeof(*stream_op)); stream_op->covered_by_poller = true; /* rewrite batch ops into a transport op */ @@ -1513,7 +1468,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } stream_op->send_trailing_metadata = true; call->sent_final_op = 1; - stream_op->send_trailing_metadata = + stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; case GRPC_OP_SEND_STATUS_FROM_SERVER: @@ -1686,8 +1641,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_closure_init(&bctl->finish_batch, finish_batch, bctl, grpc_schedule_on_exec_ctx); stream_op->on_complete = &bctl->finish_batch; - call->sent_any_op = true; - gpr_mu_unlock(&call->mu); + gpr_atm_rel_store(&call->any_ops_sent_atm, 1); execute_op(exec_ctx, call, stream_op); @@ -1718,7 +1672,6 @@ done_with_error: if (stream_op->recv_trailing_metadata) { call->requested_final_op = 0; } - gpr_mu_unlock(&call->mu); goto done; } @@ -1767,10 +1720,8 @@ uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; } grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level) { - gpr_mu_lock(&call->mu); grpc_compression_algorithm algo = compression_algorithm_for_level_locked(call, level); - gpr_mu_unlock(&call->mu); return algo; } diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index 3b0d18c789..ce06b4d004 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -264,8 +264,9 @@ typedef struct { static void destroy_made_transport_stream_op(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { made_transport_stream_op *op = arg; - grpc_closure_sched(exec_ctx, op->inner_on_complete, GRPC_ERROR_REF(error)); + grpc_closure *c = op->inner_on_complete; gpr_free(op); + grpc_closure_run(exec_ctx, c, GRPC_ERROR_REF(error)); } grpc_transport_stream_op *grpc_make_transport_stream_op( diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 79c4bab985..cf913fefe4 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -149,10 +149,22 @@ class TransportStreamOp { /// long as the TransportStreamOp object does. explicit TransportStreamOp(grpc_transport_stream_op *op) : op_(op), - send_initial_metadata_(op->send_initial_metadata), - send_trailing_metadata_(op->send_trailing_metadata), - recv_initial_metadata_(op->recv_initial_metadata), - recv_trailing_metadata_(op->recv_trailing_metadata) {} + send_initial_metadata_( + op->send_initial_metadata + ? op->payload->send_initial_metadata.send_initial_metadata + : nullptr), + send_trailing_metadata_( + op->send_trailing_metadata + ? op->payload->send_trailing_metadata.send_trailing_metadata + : nullptr), + recv_initial_metadata_( + op->recv_initial_metadata + ? op->payload->recv_initial_metadata.recv_initial_metadata + : nullptr), + recv_trailing_metadata_( + op->recv_trailing_metadata + ? op->payload->recv_trailing_metadata.recv_trailing_metadata + : nullptr) {} grpc_transport_stream_op *op() const { return op_; } @@ -160,50 +172,57 @@ class TransportStreamOp { void set_on_complete(grpc_closure *closure) { op_->on_complete = closure; } MetadataBatch *send_initial_metadata() { - return op_->send_initial_metadata == nullptr ? nullptr - : &send_initial_metadata_; + return op_->send_initial_metadata ? &send_initial_metadata_ : nullptr; } MetadataBatch *send_trailing_metadata() { - return op_->send_trailing_metadata == nullptr ? nullptr - : &send_trailing_metadata_; + return op_->send_trailing_metadata ? &send_trailing_metadata_ : nullptr; } MetadataBatch *recv_initial_metadata() { - return op_->recv_initial_metadata == nullptr ? nullptr - : &recv_initial_metadata_; + return op_->recv_initial_metadata ? &recv_initial_metadata_ : nullptr; } MetadataBatch *recv_trailing_metadata() { - return op_->recv_trailing_metadata == nullptr ? nullptr - : &recv_trailing_metadata_; + return op_->recv_trailing_metadata ? &recv_trailing_metadata_ : nullptr; } uint32_t *send_initial_metadata_flags() const { - return &op_->send_initial_metadata_flags; + return op_->send_initial_metadata + ? &op_->payload->send_initial_metadata + .send_initial_metadata_flags + : nullptr; } grpc_closure *recv_initial_metadata_ready() const { - return op_->recv_initial_metadata_ready; + return op_->recv_initial_metadata + ? op_->payload->recv_initial_metadata.recv_initial_metadata_ready + : nullptr; } void set_recv_initial_metadata_ready(grpc_closure *closure) { - op_->recv_initial_metadata_ready = closure; + op_->payload->recv_initial_metadata.recv_initial_metadata_ready = closure; } - grpc_byte_stream *send_message() const { return op_->send_message; } + grpc_byte_stream *send_message() const { + return op_->send_message ? op_->payload->send_message.send_message + : nullptr; + } void set_send_message(grpc_byte_stream *send_message) { - op_->send_message = send_message; + op_->send_message = true; + op_->payload->send_message.send_message = send_message; } /// To be called only on clients and servers, respectively. grpc_client_security_context *client_security_context() const { - return (grpc_client_security_context *)op_->context[GRPC_CONTEXT_SECURITY] + return (grpc_client_security_context *)op_->payload + ->context[GRPC_CONTEXT_SECURITY] .value; } grpc_server_security_context *server_security_context() const { - return (grpc_server_security_context *)op_->context[GRPC_CONTEXT_SECURITY] + return (grpc_server_security_context *)op_->payload + ->context[GRPC_CONTEXT_SECURITY] .value; } census_context *get_census_context() const { - return (census_context *)op_->context[GRPC_CONTEXT_TRACING].value; + return (census_context *)op_->payload->context[GRPC_CONTEXT_TRACING].value; } private: diff --git a/src/node/src/server.js b/src/node/src/server.js index 8a7eff507d..a5a0ea2642 100644 --- a/src/node/src/server.js +++ b/src/node/src/server.js @@ -728,7 +728,7 @@ var defaultHandler = { * method implementation for the provided service. */ Server.prototype.addService = function(service, implementation) { - if (!_.isObjectLike(service) || !_.isObjectLike(implementation)) { + if (!_.isObject(service) || !_.isObject(implementation)) { throw new Error('addService requires two objects as arguments'); } if (_.keys(service).length === 0) { diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 5584246ad9..d964f53e8e 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -90,6 +90,9 @@ return nil; } +// This number indicates how many bytes of overhead does Protocol Buffers encoding add onto the +// message. The number varies as different message.proto is used on different servers. The actual +// number for each interop server is overridden in corresponding derived test classes. - (int32_t)encodingOverhead { return 0; } @@ -169,8 +172,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#ifndef GRPC_COMPILE_WITH_CRONET -// TODO (mxyan): Fix this test - (void)testResponsesOverMaxSizeFailWithActionableMessage { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"ResponseOverMaxSize"]; @@ -191,7 +192,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#endif - (void)testResponsesOver4MBAreAcceptedIfOptedIn { XCTAssertNotNil(self.class.host); @@ -327,8 +327,6 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#ifndef GRPC_COMPILE_WITH_CRONET -// TODO(makdharma@): Fix this test - (void)testEmptyStreamRPC { XCTAssertNotNil(self.class.host); __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"]; @@ -342,7 +340,6 @@ }]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -#endif - (void)testCancelAfterBeginRPC { XCTAssertNotNil(self.class.host); diff --git a/src/objective-c/tests/InteropTestsLocalCleartext.m b/src/objective-c/tests/InteropTestsLocalCleartext.m index b41210f50f..4987660808 100644 --- a/src/objective-c/tests/InteropTestsLocalCleartext.m +++ b/src/objective-c/tests/InteropTestsLocalCleartext.m @@ -37,6 +37,10 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; +// The Protocol Buffers encoding overhead of local interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kLocalInteropServerOverhead = 10; + /** Tests in InteropTests.m, sending the RPCs to a local cleartext server. */ @interface InteropTestsLocalCleartext : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kLocalCleartextHost = @"localhost:5050"; } - (int32_t)encodingOverhead { - return 10; // bytes + return kLocalInteropServerOverhead; // bytes } - (void)setUp { diff --git a/src/objective-c/tests/InteropTestsLocalSSL.m b/src/objective-c/tests/InteropTestsLocalSSL.m index 1479c5896c..934d500abc 100644 --- a/src/objective-c/tests/InteropTestsLocalSSL.m +++ b/src/objective-c/tests/InteropTestsLocalSSL.m @@ -37,6 +37,10 @@ static NSString * const kLocalSSLHost = @"localhost:5051"; +// The Protocol Buffers encoding overhead of local interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kLocalInteropServerOverhead = 10; + /** Tests in InteropTests.m, sending the RPCs to a local SSL server. */ @interface InteropTestsLocalSSL : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kLocalSSLHost = @"localhost:5051"; } - (int32_t)encodingOverhead { - return 10; // bytes + return kLocalInteropServerOverhead; // bytes } - (void)setUp { diff --git a/src/objective-c/tests/InteropTestsRemote.m b/src/objective-c/tests/InteropTestsRemote.m index 70f84753bb..9fb30aa43d 100644 --- a/src/objective-c/tests/InteropTestsRemote.m +++ b/src/objective-c/tests/InteropTestsRemote.m @@ -37,6 +37,10 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The Protocol Buffers encoding overhead of remote interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kRemoteInteropServerOverhead = 12; + /** Tests in InteropTests.m, sending the RPCs to a remote SSL server. */ @interface InteropTestsRemote : InteropTests @end @@ -48,7 +52,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; } - (int32_t)encodingOverhead { - return 12; // bytes + return kRemoteInteropServerOverhead; // bytes } @end diff --git a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m index fab8ad8d25..005bac0a0d 100644 --- a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m +++ b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m @@ -37,6 +37,10 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The Protocol Buffers encoding overhead of remote interop server. Acquired +// by experiment. Adjust this when server's proto file changes. +static int32_t kRemoteInteropServerOverhead = 12; + /** Tests in InteropTests.m, sending the RPCs to a remote SSL server. */ @interface InteropTestsRemoteWithCronet : InteropTests @end @@ -47,4 +51,8 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; return kRemoteSSLHost; } +- (int32_t)encodingOverhead { + return kRemoteInteropServerOverhead; // bytes +} + @end diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index a9f20e6d2a..52d3ffd2c4 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -33,6 +33,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/profiling/basic_timers.c', 'src/core/lib/profiling/stap_timers.c', 'src/core/lib/support/alloc.c', + 'src/core/lib/support/arena.c', 'src/core/lib/support/avl.c', 'src/core/lib/support/backoff.c', 'src/core/lib/support/cmdline.c', diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index f0f11cf84b..e92c2659b7 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -33,9 +33,10 @@ import threading import grpc from grpc_health.v1 import health_pb2 +from grpc_health.v1 import health_pb2_grpc -class HealthServicer(health_pb2.HealthServicer): +class HealthServicer(health_pb2_grpc.HealthServicer): """Servicer handling RPCs for service statuses.""" def __init__(self): diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 363b4c5f99..1bc8669dad 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -34,6 +34,7 @@ import grpc from grpc.framework.foundation import logging_pool from grpc_health.v1 import health from grpc_health.v1 import health_pb2 +from grpc_health.v1 import health_pb2_grpc from tests.unit.framework.common import test_constants @@ -52,11 +53,11 @@ class HealthServicerTest(unittest.TestCase): server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) self._server = grpc.server(server_pool) port = self._server.add_insecure_port('[::]:0') - health_pb2.add_HealthServicer_to_server(servicer, self._server) + health_pb2_grpc.add_HealthServicer_to_server(servicer, self._server) self._server.start() channel = grpc.insecure_channel('localhost:%d' % port) - self._stub = health_pb2.HealthStub(channel) + self._stub = health_pb2_grpc.HealthStub(channel) def test_empty_service(self): request = health_pb2.HealthCheckRequest() diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index f8938d0abb..064119f11b 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -58,7 +58,7 @@ static int g_connections_complete = 0; static grpc_endpoint *g_connecting = NULL; static gpr_timespec test_deadline(void) { - return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); + return grpc_timeout_seconds_to_deadline(10); } static void finish_connection() { @@ -73,7 +73,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { GPR_ASSERT(g_connecting != NULL); GPR_ASSERT(error == GRPC_ERROR_NONE); - grpc_endpoint_shutdown(exec_ctx, g_connecting); + grpc_endpoint_shutdown(exec_ctx, g_connecting, + GRPC_ERROR_CREATE("must_succeed called")); grpc_endpoint_destroy(exec_ctx, g_connecting); g_connecting = NULL; finish_connection(); @@ -133,7 +134,7 @@ void test_succeeds(void) { "pollset_work", grpc_pollset_work(&exec_ctx, g_pollset, &worker, gpr_now(GPR_CLOCK_MONOTONIC), - GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5)))); + grpc_timeout_seconds_to_deadline(5)))); gpr_mu_unlock(g_mu); grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(g_mu); diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c index 7b458c90f3..0fc74599ea 100644 --- a/test/core/iomgr/tcp_server_uv_test.c +++ b/test/core/iomgr/tcp_server_uv_test.c @@ -115,7 +115,7 @@ static void server_weak_ref_set(server_weak_ref *weak_ref, static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *pollset, grpc_tcp_server_acceptor *acceptor) { - grpc_endpoint_shutdown(exec_ctx, tcp); + grpc_endpoint_shutdown(exec_ctx, tcp, GRPC_ERROR_CREATE("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); on_connect_result temp_result; @@ -203,7 +203,7 @@ static void close_cb(uv_handle_t *handle) { gpr_free(handle); } static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote, socklen_t remote_len, on_connect_result *result) { - gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10); uv_tcp_t *client_handle = gpr_malloc(sizeof(uv_tcp_t)); uv_connect_t *req = gpr_malloc(sizeof(uv_connect_t)); int nconnects_before; diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index d57a37b2a9..396ec959cd 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -62,8 +62,7 @@ static int g_number_of_writes = 0; static int g_number_of_bytes_read = 0; static int g_number_of_orphan_calls = 0; -static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - grpc_server *server) { +static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { char read_buffer[512]; ssize_t byte_count; @@ -79,7 +78,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, gpr_mu_unlock(g_mu); } -static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { +static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; @@ -88,7 +87,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { gpr_mu_unlock(g_mu); } -static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) { +static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, + void *user_data) { gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d", grpc_fd_wrapped_fd(emfd)); g_number_of_orphan_calls++; diff --git a/test/core/security/BUILD b/test/core/security/BUILD index e750c39b7c..1cb03c5cfe 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -34,7 +34,7 @@ load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer") grpc_fuzzer( name = "ssl_server_fuzzer", srcs = ["ssl_server_fuzzer.c"], - deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util"], + deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util", "//test/core/end2end:ssl_test_data"], corpus = "corpus", copts = ["-std=c99"], ) diff --git a/test/core/security/ssl_server_fuzzer.c b/test/core/security/ssl_server_fuzzer.c index f789278add..7a3612c419 100644 --- a/test/core/security/ssl_server_fuzzer.c +++ b/test/core/security/ssl_server_fuzzer.c @@ -38,6 +38,7 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/transport/security_connector.h" +#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/memory_counters.h" #include "test/core/util/mock_endpoint.h" @@ -46,10 +47,6 @@ bool squelch = true; // Turning this on will fail the leak check. bool leak_check = false; -#define SSL_CERT_PATH "src/core/lib/tsi/test_creds/server1.pem" -#define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key" -#define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem" - static void discard_write(grpc_slice slice) {} static void dont_log(gpr_log_func_args *args) {} @@ -88,12 +85,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // Load key pair and establish server SSL credentials. grpc_ssl_pem_key_cert_pair pem_key_cert_pair; grpc_slice ca_slice, cert_slice, key_slice; - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_CA_PATH, 1, &ca_slice))); - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_CERT_PATH, 1, &cert_slice))); - GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(SSL_KEY_PATH, 1, &key_slice))); + ca_slice = grpc_slice_from_static_string(test_root_cert); + cert_slice = grpc_slice_from_static_string(test_server1_cert); + key_slice = grpc_slice_from_static_string(test_server1_key); const char *ca_cert = (const char *)GRPC_SLICE_START_PTR(ca_slice); pem_key_cert_pair.private_key = (const char *)GRPC_SLICE_START_PTR(key_slice); pem_key_cert_pair.cert_chain = (const char *)GRPC_SLICE_START_PTR(cert_slice); diff --git a/test/core/support/arena_test.c b/test/core/support/arena_test.c new file mode 100644 index 0000000000..35b2bbd1b1 --- /dev/null +++ b/test/core/support/arena_test.c @@ -0,0 +1,139 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/support/arena.h" + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> +#include <grpc/support/sync.h> +#include <grpc/support/thd.h> +#include <grpc/support/useful.h> +#include <inttypes.h> +#include <string.h> + +#include "src/core/lib/support/string.h" +#include "test/core/util/test_config.h" + +static void test_noop(void) { gpr_arena_destroy(gpr_arena_create(1)); } + +static void test(const char *name, size_t init_size, const size_t *allocs, + size_t nallocs) { + gpr_strvec v; + char *s; + gpr_strvec_init(&v); + gpr_asprintf(&s, "test '%s': %" PRIdPTR " <- {", name, init_size); + gpr_strvec_add(&v, s); + for (size_t i = 0; i < nallocs; i++) { + gpr_asprintf(&s, "%" PRIdPTR ",", allocs[i]); + gpr_strvec_add(&v, s); + } + gpr_strvec_add(&v, gpr_strdup("}")); + s = gpr_strvec_flatten(&v, NULL); + gpr_strvec_destroy(&v); + gpr_log(GPR_INFO, "%s", s); + gpr_free(s); + + gpr_arena *a = gpr_arena_create(init_size); + void **ps = gpr_zalloc(sizeof(*ps) * nallocs); + for (size_t i = 0; i < nallocs; i++) { + ps[i] = gpr_arena_alloc(a, allocs[i]); + // ensure no duplicate results + for (size_t j = 0; j < i; j++) { + GPR_ASSERT(ps[i] != ps[j]); + } + // ensure writable + memset(ps[i], 1, allocs[i]); + } + gpr_arena_destroy(a); + gpr_free(ps); +} + +#define TEST(name, init_size, ...) \ + static const size_t allocs_##name[] = {__VA_ARGS__}; \ + test(#name, init_size, allocs_##name, GPR_ARRAY_SIZE(allocs_##name)) + +#define CONCURRENT_TEST_ITERATIONS 100000 +#define CONCURRENT_TEST_THREADS 100 + +typedef struct { + gpr_event ev_start; + gpr_arena *arena; +} concurrent_test_args; + +static void concurrent_test_body(void *arg) { + concurrent_test_args *a = arg; + gpr_event_wait(&a->ev_start, gpr_inf_future(GPR_CLOCK_REALTIME)); + for (size_t i = 0; i < CONCURRENT_TEST_ITERATIONS; i++) { + *(char *)gpr_arena_alloc(a->arena, 1) = (char)i; + } +} + +static void concurrent_test(void) { + gpr_log(GPR_DEBUG, "concurrent_test"); + + concurrent_test_args args; + gpr_event_init(&args.ev_start); + args.arena = gpr_arena_create(1024); + + gpr_thd_id thds[CONCURRENT_TEST_THREADS]; + + for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&opt); + gpr_thd_new(&thds[i], concurrent_test_body, &args, &opt); + } + + gpr_event_set(&args.ev_start, (void *)1); + + for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) { + gpr_thd_join(thds[i]); + } + + gpr_arena_destroy(args.arena); +} + +int main(int argc, char *argv[]) { + grpc_test_init(argc, argv); + + test_noop(); + TEST(0_1, 0, 1); + TEST(1_1, 1, 1); + TEST(1_2, 1, 2); + TEST(1_3, 1, 3); + TEST(1_inc, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + TEST(6_123, 6, 1, 2, 3); + concurrent_test(); + + return 0; +} diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 7ab0488a66..0848147158 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -31,6 +31,8 @@ * */ +#include "src/core/lib/iomgr/sockaddr.h" + #include "test/core/util/passthru_endpoint.h" #include <inttypes.h> @@ -40,9 +42,6 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> - -#include "src/core/lib/iomgr/sockaddr.h" - #include "src/core/lib/slice/slice_internal.h" typedef struct { diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index 01c07823cf..38a437f39f 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -41,7 +41,7 @@ #include <grpc/support/useful.h> #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/http2_client.h" diff --git a/test/cpp/interop/http2_client.h b/test/cpp/interop/http2_client.h index 12df5d26bc..e57d695205 100644 --- a/test/cpp/interop/http2_client.h +++ b/test/cpp/interop/http2_client.h @@ -38,7 +38,7 @@ #include <grpc++/channel.h> #include <grpc/grpc.h> -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" namespace grpc { diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc index b7f2723c39..55ba324cc7 100644 --- a/test/cpp/interop/interop_client.cc +++ b/test/cpp/interop/interop_client.cc @@ -46,8 +46,8 @@ #include <grpc/support/useful.h> #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/client_helper.h" #include "test/cpp/interop/interop_client.h" diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h index 74f4db6b78..efcb7d2860 100644 --- a/test/cpp/interop/interop_client.h +++ b/test/cpp/interop/interop_client.h @@ -38,7 +38,7 @@ #include <grpc++/channel.h> #include <grpc/grpc.h> -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" namespace grpc { diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc index 5a810b45ef..1cbca17928 100644 --- a/test/cpp/interop/interop_server.cc +++ b/test/cpp/interop/interop_server.cc @@ -48,8 +48,8 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/byte_stream.h" -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/interop/server_helper.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc index 1c2f606637..01d985068d 100644 --- a/test/cpp/interop/reconnect_interop_client.cc +++ b/test/cpp/interop/reconnect_interop_client.cc @@ -40,8 +40,8 @@ #include <grpc++/support/channel_arguments.h> #include <grpc/grpc.h> #include <grpc/support/log.h> -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/cpp/util/create_test_channel.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc index 634d0a90fc..8d1b884af9 100644 --- a/test/cpp/interop/reconnect_interop_server.cc +++ b/test/cpp/interop/reconnect_interop_server.cc @@ -47,8 +47,8 @@ #include <grpc/grpc.h> #include <grpc/support/log.h> -#include "src/proto/grpc/testing/empty.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "src/proto/grpc/testing/test.grpc.pb.h" #include "test/core/util/reconnect_server.h" #include "test/cpp/util/test_config.h" diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc new file mode 100644 index 0000000000..770c0b6d47 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_arena.cc @@ -0,0 +1,76 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Benchmark arenas */ + +extern "C" { +#include "src/core/lib/support/arena.h" +} +#include "test/cpp/microbenchmarks/helpers.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +static void BM_Arena_NoOp(benchmark::State& state) { + while (state.KeepRunning()) { + gpr_arena_destroy(gpr_arena_create(state.range(0))); + } +} +BENCHMARK(BM_Arena_NoOp)->Range(1, 1024 * 1024); + +static void BM_Arena_ManyAlloc(benchmark::State& state) { + gpr_arena* a = gpr_arena_create(state.range(0)); + const size_t realloc_after = + 1024 * 1024 * 1024 / ((state.range(1) + 15) & 0xffffff0u); + while (state.KeepRunning()) { + gpr_arena_alloc(a, state.range(1)); + // periodically recreate arena to avoid OOM + if (state.iterations() % realloc_after == 0) { + gpr_arena_destroy(a); + a = gpr_arena_create(state.range(0)); + } + } + gpr_arena_destroy(a); +} +BENCHMARK(BM_Arena_ManyAlloc)->Ranges({{1, 1024 * 1024}, {1, 32 * 1024}}); + +static void BM_Arena_Batch(benchmark::State& state) { + while (state.KeepRunning()) { + gpr_arena* a = gpr_arena_create(state.range(0)); + for (int i = 0; i < state.range(1); i++) { + gpr_arena_alloc(a, state.range(2)); + } + gpr_arena_destroy(a); + } +} +BENCHMARK(BM_Arena_Batch)->Ranges({{1, 64 * 1024}, {1, 64}, {1, 1024}}); + +BENCHMARK_MAIN(); diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 014e2b96b5..90cd529cf9 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -339,13 +339,15 @@ class SendEmptyMetadata { memset(&op_, 0, sizeof(op_)); op_.on_complete = grpc_closure_init(&closure_, DoNothing, nullptr, grpc_schedule_on_exec_ctx); + op_.send_initial_metadata = true; + op_.payload = &op_payload_; } class Op { public: Op(grpc_exec_ctx *exec_ctx, SendEmptyMetadata *p, grpc_call_stack *s) { grpc_metadata_batch_init(&batch_); - p->op_.send_initial_metadata = &batch_; + p->op_payload_.send_initial_metadata.send_initial_metadata = &batch_; } void Finish(grpc_exec_ctx *exec_ctx) { grpc_metadata_batch_destroy(exec_ctx, &batch_); @@ -360,6 +362,7 @@ class SendEmptyMetadata { const gpr_timespec start_time_ = gpr_now(GPR_CLOCK_MONOTONIC); const grpc_slice method_ = grpc_slice_from_static_string("/foo/bar"); grpc_transport_stream_op op_; + grpc_transport_stream_op_payload op_payload_; grpc_closure closure_; }; diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc index ee3dec2bce..34874b57f5 100644 --- a/test/cpp/microbenchmarks/bm_metadata.cc +++ b/test/cpp/microbenchmarks/bm_metadata.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc new file mode 100644 index 0000000000..0f3d3cef66 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -0,0 +1,254 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Test out pollset latencies */ + +#include <grpc/grpc.h> +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/useful.h> + +extern "C" { +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/pollset.h" +#include "src/core/lib/iomgr/port.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +} + +#include "test/cpp/microbenchmarks/helpers.h" +#include "third_party/benchmark/include/benchmark/benchmark.h" + +#include <string.h> + +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +#include <sys/epoll.h> +#include <sys/eventfd.h> +#include <unistd.h> +#endif + +auto& force_library_initialization = Library::get(); + +static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { + grpc_pollset_destroy(static_cast<grpc_pollset*>(ps)); +} + +static void BM_CreateDestroyPollset(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_malloc(ps_sz)); + gpr_mu* mu; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + while (state.KeepRunning()) { + memset(ps, 0, ps_sz); + grpc_pollset_init(ps, &mu); + gpr_mu_lock(mu); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(ps); + track_counters.Finish(state); +} +BENCHMARK(BM_CreateDestroyPollset); + +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) { + // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage + // what the speed of light would be if we abstracted perfectly + TrackCounters track_counters; + int epfd = epoll_create1(0); + GPR_ASSERT(epfd != -1); + size_t nev = state.range(0); + size_t nfd = state.range(1); + epoll_event* ev = new epoll_event[nev]; + std::vector<int> fds; + for (size_t i = 0; i < nfd; i++) { + fds.push_back(eventfd(0, 0)); + epoll_event ev; + ev.events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev); + } + while (state.KeepRunning()) { + epoll_wait(epfd, ev, nev, 0); + } + for (auto fd : fds) { + close(fd); + } + close(epfd); + delete[] ev; + track_counters.Finish(state); +} +BENCHMARK(BM_PollEmptyPollset_SpeedOfLight) + ->Args({1, 0}) + ->Args({1, 1}) + ->Args({1, 10}) + ->Args({1, 100}) + ->Args({1, 1000}) + ->Args({1, 10000}) + ->Args({1, 100000}) + ->Args({10, 1}) + ->Args({100, 1}) + ->Args({1000, 1}); +#endif + +static void BM_PollEmptyPollset(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz)); + gpr_mu* mu; + grpc_pollset_init(ps, &mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); + gpr_mu_lock(mu); + while (state.KeepRunning()) { + grpc_pollset_worker* worker; + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + } + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(ps); + track_counters.Finish(state); +} +BENCHMARK(BM_PollEmptyPollset); + +class Closure : public grpc_closure { + public: + virtual ~Closure() {} +}; + +template <class F> +Closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) { + struct C : public Closure { + C(F f, grpc_closure_scheduler* scheduler) : f_(f) { + grpc_closure_init(this, C::cbfn, this, scheduler); + } + static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { + C* p = static_cast<C*>(arg); + p->f_(); + } + F f_; + }; + return new C(f, scheduler); +} + +#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL +static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) { + // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage + // what the speed of light would be if we abstracted perfectly + TrackCounters track_counters; + int epfd = epoll_create1(0); + GPR_ASSERT(epfd != -1); + epoll_event ev[100]; + int fd = eventfd(0, EFD_NONBLOCK); + ev[0].events = EPOLLIN; + epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]); + while (state.KeepRunning()) { + int err; + do { + err = eventfd_write(fd, 1); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 0); + do { + err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 1); + eventfd_t value; + do { + err = eventfd_read(fd, &value); + } while (err < 0 && errno == EINTR); + GPR_ASSERT(err == 0); + } + close(fd); + close(epfd); + track_counters.Finish(state); +} +BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight); +#endif + +static void BM_SingleThreadPollOneFd(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz)); + gpr_mu* mu; + grpc_pollset_init(ps, &mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + grpc_wakeup_fd wakeup_fd; + GRPC_ERROR_UNREF(grpc_wakeup_fd_init(&wakeup_fd)); + grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read"); + grpc_pollset_add_fd(&exec_ctx, ps, wakeup); + bool done = false; + Closure* continue_closure = MakeClosure( + [&]() { + GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd)); + if (!state.KeepRunning()) { + done = true; + return; + } + GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd)); + grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); + }, + grpc_schedule_on_exec_ctx); + GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd)); + grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); + gpr_mu_lock(mu); + while (!done) { + grpc_pollset_worker* worker; + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + } + grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done"); + wakeup_fd.read_fd = 0; + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_finish(&exec_ctx); + grpc_wakeup_fd_destroy(&wakeup_fd); + gpr_free(ps); + track_counters.Finish(state); + delete continue_closure; +} +BENCHMARK(BM_SingleThreadPollOneFd); + +BENCHMARK_MAIN(); diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index baa9304cc2..25a19a5a74 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -46,7 +46,7 @@ #include <grpc/support/log.h> #include <grpc/support/time.h> -#include "src/proto/grpc/testing/payloads.grpc.pb.h" +#include "src/proto/grpc/testing/payloads.pb.h" #include "src/proto/grpc/testing/services.grpc.pb.h" #include "test/cpp/qps/histogram.h" diff --git a/test/cpp/qps/driver.h b/test/cpp/qps/driver.h index e72d30a4ef..dd32a16c87 100644 --- a/test/cpp/qps/driver.h +++ b/test/cpp/qps/driver.h @@ -36,7 +36,7 @@ #include <memory> -#include "src/proto/grpc/testing/control.grpc.pb.h" +#include "src/proto/grpc/testing/control.pb.h" #include "test/cpp/qps/histogram.h" namespace grpc { diff --git a/test/cpp/qps/histogram.h b/test/cpp/qps/histogram.h index acb415f0a1..470a394301 100644 --- a/test/cpp/qps/histogram.h +++ b/test/cpp/qps/histogram.h @@ -35,7 +35,7 @@ #define TEST_QPS_HISTOGRAM_H #include <grpc/support/histogram.h> -#include "src/proto/grpc/testing/stats.grpc.pb.h" +#include "src/proto/grpc/testing/stats.pb.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc index ddaaa7ca75..bd2c1f0ec6 100644 --- a/test/cpp/qps/qps_json_driver.cc +++ b/test/cpp/qps/qps_json_driver.cc @@ -31,6 +31,7 @@ * */ +#include <iostream> #include <memory> #include <set> diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h index 821d5935be..8fbf37a095 100644 --- a/test/cpp/qps/server.h +++ b/test/cpp/qps/server.h @@ -38,8 +38,8 @@ #include <grpc/support/cpu.h> #include <vector> -#include "src/proto/grpc/testing/control.grpc.pb.h" -#include "src/proto/grpc/testing/messages.grpc.pb.h" +#include "src/proto/grpc/testing/control.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" #include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/cpp/qps/usage_timer.h" diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index b3a06aeaf5..b58d91eea6 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -103,24 +103,25 @@ class AsyncQpsServerTest final : public grpc::testing::Server { server_ = builder.BuildAndStart(); - using namespace std::placeholders; - auto process_rpc_bound = - std::bind(process_rpc, config.payload_config(), _1, _2); + std::bind(process_rpc, config.payload_config(), std::placeholders::_1, + std::placeholders::_2); for (int i = 0; i < 15000; i++) { for (int j = 0; j < num_threads; j++) { if (request_unary_function) { - auto request_unary = - std::bind(request_unary_function, &async_service_, _1, _2, _3, - srv_cqs_[j].get(), srv_cqs_[j].get(), _4); + auto request_unary = std::bind( + request_unary_function, &async_service_, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, srv_cqs_[j].get(), + srv_cqs_[j].get(), std::placeholders::_4); contexts_.emplace_back( new ServerRpcContextUnaryImpl(request_unary, process_rpc_bound)); } if (request_streaming_function) { - auto request_streaming = - std::bind(request_streaming_function, &async_service_, _1, _2, - srv_cqs_[j].get(), srv_cqs_[j].get(), _3); + auto request_streaming = std::bind( + request_streaming_function, &async_service_, + std::placeholders::_1, std::placeholders::_2, srv_cqs_[j].get(), + srv_cqs_[j].get(), std::placeholders::_3); contexts_.emplace_back(new ServerRpcContextStreamingImpl( request_streaming, process_rpc_bound)); } diff --git a/third_party/gflags b/third_party/gflags -Subproject 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26 +Subproject f8a0efe03aa69b3336d8e228b37d4ccb17324b8 diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 131a013451..e4f56f2374 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1228,6 +1228,8 @@ src/core/lib/slice/slice_internal.h \ src/core/lib/slice/slice_string_helpers.c \ src/core/lib/slice/slice_string_helpers.h \ src/core/lib/support/alloc.c \ +src/core/lib/support/arena.c \ +src/core/lib/support/arena.h \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/backoff.h \ diff --git a/tools/fuzzer/options/api_fuzzer.options b/tools/fuzzer/options/api_fuzzer.options new file mode 100644 index 0000000000..8871ae21b6 --- /dev/null +++ b/tools/fuzzer/options/api_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = api_fuzzer.dictionary diff --git a/tools/fuzzer/options/client_fuzzer.options b/tools/fuzzer/options/client_fuzzer.options new file mode 100644 index 0000000000..fd2eebf7d2 --- /dev/null +++ b/tools/fuzzer/options/client_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/fuzzer.options b/tools/fuzzer/options/fuzzer.options new file mode 100644 index 0000000000..5d468bc6e4 --- /dev/null +++ b/tools/fuzzer/options/fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 512 diff --git a/tools/fuzzer/options/fuzzer_response.options b/tools/fuzzer/options/fuzzer_response.options new file mode 100644 index 0000000000..5dcdfac7a6 --- /dev/null +++ b/tools/fuzzer/options/fuzzer_response.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 diff --git a/tools/fuzzer/options/fuzzer_serverlist.options b/tools/fuzzer/options/fuzzer_serverlist.options new file mode 100644 index 0000000000..5dcdfac7a6 --- /dev/null +++ b/tools/fuzzer/options/fuzzer_serverlist.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 diff --git a/tools/fuzzer/options/hpack_parser_fuzzer_test.options b/tools/fuzzer/options/hpack_parser_fuzzer_test.options new file mode 100644 index 0000000000..584487fafc --- /dev/null +++ b/tools/fuzzer/options/hpack_parser_fuzzer_test.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 512 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/percent_decode_fuzzer.options b/tools/fuzzer/options/percent_decode_fuzzer.options new file mode 100644 index 0000000000..ea2785e110 --- /dev/null +++ b/tools/fuzzer/options/percent_decode_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 32 diff --git a/tools/fuzzer/options/percent_encode_fuzzer.options b/tools/fuzzer/options/percent_encode_fuzzer.options new file mode 100644 index 0000000000..ea2785e110 --- /dev/null +++ b/tools/fuzzer/options/percent_encode_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 32 diff --git a/tools/fuzzer/options/request_fuzzer.options b/tools/fuzzer/options/request_fuzzer.options new file mode 100644 index 0000000000..fd32ac16e1 --- /dev/null +++ b/tools/fuzzer/options/request_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 + diff --git a/tools/fuzzer/options/response_fuzzer.options b/tools/fuzzer/options/response_fuzzer.options new file mode 100644 index 0000000000..fd32ac16e1 --- /dev/null +++ b/tools/fuzzer/options/response_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 + diff --git a/tools/fuzzer/options/server_fuzzer.options b/tools/fuzzer/options/server_fuzzer.options new file mode 100644 index 0000000000..fd2eebf7d2 --- /dev/null +++ b/tools/fuzzer/options/server_fuzzer.options @@ -0,0 +1,3 @@ +[libfuzzer] +max_len = 2048 +dict = hpack.dictionary diff --git a/tools/fuzzer/options/ssl_server_fuzzer.options b/tools/fuzzer/options/ssl_server_fuzzer.options new file mode 100644 index 0000000000..60bd9b0b2f --- /dev/null +++ b/tools/fuzzer/options/ssl_server_fuzzer.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 2048 diff --git a/tools/fuzzer/options/uri_fuzzer_test.options b/tools/fuzzer/options/uri_fuzzer_test.options new file mode 100644 index 0000000000..5dcdfac7a6 --- /dev/null +++ b/tools/fuzzer/options/uri_fuzzer_test.options @@ -0,0 +1,2 @@ +[libfuzzer] +max_len = 128 diff --git a/tools/gcp/utils/gcr_upload.py b/tools/gcp/utils/gcr_upload.py new file mode 100755 index 0000000000..b22f8731f6 --- /dev/null +++ b/tools/gcp/utils/gcr_upload.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python2.7 +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Upload docker images to Google Container Registry.""" + +from __future__ import print_function + +import argparse +import atexit +import os +import shutil +import subprocess +import tempfile + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument('--gcr_path', + default='gcr.io/grpc-testing', + help='Path of docker images in Google Container Registry') + +argp.add_argument('--gcr_tag', + default='latest', + help='the tag string for the images to upload') + +argp.add_argument('--with_files', + default=[], + nargs='+', + help='additional files to include in the docker image') + +argp.add_argument('--with_file_dest', + default='/var/local/image_info', + help='Destination directory for with_files inside docker image') + +argp.add_argument('--images', + default=[], + nargs='+', + help='local docker images in the form of repo:tag ' + + '(i.e. grpc_interop_java:26328ad8) to upload') + +argp.add_argument('--keep', + action='store_true', + help='keep the created local images after uploading to GCR') + + +args = argp.parse_args() + +def upload_to_gcr(image): + """Tags and Pushes a docker image in Google Containger Registry. + + image: docker image name, i.e. grpc_interop_java:26328ad8 + + A docker image image_foo:tag_old will be uploaded as + <gcr_path>/image_foo:<gcr_tag> + after inserting extra with_files under with_file_dest in the image. The + original image name will be stored as label original_name:"image_foo:tag_old". + """ + tag_idx = image.find(':') + if tag_idx == -1: + print('Failed to parse docker image name %s' % image) + return False + new_tag = '%s/%s:%s' % (args.gcr_path, image[:tag_idx], args.gcr_tag) + + lines = ['FROM ' + image] + lines.append('LABEL original_name="%s"' % image) + + temp_dir = tempfile.mkdtemp() + atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir])) + + # Copy with_files inside the tmp directory, which will be the docker build + # context. + for f in args.with_files: + shutil.copy(f, temp_dir) + lines.append('COPY %s %s/' % (os.path.basename(f), args.with_file_dest)) + + # Create a Dockerfile. + with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f: + f.write('\n'.join(lines)) + + build_cmd = ['docker', 'build', '--rm', '--tag', new_tag, temp_dir] + subprocess.check_output(build_cmd) + + if not args.keep: + atexit.register(lambda: subprocess.call(['docker', 'rmi', new_tag])) + + # Upload to GCR. + if args.gcr_path: + subprocess.call(['gcloud', 'docker', '--', 'push', new_tag]) + + return True + + +for image in args.images: + upload_to_gcr(image) diff --git a/tools/internal_ci/linux/grpc_master_sanitizers.sh b/tools/internal_ci/linux/grpc_master_sanitizers.sh index 2cd0efe158..d22387fb20 100755 --- a/tools/internal_ci/linux/grpc_master_sanitizers.sh +++ b/tools/internal_ci/linux/grpc_master_sanitizers.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f sanitizers linux +tools/run_tests/run_tests_matrix.py -f sanitizers linux diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh index c7ac3bbcbe..58d3c58e70 100755 --- a/tools/internal_ci/linux/grpc_portability.sh +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f portability linux +tools/run_tests/run_tests_matrix.py -f portability linux diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index edd6f91687..80b5c4cb96 100755 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -37,4 +37,4 @@ git submodule update --init # download docker images from dockerhub export DOCKERHUB_ORGANIZATION=grpctesting -tools/run_tests/run_tests_matrix.sh -f portability linux --build_only +tools/run_tests/run_tests_matrix.py -f portability linux --build_only diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index c48a63b791..4695f829f4 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -98,6 +98,10 @@ _BM_SPECS = { 'tpl': [], 'dyn': ['request_size'], }, + 'BM_PollEmptyPollset_SpeedOfLight': { + 'tpl': [], + 'dyn': ['request_size', 'request_count'], + } } def numericalize(s): diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index e0658f4678..04702bacca 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index a7535b3852..90bbde83cf 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index d490f571c3..2547f2073c 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 9173bd7c19..69e0f44774 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -58,7 +58,7 @@ { "config": "ubsan", "environ": { - "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1" + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" } }, { diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 767c2b2e36..866d8a55c9 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -87,6 +87,21 @@ { "deps": [ "gpr", + "gpr_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "arena_test", + "src": [ + "test/core/support/arena_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", "gpr_test_util", "grpc", "grpc_test_util", @@ -2443,6 +2458,27 @@ "headers": [], "is_filegroup": false, "language": "c++", + "name": "bm_arena", + "src": [ + "test/cpp/microbenchmarks/bm_arena.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", "name": "bm_call_create", "src": [ "test/cpp/microbenchmarks/bm_call_create.cc" @@ -2641,6 +2677,27 @@ }, { "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_pollset", + "src": [ + "test/cpp/microbenchmarks/bm_pollset.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ "gpr", "grpc", "grpc++" @@ -7243,6 +7300,7 @@ "include/grpc/support/tls_pthread.h", "include/grpc/support/useful.h", "src/core/lib/profiling/timers.h", + "src/core/lib/support/arena.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", @@ -7290,6 +7348,8 @@ "src/core/lib/profiling/stap_timers.c", "src/core/lib/profiling/timers.h", "src/core/lib/support/alloc.c", + "src/core/lib/support/arena.c", + "src/core/lib/support/arena.h", "src/core/lib/support/avl.c", "src/core/lib/support/backoff.c", "src/core/lib/support/backoff.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ec55fb73b9..b81b98bc5a 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -99,6 +99,28 @@ ], "cpu_cost": 1.0, "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "arena_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], "exclude_iomgrs": [ "uv" ], @@ -1812,7 +1834,9 @@ ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [], + "exclude_iomgrs": [ + "uv" + ], "flaky": false, "gtest": false, "language": "c", @@ -2620,6 +2644,28 @@ "flaky": false, "gtest": false, "language": "c++", + "name": "bm_arena", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", "name": "bm_call_create", "platforms": [ "linux", @@ -2846,6 +2892,28 @@ ] }, { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_pollset", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { "args": [], "ci_platforms": [ "linux", diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 89d2a9b320..3bac1199a7 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # @@ -30,6 +30,8 @@ # Uploads performance benchmark result file to bigquery. +from __future__ import print_function + import argparse import calendar import json @@ -70,7 +72,7 @@ def _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, result_file): _create_results_table(bq, dataset_id, table_id) if not _insert_result(bq, dataset_id, table_id, scenario_result, flatten=False): - print 'Error uploading result to bigquery.' + print('Error uploading result to bigquery.') sys.exit(1) @@ -82,7 +84,7 @@ def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file): _create_results_table(bq, dataset_id, table_id) if not _insert_result(bq, dataset_id, table_id, scenario_result): - print 'Error uploading result to bigquery.' + print('Error uploading result to bigquery.') sys.exit(1) @@ -179,4 +181,4 @@ if args.file_format == 'netperf_latency_csv': _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, args.file_to_upload) else: _upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload) -print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload +print('Successfully uploaded %s to BigQuery.\n' % args.file_to_upload) diff --git a/tools/run_tests/python_utils/antagonist.py b/tools/run_tests/python_utils/antagonist.py index 857addfb38..111839ccf9 100755 --- a/tools/run_tests/python_utils/antagonist.py +++ b/tools/run_tests/python_utils/antagonist.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 3734f025d5..e013376295 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # @@ -30,7 +30,10 @@ """Filter out tests based on file differences compared to merge target branch""" +from __future__ import print_function + import re +import six from subprocess import check_output @@ -125,7 +128,7 @@ _WHITELIST_DICT = { } # Add all triggers to their respective test suites -for trigger, test_suites in _WHITELIST_DICT.iteritems(): +for trigger, test_suites in six.iteritems(_WHITELIST_DICT): for test_suite in test_suites: test_suite.add_trigger(trigger) diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index e9b3f7ff79..dbd32efc0e 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 9dad60408f..131772f55f 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -40,6 +40,7 @@ except (ImportError): import os import string import xml.etree.cElementTree as ET +import six def _filter_msg(msg, output_format): @@ -63,7 +64,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package, name=suite_name) - for shortname, results in resultset.iteritems(): + for shortname, results in six.iteritems(resultset): for result in results: xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) if result.elapsed_time: diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py index 654cf95a38..dd11a45e78 100755 --- a/tools/run_tests/run_build_statistics.py +++ b/tools/run_tests/run_build_statistics.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # @@ -30,6 +30,8 @@ """Tool to get build statistics from Jenkins and upload to BigQuery.""" +from __future__ import print_function + import argparse import jenkinsapi from jenkinsapi.custom_exceptions import JenkinsAPIException @@ -243,6 +245,6 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: rows = [big_query_utils.make_row(build_number, build_result)] if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, rows): - print '====> Error uploading result to bigquery.' + print('====> Error uploading result to bigquery.') sys.exit(1) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 161fa81956..29d5a9aaa0 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # @@ -44,6 +44,7 @@ import sys import tempfile import time import uuid +import six import python_utils.dockerjob as dockerjob import python_utils.jobset as jobset @@ -885,7 +886,7 @@ if not args.use_docker and servers: languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( - _LANGUAGES.iterkeys() if x == 'all' else [x] + six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) languages_http2_badserver_interop = set() @@ -925,7 +926,7 @@ if args.use_docker: else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -1053,7 +1054,7 @@ try: if not jobs: print('No jobs to run.') - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -1093,9 +1094,9 @@ finally: if not job.is_running(): print('Server "%s" has exited prematurely.' % server) - dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) + dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): if not args.manual_run: print('Removing docker image %s' % image) dockerjob.remove_image(image) diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 9471110968..57b2636e56 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2017, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index ee4102c591..35d20be5b7 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # @@ -46,6 +46,7 @@ import tempfile import time import traceback import uuid +import six import performance.scenario_config as scenario_config import python_utils.jobset as jobset @@ -502,8 +503,8 @@ args = argp.parse_args() languages = set(scenario_config.LANGUAGES[l] for l in itertools.chain.from_iterable( - scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x] - for x in args.language)) + six.iterkeys(scenario_config.LANGUAGES) if x == 'all' + else [x] for x in args.language)) # Put together set of remote hosts where to run and build @@ -572,8 +573,8 @@ for scenario in scenarios: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures - merged_resultset = dict(itertools.chain(merged_resultset.iteritems(), - resultset.iteritems())) + merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), + six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py index a94a615b88..4eea02118e 100755 --- a/tools/run_tests/run_stress_tests.py +++ b/tools/run_tests/run_stress_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # @@ -43,6 +43,7 @@ import sys import tempfile import time import uuid +import six import python_utils.dockerjob as dockerjob import python_utils.jobset as jobset @@ -239,9 +240,8 @@ servers = set( for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] for x in args.server)) -languages = set(_LANGUAGES[l] - for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys( - ) if x == 'all' else [x] for x in args.language)) +languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( + six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) docker_images = {} # languages for which to build docker images @@ -267,7 +267,7 @@ if build_jobs: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -306,7 +306,7 @@ try: if not jobs: print('No jobs to run.') - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) @@ -324,8 +324,8 @@ finally: if not job.is_running(): print('Server "%s" has exited prematurely.' % server) - dockerjob.finish_jobs([j for j in server_jobs.itervalues()]) + dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) - for image in docker_images.itervalues(): + for image in six.itervalues(docker_images): print('Removing docker image %s' % image) dockerjob.remove_image(image) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 43b8f8184e..cba91a9a63 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -54,6 +54,7 @@ import traceback import time from six.moves import urllib import uuid +import six import python_utils.jobset as jobset import python_utils.report_utils as report_utils @@ -771,7 +772,7 @@ class CSharpLanguage(object): runtime_cmd = ['mono'] specs = [] - for assembly in tests_by_assembly.iterkeys(): + for assembly in six.iterkeys(tests_by_assembly): assembly_file = 'src/csharp/%s/%s/%s%s' % (assembly, assembly_subdir, assembly, diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 9607526920..b9bf6e5037 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # @@ -30,6 +30,8 @@ """Run test matrix.""" +from __future__ import print_function + import argparse import multiprocessing import os @@ -95,19 +97,19 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', for config in configs: name = '%s_%s_%s_%s' % (language, platform, config, iomgr_platform) runtests_args = ['-l', language, - '-c', config] + '-c', config, + '--iomgr_platform', iomgr_platform] if arch or compiler: name += '_%s_%s' % (arch, compiler) runtests_args += ['--arch', arch, '--compiler', compiler] - runtests_args += extra_args if platform == 'linux': job = _docker_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) else: job = _workspace_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) - job.labels = [platform, config, language] + labels + job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) return result diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index a86db02b80..f2e0bfeb3d 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015, Google Inc. # All rights reserved. # @@ -28,6 +28,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import json import os import re diff --git a/tools/run_tests/sanity/check_test_filtering.py b/tools/run_tests/sanity/check_test_filtering.py index 290a6e2ddf..ba03f11209 100755 --- a/tools/run_tests/sanity/check_test_filtering.py +++ b/tools/run_tests/sanity/check_test_filtering.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. @@ -62,7 +62,7 @@ class TestFilteringTest(unittest.TestCase): def _get_changed_files(foo): return changed_files filter_pull_request_tests._get_changed_files = _get_changed_files - print + print() filtered_jobs = filter_pull_request_tests.filter_tests(all_jobs, "test") # Make sure sanity tests aren't being filtered out diff --git a/tools/run_tests/sanity/check_version.py b/tools/run_tests/sanity/check_version.py index e62f390818..d247260dbb 100755 --- a/tools/run_tests/sanity/check_version.py +++ b/tools/run_tests/sanity/check_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. @@ -29,6 +29,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import sys import yaml import os @@ -48,13 +50,13 @@ try: 'git rev-parse --abbrev-ref HEAD', shell=True) except: - print 'WARNING: not a git repository' + print('WARNING: not a git repository') branch_name = None if branch_name is not None: m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name) if m: - print 'RELEASE branch' + print('RELEASE branch') # version number should align with the branched version check_version = lambda version: ( version.major == int(m.group(1)) and @@ -78,7 +80,7 @@ settings = build_yaml['settings'] top_version = Version(settings['version']) if not check_version(top_version): errors += 1 - print warning % ('version', top_version) + print(warning % ('version', top_version)) for tag, value in settings.iteritems(): if re.match(r'^[a-z]+_version$', tag): @@ -86,12 +88,14 @@ for tag, value in settings.iteritems(): if tag != 'core_version': if value.major != top_version.major: errors += 1 - print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major) + print('major version mismatch on %s: %d vs %d' % (tag, value.major, + top_version.major)) if value.minor != top_version.minor: errors += 1 - print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor) + print('minor version mismatch on %s: %d vs %d' % (tag, value.minor, + top_version.minor)) if not check_version(value): errors += 1 - print warning % (tag, value) + print(warning % (tag, value)) sys.exit(errors) diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index 6d9be2e5d4..c3c3cbec76 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. @@ -29,6 +29,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + import os import sys @@ -60,7 +62,7 @@ for root, dirs, files in os.walk('src/core'): for banned, exceptions in BANNED_EXCEPT.items(): if path in exceptions: continue if banned in text: - print 'Illegal use of "%s" in %s' % (banned, path) + print('Illegal use of "%s" in %s' % (banned, path)) errors += 1 assert errors == 0 diff --git a/tools/run_tests/start_port_server.py b/tools/run_tests/start_port_server.py index e33ac12bd3..bfd72222b6 100755 --- a/tools/run_tests/start_port_server.py +++ b/tools/run_tests/start_port_server.py @@ -39,8 +39,10 @@ The path to this file is called out in test/core/util/port.c, and printed as an error message to users. """ +from __future__ import print_function + import python_utils.start_port_server as start_port_server start_port_server.start_port_server() -print "Port server started successfully" +print("Port server started successfully") diff --git a/tools/run_tests/stress_test/print_summary.py b/tools/run_tests/stress_test/print_summary.py index cb1a33961e..6f4ada2f4f 100755 --- a/tools/run_tests/stress_test/print_summary.py +++ b/tools/run_tests/stress_test/print_summary.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index e2be76e245..b190ebde7a 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2015-2016, Google Inc. # All rights reserved. # @@ -27,6 +27,9 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function + import argparse import datetime import json @@ -124,23 +127,24 @@ class DockerImage: return 'gcr.io/%s/%s' % (project_id, image_name) def build_image(self): - print 'Building docker image: %s (tag: %s)' % (self.image_name, - self.tag_name) + print('Building docker image: %s (tag: %s)' % (self.image_name, + self.tag_name)) os.environ['INTEROP_IMAGE'] = self.image_name os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name os.environ['BASE_NAME'] = self.dockerfile_dir os.environ['BUILD_TYPE'] = self.build_type - print 'DEBUG: path: ', self.build_script_path + print('DEBUG: path: ', self.build_script_path) if subprocess.call(args=[self.build_script_path]) != 0: - print 'Error in building the Docker image' + print('Error in building the Docker image') return False return True def push_to_gke_registry(self): cmd = ['gcloud', 'docker', 'push', self.tag_name] - print 'Pushing %s to the GKE registry..' % self.tag_name + print('Pushing %s to the GKE registry..' % self.tag_name) if subprocess.call(args=cmd) != 0: - print 'Error in pushing the image %s to the GKE registry' % self.tag_name + print('Error in pushing the image %s to the GKE registry' % + self.tag_name) return False return True @@ -199,11 +203,11 @@ class Gke: cmd = ['kubectl', 'proxy', '--port=%d' % port] self.p = subprocess.Popen(args=cmd) time.sleep(2) - print '\nStarted kubernetes proxy on port: %d' % port + print('\nStarted kubernetes proxy on port: %d' % port) def __del__(self): if self.p is not None: - print 'Shutting down Kubernetes proxy..' + print('Shutting down Kubernetes proxy..') self.p.kill() def __init__(self, project_id, run_id, dataset_id, summary_table_id, @@ -253,7 +257,7 @@ class Gke: for pod_name in server_pod_spec.pod_names(): server_env['POD_NAME'] = pod_name - print 'Creating server: %s' % pod_name + print('Creating server: %s' % pod_name) is_success = kubernetes_api.create_pod_and_service( 'localhost', self.kubernetes_port, @@ -267,11 +271,11 @@ class Gke: True # Headless = True for server to that GKE creates a DNS record for pod_name ) if not is_success: - print 'Error in launching server: %s' % pod_name + print('Error in launching server: %s' % pod_name) break if is_success: - print 'Successfully created server(s)' + print('Successfully created server(s)') return is_success @@ -301,7 +305,7 @@ class Gke: for pod_name in client_pod_spec.pod_names(): client_env['POD_NAME'] = pod_name - print 'Creating client: %s' % pod_name + print('Creating client: %s' % pod_name) is_success = kubernetes_api.create_pod_and_service( 'localhost', self.kubernetes_port, @@ -316,18 +320,18 @@ class Gke: ) if not is_success: - print 'Error in launching client %s' % pod_name + print('Error in launching client %s' % pod_name) break if is_success: - print 'Successfully created all client(s)' + print('Successfully created all client(s)') return is_success def _delete_pods(self, pod_name_list): is_success = True for pod_name in pod_name_list: - print 'Deleting %s' % pod_name + print('Deleting %s' % pod_name) is_success = kubernetes_api.delete_pod_and_service( 'localhost', self.kubernetes_port, @@ -335,11 +339,11 @@ class Gke: pod_name) if not is_success: - print 'Error in deleting pod %s' % pod_name + print('Error in deleting pod %s' % pod_name) break if is_success: - print 'Successfully deleted all pods' + print('Successfully deleted all pods') return is_success @@ -353,7 +357,7 @@ class Gke: class Config: def __init__(self, config_filename, gcp_project_id): - print 'Loading configuration...' + print('Loading configuration...') config_dict = self._load_config(config_filename) self.global_settings = self._parse_global_settings(config_dict, @@ -367,7 +371,7 @@ class Config: self.client_pod_specs_dict = self._parse_client_pod_specs( config_dict, self.docker_images_dict, self.client_templates_dict, self.server_pod_specs_dict) - print 'Loaded Configuaration.' + print('Loaded Configuaration.') def _parse_global_settings(self, config_dict, gcp_project_id): global_settings_dict = config_dict['globalSettings'] @@ -540,8 +544,8 @@ def run_tests(config): # run id. This is useful in debugging when looking at records in Biq query) run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) - print 'Run id:', run_id - print 'Dataset id:', dataset_id + print('Run id:', run_id) + print('Dataset id:', dataset_id) bq_helper = BigQueryHelper(run_id, '', '', config.global_settings.gcp_project_id, dataset_id, @@ -557,7 +561,7 @@ def run_tests(config): is_success = True try: - print 'Launching servers..' + print('Launching servers..') for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): if not gke.launch_servers(server_pod_spec): is_success = False # is_success is checked in the 'finally' block @@ -579,11 +583,12 @@ def run_tests(config): start_time = datetime.datetime.now() end_time = start_time + datetime.timedelta( seconds=config.global_settings.test_duration_secs) - print 'Running the test until %s' % end_time.isoformat() + print('Running the test until %s' % end_time.isoformat()) while True: if datetime.datetime.now() > end_time: - print 'Test was run for %d seconds' % config.global_settings.test_duration_secs + print('Test was run for %d seconds' % + config.global_settings.test_duration_secs) break # Check if either stress server or clients have failed (btw, the bq_helper @@ -591,11 +596,12 @@ def run_tests(config): # have a failure status) if bq_helper.check_if_any_tests_failed(): is_success = False - print 'Some tests failed.' + print('Some tests failed.') break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables # Tests running fine. Wait until next poll time to check the status - print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs + print('Sleeping for %d seconds..' % + config.global_settings.test_poll_interval_secs) time.sleep(config.global_settings.test_poll_interval_secs) # Print BiqQuery tables diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py index fdc4668222..0ec7efbbee 100755 --- a/tools/run_tests/task_runner.py +++ b/tools/run_tests/task_runner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python # Copyright 2016, Google Inc. # All rights reserved. # diff --git a/tools/ubsan_suppressions.txt b/tools/ubsan_suppressions.txt new file mode 100644 index 0000000000..9869f98a22 --- /dev/null +++ b/tools/ubsan_suppressions.txt @@ -0,0 +1,7 @@ +# boringssl stuff +nonnull-attribute:bn_wexpand +nonnull-attribute:CBB_add_bytes +nonnull-attribute:rsa_blinding_get +nonnull-attribute:ssl_copy_key_material +alignment:CRYPTO_cbc128_encrypt + diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index daafd3f350..c5bbaed60c 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -45,6 +45,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "alpn_test", "vcxproj\test\a {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "arena_test", "vcxproj\test\arena_test\arena_test.vcxproj", "{D85AC722-A88F-4280-F62E-672F571787FF}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bad_client_test", "vcxproj\test/bad_client\bad_client_test\bad_client_test.vcxproj", "{BA67B418-B699-E41A-9CC4-0279C49481A5}" ProjectSection(myProperties) = preProject lib = "True" @@ -1708,6 +1717,22 @@ Global {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|Win32.Build.0 = Release|Win32 {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|x64.ActiveCfg = Release|x64 {5BAAE7EA-A972-DD80-F190-29B9E3110BB3}.Release-DLL|x64.Build.0 = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|Win32.ActiveCfg = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|x64.ActiveCfg = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|Win32.ActiveCfg = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|x64.ActiveCfg = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|Win32.Build.0 = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug|x64.Build.0 = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|Win32.Build.0 = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release|x64.Build.0 = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Debug-DLL|x64.Build.0 = Debug|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|Win32.Build.0 = Release|Win32 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|x64.ActiveCfg = Release|x64 + {D85AC722-A88F-4280-F62E-672F571787FF}.Release-DLL|x64.Build.0 = Release|x64 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Debug|Win32.ActiveCfg = Debug|Win32 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Debug|x64.ActiveCfg = Debug|x64 {BA67B418-B699-E41A-9CC4-0279C49481A5}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj b/vsprojects/vcxproj/gpr/gpr.vcxproj index 44c21ddeb3..1b37ace69a 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj @@ -188,6 +188,7 @@ </ItemGroup> <ItemGroup> <ClInclude Include="$(SolutionDir)\..\src\core\lib\profiling\timers.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\arena.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\backoff.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\block_annotate.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\env.h" /> @@ -208,6 +209,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\alloc.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\arena.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\avl.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\backoff.c"> diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters index a5924a624a..fafd54cdf9 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters @@ -10,6 +10,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\alloc.c"> <Filter>src\core\lib\support</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\arena.c"> + <Filter>src\core\lib\support</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\avl.c"> <Filter>src\core\lib\support</Filter> </ClCompile> @@ -254,6 +257,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\lib\profiling\timers.h"> <Filter>src\core\lib\profiling</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\arena.h"> + <Filter>src\core\lib\support</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\backoff.h"> <Filter>src\core\lib\support</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj new file mode 100644 index 0000000000..5ae2f8e483 --- /dev/null +++ b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj @@ -0,0 +1,193 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\1.0.204.1.props')" /> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{D85AC722-A88F-4280-F62E-672F571787FF}</ProjectGuid> + <IgnoreWarnIntDirInTempDetected>true</IgnoreWarnIntDirInTempDetected> + <IntDir>$(SolutionDir)IntDir\$(MSBuildProjectName)\</IntDir> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '10.0'" Label="Configuration"> + <PlatformToolset>v100</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '11.0'" Label="Configuration"> + <PlatformToolset>v110</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '12.0'" Label="Configuration"> + <PlatformToolset>v120</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '14.0'" Label="Configuration"> + <PlatformToolset>v140</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <WholeProgramOptimization>true</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="$(SolutionDir)\..\vsprojects\global.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\openssl.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\winsock.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\zlib.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)'=='Debug'"> + <TargetName>arena_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Debug</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Debug</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'"> + <TargetName>arena_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Release</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Release</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\support\arena_test.c"> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr_test_util\gpr_test_util.vcxproj"> + <Project>{EAB0A629-17A9-44DB-B5FF-E91A721FE037}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj"> + <Project>{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}</Project> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <None Include="packages.config" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + </ImportGroup> + <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> + <PropertyGroup> + <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> + </PropertyGroup> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" /> + </Target> +</Project> + diff --git a/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters new file mode 100644 index 0000000000..c470f17527 --- /dev/null +++ b/vsprojects/vcxproj/test/arena_test/arena_test.vcxproj.filters @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\support\arena_test.c"> + <Filter>test\core\support</Filter> + </ClCompile> + </ItemGroup> + + <ItemGroup> + <Filter Include="test"> + <UniqueIdentifier>{130788b2-eacc-90df-a4f6-f5102a7d3370}</UniqueIdentifier> + </Filter> + <Filter Include="test\core"> + <UniqueIdentifier>{5c3e1753-6fdb-9476-f98c-a3a394fac54a}</UniqueIdentifier> + </Filter> + <Filter Include="test\core\support"> + <UniqueIdentifier>{1d3d2cc8-4e69-8b2e-6ceb-6569fcb19a86}</UniqueIdentifier> + </Filter> + </ItemGroup> +</Project> + |