diff options
77 files changed, 915 insertions, 411 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cb322814fe..093e9396cf 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,7 @@ # Auto-generated by the tools/mkowners/mkowners.py tool # Uses OWNERS files in different modules throughout the # repository as the source of truth for module ownership. -/**/OWNERS @markdroth @nicolasnoble @ctiller -/bazel/** @nicolasnoble @dgquintas @ctiller -/src/core/ext/filters/client_channel/** @markdroth @dgquintas @ctiller +/**/OWNERS @markdroth @nicolasnoble @a11r +/bazel/** @nicolasnoble @dgquintas @a11r +/src/core/ext/filters/client_channel/** @markdroth @dgquintas @a11r /tools/run_tests/performance/** @ncteisen @matt-kwong @ctiller @@ -1006,7 +1006,6 @@ grpc_cc_library( "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/exec_ctx_fwd.h", "include/grpc/impl/codegen/grpc_types.h", "include/grpc/impl/codegen/propagation_bits.h", "include/grpc/impl/codegen/status.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index eed1205268..78ccfb2132 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -594,6 +594,7 @@ add_dependencies(buildtests_cxx stress_test) add_dependencies(buildtests_cxx thread_manager_test) add_dependencies(buildtests_cxx thread_stress_test) add_dependencies(buildtests_cxx transport_pid_controller_test) +add_dependencies(buildtests_cxx vector_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() @@ -1080,7 +1081,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -1394,7 +1394,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -1680,7 +1679,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -1950,7 +1948,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -2239,7 +2236,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -2552,7 +2548,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -3038,7 +3033,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -3438,7 +3432,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -3579,7 +3572,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -3784,7 +3776,6 @@ foreach(_hdr include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h - include/grpc/impl/codegen/exec_ctx_fwd.h include/grpc/impl/codegen/grpc_types.h include/grpc/impl/codegen/propagation_bits.h include/grpc/impl/codegen/slice.h @@ -12497,6 +12488,45 @@ target_link_libraries(transport_pid_controller_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) + +add_executable(vector_test + test/core/support/vector_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(vector_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(vector_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(writes_per_rpc_test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af46246822..2f90ccf281 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,53 +2,92 @@ We definitely welcome your patches and contributions to gRPC! -If you are new to github, please start by reading [Pull Request howto](https://help.github.com/articles/about-pull-requests/) +If you are new to github, please start by reading [Pull Request +howto](https://help.github.com/articles/about-pull-requests/) ## Legal requirements In order to protect both you and ourselves, you will need to sign the -[Contributor License Agreement](https://identity.linuxfoundation.org/projects/cncf). +[Contributor License +Agreement](https://identity.linuxfoundation.org/projects/cncf). ## Running tests -Use `tools/run_tests/run_tests.py` script to run the unit tests. -See [tools/run_tests](tools/run_tests) for how to run tests for a given language. +Use `tools/run_tests/run_tests.py` script to run the unit tests. See +[tools/run_tests](tools/run_tests) for how to run tests for a given language. -Prerequisites for building and running tests are listed in [INSTALL.md](INSTALL.md) -and in `src/YOUR-LANGUAGE` (e.g. `src/csharp`) +Prerequisites for building and running tests are listed in +[INSTALL.md](INSTALL.md) and in `src/YOUR-LANGUAGE` (e.g. `src/csharp`) ## Generated project files -To ease maintenance of language- and platform- specific build systems, -many projects files are generated using templates and should not be edited -by hand. -Run `tools/buildgen/generate_projects.sh` to regenerate. -See [templates](templates) for details. +To ease maintenance of language- and platform- specific build systems, many +projects files are generated using templates and should not be edited by hand. +Run `tools/buildgen/generate_projects.sh` to regenerate. See +[templates](templates) for details. -As a rule of thumb, if you see the "sanity tests" failing you've most likely edited generated files or you didn't regenerate the projects properly (or your code formatting doesn't match our code style). +As a rule of thumb, if you see the "sanity tests" failing you've most likely +edited generated files or you didn't regenerate the projects properly (or your +code formatting doesn't match our code style). ## Guidelines for Pull Requests How to get your contributions merged smoothly and quickly. -- Create **small PRs** that are narrowly focused on **addressing a single concern**. We often times receive PRs that are trying to fix several things at a time, but only one fix is considered acceptable, nothing gets merged and both author's & review's time is wasted. Create more PRs to address different concerns and everyone will be happy. +- Create **small PRs** that are narrowly focused on **addressing a single + concern**. We often times receive PRs that are trying to fix several things + at a time, but only one fix is considered acceptable, nothing gets merged and + both author's & review's time is wasted. Create more PRs to address different + concerns and everyone will be happy. -- For speculative changes, consider opening an issue and discussing it first. If you are suggesting a behavioral or API change, consider starting with a [gRFC proposal](https://github.com/grpc/proposal). +- For speculative changes, consider opening an issue and discussing it first. + If you are suggesting a behavioral or API change, consider starting with a + [gRFC proposal](https://github.com/grpc/proposal). -- Provide a good **PR description** as a record of **what** change is being made and **why** it was made. Link to a github issue if it exists. +- Provide a good **PR description** as a record of **what** change is being made + and **why** it was made. Link to a GitHub issue if it exists. -- Don't fix code style and formatting unless you are already changing that line to address an issue. PRs with irrelevant changes won't be merged. If you do want to fix formatting or style, do that in a separate PR. +- Don't fix code style and formatting unless you are already changing that line + to address an issue. PRs with irrelevant changes won't be merged. If you do + want to fix formatting or style, do that in a separate PR. -- Unless your PR is trivial, you should expect there will be reviewer comments that you'll need to address before merging. We expect you to be reasonably responsive to those comments, otherwise the PR will be closed after 2-3 weeks of inactivity. +- Unless your PR is trivial, you should expect there will be reviewer comments + that you'll need to address before merging. We expect you to be reasonably + responsive to those comments, otherwise the PR will be closed after 2-3 weeks + of inactivity. + +- If you have non-trivial contributions, please consider adding an entry to [the + AUTHORS file](https://github.com/grpc/grpc/blob/master/AUTHORS) listing the + copyright holder for the contribution (yourself, if you are signing the + individual CLA, or your company, for corporate CLAs) in the same PR as your + contribution. This needs to be done only once, for each company, or + individual. -- Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use `rebase -i upstream/master` to curate your commit history and/or to bring in latest changes from master (but avoid rebasing in the middle of a code review). +- Maintain **clean commit history** and use **meaningful commit messages**. + PRs with messy commit history are difficult to review and won't be merged. + Use `rebase -i upstream/master` to curate your commit history and/or to + bring in latest changes from master (but avoid rebasing in the middle of + a code review). -- Keep your PR up to date with upstream/master (if there are merge conflicts, we can't really merge your change). +- Keep your PR up to date with upstream/master (if there are merge conflicts, + we can't really merge your change). -- if you are regenerating the projects using `tools/buildgen/generate_projects.sh`, make changes to generated files a separate commit with commit message `regenerate projects`. Mixing changes to generated and hand-written files make your PR difficult to review. +- If you are regenerating the projects using + `tools/buildgen/generate_projects.sh`, make changes to generated files a + separate commit with commit message `regenerate projects`. Mixing changes + to generated and hand-written files make your PR difficult to review. + Note that running this script requires the installation of Python packages + `pyyaml` and `mako` (typically installed using `pip`) as well as a recent + version of [`go`](https://golang.org/doc/install#install). -- **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on (see [tools/run_tests](tools/run_tests). Ultimately, the green signal will be provided by our testing infrastructure. The reviewer will help you if there are test failures that seem not related to the change you are making. +- **All tests need to be passing** before your change can be merged. + We recommend you **run tests locally** before creating your PR to catch + breakages early on (see [tools/run_tests](tools/run_tests). Ultimately, the + green signal will be provided by our testing infrastructure. The reviewer + will help you if there are test failures that seem not related to the change + you are making. -- Exceptions to the rules can be made if there's a compelling reason for doing so. +- Exceptions to the rules can be made if there's a compelling reason for doing + so. @@ -643,7 +643,6 @@ ZLIB_DEP = $(LIBDIR)/$(CONFIG)/libz.a ZLIB_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libz.a ZLIB_MERGE_OBJS = $(LIBZ_OBJS) CPPFLAGS += -Ithird_party/zlib -LDFLAGS += -L$(LIBDIR)/$(CONFIG)/zlib else ifeq ($(HAS_PKG_CONFIG),true) CPPFLAGS += $(shell $(PKG_CONFIG) --cflags zlib) @@ -674,7 +673,6 @@ CARES_DEP = $(LIBDIR)/$(CONFIG)/libares.a CARES_MERGE_OBJS = $(LIBARES_OBJS) CARES_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libares.a CPPFLAGS := -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS) -LDFLAGS := -L$(LIBDIR)/$(CONFIG)/c-ares $(LDFLAGS) else ifeq ($(HAS_PKG_CONFIG),true) PC_REQUIRES_GRPC += libcares @@ -1180,6 +1178,7 @@ stress_test: $(BINDIR)/$(CONFIG)/stress_test thread_manager_test: $(BINDIR)/$(CONFIG)/thread_manager_test thread_stress_test: $(BINDIR)/$(CONFIG)/thread_stress_test transport_pid_controller_test: $(BINDIR)/$(CONFIG)/transport_pid_controller_test +vector_test: $(BINDIR)/$(CONFIG)/vector_test writes_per_rpc_test: $(BINDIR)/$(CONFIG)/writes_per_rpc_test public_headers_must_be_c89: $(BINDIR)/$(CONFIG)/public_headers_must_be_c89 gen_hpack_tables: $(BINDIR)/$(CONFIG)/gen_hpack_tables @@ -1301,10 +1300,10 @@ third_party/protobuf/configure: $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure $(E) "[MAKE] Building protobuf" + $(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf $(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static $(PROTOBUF_CONFIG_OPTS)) $(Q)$(MAKE) -C third_party/protobuf clean $(Q)$(MAKE) -C third_party/protobuf - $(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf $(Q)mkdir -p $(BINDIR)/$(CONFIG)/protobuf $(Q)cp third_party/protobuf/src/.libs/libprotoc.a $(LIBDIR)/$(CONFIG)/protobuf $(Q)cp third_party/protobuf/src/.libs/libprotobuf.a $(LIBDIR)/$(CONFIG)/protobuf @@ -1620,6 +1619,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/transport_pid_controller_test \ + $(BINDIR)/$(CONFIG)/vector_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ $(BINDIR)/$(CONFIG)/boringssl_aes_test \ $(BINDIR)/$(CONFIG)/boringssl_asn1_test \ @@ -1749,6 +1749,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/transport_pid_controller_test \ + $(BINDIR)/$(CONFIG)/vector_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ $(BINDIR)/$(CONFIG)/resolver_component_test \ @@ -2167,6 +2168,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 ) $(E) "[RUN] Testing transport_pid_controller_test" $(Q) $(BINDIR)/$(CONFIG)/transport_pid_controller_test || ( echo test transport_pid_controller_test failed ; exit 1 ) + $(E) "[RUN] Testing vector_test" + $(Q) $(BINDIR)/$(CONFIG)/vector_test || ( echo test vector_test failed ; exit 1 ) $(E) "[RUN] Testing writes_per_rpc_test" $(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 ) $(E) "[RUN] Testing resolver_component_tests_runner_invoker_unsecure" @@ -3226,7 +3229,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -3540,7 +3542,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -3827,7 +3828,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -4088,7 +4088,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -4354,7 +4353,6 @@ PUBLIC_HEADERS_C += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -4646,7 +4644,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -5133,7 +5130,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -5526,7 +5522,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -5644,7 +5639,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -5854,7 +5848,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ - include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/grpc_types.h \ include/grpc/impl/codegen/propagation_bits.h \ include/grpc/impl/codegen/slice.h \ @@ -17270,6 +17263,49 @@ endif endif +VECTOR_TEST_SRC = \ + test/core/support/vector_test.cc \ + +VECTOR_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(VECTOR_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/vector_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/vector_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/vector_test: $(PROTOBUF_DEP) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(VECTOR_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/vector_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/support/vector_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_vector_test: $(VECTOR_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(VECTOR_TEST_OBJS:.o=.dep) +endif +endif + + WRITES_PER_RPC_TEST_SRC = \ test/cpp/performance/writes_per_rpc_test.cc \ @@ -13,5 +13,5 @@ # lead to a bus factor of one to changes to that code @markdroth **/OWNERS @nicolasnoble **/OWNERS -@ctiller **/OWNERS +@a11r **/OWNERS diff --git a/bazel/OWNERS b/bazel/OWNERS index 8fc7502a91..f6ad849be3 100644 --- a/bazel/OWNERS +++ b/bazel/OWNERS @@ -1,5 +1,5 @@ set noparent @nicolasnoble @dgquintas -@ctiller +@a11r diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index d146ca9c2c..2560519284 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -30,9 +30,12 @@ def _get_external_deps(external_deps): ret = [] for dep in external_deps: if dep == "nanopb": - ret.append("//third_party/nanopb") + ret += ["//third_party/nanopb"] + elif dep == "cares": + ret += select({"//:grpc_no_ares": [], + "//conditions:default": ["//external:cares"],}) else: - ret.append("//external:" + dep) + ret += ["//external:" + dep] return ret def _maybe_update_cc_library_hdrs(hdrs): diff --git a/build.yaml b/build.yaml index fef7d6189f..db2ff8828b 100644 --- a/build.yaml +++ b/build.yaml @@ -399,6 +399,7 @@ filegroups: - src/core/lib/support/debug_location.h - src/core/lib/support/ref_counted.h - src/core/lib/support/ref_counted_ptr.h + - src/core/lib/support/vector.h - src/core/lib/surface/alarm_internal.h - src/core/lib/surface/api_trace.h - src/core/lib/surface/call.h @@ -485,7 +486,6 @@ filegroups: - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h - - include/grpc/impl/codegen/exec_ctx_fwd.h - include/grpc/impl/codegen/grpc_types.h - include/grpc/impl/codegen/propagation_bits.h - include/grpc/impl/codegen/slice.h @@ -4798,6 +4798,20 @@ targets: - grpc - gpr_test_util - gpr +- name: vector_test + gtest: true + build: test + language: c++ + src: + - test/core/support/vector_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + uses: + - grpc++_test - name: writes_per_rpc_test gtest: true cpu_cost: 0.5 diff --git a/doc/environment_variables.md b/doc/environment_variables.md index 40af758f69..4796ad067d 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -127,3 +127,8 @@ some configuration as environment variables that can be set. there is no active polling thread. They help reconnect disconnected client channels (mostly due to idleness), so that the next RPC on this channel won't fail. Set to 0 to turn off the backup polls. + +* GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL + if set, flow control will be effectively disabled. Max out all values and + assume the remote peer does the same. Thus we can ignore any flow control + bookkeeping, error checking, and decision making diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c127660dd5..358fad3d98 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -152,7 +152,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', - 'include/grpc/impl/codegen/exec_ctx_fwd.h', 'include/grpc/impl/codegen/grpc_types.h', 'include/grpc/impl/codegen/propagation_bits.h', 'include/grpc/impl/codegen/slice.h', @@ -423,6 +422,7 @@ Pod::Spec.new do |s| 'src/core/lib/support/debug_location.h', 'src/core/lib/support/ref_counted.h', 'src/core/lib/support/ref_counted_ptr.h', + 'src/core/lib/support/vector.h', 'src/core/lib/surface/alarm_internal.h', 'src/core/lib/surface/api_trace.h', 'src/core/lib/surface/call.h', @@ -903,6 +903,7 @@ Pod::Spec.new do |s| 'src/core/lib/support/debug_location.h', 'src/core/lib/support/ref_counted.h', 'src/core/lib/support/ref_counted_ptr.h', + 'src/core/lib/support/vector.h', 'src/core/lib/surface/alarm_internal.h', 'src/core/lib/surface/api_trace.h', 'src/core/lib/surface/call.h', diff --git a/grpc.gemspec b/grpc.gemspec index d185995261..7547bc85de 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -149,7 +149,6 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) - s.files += %w( include/grpc/impl/codegen/exec_ctx_fwd.h ) s.files += %w( include/grpc/impl/codegen/grpc_types.h ) s.files += %w( include/grpc/impl/codegen/propagation_bits.h ) s.files += %w( include/grpc/impl/codegen/slice.h ) @@ -349,6 +348,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/debug_location.h ) s.files += %w( src/core/lib/support/ref_counted.h ) s.files += %w( src/core/lib/support/ref_counted_ptr.h ) + s.files += %w( src/core/lib/support/vector.h ) s.files += %w( src/core/lib/surface/alarm_internal.h ) s.files += %w( src/core/lib/surface/api_trace.h ) s.files += %w( src/core/lib/surface/call.h ) diff --git a/include/grpc/impl/codegen/exec_ctx_fwd.h b/include/grpc/impl/codegen/exec_ctx_fwd.h deleted file mode 100644 index 005ff14e7e..0000000000 --- a/include/grpc/impl/codegen/exec_ctx_fwd.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H -#define GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H - -/* forward declaration for exec_ctx.h */ -struct grpc_exec_ctx; -typedef struct grpc_exec_ctx grpc_exec_ctx; - -#endif /* GRPC_IMPL_CODEGEN_EXEC_CTX_FWD_H */ diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index fcbc8ac5a1..d481a70ab9 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -22,7 +22,6 @@ #include <grpc/impl/codegen/port_platform.h> #include <grpc/impl/codegen/compression_types.h> -#include <grpc/impl/codegen/exec_ctx_fwd.h> #include <grpc/impl/codegen/gpr_types.h> #include <grpc/impl/codegen/slice.h> #include <grpc/impl/codegen/status.h> diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h index ad026b685e..a3cd1f1bbe 100644 --- a/include/grpc/impl/codegen/slice.h +++ b/include/grpc/impl/codegen/slice.h @@ -23,7 +23,6 @@ #include <stddef.h> -#include <grpc/impl/codegen/exec_ctx_fwd.h> #include <grpc/impl/codegen/gpr_slice.h> typedef struct grpc_slice grpc_slice; diff --git a/include/grpc/module.modulemap b/include/grpc/module.modulemap index 67136cba8a..da95515d8e 100644 --- a/include/grpc/module.modulemap +++ b/include/grpc/module.modulemap @@ -30,7 +30,6 @@ framework module grpc { header "impl/codegen/byte_buffer_reader.h" header "impl/codegen/compression_types.h" header "impl/codegen/connectivity_state.h" - header "impl/codegen/exec_ctx_fwd.h" header "impl/codegen/grpc_types.h" header "impl/codegen/propagation_bits.h" header "impl/codegen/slice.h" diff --git a/package.xml b/package.xml index b4d8c88693..ff3d0797ab 100644 --- a/package.xml +++ b/package.xml @@ -161,7 +161,6 @@ <file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer_reader.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/compression_types.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/connectivity_state.h" role="src" /> - <file baseinstalldir="/" name="include/grpc/impl/codegen/exec_ctx_fwd.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/grpc_types.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/propagation_bits.h" role="src" /> <file baseinstalldir="/" name="include/grpc/impl/codegen/slice.h" role="src" /> @@ -361,6 +360,7 @@ <file baseinstalldir="/" name="src/core/lib/support/debug_location.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/ref_counted.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/support/ref_counted_ptr.h" role="src" /> + <file baseinstalldir="/" name="src/core/lib/support/vector.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/surface/alarm_internal.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/surface/api_trace.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/surface/call.h" role="src" /> diff --git a/src/core/ext/filters/client_channel/OWNERS b/src/core/ext/filters/client_channel/OWNERS index 773bc73179..8f5e92808e 100644 --- a/src/core/ext/filters/client_channel/OWNERS +++ b/src/core/ext/filters/client_channel/OWNERS @@ -1,4 +1,4 @@ set noparent @markdroth @dgquintas -@ctiller +@a11r diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 5849ac9d2d..272b3617b2 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -340,7 +340,9 @@ static void pending_pick_set_metadata_and_context(pending_pick* pp) { pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy = destroy_client_stats; } else { - grpc_grpclb_client_stats_unref(pp->client_stats); + if (pp->client_stats != nullptr) { + grpc_grpclb_client_stats_unref(pp->client_stats); + } } } @@ -927,7 +929,9 @@ static void glb_shutdown_locked(grpc_lb_policy* pol, pending_pick* next = pp->next; if (new_policy != nullptr) { // Hand pick over to new policy. - grpc_grpclb_client_stats_unref(pp->client_stats); + if (pp->client_stats != nullptr) { + grpc_grpclb_client_stats_unref(pp->client_stats); + } pp->pick->on_complete = pp->original_on_complete; if (grpc_lb_policy_pick_locked(new_policy, pp->pick)) { // Synchronous return; schedule callback. diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc index 97c1878f34..3aca61fdac 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc @@ -18,8 +18,16 @@ #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/support/env.h" #include "src/core/lib/transport/metadata.h" -void grpc_chttp2_plugin_init(void) {} +void grpc_chttp2_plugin_init(void) { + g_flow_control_enabled = true; + char* env_variable = gpr_getenv("GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL"); + if (env_variable != nullptr) { + g_flow_control_enabled = false; + gpr_free(env_variable); + } +} void grpc_chttp2_plugin_shutdown(void) {} diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 7c77de2168..835de6aa0f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -152,6 +152,10 @@ static void keepalive_watchdog_fired_locked(void* arg, grpc_error* error); static void reset_byte_stream(void* arg, grpc_error* error); +// Flow control default enabled. Can be disabled by setting +// GRPC_EXPERIMENTAL_DISABLE_FLOW_CONTROL +bool g_flow_control_enabled = true; + /******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -517,7 +521,13 @@ static void init_transport(grpc_chttp2_transport* t, } } - t->flow_control.Init(t, enable_bdp); + if (g_flow_control_enabled) { + t->flow_control.Init<grpc_core::chttp2::TransportFlowControl>(t, + enable_bdp); + } else { + t->flow_control.Init<grpc_core::chttp2::TransportFlowControlDisabled>(t); + enable_bdp = false; + } /* No pings allowed before receiving a header or data frame. */ t->ping_state.pings_before_data_required = 0; @@ -682,7 +692,14 @@ static int init_stream(grpc_transport* gt, grpc_stream* gs, post_destructive_reclaimer(t); } - s->flow_control.Init(t->flow_control.get(), s); + if (t->flow_control->flow_control_enabled()) { + s->flow_control.Init<grpc_core::chttp2::StreamFlowControl>( + static_cast<grpc_core::chttp2::TransportFlowControl*>( + t->flow_control.get()), + s); + } else { + s->flow_control.Init<grpc_core::chttp2::StreamFlowControlDisabled>(); + } GPR_TIMER_END("init_stream", 0); return 0; @@ -2402,8 +2419,11 @@ static void read_action_locked(void* tp, grpc_error* error) { grpc_error* errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE, GRPC_ERROR_NONE}; for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) { - t->flow_control->bdp_estimator()->AddIncomingBytes( - (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i])); + grpc_core::BdpEstimator* bdp_est = t->flow_control->bdp_estimator(); + if (bdp_est) { + bdp_est->AddIncomingBytes( + (int64_t)GRPC_SLICE_LENGTH(t->read_buffer.slices[i])); + } errors[1] = grpc_chttp2_perform_read(t, t->read_buffer.slices[i]); } if (errors[1] != GRPC_ERROR_NONE) { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 596ababb19..34519ceec9 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -27,6 +27,8 @@ extern grpc_core::TraceFlag grpc_http_trace; extern grpc_core::TraceFlag grpc_trace_http2_stream_state; extern grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount; +extern bool g_flow_control_enabled; + grpc_transport* grpc_create_chttp2_transport( const grpc_channel_args* channel_args, grpc_endpoint* ep, bool is_client); diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index ca48cc7e0a..3013db23f7 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -149,6 +149,25 @@ void FlowControlAction::Trace(grpc_chttp2_transport* t) const { gpr_free(mf_str); } +TransportFlowControlDisabled::TransportFlowControlDisabled( + grpc_chttp2_transport* t) { + remote_window_ = kMaxWindow; + target_initial_window_size_ = kMaxWindow; + announced_window_ = kMaxWindow; + t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE] = + kFrameSize; + t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; + t->settings[GRPC_SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; + t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] = + kMaxWindow; +} + TransportFlowControl::TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe) : t_(t), @@ -318,7 +337,7 @@ double TransportFlowControl::SmoothLogBdp(double value) { } FlowControlAction::Urgency TransportFlowControl::DeltaUrgency( - int32_t value, grpc_chttp2_setting_id setting_id) { + int64_t value, grpc_chttp2_setting_id setting_id) { int64_t delta = (int64_t)value - (int64_t)t_->settings[GRPC_LOCAL_SETTINGS][setting_id]; // TODO(ncteisen): tune this @@ -344,7 +363,7 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() { action.set_send_initial_window_update( DeltaUrgency(target_initial_window_size_, GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE), - target_initial_window_size_); + (uint32_t)target_initial_window_size_); // get bandwidth estimate and update max_frame accordingly. double bw_dbl = bdp_estimator_.EstimateBandwidth(); @@ -354,7 +373,7 @@ FlowControlAction TransportFlowControl::PeriodicUpdate() { target_initial_window_size_), 16384, 16777215); action.set_send_max_frame_size_update( - DeltaUrgency(frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE), + DeltaUrgency((int64_t)frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE), frame_size); } return UpdateAction(action); diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h index 8306047dbc..38b9f50c8d 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.h +++ b/src/core/ext/transport/chttp2/transport/flow_control.h @@ -24,6 +24,7 @@ #include <grpc/support/useful.h> #include "src/core/ext/transport/chttp2/transport/http2_settings.h" +#include "src/core/lib/support/abstract.h" #include "src/core/lib/support/manual_constructor.h" #include "src/core/lib/transport/bdp_estimator.h" #include "src/core/lib/transport/pid_controller.h" @@ -43,10 +44,16 @@ namespace grpc_core { namespace chttp2 { static constexpr uint32_t kDefaultWindow = 65535; +static constexpr int64_t kMaxWindow = (int64_t)((1u << 31) - 1); +// TODO(ncteisen): Tune this +static constexpr uint32_t kFrameSize = 1024 * 1024; class TransportFlowControl; class StreamFlowControl; +// Encapsulates a collections of actions the transport needs to take with +// regard to flow control. Each action comes with urgencies that tell the +// transport how quickly the action must take place. class FlowControlAction { public: enum class Urgency : uint8_t { @@ -132,36 +139,122 @@ class FlowControlTrace { int64_t announced_window_delta_; }; -class TransportFlowControl { +// Fat interface with all methods a flow control implementation needs to +// support. gRPC C Core does not support pure virtual functions, so instead +// we abort in any methods which require implementation in the base class. +class TransportFlowControlBase { + public: + TransportFlowControlBase() {} + virtual ~TransportFlowControlBase() {} + + // Is flow control enabled? This is needed in other codepaths like the checks + // in parsing and in writing. + virtual bool flow_control_enabled() const { abort(); } + + // Called to check if the transport needs to send a WINDOW_UPDATE frame + virtual uint32_t MaybeSendUpdate(bool writing_anyway) { abort(); } + + // Using the protected members, returns and Action to be taken by the + // tranport. + virtual FlowControlAction MakeAction() { abort(); } + + // Using the protected members, returns and Action to be taken by the + // tranport. Also checks for updates to our BDP estimate and acts + // accordingly. + virtual FlowControlAction PeriodicUpdate() { abort(); } + + // Called to do bookkeeping when a stream owned by this transport sends + // data on the wire + virtual void StreamSentData(int64_t size) { abort(); } + + // Called to do bookkeeping when a stream owned by this transport receives + // data from the wire. Also does error checking for frame size. + virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); } + + // Called to do bookkeeping when we receive a WINDOW_UPDATE frame. + virtual void RecvUpdate(uint32_t size) { abort(); } + + // Returns the BdpEstimator held by this object. Caller is responsible for + // checking for nullptr. TODO(ncteisen): consider fully encapsulating all + // bdp estimator actions inside TransportFlowControl + virtual BdpEstimator* bdp_estimator() { return nullptr; } + + // Getters + int64_t remote_window() const { return remote_window_; } + virtual int64_t target_window() const { return target_initial_window_size_; } + int64_t announced_window() const { return announced_window_; } + + // Used in certain benchmarks in which we don't want FlowControl to be a + // factor + virtual void TestOnlyForceHugeWindow() {} + + GRPC_ABSTRACT_BASE_CLASS + + protected: + friend class ::grpc::testing::TrickledCHTTP2; + int64_t remote_window_ = kDefaultWindow; + int64_t target_initial_window_size_ = kDefaultWindow; + int64_t announced_window_ = kDefaultWindow; +}; + +// Implementation of flow control that does NOTHING. Always returns maximum +// values, never initiates writes, and assumes that the remote peer is doing +// the same. To be used to narrow down on flow control as the cause of negative +// performance. +class TransportFlowControlDisabled final : public TransportFlowControlBase { + public: + // Maxes out all values + TransportFlowControlDisabled(grpc_chttp2_transport* t); + + bool flow_control_enabled() const override { return false; } + + // Never do anything. + uint32_t MaybeSendUpdate(bool writing_anyway) override { return 0; } + FlowControlAction MakeAction() override { return FlowControlAction(); } + FlowControlAction PeriodicUpdate() override { return FlowControlAction(); } + void StreamSentData(int64_t size) override {} + grpc_error* RecvData(int64_t incoming_frame_size) override { + return GRPC_ERROR_NONE; + } + void RecvUpdate(uint32_t size) override {} +}; + +// Implementation of flow control that abides to HTTP/2 spec and attempts +// to be as performant as possible. +class TransportFlowControl final : public TransportFlowControlBase { public: TransportFlowControl(const grpc_chttp2_transport* t, bool enable_bdp_probe); ~TransportFlowControl() {} + bool flow_control_enabled() const override { return true; } + bool bdp_probe() const { return enable_bdp_probe_; } // returns an announce if we should send a transport update to our peer, // else returns zero; writing_anyway indicates if a write would happen // regardless of the send - if it is false and this function returns non-zero, // this announce will cause a write to occur - uint32_t MaybeSendUpdate(bool writing_anyway); + uint32_t MaybeSendUpdate(bool writing_anyway) override; // Reads the flow control data and returns and actionable struct that will // tell chttp2 exactly what it needs to do - FlowControlAction MakeAction() { return UpdateAction(FlowControlAction()); } + FlowControlAction MakeAction() override { + return UpdateAction(FlowControlAction()); + } // Call periodically (at a low-ish rate, 100ms - 10s makes sense) // to perform more complex flow control calculations and return an action // to let chttp2 change its parameters - FlowControlAction PeriodicUpdate(); + FlowControlAction PeriodicUpdate() override; - void StreamSentData(int64_t size) { remote_window_ -= size; } + void StreamSentData(int64_t size) override { remote_window_ -= size; } grpc_error* ValidateRecvData(int64_t incoming_frame_size); void CommitRecvData(int64_t incoming_frame_size) { announced_window_ -= incoming_frame_size; } - grpc_error* RecvData(int64_t incoming_frame_size) { + grpc_error* RecvData(int64_t incoming_frame_size) override { FlowControlTrace trace(" data recv", this, nullptr); grpc_error* error = ValidateRecvData(incoming_frame_size); if (error != GRPC_ERROR_NONE) return error; @@ -170,18 +263,18 @@ class TransportFlowControl { } // we have received a WINDOW_UPDATE frame for a transport - void RecvUpdate(uint32_t size) { + void RecvUpdate(uint32_t size) override { FlowControlTrace trace("t updt recv", this, nullptr); remote_window_ += size; } - int64_t remote_window() const { return remote_window_; } - int64_t target_window() const { + // See comment above announced_stream_total_over_incoming_window_ for the + // logic behind this decision. + int64_t target_window() const override { return (uint32_t)GPR_MIN((int64_t)((1u << 31) - 1), announced_stream_total_over_incoming_window_ + target_initial_window_size_); } - int64_t announced_window() const { return announced_window_; } const grpc_chttp2_transport* transport() const { return t_; } @@ -201,18 +294,17 @@ class TransportFlowControl { } } - BdpEstimator* bdp_estimator() { return &bdp_estimator_; } + BdpEstimator* bdp_estimator() override { return &bdp_estimator_; } - void TestOnlyForceHugeWindow() { + void TestOnlyForceHugeWindow() override { announced_window_ = 1024 * 1024 * 1024; remote_window_ = 1024 * 1024 * 1024; } private: - friend class ::grpc::testing::TrickledCHTTP2; double TargetLogBdp(); double SmoothLogBdp(double value); - FlowControlAction::Urgency DeltaUrgency(int32_t value, + FlowControlAction::Urgency DeltaUrgency(int64_t value, grpc_chttp2_setting_id setting_id); FlowControlAction UpdateAction(FlowControlAction action) { @@ -225,9 +317,6 @@ class TransportFlowControl { const grpc_chttp2_transport* const t_; - /** Our bookkeeping for the remote peer's available window */ - int64_t remote_window_ = kDefaultWindow; - /** calculating what we should give for local window: we track the total amount of flow control over initial window size across all streams: this is data that we want to receive right now (it @@ -239,13 +328,6 @@ class TransportFlowControl { int64_t announced_stream_total_over_incoming_window_ = 0; int64_t announced_stream_total_under_incoming_window_ = 0; - /** This is out window according to what we have sent to our remote peer. The - * difference between this and target window is what we use to decide when - * to send WINDOW_UPDATE frames. */ - int64_t announced_window_ = kDefaultWindow; - - int32_t target_initial_window_size_ = kDefaultWindow; - /** should we probe bdp? */ const bool enable_bdp_probe_; @@ -257,39 +339,117 @@ class TransportFlowControl { grpc_millis last_pid_update_ = 0; }; -class StreamFlowControl { +// Fat interface with all methods a stream flow control implementation needs +// to support. gRPC C Core does not support pure virtual functions, so instead +// we abort in any methods which require implementation in the base class. +class StreamFlowControlBase { + public: + StreamFlowControlBase() {} + virtual ~StreamFlowControlBase() {} + + // Updates an action using the protected members. + virtual FlowControlAction UpdateAction(FlowControlAction action) { abort(); } + + // Using the protected members, returns an Action for this stream to be + // taken by the tranport. + virtual FlowControlAction MakeAction() { abort(); } + + // Bookkeeping for when data is sent on this stream. + virtual void SentData(int64_t outgoing_frame_size) { abort(); } + + // Bookkeeping and error checking for when data is received by this stream. + virtual grpc_error* RecvData(int64_t incoming_frame_size) { abort(); } + + // Called to check if this stream needs to send a WINDOW_UPDATE frame. + virtual uint32_t MaybeSendUpdate() { abort(); } + + // Bookkeeping for receiving a WINDOW_UPDATE from for this stream. + virtual void RecvUpdate(uint32_t size) { abort(); } + + // Bookkeeping for when a call pulls bytes out of the transport. At this + // point we consider the data 'used' and can thus let out peer know we are + // ready for more data. + virtual void IncomingByteStreamUpdate(size_t max_size_hint, + size_t have_already) { + abort(); + } + + // Used in certain benchmarks in which we don't want FlowControl to be a + // factor + virtual void TestOnlyForceHugeWindow() {} + + // Getters + int64_t remote_window_delta() { return remote_window_delta_; } + int64_t local_window_delta() { return local_window_delta_; } + int64_t announced_window_delta() { return announced_window_delta_; } + + GRPC_ABSTRACT_BASE_CLASS + + protected: + friend class ::grpc::testing::TrickledCHTTP2; + int64_t remote_window_delta_ = 0; + int64_t local_window_delta_ = 0; + int64_t announced_window_delta_ = 0; +}; + +// Implementation of flow control that does NOTHING. Always returns maximum +// values, never initiates writes, and assumes that the remote peer is doing +// the same. To be used to narrow down on flow control as the cause of negative +// performance. +class StreamFlowControlDisabled : public StreamFlowControlBase { + public: + FlowControlAction UpdateAction(FlowControlAction action) override { + return action; + } + FlowControlAction MakeAction() override { return FlowControlAction(); } + void SentData(int64_t outgoing_frame_size) override {} + grpc_error* RecvData(int64_t incoming_frame_size) override { + return GRPC_ERROR_NONE; + } + uint32_t MaybeSendUpdate() override { return 0; } + void RecvUpdate(uint32_t size) override {} + void IncomingByteStreamUpdate(size_t max_size_hint, + size_t have_already) override {} +}; + +// Implementation of flow control that abides to HTTP/2 spec and attempts +// to be as performant as possible. +class StreamFlowControl final : public StreamFlowControlBase { public: StreamFlowControl(TransportFlowControl* tfc, const grpc_chttp2_stream* s); ~StreamFlowControl() { tfc_->PreUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); } - FlowControlAction UpdateAction(FlowControlAction action); - FlowControlAction MakeAction() { return UpdateAction(tfc_->MakeAction()); } + FlowControlAction UpdateAction(FlowControlAction action) override; + FlowControlAction MakeAction() override { + return UpdateAction(tfc_->MakeAction()); + } // we have sent data on the wire, we must track this in our bookkeeping for // the remote peer's flow control. - void SentData(int64_t outgoing_frame_size) { + void SentData(int64_t outgoing_frame_size) override { FlowControlTrace tracer(" data sent", tfc_, this); tfc_->StreamSentData(outgoing_frame_size); remote_window_delta_ -= outgoing_frame_size; } // we have received data from the wire - grpc_error* RecvData(int64_t incoming_frame_size); + grpc_error* RecvData(int64_t incoming_frame_size) override; // returns an announce if we should send a stream update to our peer, else // returns zero - uint32_t MaybeSendUpdate(); + uint32_t MaybeSendUpdate() override; // we have received a WINDOW_UPDATE frame for a stream - void RecvUpdate(uint32_t size) { + void RecvUpdate(uint32_t size) override { FlowControlTrace trace("s updt recv", tfc_, this); remote_window_delta_ += size; } // the application is asking for a certain amount of bytes - void IncomingByteStreamUpdate(size_t max_size_hint, size_t have_already); + void IncomingByteStreamUpdate(size_t max_size_hint, + size_t have_already) override; int64_t remote_window_delta() const { return remote_window_delta_; } int64_t local_window_delta() const { return local_window_delta_; } @@ -297,14 +457,13 @@ class StreamFlowControl { const grpc_chttp2_stream* stream() const { return s_; } - void TestOnlyForceHugeWindow() { + void TestOnlyForceHugeWindow() override { announced_window_delta_ = 1024 * 1024 * 1024; local_window_delta_ = 1024 * 1024 * 1024; remote_window_delta_ = 1024 * 1024 * 1024; } private: - friend class ::grpc::testing::TrickledCHTTP2; TransportFlowControl* const tfc_; const grpc_chttp2_stream* const s_; @@ -313,21 +472,6 @@ class StreamFlowControl { announced_window_delta_ += change; tfc->PostUpdateAnnouncedWindowOverIncomingWindow(announced_window_delta_); } - - /** window available for us to send to peer, over or under the initial - * window - * size of the transport... ie: - * remote_window = remote_window_delta + transport.initial_window_size */ - int64_t remote_window_delta_ = 0; - - /** window available for peer to send to us (as a delta on - * transport.initial_window_size) - * local_window = local_window_delta + transport.initial_window_size */ - int64_t local_window_delta_ = 0; - - /** window available for peer to send to us over this stream that we have - * announced to the peer */ - int64_t announced_window_delta_ = 0; }; } // namespace chttp2 diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.cc b/src/core/ext/transport/chttp2/transport/frame_settings.cc index c6c2a6c301..0d245f4ba1 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.cc +++ b/src/core/ext/transport/chttp2/transport/frame_settings.cc @@ -186,6 +186,12 @@ grpc_error* grpc_chttp2_settings_parser_parse(void* p, grpc_chttp2_transport* t, if (grpc_wire_id_to_setting_id(parser->id, &id)) { const grpc_chttp2_setting_parameters* sp = &grpc_chttp2_settings_parameters[id]; + // If flow control is disabled we skip these. + if (!t->flow_control->flow_control_enabled() && + (id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE || + id == GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE)) { + continue; + } if (parser->value < sp->min_value || parser->value > sp->max_value) { switch (sp->invalid_value_behavior) { case GRPC_CHTTP2_CLAMP_INVALID_VALUE: diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 932f5ba83d..8852d9ee2b 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -351,7 +351,10 @@ struct grpc_chttp2_transport { /** parser for goaway frames */ grpc_chttp2_goaway_parser goaway_parser; - grpc_core::ManualConstructor<grpc_core::chttp2::TransportFlowControl> + grpc_core::PolymorphicManualConstructor< + grpc_core::chttp2::TransportFlowControlBase, + grpc_core::chttp2::TransportFlowControl, + grpc_core::chttp2::TransportFlowControlDisabled> flow_control; /** initial window change. This is tracked as we parse settings frames from * the remote peer. If there is a positive delta, then we will make all @@ -525,7 +528,10 @@ struct grpc_chttp2_stream { bool sent_initial_metadata; bool sent_trailing_metadata; - grpc_core::ManualConstructor<grpc_core::chttp2::StreamFlowControl> + grpc_core::PolymorphicManualConstructor< + grpc_core::chttp2::StreamFlowControlBase, + grpc_core::chttp2::StreamFlowControl, + grpc_core::chttp2::StreamFlowControlDisabled> flow_control; grpc_slice_buffer flow_controlled_buffer; diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index a56f89cc75..59dc38ef98 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -186,9 +186,10 @@ grpc_error* grpc_chttp2_perform_read(grpc_chttp2_transport* t, return GRPC_ERROR_NONE; } goto dts_fh_0; /* loop */ - } else if (t->incoming_frame_size > - t->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { + } else if (t->flow_control->flow_control_enabled() && + t->incoming_frame_size > + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { char* msg; gpr_asprintf(&msg, "Frame size %d is larger than max frame size %d", t->incoming_frame_size, diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index c95d02541a..3aad8c5823 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -183,6 +183,7 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { + GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } @@ -198,6 +199,7 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport* t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { + GPR_ASSERT(t->flow_control->flow_control_enabled()); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index 88af76006a..4c58c0e4bf 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -22,7 +22,6 @@ #include <grpc/support/port_platform.h> #include <assert.h> -#include <grpc/impl/codegen/exec_ctx_fwd.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> #include <stdbool.h> diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 3f238c660a..c7cde7ea59 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -19,7 +19,6 @@ #ifndef GRPC_CORE_LIB_IOMGR_IOMGR_H #define GRPC_CORE_LIB_IOMGR_IOMGR_H -#include <grpc/impl/codegen/exec_ctx_fwd.h> #include "src/core/lib/iomgr/port.h" /** Initializes the iomgr. */ diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc index 4a97f3353d..946846a8b8 100644 --- a/src/core/lib/iomgr/udp_server.cc +++ b/src/core/lib/iomgr/udp_server.cc @@ -21,6 +21,10 @@ #define _GNU_SOURCE #endif +#ifndef SO_RXQ_OVFL +#define SO_RXQ_OVFL 40 +#endif + #include "src/core/lib/iomgr/port.h" #ifdef GRPC_POSIX_SOCKET @@ -280,11 +284,10 @@ static int bind_socket(grpc_socket_factory* socket_factory, int sockfd, /* Prepare a recently-created socket for listening. */ static int prepare_socket(grpc_socket_factory* socket_factory, int fd, - const grpc_resolved_address* addr) { + const grpc_resolved_address* addr, int rcv_buf_size, + int snd_buf_size) { grpc_resolved_address sockname_temp; struct sockaddr* addr_ptr = (struct sockaddr*)addr->addr; - /* Set send/receive socket buffers to 1 MB */ - int buffer_size_bytes = 1024 * 1024; if (fd < 0) { goto error; @@ -325,18 +328,25 @@ static int prepare_socket(grpc_socket_factory* socket_factory, int fd, goto error; } - if (grpc_set_socket_sndbuf(fd, buffer_size_bytes) != GRPC_ERROR_NONE) { + if (grpc_set_socket_sndbuf(fd, snd_buf_size) != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Failed to set send buffer size to %d bytes", - buffer_size_bytes); + snd_buf_size); goto error; } - if (grpc_set_socket_rcvbuf(fd, buffer_size_bytes) != GRPC_ERROR_NONE) { + if (grpc_set_socket_rcvbuf(fd, rcv_buf_size) != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Failed to set receive buffer size to %d bytes", - buffer_size_bytes); + rcv_buf_size); goto error; } + { + int get_overflow = 1; + if (0 != setsockopt(fd, SOL_SOCKET, SO_RXQ_OVFL, &get_overflow, + sizeof(get_overflow))) { + gpr_log(GPR_INFO, "Failed to set socket overflow support"); + } + } return grpc_sockaddr_get_port(&sockname_temp); error: @@ -451,6 +461,7 @@ static void on_write(void* arg, grpc_error* error) { static int add_socket_to_server(grpc_udp_server* s, int fd, const grpc_resolved_address* addr, + int rcv_buf_size, int snd_buf_size, grpc_udp_server_start_cb start_cb, grpc_udp_server_read_cb read_cb, grpc_udp_server_write_cb write_cb, @@ -460,7 +471,8 @@ static int add_socket_to_server(grpc_udp_server* s, int fd, char* addr_str; char* name; - port = prepare_socket(s->socket_factory, fd, addr); + port = + prepare_socket(s->socket_factory, fd, addr, rcv_buf_size, snd_buf_size); if (port >= 0) { grpc_sockaddr_to_string(&addr_str, addr, 1); gpr_asprintf(&name, "udp-server-listener:%s", addr_str); @@ -495,6 +507,7 @@ static int add_socket_to_server(grpc_udp_server* s, int fd, int grpc_udp_server_add_port(grpc_udp_server* s, const grpc_resolved_address* addr, + int rcv_buf_size, int snd_buf_size, grpc_udp_server_start_cb start_cb, grpc_udp_server_read_cb read_cb, grpc_udp_server_write_cb write_cb, @@ -545,8 +558,9 @@ int grpc_udp_server_add_port(grpc_udp_server* s, // TODO(rjshade): Test and propagate the returned grpc_error*: GRPC_ERROR_UNREF(grpc_create_dualstack_socket_using_factory( s->socket_factory, addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode, &fd)); - allocated_port1 = add_socket_to_server(s, fd, addr, start_cb, read_cb, - write_cb, orphan_cb); + allocated_port1 = + add_socket_to_server(s, fd, addr, rcv_buf_size, snd_buf_size, start_cb, + read_cb, write_cb, orphan_cb); if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) { goto done; } @@ -569,7 +583,8 @@ int grpc_udp_server_add_port(grpc_udp_server* s, addr = &addr4_copy; } allocated_port2 = - add_socket_to_server(s, fd, addr, start_cb, read_cb, write_cb, orphan_cb); + add_socket_to_server(s, fd, addr, rcv_buf_size, snd_buf_size, start_cb, + read_cb, write_cb, orphan_cb); done: gpr_free(allocated_addr); diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index a469ab9be5..c1aa49f15d 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -68,6 +68,7 @@ int grpc_udp_server_get_fd(grpc_udp_server* s, unsigned port_index); all of the multiple socket port matching logic in one place */ int grpc_udp_server_add_port(grpc_udp_server* s, const grpc_resolved_address* addr, + int rcv_buf_size, int snd_buf_size, grpc_udp_server_start_cb start_cb, grpc_udp_server_read_cb read_cb, grpc_udp_server_write_cb write_cb, diff --git a/src/core/lib/support/vector.h b/src/core/lib/support/vector.h index 4a7db80676..2f249a5b9e 100644 --- a/src/core/lib/support/vector.h +++ b/src/core/lib/support/vector.h @@ -19,13 +19,93 @@ #ifndef GRPC_CORE_LIB_SUPPORT_VECTOR_H #define GRPC_CORE_LIB_SUPPORT_VECTOR_H -#include "absl/container/inlined_vector.h" +#include <cassert> + #include "src/core/lib/support/memory.h" namespace grpc_core { +// NOTE: We eventually want to use absl::InlinedVector here. However, +// there are currently build problems that prevent us from using absl. +// In the interim, we define a custom implementation as a place-holder, +// with the intent to eventually replace this with the absl +// implementation. +// +// This place-holder implementation does not implement the full set of +// functionality from the absl version; it has just the methods that we +// currently happen to need in gRPC. If additional functionality is +// needed before this gets replaced with the absl version, it can be +// added, with the following proviso: +// +// ANY METHOD ADDED HERE MUST COMPLY WITH THE INTERFACE IN THE absl +// IMPLEMENTATION! +// +// TODO(ctiller, nnoble, roth): Replace this with absl::InlinedVector +// once we integrate absl into the gRPC build system in a usable way. template <typename T, size_t N> -using InlinedVector = absl::InlinedVector<T, N, Allocator<T>>; +class InlinedVector { + public: + InlinedVector() {} + ~InlinedVector() { + for (size_t i = 0; i < size_ && i < N; ++i) { + T& value = *reinterpret_cast<T*>(inline_ + i); + value.~T(); + } + if (size_ > N) { // Avoid subtracting two signed values. + for (size_t i = 0; i < size_ - N; ++i) { + dynamic_[i].~T(); + } + } + gpr_free(dynamic_); + } + + // For now, we do not support copying. + InlinedVector(const InlinedVector&) = delete; + InlinedVector& operator=(const InlinedVector&) = delete; + + T& operator[](size_t offset) { + assert(offset < size_); + if (offset < N) { + return *reinterpret_cast<T*>(inline_ + offset); + } else { + return dynamic_[offset - N]; + } + } + + template <typename... Args> + void emplace_back(Args&&... args) { + if (size_ < N) { + new (&inline_[size_]) T(std::forward<Args>(args)...); + } else { + if (size_ - N == dynamic_capacity_) { + size_t new_capacity = + dynamic_capacity_ == 0 ? 2 : dynamic_capacity_ * 2; + T* new_dynamic = static_cast<T*>(gpr_malloc(sizeof(T) * new_capacity)); + for (size_t i = 0; i < dynamic_capacity_; ++i) { + new (&new_dynamic[i]) T(std::move(dynamic_[i])); + dynamic_[i].~T(); + } + gpr_free(dynamic_); + dynamic_ = new_dynamic; + dynamic_capacity_ = new_capacity; + } + new (&dynamic_[size_ - N]) T(std::forward<Args>(args)...); + } + ++size_; + } + + void push_back(const T& value) { emplace_back(value); } + + void push_back(T&& value) { emplace_back(std::move(value)); } + + size_t size() const { return size_; } + + private: + typename std::aligned_storage<sizeof(T)>::type inline_[N]; + T* dynamic_ = nullptr; + size_t size_ = 0; + size_t dynamic_capacity_ = 0; +}; } // namespace grpc_core diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs index e39da9c1c2..e7b30cd1e9 100644 --- a/src/csharp/Grpc.Core/Channel.cs +++ b/src/csharp/Grpc.Core/Channel.cs @@ -130,15 +130,8 @@ namespace Grpc.Core // cached handler for watch connectivity state static readonly BatchCompletionDelegate WatchConnectivityStateHandler = (success, ctx, state) => { - var tcs = (TaskCompletionSource<object>) state; - if (success) - { - tcs.SetResult(null); - } - else - { - tcs.SetCanceled(); - } + var tcs = (TaskCompletionSource<bool>) state; + tcs.SetResult(success); }; /// <summary> @@ -146,11 +139,24 @@ namespace Grpc.Core /// given lastObservedState. /// If deadline is reached or and error occurs, returned task is cancelled. /// </summary> - public Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null) + public async Task WaitForStateChangedAsync(ChannelState lastObservedState, DateTime? deadline = null) + { + var result = await WaitForStateChangedInternalAsync(lastObservedState, deadline).ConfigureAwait(false); + if (!result) + { + throw new TaskCanceledException("Reached deadline."); + } + } + + /// <summary> + /// Returned tasks completes once channel state has become different from + /// given lastObservedState (<c>true</c> is returned) or if the wait has timed out (<c>false</c> is returned). + /// </summary> + internal Task<bool> WaitForStateChangedInternalAsync(ChannelState lastObservedState, DateTime? deadline = null) { GrpcPreconditions.CheckArgument(lastObservedState != ChannelState.Shutdown, "Shutdown is a terminal state. No further state changes can occur."); - var tcs = new TaskCompletionSource<object>(); + var tcs = new TaskCompletionSource<bool>(); var deadlineTimespec = deadline.HasValue ? Timespec.FromDateTime(deadline.Value) : Timespec.InfFuture; lock (myLock) { @@ -320,14 +326,8 @@ namespace Grpc.Core } } - try - { - await WaitForStateChangedAsync(lastState, DateTime.UtcNow.AddSeconds(1)).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - // ignore timeout - } + // ignore the result + await WaitForStateChangedInternalAsync(lastState, DateTime.UtcNow.AddSeconds(1)).ConfigureAwait(false); lastState = State; } } diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index 24be042f61..bfc7208310 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -27,7 +27,6 @@ from grpc.framework.foundation import callable_util _USER_AGENT = 'grpc-python/{}'.format(_grpcio_metadata.__version__) _EMPTY_FLAGS = 0 -_INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) _UNARY_UNARY_INITIAL_DUE = ( cygrpc.OperationType.send_initial_metadata, @@ -61,11 +60,7 @@ _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = ( def _deadline(timeout): - if timeout is None: - return None, _INFINITE_FUTURE - else: - deadline = time.time() + timeout - return deadline, cygrpc.Timespec(deadline) + return None if timeout is None else time.time() + timeout def _unknown_code_details(unknown_cygrpc_code, details): @@ -420,15 +415,15 @@ class _Rendezvous(grpc.RpcError, grpc.Future, grpc.Call): def _start_unary_request(request, timeout, request_serializer): - deadline, deadline_timespec = _deadline(timeout) + deadline = _deadline(timeout) serialized_request = _common.serialize(request, request_serializer) if serialized_request is None: state = _RPCState((), (), (), grpc.StatusCode.INTERNAL, 'Exception serializing request!') rendezvous = _Rendezvous(state, None, None, deadline) - return deadline, deadline_timespec, None, rendezvous + return deadline, None, rendezvous else: - return deadline, deadline_timespec, serialized_request, None + return deadline, serialized_request, None def _end_unary_response_blocking(state, call, with_call, deadline): @@ -453,10 +448,10 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): self._response_deserializer = response_deserializer def _prepare(self, request, timeout, metadata): - deadline, deadline_timespec, serialized_request, rendezvous = ( - _start_unary_request(request, timeout, self._request_serializer)) + deadline, serialized_request, rendezvous = (_start_unary_request( + request, timeout, self._request_serializer)) if serialized_request is None: - return None, None, None, None, rendezvous + return None, None, None, rendezvous else: state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None) operations = ( @@ -467,18 +462,17 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): cygrpc.ReceiveMessageOperation(_EMPTY_FLAGS), cygrpc.ReceiveStatusOnClientOperation(_EMPTY_FLAGS), ) - return state, operations, deadline, deadline_timespec, None + return state, operations, deadline, None def _blocking(self, request, timeout, metadata, credentials): - state, operations, deadline, deadline_timespec, rendezvous = self._prepare( + state, operations, deadline, rendezvous = self._prepare( request, timeout, metadata) if rendezvous: raise rendezvous else: completion_queue = cygrpc.CompletionQueue() call = self._channel.create_call(None, 0, completion_queue, - self._method, None, - deadline_timespec) + self._method, None, deadline) if credentials is not None: call.set_credentials(credentials._credentials) call_error = call.start_client_batch(operations, None) @@ -498,13 +492,13 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): return _end_unary_response_blocking(state, call, True, deadline) def future(self, request, timeout=None, metadata=None, credentials=None): - state, operations, deadline, deadline_timespec, rendezvous = self._prepare( + state, operations, deadline, rendezvous = self._prepare( request, timeout, metadata) if rendezvous: return rendezvous else: call, drive_call = self._managed_call(None, 0, self._method, None, - deadline_timespec) + deadline) if credentials is not None: call.set_credentials(credentials._credentials) event_handler = _event_handler(state, call, @@ -530,14 +524,14 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): self._response_deserializer = response_deserializer def __call__(self, request, timeout=None, metadata=None, credentials=None): - deadline, deadline_timespec, serialized_request, rendezvous = ( - _start_unary_request(request, timeout, self._request_serializer)) + deadline, serialized_request, rendezvous = (_start_unary_request( + request, timeout, self._request_serializer)) if serialized_request is None: raise rendezvous else: state = _RPCState(_UNARY_STREAM_INITIAL_DUE, None, None, None, None) call, drive_call = self._managed_call(None, 0, self._method, None, - deadline_timespec) + deadline) if credentials is not None: call.set_credentials(credentials._credentials) event_handler = _event_handler(state, call, @@ -573,11 +567,11 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): self._response_deserializer = response_deserializer def _blocking(self, request_iterator, timeout, metadata, credentials): - deadline, deadline_timespec = _deadline(timeout) + deadline = _deadline(timeout) state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None) completion_queue = cygrpc.CompletionQueue() call = self._channel.create_call(None, 0, completion_queue, - self._method, None, deadline_timespec) + self._method, None, deadline) if credentials is not None: call.set_credentials(credentials._credentials) with state.condition: @@ -624,10 +618,10 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): timeout=None, metadata=None, credentials=None): - deadline, deadline_timespec = _deadline(timeout) + deadline = _deadline(timeout) state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None) call, drive_call = self._managed_call(None, 0, self._method, None, - deadline_timespec) + deadline) if credentials is not None: call.set_credentials(credentials._credentials) event_handler = _event_handler(state, call, self._response_deserializer) @@ -665,10 +659,10 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): timeout=None, metadata=None, credentials=None): - deadline, deadline_timespec = _deadline(timeout) + deadline = _deadline(timeout) state = _RPCState(_STREAM_STREAM_INITIAL_DUE, None, None, None, None) call, drive_call = self._managed_call(None, 0, self._method, None, - deadline_timespec) + deadline) if credentials is not None: call.set_credentials(credentials._credentials) event_handler = _event_handler(state, call, self._response_deserializer) @@ -737,7 +731,8 @@ def _channel_managed_call_management(state): flags: An integer bitfield of call flags. method: The RPC method. host: A host string for the created call. - deadline: A cygrpc.Timespec to be the deadline of the created call. + deadline: A float to be the deadline of the created call or None if the + call is to have an infinite deadline. Returns: A cygrpc.Call with which to conduct an RPC and a function to call if @@ -827,8 +822,8 @@ def _poll_connectivity(state, channel, initial_try_to_connect): completion_queue = cygrpc.CompletionQueue() while True: channel.watch_connectivity_state(connectivity, - cygrpc.Timespec(time.time() + 0.2), - completion_queue, None) + time.time() + 0.2, completion_queue, + None) event = completion_queue.poll() with state.lock: if not state.callbacks_and_connectivities and not state.try_to_connect: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 443d534d7e..efe5f2e0db 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -42,7 +42,7 @@ cdef class Channel: def create_call(self, Call parent, int flags, CompletionQueue queue not None, - method, host, Timespec deadline not None): + method, host, object deadline): if queue.is_shutting_down: raise ValueError("queue must not be shutting down or shutdown") cdef grpc_slice method_slice = _slice_from_bytes(method) @@ -56,14 +56,13 @@ cdef class Channel: cdef grpc_call *parent_call = NULL if parent is not None: parent_call = parent.c_call - with nogil: - operation_call.c_call = grpc_channel_create_call( - self.c_channel, parent_call, flags, - queue.c_completion_queue, method_slice, host_slice_ptr, - deadline.c_time, NULL) - grpc_slice_unref(method_slice) - if host_slice_ptr: - grpc_slice_unref(host_slice) + operation_call.c_call = grpc_channel_create_call( + self.c_channel, parent_call, flags, + queue.c_completion_queue, method_slice, host_slice_ptr, + _timespec_from_time(deadline), NULL) + grpc_slice_unref(method_slice) + if host_slice_ptr: + grpc_slice_unref(host_slice) return operation_call def check_connectivity_state(self, bint try_to_connect): @@ -75,13 +74,12 @@ cdef class Channel: def watch_connectivity_state( self, grpc_connectivity_state last_observed_state, - Timespec deadline not None, CompletionQueue queue not None, tag): + object deadline, CompletionQueue queue not None, tag): cdef _ConnectivityTag connectivity_tag = _ConnectivityTag(tag) cpython.Py_INCREF(connectivity_tag) - with nogil: - grpc_channel_watch_connectivity_state( - self.c_channel, last_observed_state, deadline.c_time, - queue.c_completion_queue, <cpython.PyObject *>connectivity_tag) + grpc_channel_watch_connectivity_state( + self.c_channel, last_observed_state, _timespec_from_time(deadline), + queue.c_completion_queue, <cpython.PyObject *>connectivity_tag) def target(self): cdef char *target = NULL diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index e259789b35..40496d1124 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -52,17 +52,18 @@ cdef class CompletionQueue: cpython.Py_DECREF(tag) return tag.event(event) - def poll(self, Timespec deadline=None): + def poll(self, deadline=None): # We name this 'poll' to avoid problems with CPython's expectations for # 'special' methods (like next and __next__). cdef gpr_timespec c_increment cdef gpr_timespec c_timeout cdef gpr_timespec c_deadline + if deadline is None: + c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME) + else: + c_deadline = _timespec_from_time(deadline) with nogil: c_increment = gpr_time_from_millis(_INTERRUPT_CHECK_PERIOD_MS, GPR_TIMESPAN) - c_deadline = gpr_inf_future(GPR_CLOCK_REALTIME) - if deadline is not None: - c_deadline = deadline.c_time while True: c_timeout = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c_increment) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi index 7b2482d947..297bbadfe0 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi @@ -18,11 +18,6 @@ cdef grpc_slice _copy_slice(grpc_slice slice) nogil cdef grpc_slice _slice_from_bytes(bytes value) nogil -cdef class Timespec: - - cdef gpr_timespec c_time - - cdef class CallDetails: cdef grpc_call_details c_details diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index bc2cd0338e..b2343b53d6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -123,74 +123,6 @@ class CompressionLevel: high = GRPC_COMPRESS_LEVEL_HIGH -cdef class Timespec: - - def __cinit__(self, time): - if time is None: - with nogil: - self.c_time = gpr_now(GPR_CLOCK_REALTIME) - return - if isinstance(time, int): - time = float(time) - if isinstance(time, float): - if time == float("+inf"): - with nogil: - self.c_time = gpr_inf_future(GPR_CLOCK_REALTIME) - elif time == float("-inf"): - with nogil: - self.c_time = gpr_inf_past(GPR_CLOCK_REALTIME) - else: - self.c_time.seconds = time - self.c_time.nanoseconds = (time - float(self.c_time.seconds)) * 1e9 - self.c_time.clock_type = GPR_CLOCK_REALTIME - elif isinstance(time, Timespec): - self.c_time = (<Timespec>time).c_time - else: - raise TypeError("expected time to be float, int, or Timespec, not {}" - .format(type(time))) - - @property - def seconds(self): - # TODO(atash) ensure that everywhere a Timespec is created that it's - # converted to GPR_CLOCK_REALTIME then and not every time someone wants to - # read values off in Python. - cdef gpr_timespec real_time - with nogil: - real_time = ( - gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME)) - return real_time.seconds - - @property - def nanoseconds(self): - cdef gpr_timespec real_time = ( - gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME)) - return real_time.nanoseconds - - def __float__(self): - cdef gpr_timespec real_time = ( - gpr_convert_clock_type(self.c_time, GPR_CLOCK_REALTIME)) - return <double>real_time.seconds + <double>real_time.nanoseconds / 1e9 - - def __richcmp__(Timespec self not None, Timespec other not None, int op): - cdef gpr_timespec self_c_time = self.c_time - cdef gpr_timespec other_c_time = other.c_time - cdef int result = gpr_time_cmp(self_c_time, other_c_time) - if op == 0: # < - return result < 0 - elif op == 2: # == - return result == 0 - elif op == 4: # > - return result > 0 - elif op == 1: # <= - return result <= 0 - elif op == 3: # != - return result != 0 - elif op == 5: # >= - return result >= 0 - else: - raise ValueError('__richcmp__ `op` contract violated') - - cdef class CallDetails: def __cinit__(self): @@ -213,9 +145,7 @@ cdef class CallDetails: @property def deadline(self): - timespec = Timespec(float("-inf")) - timespec.c_time = self.c_details.deadline - return timespec + return _time_from_timespec(self.c_details.deadline) cdef class SslPemKeyCertPair: diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index c19beccde6..e5d28a85d5 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -106,7 +106,7 @@ cdef class Server: with nogil: grpc_server_start(self.c_server) # Ensure the core has gotten a chance to do the start-up work - self.backup_shutdown_queue.poll(Timespec(None)) + self.backup_shutdown_queue.poll(deadline=time.time()) def add_http2_port(self, bytes address, ServerCredentials server_credentials=None): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi new file mode 100644 index 0000000000..ce67c61eaf --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/time.pxd.pxi @@ -0,0 +1,19 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef gpr_timespec _timespec_from_time(object time) + + +cdef double _time_from_timespec(gpr_timespec timespec) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi new file mode 100644 index 0000000000..7a668680b8 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/time.pyx.pxi @@ -0,0 +1,30 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef gpr_timespec _timespec_from_time(object time): + cdef gpr_timespec timespec + if time is None: + return gpr_inf_future(GPR_CLOCK_REALTIME) + else: + timespec.seconds = time + timespec.nanoseconds = (time - float(timespec.seconds)) * 1e9 + timespec.clock_type = GPR_CLOCK_REALTIME + return timespec + + +cdef double _time_from_timespec(gpr_timespec timespec): + cdef gpr_timespec real_timespec = gpr_convert_clock_type( + timespec, GPR_CLOCK_REALTIME) + return <double>real_timespec.seconds + <double>real_timespec.nanoseconds / 1e9 diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pxd b/src/python/grpcio/grpc/_cython/cygrpc.pxd index b32fa518fc..01e2da6d54 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pxd +++ b/src/python/grpcio/grpc/_cython/cygrpc.pxd @@ -25,3 +25,4 @@ include "_cygrpc/records.pxd.pxi" include "_cygrpc/security.pxd.pxi" include "_cygrpc/server.pxd.pxi" include "_cygrpc/tag.pxd.pxi" +include "_cygrpc/time.pxd.pxi" diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index 5106394708..d8ac84a317 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -32,6 +32,7 @@ include "_cygrpc/records.pyx.pxi" include "_cygrpc/security.pyx.pxi" include "_cygrpc/server.pyx.pxi" include "_cygrpc/tag.pyx.pxi" +include "_cygrpc/time.pyx.pxi" # # initialize gRPC diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index a89987a86d..56122fee11 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -220,8 +220,7 @@ class _Context(grpc.ServicerContext): return self._state.client is not _CANCELLED and not self._state.statused def time_remaining(self): - return max( - float(self._rpc_event.call_details.deadline) - time.time(), 0) + return max(self._rpc_event.call_details.deadline - time.time(), 0) def cancel(self): self._rpc_event.call.cancel() diff --git a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py index b81d6fbc61..2ca1fa82f4 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_cancel_many_calls_test.py @@ -20,7 +20,6 @@ from grpc._cython import cygrpc from grpc.framework.foundation import logging_pool from tests.unit.framework.common import test_constants -_INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) _EMPTY_FLAGS = 0 _EMPTY_METADATA = () @@ -171,9 +170,9 @@ class CancelManyCallsTest(unittest.TestCase): with client_condition: client_calls = [] for index in range(test_constants.RPC_CONCURRENCY): - client_call = channel.create_call( - None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', - None, _INFINITE_FUTURE) + client_call = channel.create_call(None, _EMPTY_FLAGS, + client_completion_queue, + b'/twinkies', None, None) operations = ( cygrpc.SendInitialMetadataOperation(_EMPTY_METADATA, _EMPTY_FLAGS), diff --git a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py index 4eeb34b92e..c22c77ddbd 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_channel_test.py @@ -31,9 +31,9 @@ def _connectivity_loop(channel, completion_queue): for _ in range(100): connectivity = channel.check_connectivity_state(True) channel.watch_connectivity_state(connectivity, - cygrpc.Timespec(time.time() + 0.2), - completion_queue, None) - completion_queue.poll(deadline=cygrpc.Timespec(float('+inf'))) + time.time() + 0.2, completion_queue, + None) + completion_queue.poll() def _create_loop_destroy(): diff --git a/src/python/grpcio_tests/tests/unit/_cython/_common.py b/src/python/grpcio_tests/tests/unit/_cython/_common.py index ffd226fa95..d4b01ca38b 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_common.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py @@ -20,7 +20,6 @@ from grpc._cython import cygrpc RPC_COUNT = 4000 -INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) EMPTY_FLAGS = 0 INVOCATION_METADATA = ( diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py index 4ef4ad33e5..7caa98f72d 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py @@ -41,9 +41,9 @@ class Test(_common.RpcTest, unittest.TestCase): server_request_call_tag, }) - client_call = self.channel.create_call( - None, _common.EMPTY_FLAGS, self.client_completion_queue, - b'/twinkies', None, _common.INFINITE_FUTURE) + client_call = self.channel.create_call(None, _common.EMPTY_FLAGS, + self.client_completion_queue, + b'/twinkies', None, None) client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' client_complete_rpc_tag = 'client_complete_rpc_tag' with self.client_condition: diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py index 85395c9680..8582a39c01 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py @@ -36,9 +36,9 @@ class Test(_common.RpcTest, unittest.TestCase): server_request_call_tag, }) - client_call = self.channel.create_call( - None, _common.EMPTY_FLAGS, self.client_completion_queue, - b'/twinkies', None, _common.INFINITE_FUTURE) + client_call = self.channel.create_call(None, _common.EMPTY_FLAGS, + self.client_completion_queue, + b'/twinkies', None, None) client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' client_complete_rpc_tag = 'client_complete_rpc_tag' with self.client_condition: diff --git a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py index 82ef25b2a7..ecd23afda7 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/_read_some_but_not_all_responses_test.py @@ -18,7 +18,6 @@ import unittest from grpc._cython import cygrpc -_INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) _EMPTY_FLAGS = 0 _EMPTY_METADATA = () @@ -156,7 +155,7 @@ class ReadSomeButNotAllResponsesTest(unittest.TestCase): client_call = channel.create_call(None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', - None, _INFINITE_FUTURE) + None, None) client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' client_complete_rpc_tag = 'client_complete_rpc_tag' with client_condition: diff --git a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py index 5f9b74ba98..561adf7dff 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py +++ b/src/python/grpcio_tests/tests/unit/_cython/cygrpc_test.py @@ -37,21 +37,6 @@ def _metadata_plugin(context, callback): class TypeSmokeTest(unittest.TestCase): - def testTimespec(self): - now = time.time() - now_timespec_a = cygrpc.Timespec(now) - now_timespec_b = cygrpc.Timespec(now) - self.assertAlmostEqual(now, float(now_timespec_a), places=8) - self.assertEqual(now_timespec_a, now_timespec_b) - self.assertLess(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) - self.assertGreater(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) - self.assertGreaterEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) - self.assertGreaterEqual(cygrpc.Timespec(now), cygrpc.Timespec(now)) - self.assertLessEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) - self.assertLessEqual(cygrpc.Timespec(now), cygrpc.Timespec(now)) - self.assertNotEqual(cygrpc.Timespec(now - 1), cygrpc.Timespec(now)) - self.assertNotEqual(cygrpc.Timespec(now + 1), cygrpc.Timespec(now)) - def testCompletionQueueUpDown(self): completion_queue = cygrpc.CompletionQueue() del completion_queue @@ -147,7 +132,7 @@ class ServerClientMixin(object): try: call_result = call.start_client_batch(operations, tag) self.assertEqual(cygrpc.CallError.ok, call_result) - event = queue.poll(deadline) + event = queue.poll(deadline=deadline) self.assertEqual(cygrpc.CompletionType.operation_complete, event.completion_type) self.assertTrue(event.success) @@ -176,8 +161,6 @@ class ServerClientMixin(object): RESPONSE = b'his name is robert paulson' METHOD = b'twinkies' - cygrpc_deadline = cygrpc.Timespec(DEADLINE) - server_request_tag = object() request_call_result = self.server.request_call( self.server_completion_queue, self.server_completion_queue, @@ -188,7 +171,7 @@ class ServerClientMixin(object): client_call_tag = object() client_call = self.client_channel.create_call( None, 0, self.client_completion_queue, METHOD, self.host_argument, - cygrpc_deadline) + DEADLINE) client_initial_metadata = ( ( CLIENT_METADATA_ASCII_KEY, @@ -210,9 +193,9 @@ class ServerClientMixin(object): ], client_call_tag) self.assertEqual(cygrpc.CallError.ok, client_start_batch_result) client_event_future = test_utilities.CompletionQueuePollFuture( - self.client_completion_queue, cygrpc_deadline) + self.client_completion_queue, DEADLINE) - request_event = self.server_completion_queue.poll(cygrpc_deadline) + request_event = self.server_completion_queue.poll(deadline=DEADLINE) self.assertEqual(cygrpc.CompletionType.operation_complete, request_event.completion_type) self.assertIsInstance(request_event.call, cygrpc.Call) @@ -223,7 +206,7 @@ class ServerClientMixin(object): self.assertEqual(METHOD, request_event.call_details.method) self.assertEqual(self.expected_host, request_event.call_details.host) self.assertLess( - abs(DEADLINE - float(request_event.call_details.deadline)), + abs(DEADLINE - request_event.call_details.deadline), DEADLINE_TOLERANCE) server_call_tag = object() @@ -248,7 +231,7 @@ class ServerClientMixin(object): ], server_call_tag) self.assertEqual(cygrpc.CallError.ok, server_start_batch_result) - server_event = self.server_completion_queue.poll(cygrpc_deadline) + server_event = self.server_completion_queue.poll(deadline=DEADLINE) client_event = client_event_future.result() self.assertEqual(6, len(client_event.batch_operations)) @@ -310,7 +293,6 @@ class ServerClientMixin(object): DEADLINE_TOLERANCE = 0.25 METHOD = b'twinkies' - cygrpc_deadline = cygrpc.Timespec(DEADLINE) empty_metadata = () server_request_tag = object() @@ -319,26 +301,26 @@ class ServerClientMixin(object): server_request_tag) client_call = self.client_channel.create_call( None, 0, self.client_completion_queue, METHOD, self.host_argument, - cygrpc_deadline) + DEADLINE) # Prologue def perform_client_operations(operations, description): return self._perform_operations(operations, client_call, self.client_completion_queue, - cygrpc_deadline, description) + DEADLINE, description) client_event_future = perform_client_operations([ cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS), cygrpc.ReceiveInitialMetadataOperation(_EMPTY_FLAGS), ], "Client prologue") - request_event = self.server_completion_queue.poll(cygrpc_deadline) + request_event = self.server_completion_queue.poll(deadline=DEADLINE) server_call = request_event.call def perform_server_operations(operations, description): return self._perform_operations(operations, server_call, self.server_completion_queue, - cygrpc_deadline, description) + DEADLINE, description) server_event_future = perform_server_operations([ cygrpc.SendInitialMetadataOperation(empty_metadata, _EMPTY_FLAGS), diff --git a/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py b/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py index 8e91161f80..4a00b9ef2f 100644 --- a/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py +++ b/src/python/grpcio_tests/tests/unit/_cython/test_utilities.py @@ -49,4 +49,4 @@ class CompletionQueuePollFuture(SimpleFuture): def __init__(self, completion_queue, deadline): super(CompletionQueuePollFuture, - self).__init__(lambda: completion_queue.poll(deadline)) + self).__init__(lambda: completion_queue.poll(deadline=deadline)) diff --git a/summerofcode/ideas.md b/summerofcode/ideas.md index d87cf1b8fa..49991f0c75 100644 --- a/summerofcode/ideas.md +++ b/summerofcode/ideas.md @@ -19,15 +19,8 @@ gRPC C Core: 1. Port gRPC to one of the major BSD platforms ([FreeBSD](https://freebsd.org), [NetBSD](https://netbsd.org), and [OpenBSD](https://openbsd.org)) and create packages for them. Add [kqueue](https://www.freebsd.org/cgi/man.cgi?query=kqueue) support in the process. * **Required skills:** C programming language, BSD operating system. - * **Likely mentors:** [Craig Tiller](https://github.com/ctiller), - [Nicolas Noble](https://github.com/nicolasnoble), + * **Likely mentors:** [Nicolas Noble](https://github.com/nicolasnoble), [Vijay Pai](https://github.com/vjpai). -1. Fix gRPC C-core's URI parser. The current parser does not qualify as a standard parser according to [RFC3986]( https://tools.ietf.org/html/rfc3986). Write test suites to verify this and make changes necessary to make the URI parser compliant. - * **Required skills:** C programming language, HTTP standard compliance. - * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). -1. HPACK compression efficiency evaluation - Figure out how to benchmark gRPC's compression efficiency (both in terms of bytes on the wire and cpu cycles). Implement benchmarks. Potentially extend this to other full-stack gRPC implementations (Java and Go). - * **Required skills:** C programming language, software performance benchmarking, potentially Java and Go. - * **Likely mentors:** [Craig Tiller](https://github.com/ctiller). gRPC Python: @@ -38,7 +31,7 @@ gRPC Python: 1. Develop and test Python 3.5 Support for gRPC. Make necessary changes to port gRPC and package it for supported platforms. * **Required skills:** Python programming language, Python 3.5 interpreter. * **Likely mentors:** [Nathaniel Manista](https://github.com/nathanielmanistaatgoogle), [Masood Malekghassemi](https://github.com/soltanmm). - + gRPC Ruby/Java: 1. [jRuby](http://jruby.org) support for gRPC. Develop a jRuby wrapper for gRPC based on grpc-java and ensure that it is API compatible with the existing Ruby implementation and passes all tests. diff --git a/templates/Makefile.template b/templates/Makefile.template index 954bea7e17..b8e26b646b 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -563,7 +563,6 @@ ZLIB_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libz.a ZLIB_MERGE_OBJS = $(LIBZ_OBJS) CPPFLAGS += -Ithird_party/zlib - LDFLAGS += -L$(LIBDIR)/$(CONFIG)/zlib else ifeq ($(HAS_PKG_CONFIG),true) CPPFLAGS += $(shell $(PKG_CONFIG) --cflags zlib) @@ -594,7 +593,6 @@ CARES_MERGE_OBJS = $(LIBARES_OBJS) CARES_MERGE_LIBS = $(LIBDIR)/$(CONFIG)/libares.a CPPFLAGS := -Ithird_party/cares -Ithird_party/cares/cares $(CPPFLAGS) - LDFLAGS := -L$(LIBDIR)/$(CONFIG)/c-ares $(LDFLAGS) else ifeq ($(HAS_PKG_CONFIG),true) PC_REQUIRES_GRPC += libcares @@ -896,10 +894,10 @@ $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure $(E) "[MAKE] Building protobuf" + $(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf $(Q)(cd third_party/protobuf ; CC="$(CC)" CXX="$(CXX)" LDFLAGS="$(LDFLAGS_$(CONFIG)) -g $(PROTOBUF_LDFLAGS_EXTRA)" CPPFLAGS="$(PIC_CPPFLAGS) $(CPPFLAGS_$(CONFIG)) -g $(PROTOBUF_CPPFLAGS_EXTRA)" ./configure --disable-shared --enable-static $(PROTOBUF_CONFIG_OPTS)) $(Q)$(MAKE) -C third_party/protobuf clean $(Q)$(MAKE) -C third_party/protobuf - $(Q)mkdir -p $(LIBDIR)/$(CONFIG)/protobuf $(Q)mkdir -p $(BINDIR)/$(CONFIG)/protobuf $(Q)cp third_party/protobuf/src/.libs/libprotoc.a $(LIBDIR)/$(CONFIG)/protobuf $(Q)cp third_party/protobuf/src/.libs/libprotobuf.a $(LIBDIR)/$(CONFIG)/protobuf diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc index 261ddd93ec..61c658b95a 100644 --- a/test/core/end2end/tests/filter_status_code.cc +++ b/test/core/end2end/tests/filter_status_code.cc @@ -30,11 +30,14 @@ #include <grpc/support/useful.h> #include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" #include "test/core/end2end/cq_verifier.h" static bool g_enable_filter = false; static gpr_mu g_mu; +static grpc_call_stack* g_client_call_stack; +static grpc_call_stack* g_server_call_stack; static bool g_client_code_recv; static bool g_server_code_recv; static gpr_cv g_client_code_cv; @@ -117,6 +120,8 @@ static void test_request(grpc_end2end_test_config config) { int was_cancelled = 2; gpr_mu_lock(&g_mu); + g_client_call_stack = nullptr; + g_server_call_stack = nullptr; g_client_status_code = GRPC_STATUS_OK; g_server_status_code = GRPC_STATUS_OK; gpr_mu_unlock(&g_mu); @@ -127,6 +132,9 @@ static void test_request(grpc_end2end_test_config config) { grpc_slice_from_static_string("/foo"), get_host_override_slice("foo.test.google.fr", config), deadline, nullptr); GPR_ASSERT(c); + gpr_mu_lock(&g_mu); + g_client_call_stack = grpc_call_get_call_stack(c); + gpr_mu_unlock(&g_mu); grpc_metadata_array_init(&initial_metadata_recv); grpc_metadata_array_init(&trailing_metadata_recv); @@ -168,6 +176,10 @@ static void test_request(grpc_end2end_test_config config) { CQ_EXPECT_COMPLETION(cqv, tag(101), 1); cq_verify(cqv); + gpr_mu_lock(&g_mu); + g_server_call_stack = grpc_call_get_call_stack(s); + gpr_mu_unlock(&g_mu); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -215,49 +227,62 @@ static void test_request(grpc_end2end_test_config config) { // Perform checks after test tear-down // Guards against the case that there's outstanding channel-related work on a // call prior to verification - // TODO(https://github.com/grpc/grpc/issues/13915) enable this for windows -#ifndef GPR_WINDOWS gpr_mu_lock(&g_mu); if (!g_client_code_recv) { GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu, - grpc_timeout_seconds_to_deadline(3))); + grpc_timeout_seconds_to_deadline(3)) == 0); } if (!g_server_code_recv) { - GPR_ASSERT(gpr_cv_wait(&g_client_code_cv, &g_mu, - grpc_timeout_seconds_to_deadline(3))); + GPR_ASSERT(gpr_cv_wait(&g_server_code_cv, &g_mu, + grpc_timeout_seconds_to_deadline(3)) == 0); } GPR_ASSERT(g_client_status_code == GRPC_STATUS_UNIMPLEMENTED); GPR_ASSERT(g_server_status_code == GRPC_STATUS_UNIMPLEMENTED); gpr_mu_unlock(&g_mu); -#endif // GPR_WINDOWS } /******************************************************************************* * Test status_code filter */ +typedef struct final_status_data { + grpc_call_stack* call; +} final_status_data; + static grpc_error* init_call_elem(grpc_call_element* elem, const grpc_call_element_args* args) { + final_status_data* data = (final_status_data*)elem->call_data; + data->call = args->call_stack; return GRPC_ERROR_NONE; } static void client_destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, grpc_closure* ignored) { + final_status_data* data = (final_status_data*)elem->call_data; gpr_mu_lock(&g_mu); - g_client_status_code = final_info->final_status; - g_client_code_recv = true; - gpr_cv_signal(&g_client_code_cv); + // Some fixtures, like proxies, will spawn intermidiate calls + // We only want the results from our explicit calls + if (data->call == g_client_call_stack) { + g_client_status_code = final_info->final_status; + g_client_code_recv = true; + gpr_cv_signal(&g_client_code_cv); + } gpr_mu_unlock(&g_mu); } static void server_destroy_call_elem(grpc_call_element* elem, const grpc_call_final_info* final_info, grpc_closure* ignored) { + final_status_data* data = (final_status_data*)elem->call_data; gpr_mu_lock(&g_mu); - g_server_status_code = final_info->final_status; - g_server_code_recv = true; - gpr_cv_signal(&g_server_code_cv); + // Some fixtures, like proxies, will spawn intermidiate calls + // We only want the results from our explicit calls + if (data->call == g_server_call_stack) { + g_server_status_code = final_info->final_status; + g_server_code_recv = true; + gpr_cv_signal(&g_server_code_cv); + } gpr_mu_unlock(&g_mu); } @@ -271,7 +296,7 @@ static void destroy_channel_elem(grpc_channel_element* elem) {} static const grpc_channel_filter test_client_filter = { grpc_call_next_op, grpc_channel_next_op, - 0, + sizeof(final_status_data), init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, client_destroy_call_elem, @@ -284,7 +309,7 @@ static const grpc_channel_filter test_client_filter = { static const grpc_channel_filter test_server_filter = { grpc_call_next_op, grpc_channel_next_op, - 0, + sizeof(final_status_data), init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, server_destroy_call_elem, diff --git a/test/core/fling/client.cc b/test/core/fling/client.cc index 69fb6dc7c7..28e62e0e83 100644 --- a/test/core/fling/client.cc +++ b/test/core/fling/client.cc @@ -186,8 +186,10 @@ int main(int argc, char** argv) { } if (!sc.name) { fprintf(stderr, "unsupported scenario '%s'. Valid are:", scenario_name); + fflush(stderr); for (i = 0; i < GPR_ARRAY_SIZE(scenarios); i++) { fprintf(stderr, " %s", scenarios[i].name); + fflush(stderr); } return 1; } diff --git a/test/core/iomgr/udp_server_test.cc b/test/core/iomgr/udp_server_test.cc index dc1248bc1c..09f0283013 100644 --- a/test/core/iomgr/udp_server_test.cc +++ b/test/core/iomgr/udp_server_test.cc @@ -51,6 +51,9 @@ static int g_number_of_bytes_read = 0; static int g_number_of_orphan_calls = 0; static int g_number_of_starts = 0; +int rcv_buf_size = 1024; +int snd_buf_size = 1024; + static void on_start(grpc_fd* emfd, void* user_data) { g_number_of_starts++; } static bool on_read(grpc_fd* emfd) { @@ -177,8 +180,9 @@ static void test_no_op_with_port(void) { memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_in); addr->sin_family = AF_INET; - GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read, - on_write, on_fd_orphaned)); + GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, + snd_buf_size, on_start, on_read, on_write, + on_fd_orphaned)); grpc_udp_server_destroy(s, nullptr); @@ -207,8 +211,9 @@ static void test_no_op_with_port_and_socket_factory(void) { memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_in); addr->sin_family = AF_INET; - GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read, - on_write, on_fd_orphaned)); + GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, + snd_buf_size, on_start, on_read, on_write, + on_fd_orphaned)); GPR_ASSERT(socket_factory->number_of_socket_calls == 1); GPR_ASSERT(socket_factory->number_of_bind_calls == 1); @@ -233,8 +238,9 @@ static void test_no_op_with_port_and_start(void) { memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_in); addr->sin_family = AF_INET; - GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read, - on_write, on_fd_orphaned)); + GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, + snd_buf_size, on_start, on_read, on_write, + on_fd_orphaned)); grpc_udp_server_start(s, nullptr, 0, nullptr); GPR_ASSERT(g_number_of_starts == 1); @@ -265,8 +271,9 @@ static void test_receive(int number_of_clients) { memset(&resolved_addr, 0, sizeof(resolved_addr)); resolved_addr.len = sizeof(struct sockaddr_storage); addr->ss_family = AF_INET; - GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, on_start, on_read, - on_write, on_fd_orphaned)); + GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, + snd_buf_size, on_start, on_read, on_write, + on_fd_orphaned)); svrfd = grpc_udp_server_get_fd(s, 0); GPR_ASSERT(svrfd >= 0); diff --git a/test/core/network_benchmarks/low_level_ping_pong.cc b/test/core/network_benchmarks/low_level_ping_pong.cc index 96b0745f52..fb982a10fd 100644 --- a/test/core/network_benchmarks/low_level_ping_pong.cc +++ b/test/core/network_benchmarks/low_level_ping_pong.cc @@ -535,6 +535,7 @@ void print_usage(char* argv0) { fprintf(stderr, " tcp: fds are endpoints of a TCP connection\n"); fprintf(stderr, " socketpair: fds come from socketpair()\n"); fprintf(stderr, " pipe: fds come from pipe()\n"); + fflush(stderr); } typedef struct test_strategy { @@ -565,6 +566,7 @@ int create_socket(const char* socket_type, fd_pair* client_fds, create_sockets_pipe(client_fds, server_fds); } else { fprintf(stderr, "Invalid socket type %s\n", socket_type); + fflush(stderr); return -1; } return 0; @@ -657,6 +659,7 @@ int main(int argc, char** argv) { } if (msg_size <= 0) { fprintf(stderr, "msg_size must be > 0\n"); + fflush(stderr); print_usage(argv[0]); return -1; } @@ -668,6 +671,7 @@ int main(int argc, char** argv) { } if (strategy == nullptr) { fprintf(stderr, "Invalid read strategy %s\n", read_strategy); + fflush(stderr); return -1; } diff --git a/test/core/security/create_jwt.cc b/test/core/security/create_jwt.cc index 867a8ba575..56ae9c891c 100644 --- a/test/core/security/create_jwt.cc +++ b/test/core/security/create_jwt.cc @@ -39,6 +39,7 @@ void create_jwt(const char* json_key_file_path, const char* service_url, grpc_slice_unref(json_key_data); if (!grpc_auth_json_key_is_valid(&key)) { fprintf(stderr, "Could not parse json key.\n"); + fflush(stderr); exit(1); } jwt = grpc_jwt_encode_and_sign( @@ -47,6 +48,7 @@ void create_jwt(const char* json_key_file_path, const char* service_url, grpc_auth_json_key_destruct(&key); if (jwt == nullptr) { fprintf(stderr, "Could not create JWT.\n"); + fflush(stderr); exit(1); } fprintf(stdout, "%s\n", jwt); @@ -72,16 +74,19 @@ int main(int argc, char** argv) { if (json_key_file_path == nullptr) { fprintf(stderr, "Missing --json_key option.\n"); + fflush(stderr); exit(1); } if (scope != nullptr) { if (service_url != nullptr) { fprintf(stderr, "Options --scope and --service_url are mutually exclusive.\n"); + fflush(stderr); exit(1); } } else if (service_url == nullptr) { fprintf(stderr, "Need one of --service_url or --scope options.\n"); + fflush(stderr); exit(1); } diff --git a/test/core/security/print_google_default_creds_token.cc b/test/core/security/print_google_default_creds_token.cc index b3742f58a8..d71116d8f6 100644 --- a/test/core/security/print_google_default_creds_token.cc +++ b/test/core/security/print_google_default_creds_token.cc @@ -45,6 +45,7 @@ static void on_metadata_response(void* arg, grpc_error* error) { synchronizer* sync = static_cast<synchronizer*>(arg); if (error != GRPC_ERROR_NONE) { fprintf(stderr, "Fetching token failed: %s\n", grpc_error_string(error)); + fflush(stderr); } else { char* token; GPR_ASSERT(sync->md_array.size == 1); @@ -81,6 +82,7 @@ int main(int argc, char** argv) { creds = grpc_google_default_credentials_create(); if (creds == nullptr) { fprintf(stderr, "\nCould not find default credentials.\n\n"); + fflush(stderr); result = 1; goto end; } diff --git a/test/core/security/verify_jwt.cc b/test/core/security/verify_jwt.cc index e039970c67..5d32ce0cdb 100644 --- a/test/core/security/verify_jwt.cc +++ b/test/core/security/verify_jwt.cc @@ -39,6 +39,7 @@ typedef struct { static void print_usage_and_exit(gpr_cmdline* cl, const char* argv0) { char* usage = gpr_cmdline_usage_string(cl, argv0); fprintf(stderr, "%s", usage); + fflush(stderr); gpr_free(usage); gpr_cmdline_destroy(cl); exit(1); @@ -62,6 +63,7 @@ static void on_jwt_verification_done(void* user_data, GPR_ASSERT(claims == nullptr); fprintf(stderr, "Verification failed with error %s\n", grpc_jwt_verifier_status_to_string(status)); + fflush(stderr); } gpr_mu_lock(sync->mu); diff --git a/test/core/support/cpu_test.cc b/test/core/support/cpu_test.cc index 334c4318e1..87cdc0fb50 100644 --- a/test/core/support/cpu_test.cc +++ b/test/core/support/cpu_test.cc @@ -119,13 +119,16 @@ static void cpu_test(void) { } gpr_mu_unlock(&ct.mu); fprintf(stderr, "Saw cores ["); + fflush(stderr); for (i = 0; i < ct.ncores; i++) { if (ct.used[i]) { fprintf(stderr, "%d,", i); + fflush(stderr); cores_seen++; } } fprintf(stderr, "] (%d/%d)\n", cores_seen, ct.ncores); + fflush(stderr); gpr_free(ct.used); } diff --git a/test/core/support/spinlock_test.cc b/test/core/support/spinlock_test.cc index 58d5fcd42b..ea0dbbf7c6 100644 --- a/test/core/support/spinlock_test.cc +++ b/test/core/support/spinlock_test.cc @@ -95,15 +95,18 @@ static void test(const char* name, void (*body)(void* m), int timeout_s, gpr_timespec deadline = gpr_time_add( start, gpr_time_from_micros((int64_t)timeout_s * 1000000, GPR_TIMESPAN)); fprintf(stderr, "%s:", name); + fflush(stderr); while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) { if (iterations < INT64_MAX / 2) iterations <<= 1; fprintf(stderr, " %ld", (long)iterations); + fflush(stderr); m = test_new(10, iterations, incr_step); test_create_threads(m, body); test_wait(m); if (m->counter != m->thread_count * m->iterations * m->incr_step) { fprintf(stderr, "counter %ld threads %d iterations %ld\n", (long)m->counter, m->thread_count, (long)m->iterations); + fflush(stderr); GPR_ASSERT(0); } test_destroy(m); @@ -111,6 +114,7 @@ static void test(const char* name, void (*body)(void* m), int timeout_s, time_taken = gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start); fprintf(stderr, " done %lld.%09d s\n", (long long)time_taken.tv_sec, (int)time_taken.tv_nsec); + fflush(stderr); } /* Increment m->counter on each iteration; then mark thread as done. */ diff --git a/test/core/support/sync_test.cc b/test/core/support/sync_test.cc index fb7ec44754..768f96d093 100644 --- a/test/core/support/sync_test.cc +++ b/test/core/support/sync_test.cc @@ -231,16 +231,17 @@ static void mark_thread_done(struct test* m) { */ static void test(const char* name, void (*body)(void* m), void (*extra)(void* m), int timeout_s, int incr_step) { - int64_t iterations = 1024; + int64_t iterations = 256; struct test* m; gpr_timespec start = gpr_now(GPR_CLOCK_REALTIME); gpr_timespec time_taken; gpr_timespec deadline = gpr_time_add( start, gpr_time_from_micros((int64_t)timeout_s * 1000000, GPR_TIMESPAN)); fprintf(stderr, "%s:", name); + fflush(stderr); while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) { - iterations <<= 1; fprintf(stderr, " %ld", (long)iterations); + fflush(stderr); m = test_new(10, iterations, incr_step); if (extra != nullptr) { gpr_thd_id id; @@ -252,13 +253,16 @@ static void test(const char* name, void (*body)(void* m), if (m->counter != m->threads * m->iterations * m->incr_step) { fprintf(stderr, "counter %ld threads %d iterations %ld\n", (long)m->counter, m->threads, (long)m->iterations); + fflush(stderr); GPR_ASSERT(0); } test_destroy(m); + iterations <<= 1; } time_taken = gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start); fprintf(stderr, " done %lld.%09d s\n", (long long)time_taken.tv_sec, (int)time_taken.tv_nsec); + fflush(stderr); } /* Increment m->counter on each iteration; then mark thread as done. */ diff --git a/test/core/support/time_test.cc b/test/core/support/time_test.cc index 608169274f..b2b4dce58e 100644 --- a/test/core/support/time_test.cc +++ b/test/core/support/time_test.cc @@ -66,21 +66,28 @@ static void test_values(void) { x = gpr_inf_future(GPR_CLOCK_REALTIME); fprintf(stderr, "far future "); + fflush(stderr); i_to_s(x.tv_sec, 16, 16, &to_fp, stderr); fprintf(stderr, "\n"); GPR_ASSERT(x.tv_sec == INT64_MAX); fprintf(stderr, "far future "); + fflush(stderr); ts_to_s(x, &to_fp, stderr); fprintf(stderr, "\n"); + fflush(stderr); x = gpr_inf_past(GPR_CLOCK_REALTIME); fprintf(stderr, "far past "); + fflush(stderr); i_to_s(x.tv_sec, 16, 16, &to_fp, stderr); fprintf(stderr, "\n"); + fflush(stderr); GPR_ASSERT(x.tv_sec == INT64_MIN); fprintf(stderr, "far past "); + fflush(stderr); ts_to_s(x, &to_fp, stderr); fprintf(stderr, "\n"); + fflush(stderr); for (i = 1; i != 1000 * 1000 * 1000; i *= 10) { x = gpr_time_from_micros(i, GPR_TIMESPAN); @@ -135,15 +142,19 @@ static void test_add_sub(void) { if (gpr_time_cmp(gpr_time_from_micros(sum * k, GPR_TIMESPAN), sumt) != 0) { fprintf(stderr, "i %d j %d sum %d sumt ", i, j, sum); + fflush(stderr); ts_to_s(sumt, &to_fp, stderr); fprintf(stderr, "\n"); + fflush(stderr); GPR_ASSERT(0); } if (gpr_time_cmp(gpr_time_from_micros(diff * k, GPR_TIMESPAN), difft) != 0) { fprintf(stderr, "i %d j %d diff %d diff ", i, j, diff); + fflush(stderr); ts_to_s(sumt, &to_fp, stderr); fprintf(stderr, "\n"); + fflush(stderr); GPR_ASSERT(0); } } diff --git a/test/core/support/vector_test.cc b/test/core/support/vector_test.cc index aad9f3be90..82607a1b26 100644 --- a/test/core/support/vector_test.cc +++ b/test/core/support/vector_test.cc @@ -18,18 +18,50 @@ #include "src/core/lib/support/vector.h" #include <gtest/gtest.h> +#include "src/core/lib/support/memory.h" #include "test/core/util/test_config.h" namespace grpc_core { namespace testing { TEST(InlinedVectorTest, CreateAndIterate) { - InlinedVector<int, 1> v{1, 2, 3}; - int sum = 0; - for (auto i : v) { - sum += i; + const int kNumElements = 9; + InlinedVector<int, 2> v; + for (int i = 0; i < kNumElements; ++i) { + v.push_back(i); } - EXPECT_EQ(6, sum); + EXPECT_EQ(static_cast<size_t>(kNumElements), v.size()); + for (int i = 0; i < kNumElements; ++i) { + EXPECT_EQ(i, v[i]); + } +} + +TEST(InlinedVectorTest, ValuesAreInlined) { + const int kNumElements = 5; + InlinedVector<int, 10> v; + for (int i = 0; i < kNumElements; ++i) { + v.push_back(i); + } + EXPECT_EQ(static_cast<size_t>(kNumElements), v.size()); + for (int i = 0; i < kNumElements; ++i) { + EXPECT_EQ(i, v[i]); + } +} + +TEST(InlinedVectorTest, PushBackWithMove) { + InlinedVector<UniquePtr<int>, 1> v; + UniquePtr<int> i = MakeUnique<int>(3); + v.push_back(std::move(i)); + EXPECT_EQ(nullptr, i.get()); + EXPECT_EQ(1UL, v.size()); + EXPECT_EQ(3, *v[0]); +} + +TEST(InlinedVectorTest, EmplaceBack) { + InlinedVector<UniquePtr<int>, 1> v; + v.emplace_back(New<int>(3)); + EXPECT_EQ(1UL, v.size()); + EXPECT_EQ(3, *v[0]); } } // namespace testing diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 8d2384ba61..7fd36a241a 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -29,7 +29,6 @@ #include <grpc/impl/codegen/byte_buffer_reader.h> #include <grpc/impl/codegen/compression_types.h> #include <grpc/impl/codegen/connectivity_state.h> -#include <grpc/impl/codegen/exec_ctx_fwd.h> #include <grpc/impl/codegen/fork.h> #include <grpc/impl/codegen/gpr_slice.h> #include <grpc/impl/codegen/gpr_types.h> diff --git a/test/core/util/debugger_macros.cc b/test/core/util/debugger_macros.cc index f1e4ffd3af..bb96fc7054 100644 --- a/test/core/util/debugger_macros.cc +++ b/test/core/util/debugger_macros.cc @@ -39,6 +39,7 @@ grpc_stream* grpc_transport_stream_from_call(grpc_call* call) { grpc_subchannel_call* scc = grpc_client_channel_get_subchannel_call(el); if (scc == nullptr) { fprintf(stderr, "No subchannel-call"); + fflush(stderr); return nullptr; } cs = grpc_subchannel_call_get_call_stack(scc); @@ -46,6 +47,7 @@ grpc_stream* grpc_transport_stream_from_call(grpc_call* call) { return grpc_connected_channel_get_stream(el); } else { fprintf(stderr, "Unrecognized filter: %s", el->filter->name); + fflush(stderr); return nullptr; } } diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index e62278cb9f..5bdbcd71f5 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -887,7 +887,6 @@ include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/gpr_slice.h \ include/grpc/impl/codegen/gpr_types.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index d09b325c97..d9184f49a2 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -888,7 +888,6 @@ include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/gpr_slice.h \ include/grpc/impl/codegen/gpr_types.h \ @@ -1047,6 +1046,7 @@ src/core/lib/support/string_windows.h \ src/core/lib/support/thd_internal.h \ src/core/lib/support/time_precise.h \ src/core/lib/support/tmpfile.h \ +src/core/lib/support/vector.h \ src/core/lib/surface/alarm_internal.h \ src/core/lib/surface/api_trace.h \ src/core/lib/surface/call.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 6ce9041747..916d3b1e49 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -818,7 +818,6 @@ include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/gpr_slice.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1aff0075a6..3d3c6711d0 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -818,7 +818,6 @@ include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ -include/grpc/impl/codegen/exec_ctx_fwd.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/fork.h \ include/grpc/impl/codegen/gpr_slice.h \ @@ -1333,6 +1332,7 @@ src/core/lib/support/tmpfile.h \ src/core/lib/support/tmpfile_msys.cc \ src/core/lib/support/tmpfile_posix.cc \ src/core/lib/support/tmpfile_windows.cc \ +src/core/lib/support/vector.h \ src/core/lib/support/wrap_memcpy.cc \ src/core/lib/surface/README.md \ src/core/lib/surface/alarm.cc \ diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 71d3a79023..dbbf8a5a09 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -77,6 +77,9 @@ LANG_RELEASE_MATRIX = { { 'v1.7.2': None }, + { + 'v1.8.0': None + }, ], 'go': [ { @@ -103,6 +106,9 @@ LANG_RELEASE_MATRIX = { { 'v1.8.2': None }, + { + 'v1.9.1': None + }, ], 'java': [ { @@ -132,6 +138,9 @@ LANG_RELEASE_MATRIX = { { 'v1.8.0': None }, + { + 'v1.9.0': None + }, ], 'python': [ { @@ -155,6 +164,9 @@ LANG_RELEASE_MATRIX = { { 'v1.7.2': None }, + { + 'v1.8.1': None # first python 1.8 release is 1.8.1 + }, ], 'node': [ { @@ -204,6 +216,9 @@ LANG_RELEASE_MATRIX = { { 'v1.7.2': None }, + { + 'v1.8.0': None + }, ], 'php': [ { @@ -227,6 +242,9 @@ LANG_RELEASE_MATRIX = { { 'v1.7.2': None }, + { + 'v1.8.0': None + }, ], 'csharp': [ #{'v1.0.1': None}, @@ -248,5 +266,8 @@ LANG_RELEASE_MATRIX = { { 'v1.7.2': None }, + { + 'v1.8.0': None + }, ], } diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index d432bd0e53..51f0ac7ca6 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -4268,6 +4268,25 @@ "gpr_test_util", "grpc", "grpc++", + "grpc++_test", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "vector_test", + "src": [ + "test/core/support/vector_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", "grpc++_test_util", "grpc_test_util" ], @@ -8250,6 +8269,7 @@ "src/core/lib/support/debug_location.h", "src/core/lib/support/ref_counted.h", "src/core/lib/support/ref_counted_ptr.h", + "src/core/lib/support/vector.h", "src/core/lib/surface/alarm_internal.h", "src/core/lib/surface/api_trace.h", "src/core/lib/surface/call.h", @@ -8389,6 +8409,7 @@ "src/core/lib/support/debug_location.h", "src/core/lib/support/ref_counted.h", "src/core/lib/support/ref_counted_ptr.h", + "src/core/lib/support/vector.h", "src/core/lib/surface/alarm_internal.h", "src/core/lib/surface/api_trace.h", "src/core/lib/surface/call.h", @@ -8505,7 +8526,6 @@ "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/exec_ctx_fwd.h", "include/grpc/impl/codegen/grpc_types.h", "include/grpc/impl/codegen/propagation_bits.h", "include/grpc/impl/codegen/slice.h", @@ -8519,7 +8539,6 @@ "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", - "include/grpc/impl/codegen/exec_ctx_fwd.h", "include/grpc/impl/codegen/grpc_types.h", "include/grpc/impl/codegen/propagation_bits.h", "include/grpc/impl/codegen/slice.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 98517cba2e..6b83cecd41 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4510,6 +4510,30 @@ "ci_platforms": [ "linux", "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "vector_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", "posix" ], "cpu_cost": 0.5, diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index 6a704eb2e0..57ae5c5acc 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -69,13 +69,10 @@ target_headers_transitive = get_headers_transitive() def target_has_header(target, name): - if name.startswith('absl/'): return True - # print target['name'], name - if name in target['headers']: + if name in target_headers_transitive[target['name']]: + return True + if name.startswith('absl/'): return True - for dep in target['deps']: - if target_has_header(get_target(dep), name): - return True if name in [ 'src/core/lib/profiling/stap_probes.h', 'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h' |