diff options
115 files changed, 1852 insertions, 673 deletions
@@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "glider" +g_stands_for = "gao" -core_version = "6.0.0-pre1" +core_version = "6.0.0-dev" -version = "1.15.0-pre1" +version = "1.16.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", @@ -696,6 +696,7 @@ grpc_cc_library( "src/core/lib/http/format_request.cc", "src/core/lib/http/httpcli.cc", "src/core/lib/http/parser.cc", + "src/core/lib/iomgr/buffer_list.cc", "src/core/lib/iomgr/call_combiner.cc", "src/core/lib/iomgr/combiner.cc", "src/core/lib/iomgr/endpoint.cc", @@ -716,6 +717,7 @@ grpc_cc_library( "src/core/lib/iomgr/gethostname_fallback.cc", "src/core/lib/iomgr/gethostname_host_name_max.cc", "src/core/lib/iomgr/gethostname_sysconf.cc", + "src/core/lib/iomgr/internal_errqueue.cc", "src/core/lib/iomgr/iocp_windows.cc", "src/core/lib/iomgr/iomgr.cc", "src/core/lib/iomgr/iomgr_custom.cc", @@ -845,6 +847,7 @@ grpc_cc_library( "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/block_annotate.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", @@ -862,6 +865,7 @@ grpc_cc_library( "src/core/lib/iomgr/executor.h", "src/core/lib/iomgr/gethostname.h", "src/core/lib/iomgr/gevent_util.h", + "src/core/lib/iomgr/internal_errqueue.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_custom.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index acd17ebdc8..a21bb8b5fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.15.0-pre1") +set(PACKAGE_VERSION "1.16.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") @@ -230,6 +230,9 @@ add_dependencies(buildtests_c avl_test) add_dependencies(buildtests_c bad_server_response_test) add_dependencies(buildtests_c bin_decoder_test) add_dependencies(buildtests_c bin_encoder_test) +if(_gRPC_PLATFORM_LINUX) +add_dependencies(buildtests_c buffer_list_test) +endif() add_dependencies(buildtests_c channel_create_test) add_dependencies(buildtests_c chttp2_hpack_encoder_test) add_dependencies(buildtests_c chttp2_stream_map_test) @@ -959,6 +962,7 @@ add_library(grpc src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -979,6 +983,7 @@ add_library(grpc src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -1365,6 +1370,7 @@ add_library(grpc_cronet src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -1385,6 +1391,7 @@ add_library(grpc_cronet src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -1757,6 +1764,7 @@ add_library(grpc_test_util src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -1777,6 +1785,7 @@ add_library(grpc_test_util src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -2065,6 +2074,7 @@ add_library(grpc_test_util_unsecure src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -2085,6 +2095,7 @@ add_library(grpc_test_util_unsecure src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -2352,6 +2363,7 @@ add_library(grpc_unsecure src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -2372,6 +2384,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -3192,6 +3205,7 @@ add_library(grpc++_cronet src/core/lib/http/format_request.cc src/core/lib/http/httpcli.cc src/core/lib/http/parser.cc + src/core/lib/iomgr/buffer_list.cc src/core/lib/iomgr/call_combiner.cc src/core/lib/iomgr/combiner.cc src/core/lib/iomgr/endpoint.cc @@ -3212,6 +3226,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/gethostname_fallback.cc src/core/lib/iomgr/gethostname_host_name_max.cc src/core/lib/iomgr/gethostname_sysconf.cc + src/core/lib/iomgr/internal_errqueue.cc src/core/lib/iomgr/iocp_windows.cc src/core/lib/iomgr/iomgr.cc src/core/lib/iomgr/iomgr_custom.cc @@ -5837,6 +5852,37 @@ target_link_libraries(bin_encoder_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX) + +add_executable(buffer_list_test + test/core/iomgr/buffer_list_test.cc +) + + +target_include_directories(buffer_list_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} +) + +target_link_libraries(buffer_list_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) add_executable(channel_create_test test/core/surface/channel_create_test.cc @@ -436,9 +436,9 @@ E = @echo Q = @ endif -CORE_VERSION = 6.0.0-pre1 -CPP_VERSION = 1.15.0-pre1 -CSHARP_VERSION = 1.15.0-pre1 +CORE_VERSION = 6.0.0-dev +CPP_VERSION = 1.16.0-dev +CSHARP_VERSION = 1.16.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) @@ -978,6 +978,7 @@ avl_test: $(BINDIR)/$(CONFIG)/avl_test bad_server_response_test: $(BINDIR)/$(CONFIG)/bad_server_response_test bin_decoder_test: $(BINDIR)/$(CONFIG)/bin_decoder_test bin_encoder_test: $(BINDIR)/$(CONFIG)/bin_encoder_test +buffer_list_test: $(BINDIR)/$(CONFIG)/buffer_list_test channel_create_test: $(BINDIR)/$(CONFIG)/channel_create_test check_epollexclusive: $(BINDIR)/$(CONFIG)/check_epollexclusive chttp2_hpack_encoder_test: $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test @@ -1434,6 +1435,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/bad_server_response_test \ $(BINDIR)/$(CONFIG)/bin_decoder_test \ $(BINDIR)/$(CONFIG)/bin_encoder_test \ + $(BINDIR)/$(CONFIG)/buffer_list_test \ $(BINDIR)/$(CONFIG)/channel_create_test \ $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test \ $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \ @@ -1950,6 +1952,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/bin_decoder_test || ( echo test bin_decoder_test failed ; exit 1 ) $(E) "[RUN] Testing bin_encoder_test" $(Q) $(BINDIR)/$(CONFIG)/bin_encoder_test || ( echo test bin_encoder_test failed ; exit 1 ) + $(E) "[RUN] Testing buffer_list_test" + $(Q) $(BINDIR)/$(CONFIG)/buffer_list_test || ( echo test buffer_list_test failed ; exit 1 ) $(E) "[RUN] Testing channel_create_test" $(Q) $(BINDIR)/$(CONFIG)/channel_create_test || ( echo test channel_create_test failed ; exit 1 ) $(E) "[RUN] Testing chttp2_hpack_encoder_test" @@ -3460,6 +3464,7 @@ LIBGRPC_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -3480,6 +3485,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -3865,6 +3871,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -3885,6 +3892,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -4255,6 +4263,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -4275,6 +4284,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -4554,6 +4564,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -4574,6 +4585,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -4819,6 +4831,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -4839,6 +4852,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -5647,6 +5661,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -5667,6 +5682,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ @@ -10701,6 +10717,38 @@ endif endif +BUFFER_LIST_TEST_SRC = \ + test/core/iomgr/buffer_list_test.cc \ + +BUFFER_LIST_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BUFFER_LIST_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/buffer_list_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/buffer_list_test: $(BUFFER_LIST_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(BUFFER_LIST_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/buffer_list_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/buffer_list_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_buffer_list_test: $(BUFFER_LIST_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BUFFER_LIST_TEST_OBJS:.o=.dep) +endif +endif + + CHANNEL_CREATE_TEST_SRC = \ test/core/surface/channel_create_test.cc \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 73147bf3ac..b3f97650e8 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -24,178 +24,203 @@ # # The set of pollers to test against if a test exercises polling -POLLERS = ['epollex', 'epollsig', 'epoll1', 'poll', 'poll-cv'] +POLLERS = ["epollex", "epollsig", "epoll1", "poll", "poll-cv"] def if_not_windows(a): - return select({ - "//:windows": [], - "//:windows_msvc": [], - "//conditions:default": a, - }) + return select({ + "//:windows": [], + "//:windows_msvc": [], + "//conditions:default": a, + }) def _get_external_deps(external_deps): - ret = [] - for dep in external_deps: - if dep == "address_sorting": - ret += ["//third_party/address_sorting"] - elif dep == "cares": - ret += select({"//:grpc_no_ares": [], - "//conditions:default": ["//external:cares"],}) - else: - ret += ["//external:" + dep] - return ret + ret = [] + for dep in external_deps: + if dep == "address_sorting": + ret += ["//third_party/address_sorting"] + elif dep == "cares": + ret += select({ + "//:grpc_no_ares": [], + "//conditions:default": ["//external:cares"], + }) + else: + ret += ["//external:" + dep] + return ret def _maybe_update_cc_library_hdrs(hdrs): - ret = [] - hdrs_to_update = { - "third_party/objective_c/Cronet/bidirectional_stream_c.h": "//third_party:objective_c/Cronet/bidirectional_stream_c.h", - } - for h in hdrs: - if h in hdrs_to_update.keys(): - ret.append(hdrs_to_update[h]) - else: - ret.append(h) - return ret - -def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [], - external_deps = [], deps = [], standalone = False, - language = "C++", testonly = False, visibility = None, - alwayslink = 0, data = []): - copts = [] - if language.upper() == "C": - copts = if_not_windows(["-std=c99"]) - native.cc_library( - name = name, - srcs = srcs, - defines = select({"//:grpc_no_ares": ["GRPC_ARES=0"], - "//conditions:default": [],}) + - select({"//:remote_execution": ["GRPC_PORT_ISOLATED_RUNTIME=1"], - "//conditions:default": [],}) + - select({"//:grpc_allow_exceptions": ["GRPC_ALLOW_EXCEPTIONS=1"], - "//:grpc_disallow_exceptions": - ["GRPC_ALLOW_EXCEPTIONS=0"], - "//conditions:default": [],}), - hdrs = _maybe_update_cc_library_hdrs(hdrs + public_hdrs), - deps = deps + _get_external_deps(external_deps), - copts = copts, - visibility = visibility, - testonly = testonly, - linkopts = if_not_windows(["-pthread"]), - includes = [ - "include" - ], - alwayslink = alwayslink, - data = data, - ) + ret = [] + hdrs_to_update = { + "third_party/objective_c/Cronet/bidirectional_stream_c.h": "//third_party:objective_c/Cronet/bidirectional_stream_c.h", + } + for h in hdrs: + if h in hdrs_to_update.keys(): + ret.append(hdrs_to_update[h]) + else: + ret.append(h) + return ret + +def grpc_cc_library( + name, + srcs = [], + public_hdrs = [], + hdrs = [], + external_deps = [], + deps = [], + standalone = False, + language = "C++", + testonly = False, + visibility = None, + alwayslink = 0, + data = []): + copts = [] + if language.upper() == "C": + copts = if_not_windows(["-std=c99"]) + native.cc_library( + name = name, + srcs = srcs, + defines = select({ + "//:grpc_no_ares": ["GRPC_ARES=0"], + "//conditions:default": [], + }) + + select({ + "//:remote_execution": ["GRPC_PORT_ISOLATED_RUNTIME=1"], + "//conditions:default": [], + }) + + select({ + "//:grpc_allow_exceptions": ["GRPC_ALLOW_EXCEPTIONS=1"], + "//:grpc_disallow_exceptions": ["GRPC_ALLOW_EXCEPTIONS=0"], + "//conditions:default": [], + }), + hdrs = _maybe_update_cc_library_hdrs(hdrs + public_hdrs), + deps = deps + _get_external_deps(external_deps), + copts = copts, + visibility = visibility, + testonly = testonly, + linkopts = if_not_windows(["-pthread"]), + includes = [ + "include", + ], + alwayslink = alwayslink, + data = data, + ) def grpc_proto_plugin(name, srcs = [], deps = []): - native.cc_binary( - name = name, - srcs = srcs, - deps = deps, - ) + native.cc_binary( + name = name, + srcs = srcs, + deps = deps, + ) load("//:bazel/cc_grpc_library.bzl", "cc_grpc_library") -def grpc_proto_library(name, srcs = [], deps = [], well_known_protos = False, - has_services = True, use_external = False, generate_mocks = False): - cc_grpc_library( - name = name, - srcs = srcs, - deps = deps, - well_known_protos = well_known_protos, - proto_only = not has_services, - use_external = use_external, - generate_mocks = generate_mocks, - ) +def grpc_proto_library( + name, + srcs = [], + deps = [], + well_known_protos = False, + has_services = True, + use_external = False, + generate_mocks = False): + cc_grpc_library( + name = name, + srcs = srcs, + deps = deps, + well_known_protos = well_known_protos, + proto_only = not has_services, + use_external = use_external, + generate_mocks = generate_mocks, + ) def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = "moderate", tags = []): - copts = [] - if language.upper() == "C": - copts = if_not_windows(["-std=c99"]) - args = { - 'name': name, - 'srcs': srcs, - 'args': args, - 'data': data, - 'deps': deps + _get_external_deps(external_deps), - 'copts': copts, - 'linkopts': if_not_windows(["-pthread"]), - 'size': size, - 'timeout': timeout, - } - if uses_polling: - native.cc_test(testonly=True, tags=['manual'], **args) - for poller in POLLERS: - native.sh_test( - name = name + '@poller=' + poller, - data = [name], - srcs = [ - '//test/core/util:run_with_poller_sh', - ], - size = size, - timeout = timeout, - args = [ - poller, - '$(location %s)' % name, - ] + args['args'], - tags = tags, - ) - else: - native.cc_test(**args) + copts = [] + if language.upper() == "C": + copts = if_not_windows(["-std=c99"]) + args = { + "name": name, + "srcs": srcs, + "args": args, + "data": data, + "deps": deps + _get_external_deps(external_deps), + "copts": copts, + "linkopts": if_not_windows(["-pthread"]), + "size": size, + "timeout": timeout, + } + if uses_polling: + native.cc_test(testonly = True, tags = ["manual"], **args) + for poller in POLLERS: + native.sh_test( + name = name + "@poller=" + poller, + data = [name] + data, + srcs = [ + "//test/core/util:run_with_poller_sh", + ], + size = size, + timeout = timeout, + args = [ + poller, + "$(location %s)" % name, + ] + args["args"], + tags = tags, + ) + else: + native.cc_test(**args) def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkopts = []): - copts = [] - if language.upper() == "C": - copts = ["-std=c99"] - native.cc_binary( - name = name, - srcs = srcs, - args = args, - data = data, - testonly = testonly, - linkshared = linkshared, - deps = deps + _get_external_deps(external_deps), - copts = copts, - linkopts = if_not_windows(["-pthread"]) + linkopts, - ) - -def grpc_generate_one_off_targets(): pass + copts = [] + if language.upper() == "C": + copts = ["-std=c99"] + native.cc_binary( + name = name, + srcs = srcs, + args = args, + data = data, + testonly = testonly, + linkshared = linkshared, + deps = deps + _get_external_deps(external_deps), + copts = copts, + linkopts = if_not_windows(["-pthread"]) + linkopts, + ) + +def grpc_generate_one_off_targets(): + pass def grpc_sh_test(name, srcs, args = [], data = []): - native.sh_test( - name = name, - srcs = srcs, - args = args, - data = data) + native.sh_test( + name = name, + srcs = srcs, + args = args, + data = data, + ) def grpc_sh_binary(name, srcs, data = []): - native.sh_binary( - name = name, - srcs = srcs, - data = data) + native.sh_binary( + name = name, + srcs = srcs, + data = data, + ) def grpc_py_binary(name, srcs, data = [], deps = [], external_deps = [], testonly = False): - native.py_binary( - name = name, - srcs = srcs, - testonly = testonly, - data = data, - deps = deps + _get_external_deps(external_deps) - ) + native.py_binary( + name = name, + srcs = srcs, + testonly = testonly, + data = data, + deps = deps + _get_external_deps(external_deps), + ) def grpc_package(name, visibility = "private", features = []): - if visibility == "tests": - visibility = ["//test:__subpackages__"] - elif visibility == "public": - visibility = ["//visibility:public"] - elif visibility == "private": - visibility = [] - else: - fail("Unknown visibility " + visibility) - - if len(visibility) != 0: - native.package( - default_visibility = visibility, - features = features - ) + if visibility == "tests": + visibility = ["//test:__subpackages__"] + elif visibility == "public": + visibility = ["//visibility:public"] + elif visibility == "private": + visibility = [] + else: + fail("Unknown visibility " + visibility) + + if len(visibility) != 0: + native.package( + default_visibility = visibility, + features = features, + ) diff --git a/build.yaml b/build.yaml index 7176c549a7..a50a0a4ab6 100644 --- a/build.yaml +++ b/build.yaml @@ -12,9 +12,9 @@ settings: '#08': Use "-preN" suffixes to identify pre-release versions '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here - core_version: 6.0.0-pre1 - g_stands_for: glider - version: 1.15.0-pre1 + core_version: 6.0.0-dev + g_stands_for: gao + version: 1.16.0-dev filegroups: - name: alts_proto headers: @@ -256,6 +256,7 @@ filegroups: - src/core/lib/http/format_request.cc - src/core/lib/http/httpcli.cc - src/core/lib/http/parser.cc + - src/core/lib/iomgr/buffer_list.cc - src/core/lib/iomgr/call_combiner.cc - src/core/lib/iomgr/combiner.cc - src/core/lib/iomgr/endpoint.cc @@ -276,6 +277,7 @@ filegroups: - src/core/lib/iomgr/gethostname_fallback.cc - src/core/lib/iomgr/gethostname_host_name_max.cc - src/core/lib/iomgr/gethostname_sysconf.cc + - src/core/lib/iomgr/internal_errqueue.cc - src/core/lib/iomgr/iocp_windows.cc - src/core/lib/iomgr/iomgr.cc - src/core/lib/iomgr/iomgr_custom.cc @@ -434,6 +436,7 @@ filegroups: - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h - src/core/lib/iomgr/block_annotate.h + - src/core/lib/iomgr/buffer_list.h - src/core/lib/iomgr/call_combiner.h - src/core/lib/iomgr/closure.h - src/core/lib/iomgr/combiner.h @@ -449,6 +452,7 @@ filegroups: - src/core/lib/iomgr/exec_ctx.h - src/core/lib/iomgr/executor.h - src/core/lib/iomgr/gethostname.h + - src/core/lib/iomgr/internal_errqueue.h - src/core/lib/iomgr/iocp_windows.h - src/core/lib/iomgr/iomgr.h - src/core/lib/iomgr/iomgr_custom.h @@ -2139,6 +2143,20 @@ targets: - grpc_test_util - grpc uses_polling: false +- name: buffer_list_test + build: test + language: c + src: + - test/core/iomgr/buffer_list_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + exclude_iomgrs: + - uv + platforms: + - linux - name: channel_create_test build: test language: c @@ -108,6 +108,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/http/format_request.cc \ src/core/lib/http/httpcli.cc \ src/core/lib/http/parser.cc \ + src/core/lib/iomgr/buffer_list.cc \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/combiner.cc \ src/core/lib/iomgr/endpoint.cc \ @@ -128,6 +129,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ + src/core/lib/iomgr/internal_errqueue.cc \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iomgr.cc \ src/core/lib/iomgr/iomgr_custom.cc \ diff --git a/config.w32 b/config.w32 index a9d1e6c9d0..ad91ee40bd 100644 --- a/config.w32 +++ b/config.w32 @@ -83,6 +83,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\http\\format_request.cc " + "src\\core\\lib\\http\\httpcli.cc " + "src\\core\\lib\\http\\parser.cc " + + "src\\core\\lib\\iomgr\\buffer_list.cc " + "src\\core\\lib\\iomgr\\call_combiner.cc " + "src\\core\\lib\\iomgr\\combiner.cc " + "src\\core\\lib\\iomgr\\endpoint.cc " + @@ -103,6 +104,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\iomgr\\gethostname_fallback.cc " + "src\\core\\lib\\iomgr\\gethostname_host_name_max.cc " + "src\\core\\lib\\iomgr\\gethostname_sysconf.cc " + + "src\\core\\lib\\iomgr\\internal_errqueue.cc " + "src\\core\\lib\\iomgr\\iocp_windows.cc " + "src\\core\\lib\\iomgr\\iomgr.cc " + "src\\core\\lib\\iomgr\\iomgr_custom.cc " + diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index b3ed9bd713..c89a6cb572 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -14,4 +14,5 @@ - 1.12 'g' stands for ['glorious'](https://github.com/grpc/grpc/tree/v1.12.x) - 1.13 'g' stands for ['gloriosa'](https://github.com/grpc/grpc/tree/v1.13.x) - 1.14 'g' stands for ['gladiolus'](https://github.com/grpc/grpc/tree/v1.14.x) -- 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/master) +- 1.15 'g' stands for ['glider'](https://github.com/grpc/grpc/tree/v1.15.x) +- 1.16 'g' stands for ['gao'](https://github.com/grpc/grpc/tree/master) diff --git a/doc/naming.md b/doc/naming.md index 676aa9f298..581c550567 100644 --- a/doc/naming.md +++ b/doc/naming.md @@ -14,34 +14,48 @@ be plugged in. ### Name Syntax A fully qualified, self contained name used for gRPC channel construction -uses the syntax: - -``` -scheme://authority/endpoint_name -``` - -Here, `scheme` indicates the name-system to be used. Currently, we -support the following schemes: - -- `dns` - -- `ipv4` (IPv4 address) - -- `ipv6` (IPv6 address) - -- `unix` (path to unix domain socket -- unix systems only) +uses URI syntax as defined in [RFC 3986](https://tools.ietf.org/html/rfc3986). + +The URI scheme indicates what resolver plugin to use. If no scheme +prefix is specified or the scheme is unknown, the `dns` scheme is used +by default. + +The URI path indicates the name to be resolved. + +Most gRPC implementations support the following URI schemes: + +- `dns:[//authority/]host[:port]` -- DNS (default) + - `host` is the host to resolve via DNS. + - `port` is the port to return for each address. If not specified, + 443 is used (but some implementations default to 80 for insecure + channels). + - `authority` indicates the DNS server to use, although this is only + supported by some implementations. (In C-core, the default DNS + resolver does not support this, but the c-ares based resolver + supports specifying this in the form "IP:port".) + +- `unix:path` or `unix://absolute_path` -- Unix domain sockets (Unix systems only) + - `path` indicates the location of the desired socket. + - In the first form, the path may be relative or absolute; in the + second form, the path must be absolute (i.e., there will actually be + three slashes, two prior to the path and another to begin the + absolute path). + +The following schemes are supported by the gRPC C-core implementation, +but may not be supported in other languages: + +- `ipv4:address[:port][,address[:port],...]` -- IPv4 addresses + - Can specify multiple comma-delimited addresses of the form `address[:port]`: + - `address` is the IPv4 address to use. + - `port` is the port to use. If not specified, 443 is used. + +- `ipv6:address[:port][,address[:port],...]` -- IPv6 addresses + - Can specify multiple comma-delimited addresses of the form `address[:port]`: + - `address` is the IPv6 address to use. + - `port` is the port to use. If not specified, 443 is used. In the future, additional schemes such as `etcd` could be added. -The `authority` indicates some scheme-specific bootstrap information, e.g., -for DNS, the authority may include the IP[:port] of the DNS server to -use. Often, a DNS name may be used as the authority, since the ability to -resolve DNS names is already built into all gRPC client libraries. - -Finally, the `endpoint_name` indicates a concrete name to be looked up -in a given name-system identified by the scheme and the authority. The -syntax of the endpoint name is dictated by the scheme in use. - ### Resolver Plugins The gRPC client library will use the specified scheme to pick the right diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 9f61632ffa..581b9246bc 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.15.0-pre1' + # version = '1.16.0-dev' version = '0.0.3' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.15.0-pre1' + grpc_version = '1.16.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', @@ -382,6 +382,7 @@ Pod::Spec.new do |s| 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', 'src/core/lib/iomgr/block_annotate.h', + 'src/core/lib/iomgr/buffer_list.h', 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', @@ -397,6 +398,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', 'src/core/lib/iomgr/gethostname.h', + 'src/core/lib/iomgr/internal_errqueue.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_custom.h', @@ -570,6 +572,7 @@ Pod::Spec.new do |s| 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', 'src/core/lib/iomgr/block_annotate.h', + 'src/core/lib/iomgr/buffer_list.h', 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', @@ -585,6 +588,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', 'src/core/lib/iomgr/gethostname.h', + 'src/core/lib/iomgr/internal_errqueue.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_custom.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 509c0ed6a0..5a82a4200a 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.15.0-pre1' + version = '1.16.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' @@ -394,6 +394,7 @@ Pod::Spec.new do |s| 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', 'src/core/lib/iomgr/block_annotate.h', + 'src/core/lib/iomgr/buffer_list.h', 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', @@ -409,6 +410,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', 'src/core/lib/iomgr/gethostname.h', + 'src/core/lib/iomgr/internal_errqueue.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_custom.h', @@ -538,6 +540,7 @@ Pod::Spec.new do |s| 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -558,6 +561,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', @@ -993,6 +997,7 @@ Pod::Spec.new do |s| 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', 'src/core/lib/iomgr/block_annotate.h', + 'src/core/lib/iomgr/buffer_list.h', 'src/core/lib/iomgr/call_combiner.h', 'src/core/lib/iomgr/closure.h', 'src/core/lib/iomgr/combiner.h', @@ -1008,6 +1013,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/exec_ctx.h', 'src/core/lib/iomgr/executor.h', 'src/core/lib/iomgr/gethostname.h', + 'src/core/lib/iomgr/internal_errqueue.h', 'src/core/lib/iomgr/iocp_windows.h', 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_custom.h', diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 029e64e496..75fd592e75 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.15.0-pre1' + version = '1.16.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index e9bf60dd6f..ecd786e7b3 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.15.0-pre1' + version = '1.16.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 5692320c8e..9164b2169a 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.15.0-pre1' + version = '1.16.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/grpc.gemspec b/grpc.gemspec index 5c38071d15..c8e58faec9 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -330,6 +330,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/http/httpcli.h ) s.files += %w( src/core/lib/http/parser.h ) s.files += %w( src/core/lib/iomgr/block_annotate.h ) + s.files += %w( src/core/lib/iomgr/buffer_list.h ) s.files += %w( src/core/lib/iomgr/call_combiner.h ) s.files += %w( src/core/lib/iomgr/closure.h ) s.files += %w( src/core/lib/iomgr/combiner.h ) @@ -345,6 +346,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/exec_ctx.h ) s.files += %w( src/core/lib/iomgr/executor.h ) s.files += %w( src/core/lib/iomgr/gethostname.h ) + s.files += %w( src/core/lib/iomgr/internal_errqueue.h ) s.files += %w( src/core/lib/iomgr/iocp_windows.h ) s.files += %w( src/core/lib/iomgr/iomgr.h ) s.files += %w( src/core/lib/iomgr/iomgr_custom.h ) @@ -474,6 +476,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/http/format_request.cc ) s.files += %w( src/core/lib/http/httpcli.cc ) s.files += %w( src/core/lib/http/parser.cc ) + s.files += %w( src/core/lib/iomgr/buffer_list.cc ) s.files += %w( src/core/lib/iomgr/call_combiner.cc ) s.files += %w( src/core/lib/iomgr/combiner.cc ) s.files += %w( src/core/lib/iomgr/endpoint.cc ) @@ -494,6 +497,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/gethostname_fallback.cc ) s.files += %w( src/core/lib/iomgr/gethostname_host_name_max.cc ) s.files += %w( src/core/lib/iomgr/gethostname_sysconf.cc ) + s.files += %w( src/core/lib/iomgr/internal_errqueue.cc ) s.files += %w( src/core/lib/iomgr/iocp_windows.cc ) s.files += %w( src/core/lib/iomgr/iomgr.cc ) s.files += %w( src/core/lib/iomgr/iomgr_custom.cc ) @@ -300,6 +300,7 @@ 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -320,6 +321,7 @@ 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', @@ -660,6 +662,7 @@ 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -680,6 +683,7 @@ 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', @@ -893,6 +897,7 @@ 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -913,6 +918,7 @@ 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', @@ -1104,6 +1110,7 @@ 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -1124,6 +1131,7 @@ 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', diff --git a/include/grpc/grpc_security_constants.h b/include/grpc/grpc_security_constants.h index 944a1e927f..f935557f2d 100644 --- a/include/grpc/grpc_security_constants.h +++ b/include/grpc/grpc_security_constants.h @@ -57,46 +57,51 @@ typedef enum { } grpc_ssl_certificate_config_reload_status; typedef enum { - /** Server does not request client certificate. A client can present a self - signed or signed certificates if it wishes to do so and they would be - accepted. */ + /** Server does not request client certificate. + The certificate presented by the client is not checked by the server at + all. (A client may present a self signed or signed certificate or not + present a certificate at all and any of those option would be accepted) */ GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE, /** Server requests client certificate but does not enforce that the client presents a certificate. If the client presents a certificate, the client authentication is left to - the application based on the metadata like certificate etc. + the application (the necessary metadata will be available to the + application via authentication context properties, see grpc_auth_context). - The key cert pair should still be valid for the SSL connection to be - established. */ + The client's key certificate pair must be valid for the SSL connection to + be established. */ GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_BUT_DONT_VERIFY, /** Server requests client certificate but does not enforce that the client presents a certificate. If the client presents a certificate, the client authentication is done by - grpc framework (The client needs to either present a signed cert or skip no - certificate for a successful connection). + the gRPC framework. (For a successful connection the client needs to either + present a certificate that can be verified against the root certificate + configured by the server or not present a certificate at all) - The key cert pair should still be valid for the SSL connection to be - established. */ + The client's key certificate pair must be valid for the SSL connection to + be established. */ GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY, - /** Server requests client certificate but enforces that the client presents a + /** Server requests client certificate and enforces that the client presents a certificate. If the client presents a certificate, the client authentication is left to - the application based on the metadata like certificate etc. + the application (the necessary metadata will be available to the + application via authentication context properties, see grpc_auth_context). - The key cert pair should still be valid for the SSL connection to be - established. */ + The client's key certificate pair must be valid for the SSL connection to + be established. */ GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_BUT_DONT_VERIFY, - /** Server requests client certificate but enforces that the client presents a + /** Server requests client certificate and enforces that the client presents a certificate. - The cerificate presented by the client is verified by grpc framework (The - client needs to present signed certs for a successful connection). + The cerificate presented by the client is verified by the gRPC framework. + (For a successful connection the client needs to present a certificate that + can be verified against the root certificate configured by the server) - The key cert pair should still be valid for the SSL connection to be - established. */ + The client's key certificate pair must be valid for the SSL connection to + be established. */ GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY } grpc_ssl_client_certificate_request_type; diff --git a/package.xml b/package.xml index 49aae9dd00..9c0e078e92 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ <date>2018-01-19</date> <time>16:06:07</time> <version> - <release>1.15.0RC1</release> - <api>1.15.0RC1</api> + <release>1.16.0dev</release> + <api>1.16.0dev</api> </version> <stability> <release>beta</release> @@ -335,6 +335,7 @@ <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/block_annotate.h" role="src" /> + <file baseinstalldir="/" name="src/core/lib/iomgr/buffer_list.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/call_combiner.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/closure.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.h" role="src" /> @@ -350,6 +351,7 @@ <file baseinstalldir="/" name="src/core/lib/iomgr/exec_ctx.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/executor.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/gethostname.h" role="src" /> + <file baseinstalldir="/" name="src/core/lib/iomgr/internal_errqueue.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iocp_windows.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iomgr.h" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iomgr_custom.h" role="src" /> @@ -479,6 +481,7 @@ <file baseinstalldir="/" name="src/core/lib/http/format_request.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/http/httpcli.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/http/parser.cc" role="src" /> + <file baseinstalldir="/" name="src/core/lib/iomgr/buffer_list.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/call_combiner.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.cc" role="src" /> @@ -499,6 +502,7 @@ <file baseinstalldir="/" name="src/core/lib/iomgr/gethostname_fallback.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/gethostname_host_name_max.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/gethostname_sysconf.cc" role="src" /> + <file baseinstalldir="/" name="src/core/lib/iomgr/internal_errqueue.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iocp_windows.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iomgr.cc" role="src" /> <file baseinstalldir="/" name="src/core/lib/iomgr/iomgr_custom.cc" role="src" /> diff --git a/src/core/ext/filters/client_channel/README.md b/src/core/ext/filters/client_channel/README.md index 7c209db12e..9676a4535b 100644 --- a/src/core/ext/filters/client_channel/README.md +++ b/src/core/ext/filters/client_channel/README.md @@ -46,20 +46,4 @@ construction arguments for concrete grpc_subchannel instances. Naming for GRPC =============== -Names in GRPC are represented by a URI (as defined in -[RFC 3986](https://tools.ietf.org/html/rfc3986)). - -The following schemes are currently supported: - -dns:///host:port - dns schemes are currently supported so long as authority is - empty (authority based dns resolution is expected in a future - release) - -unix:path - the unix scheme is used to create and connect to unix domain - sockets - the authority must be empty, and the path - represents the absolute or relative path to the desired - socket - -ipv4:host:port - a pre-resolved ipv4 dotted decimal address/port combination - -ipv6:[host]:port - a pre-resolved ipv6 address/port combination +See [/doc/naming.md](gRPC name resolution). diff --git a/src/core/ext/filters/client_channel/http_connect_handshaker.cc b/src/core/ext/filters/client_channel/http_connect_handshaker.cc index 4e8b8b71db..7ce8da8c00 100644 --- a/src/core/ext/filters/client_channel/http_connect_handshaker.cc +++ b/src/core/ext/filters/client_channel/http_connect_handshaker.cc @@ -320,7 +320,7 @@ static void http_connect_handshaker_do_handshake( // Take a new ref to be held by the write callback. gpr_ref(&handshaker->refcount); grpc_endpoint_write(args->endpoint, &handshaker->write_buffer, - &handshaker->request_done_closure); + &handshaker->request_done_closure, nullptr); gpr_mu_unlock(&handshaker->mu); } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 9120abfa3c..602d6e92f9 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -126,7 +126,6 @@ class PickFirst : public LoadBalancingPolicy { void ShutdownLocked() override; void StartPickingLocked(); - void DestroyUnselectedSubchannelsLocked(); void UpdateChildRefsLocked(); // All our subchannels. @@ -250,14 +249,9 @@ void PickFirst::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, void PickFirst::StartPickingLocked() { started_picking_ = true; - if (subchannel_list_ != nullptr) { - for (size_t i = 0; i < subchannel_list_->num_subchannels(); ++i) { - if (subchannel_list_->subchannel(i)->subchannel() != nullptr) { - subchannel_list_->subchannel(i) - ->CheckConnectivityStateAndStartWatchingLocked(); - break; - } - } + if (subchannel_list_ != nullptr && subchannel_list_->num_subchannels() > 0) { + subchannel_list_->subchannel(0) + ->CheckConnectivityStateAndStartWatchingLocked(); } } @@ -294,15 +288,6 @@ bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { return false; } -void PickFirst::DestroyUnselectedSubchannelsLocked() { - for (size_t i = 0; i < subchannel_list_->num_subchannels(); ++i) { - PickFirstSubchannelData* sd = subchannel_list_->subchannel(i); - if (selected_ != sd) { - sd->UnrefSubchannelLocked("selected_different_subchannel"); - } - } -} - grpc_connectivity_state PickFirst::CheckConnectivityLocked(grpc_error** error) { return grpc_connectivity_state_get(&state_tracker_, error); } @@ -419,7 +404,6 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { if (sd->CheckConnectivityStateLocked(&error) == GRPC_CHANNEL_READY) { selected_ = sd; subchannel_list_ = std::move(subchannel_list); - DestroyUnselectedSubchannelsLocked(); sd->StartConnectivityWatchLocked(); // If there was a previously pending update (which may or may // not have contained the currently selected subchannel), drop @@ -504,7 +488,6 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE); // In transient failure. Rely on re-resolution to recover. p->selected_ = nullptr; - UnrefSubchannelLocked("pf_selected_shutdown"); StopConnectivityWatchLocked(); } else { grpc_connectivity_state_set(&p->state_tracker_, connectivity_state, @@ -535,11 +518,9 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( case GRPC_CHANNEL_TRANSIENT_FAILURE: { StopConnectivityWatchLocked(); PickFirstSubchannelData* sd = this; - do { - size_t next_index = - (sd->Index() + 1) % subchannel_list()->num_subchannels(); - sd = subchannel_list()->subchannel(next_index); - } while (sd->subchannel() == nullptr); + size_t next_index = + (sd->Index() + 1) % subchannel_list()->num_subchannels(); + sd = subchannel_list()->subchannel(next_index); // Case 1: Only set state to TRANSIENT_FAILURE if we've tried // all subchannels. if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) { @@ -600,8 +581,6 @@ void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() { if (grpc_lb_pick_first_trace.enabled()) { gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, subchannel()); } - // Drop all other subchannels, since we are now connected. - p->DestroyUnselectedSubchannelsLocked(); // Update any calls that were waiting for a pick. PickState* pick; while ((pick = p->pending_picks_)) { diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 0fa2f04e73..91ddaec8b8 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -102,11 +102,6 @@ class SubchannelData { return pending_connectivity_state_unsafe_; } - // Unrefs the subchannel. May be used if an individual subchannel is - // no longer needed even though the subchannel list as a whole is not - // being unreffed. - virtual void UnrefSubchannelLocked(const char* reason); - // Resets the connection backoff. // TODO(roth): This method should go away when we move the backoff // code out of the subchannel and into the LB policies. @@ -154,6 +149,10 @@ class SubchannelData { grpc_connectivity_state connectivity_state, grpc_error* error) GRPC_ABSTRACT; + // Unrefs the subchannel. May be overridden by subclasses that need + // to perform extra cleanup when unreffing the subchannel. + virtual void UnrefSubchannelLocked(const char* reason); + private: // Updates connected_subchannel_ based on pending_connectivity_state_unsafe_. // Returns true if the connectivity state should be reported. diff --git a/src/core/ext/filters/client_channel/resolver.h b/src/core/ext/filters/client_channel/resolver.h index 48f2e89095..e9acbb7c41 100644 --- a/src/core/ext/filters/client_channel/resolver.h +++ b/src/core/ext/filters/client_channel/resolver.h @@ -81,18 +81,7 @@ class Resolver : public InternallyRefCountedWithTracing<Resolver> { /// /// If this causes new data to become available, then the currently /// pending call to \a NextLocked() will return the new result. - /// - /// Note: Currently, all resolvers are required to return a new result - /// shortly after this method is called. For pull-based mechanisms, if - /// the implementation decides to delay querying the name service, it - /// should immediately return a new copy of the previously returned - /// result (and it can then return the updated data later, when it - /// actually does query the name service). For push-based mechanisms, - /// the implementation should immediately return a new copy of the - /// last-seen result. - /// TODO(roth): Remove this requirement once we fix pick_first to not - /// throw away unselected subchannels. - virtual void RequestReresolutionLocked() GRPC_ABSTRACT; + virtual void RequestReresolutionLocked() {} /// Resets the re-resolution backoff, if any. /// This needs to be implemented only by pull-based implementations; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index f2bb5f3c71..dfa52867d8 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -373,13 +373,7 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) { void AresDnsResolver::MaybeStartResolvingLocked() { // If there is an existing timer, the time it fires is the earliest time we // can start the next resolution. - if (have_next_resolution_timer_) { - // TODO(dgq): remove the following two lines once Pick First stops - // discarding subchannels after selecting. - ++resolved_version_; - MaybeFinishNextLocked(); - return; - } + if (have_next_resolution_timer_) return; if (last_resolution_timestamp_ >= 0) { const grpc_millis earliest_next_resolution = last_resolution_timestamp_ + min_time_between_resolutions_; @@ -401,10 +395,6 @@ void AresDnsResolver::MaybeStartResolvingLocked() { self.release(); grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution, &on_next_resolution_); - // TODO(dgq): remove the following two lines once Pick First stops - // discarding subchannels after selecting. - ++resolved_version_; - MaybeFinishNextLocked(); return; } } diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index 282caf215c..65ff1ec1a5 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -247,13 +247,7 @@ void NativeDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) { void NativeDnsResolver::MaybeStartResolvingLocked() { // If there is an existing timer, the time it fires is the earliest time we // can start the next resolution. - if (have_next_resolution_timer_) { - // TODO(dgq): remove the following two lines once Pick First stops - // discarding subchannels after selecting. - ++resolved_version_; - MaybeFinishNextLocked(); - return; - } + if (have_next_resolution_timer_) return; if (last_resolution_timestamp_ >= 0) { const grpc_millis earliest_next_resolution = last_resolution_timestamp_ + min_time_between_resolutions_; @@ -275,10 +269,6 @@ void NativeDnsResolver::MaybeStartResolvingLocked() { self.release(); grpc_timer_init(&next_resolution_timer_, ms_until_next_resolution, &on_next_resolution_); - // TODO(dgq): remove the following two lines once Pick First stops - // discarding subchannels after selecting. - ++resolved_version_; - MaybeFinishNextLocked(); return; } } diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc index 99a33f2277..144ac24a56 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc @@ -73,11 +73,6 @@ class FakeResolver : public Resolver { // Results to use for the pretended re-resolution in // RequestReresolutionLocked(). grpc_channel_args* reresolution_results_ = nullptr; - // TODO(juanlishen): This can go away once pick_first is changed to not throw - // away its subchannels, since that will eliminate its dependence on - // channel_saw_error_locked() causing an immediate resolver return. - // A copy of the most-recently used resolution results. - grpc_channel_args* last_used_results_ = nullptr; // pending next completion, or NULL grpc_closure* next_completion_ = nullptr; // target result address for next completion @@ -96,7 +91,6 @@ FakeResolver::FakeResolver(const ResolverArgs& args) : Resolver(args.combiner) { FakeResolver::~FakeResolver() { grpc_channel_args_destroy(next_results_); grpc_channel_args_destroy(reresolution_results_); - grpc_channel_args_destroy(last_used_results_); grpc_channel_args_destroy(channel_args_); } @@ -109,17 +103,11 @@ void FakeResolver::NextLocked(grpc_channel_args** target_result, } void FakeResolver::RequestReresolutionLocked() { - // A resolution must have been returned before an error is seen. - GPR_ASSERT(last_used_results_ != nullptr); - grpc_channel_args_destroy(next_results_); if (reresolution_results_ != nullptr) { + grpc_channel_args_destroy(next_results_); next_results_ = grpc_channel_args_copy(reresolution_results_); - } else { - // If reresolution_results is unavailable, re-resolve with the most-recently - // used results to avoid a no-op re-resolution. - next_results_ = grpc_channel_args_copy(last_used_results_); + MaybeFinishNextLocked(); } - MaybeFinishNextLocked(); } void FakeResolver::MaybeFinishNextLocked() { @@ -161,8 +149,6 @@ void FakeResolverResponseGenerator::SetResponseLocked(void* arg, FakeResolver* resolver = closure_arg->generator->resolver_; grpc_channel_args_destroy(resolver->next_results_); resolver->next_results_ = closure_arg->response; - grpc_channel_args_destroy(resolver->last_used_results_); - resolver->last_used_results_ = grpc_channel_args_copy(closure_arg->response); resolver->MaybeFinishNextLocked(); Delete(closure_arg); } diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h index e5175f9b7b..708eaf1147 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h @@ -53,7 +53,8 @@ class FakeResolverResponseGenerator // The new re-resolution response replaces any previous re-resolution // response that may have been set by a previous call. // If the re-resolution response is set to NULL, then the fake - // resolver will return the last value set via \a SetResponse(). + // resolver will not return anything when \a RequestReresolutionLocked() + // is called. void SetReresolutionResponse(grpc_channel_args* response); // Tells the resolver to return a transient failure (signalled by diff --git a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc index f74ac5aebe..801734764b 100644 --- a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc @@ -50,8 +50,6 @@ class SockaddrResolver : public Resolver { void NextLocked(grpc_channel_args** result, grpc_closure* on_complete) override; - void RequestReresolutionLocked() override; - void ShutdownLocked() override; private: @@ -90,11 +88,6 @@ void SockaddrResolver::NextLocked(grpc_channel_args** target_result, MaybeFinishNextLocked(); } -void SockaddrResolver::RequestReresolutionLocked() { - published_ = false; - MaybeFinishNextLocked(); -} - void SockaddrResolver::ShutdownLocked() { if (next_completion_ != nullptr) { *target_result_ = nullptr; diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc index 1fe8288bd0..431472609e 100644 --- a/src/core/ext/filters/max_age/max_age_filter.cc +++ b/src/core/ext/filters/max_age/max_age_filter.cc @@ -429,8 +429,7 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem, ? GRPC_MILLIS_INF_FUTURE : DEFAULT_MAX_CONNECTION_IDLE_MS; chand->idle_state = MAX_IDLE_STATE_INIT; - gpr_atm_no_barrier_store(&chand->last_enter_idle_time_millis, - GRPC_MILLIS_INF_PAST); + gpr_atm_no_barrier_store(&chand->last_enter_idle_time_millis, GPR_ATM_MIN); for (size_t i = 0; i < args->channel_args->num_args; ++i) { if (0 == strcmp(args->channel_args->args[i].key, GRPC_ARG_MAX_CONNECTION_AGE_MS)) { diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc index dfed824cd5..5bdcb387c9 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc @@ -50,7 +50,7 @@ grpc_channel* grpc_insecure_channel_create_from_fd( GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0); grpc_endpoint* client = grpc_tcp_client_create_from_fd( - grpc_fd_create(fd, "client", false), args, "fd-client"); + grpc_fd_create(fd, "client", true), args, "fd-client"); grpc_transport* transport = grpc_create_chttp2_transport(final_args, client, true); diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc index a0228785ee..e4bd91d07b 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc @@ -44,7 +44,7 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server* server, gpr_asprintf(&name, "fd:%d", fd); grpc_endpoint* server_endpoint = - grpc_tcp_create(grpc_fd_create(fd, name, false), + grpc_tcp_create(grpc_fd_create(fd, name, true), grpc_server_get_channel_args(server), name); gpr_free(name); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 36511fa608..027a57d606 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1029,7 +1029,8 @@ static void write_action(void* gt, grpc_error* error) { grpc_endpoint_write( t->ep, &t->outbuf, GRPC_CLOSURE_INIT(&t->write_action_end_locked, write_action_end_locked, t, - grpc_combiner_scheduler(t->combiner))); + grpc_combiner_scheduler(t->combiner)), + nullptr); } /* Callback from the grpc_endpoint after bytes have been written by calling diff --git a/src/core/lib/http/httpcli.cc b/src/core/lib/http/httpcli.cc index 12060074c5..3bd7a2ce59 100644 --- a/src/core/lib/http/httpcli.cc +++ b/src/core/lib/http/httpcli.cc @@ -163,7 +163,7 @@ static void done_write(void* arg, grpc_error* error) { static void start_write(internal_request* req) { grpc_slice_ref_internal(req->request_text); grpc_slice_buffer_add(&req->outgoing, req->request_text); - grpc_endpoint_write(req->ep, &req->outgoing, &req->done_write); + grpc_endpoint_write(req->ep, &req->outgoing, &req->done_write, nullptr); } static void on_handshake_done(void* arg, grpc_endpoint* ep) { diff --git a/src/core/lib/iomgr/buffer_list.cc b/src/core/lib/iomgr/buffer_list.cc new file mode 100644 index 0000000000..6ada23db1c --- /dev/null +++ b/src/core/lib/iomgr/buffer_list.cc @@ -0,0 +1,134 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <grpc/support/port_platform.h> + +#include "src/core/lib/iomgr/buffer_list.h" +#include "src/core/lib/iomgr/port.h" + +#include <grpc/support/log.h> + +#ifdef GRPC_LINUX_ERRQUEUE +#include <time.h> + +#include "src/core/lib/gprpp/memory.h" + +namespace grpc_core { +void TracedBuffer::AddNewEntry(TracedBuffer** head, uint32_t seq_no, + void* arg) { + GPR_DEBUG_ASSERT(head != nullptr); + TracedBuffer* new_elem = New<TracedBuffer>(seq_no, arg); + /* Store the current time as the sendmsg time. */ + new_elem->ts_.sendmsg_time = gpr_now(GPR_CLOCK_REALTIME); + if (*head == nullptr) { + *head = new_elem; + return; + } + /* Append at the end. */ + TracedBuffer* ptr = *head; + while (ptr->next_ != nullptr) { + ptr = ptr->next_; + } + ptr->next_ = new_elem; +} + +namespace { +/** Fills gpr_timespec gts based on values from timespec ts */ +void fill_gpr_from_timestamp(gpr_timespec* gts, const struct timespec* ts) { + gts->tv_sec = ts->tv_sec; + gts->tv_nsec = static_cast<int32_t>(ts->tv_nsec); + gts->clock_type = GPR_CLOCK_REALTIME; +} + +/** The saved callback function that will be invoked when we get all the + * timestamps that we are going to get for a TracedBuffer. */ +void (*timestamps_callback)(void*, grpc_core::Timestamps*, + grpc_error* shutdown_err); +} /* namespace */ + +void TracedBuffer::ProcessTimestamp(TracedBuffer** head, + struct sock_extended_err* serr, + struct scm_timestamping* tss) { + GPR_DEBUG_ASSERT(head != nullptr); + TracedBuffer* elem = *head; + TracedBuffer* next = nullptr; + while (elem != nullptr) { + /* The byte number refers to the sequence number of the last byte which this + * timestamp relates to. */ + if (serr->ee_data >= elem->seq_no_) { + switch (serr->ee_info) { + case SCM_TSTAMP_SCHED: + fill_gpr_from_timestamp(&(elem->ts_.scheduled_time), &(tss->ts[0])); + elem = elem->next_; + break; + case SCM_TSTAMP_SND: + fill_gpr_from_timestamp(&(elem->ts_.sent_time), &(tss->ts[0])); + elem = elem->next_; + break; + case SCM_TSTAMP_ACK: + fill_gpr_from_timestamp(&(elem->ts_.acked_time), &(tss->ts[0])); + /* Got all timestamps. Do the callback and free this TracedBuffer. + * The thing below can be passed by value if we don't want the + * restriction on the lifetime. */ + timestamps_callback(elem->arg_, &(elem->ts_), GRPC_ERROR_NONE); + next = elem->next_; + Delete<TracedBuffer>(elem); + *head = elem = next; + break; + default: + abort(); + } + } else { + break; + } + } +} + +void TracedBuffer::Shutdown(TracedBuffer** head, grpc_error* shutdown_err) { + GPR_DEBUG_ASSERT(head != nullptr); + TracedBuffer* elem = *head; + while (elem != nullptr) { + if (timestamps_callback) { + timestamps_callback(elem->arg_, &(elem->ts_), shutdown_err); + } + auto* next = elem->next_; + Delete<TracedBuffer>(elem); + elem = next; + } + *head = nullptr; + GRPC_ERROR_UNREF(shutdown_err); +} + +void grpc_tcp_set_write_timestamps_callback(void (*fn)(void*, + grpc_core::Timestamps*, + grpc_error* error)) { + timestamps_callback = fn; +} +} /* namespace grpc_core */ + +#else /* GRPC_LINUX_ERRQUEUE */ + +namespace grpc_core { +void grpc_tcp_set_write_timestamps_callback(void (*fn)(void*, + grpc_core::Timestamps*, + grpc_error* error)) { + gpr_log(GPR_DEBUG, "Timestamps callback is not enabled for this platform"); +} +} /* namespace grpc_core */ + +#endif /* GRPC_LINUX_ERRQUEUE */ diff --git a/src/core/lib/iomgr/buffer_list.h b/src/core/lib/iomgr/buffer_list.h new file mode 100644 index 0000000000..cbbf50a657 --- /dev/null +++ b/src/core/lib/iomgr/buffer_list.h @@ -0,0 +1,96 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H +#define GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H + +#include <grpc/support/port_platform.h> + +#include "src/core/lib/iomgr/port.h" + +#include <grpc/support/time.h> + +#include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/internal_errqueue.h" + +namespace grpc_core { +struct Timestamps { + /* TODO(yashykt): This would also need to store OPTSTAT once support is added + */ + gpr_timespec sendmsg_time; + gpr_timespec scheduled_time; + gpr_timespec sent_time; + gpr_timespec acked_time; +}; + +/** TracedBuffer is a class to keep track of timestamps for a specific buffer in + * the TCP layer. We are only tracking timestamps for Linux kernels and hence + * this class would only be used by Linux platforms. For all other platforms, + * TracedBuffer would be an empty class. + * + * The timestamps collected are according to grpc_core::Timestamps declared + * above. + * + * A TracedBuffer list is kept track of using the head element of the list. If + * the head element of the list is nullptr, then the list is empty. + */ +#ifdef GRPC_LINUX_ERRQUEUE +class TracedBuffer { + public: + /** Add a new entry in the TracedBuffer list pointed to by head. Also saves + * sendmsg_time with the current timestamp. */ + static void AddNewEntry(grpc_core::TracedBuffer** head, uint32_t seq_no, + void* arg); + + /** Processes a received timestamp based on sock_extended_err and + * scm_timestamping structures. It will invoke the timestamps callback if the + * timestamp type is SCM_TSTAMP_ACK. */ + static void ProcessTimestamp(grpc_core::TracedBuffer** head, + struct sock_extended_err* serr, + struct scm_timestamping* tss); + + /** Cleans the list by calling the callback for each traced buffer in the list + * with timestamps that it has. */ + static void Shutdown(grpc_core::TracedBuffer** head, + grpc_error* shutdown_err); + + private: + GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW + + TracedBuffer(int seq_no, void* arg) + : seq_no_(seq_no), arg_(arg), next_(nullptr) {} + + uint32_t seq_no_; /* The sequence number for the last byte in the buffer */ + void* arg_; /* The arg to pass to timestamps_callback */ + grpc_core::Timestamps ts_; /* The timestamps corresponding to this buffer */ + grpc_core::TracedBuffer* next_; /* The next TracedBuffer in the list */ +}; +#else /* GRPC_LINUX_ERRQUEUE */ +class TracedBuffer {}; +#endif /* GRPC_LINUX_ERRQUEUE */ + +/** Sets the callback function to call when timestamps for a write are + * collected. The callback does not own a reference to error. */ +void grpc_tcp_set_write_timestamps_callback(void (*fn)(void*, + grpc_core::Timestamps*, + grpc_error* error)); + +}; /* namespace grpc_core */ + +#endif /* GRPC_CORE_LIB_IOMGR_BUFFER_LIST_H */ diff --git a/src/core/lib/iomgr/endpoint.cc b/src/core/lib/iomgr/endpoint.cc index 92e7930111..44fb47e19d 100644 --- a/src/core/lib/iomgr/endpoint.cc +++ b/src/core/lib/iomgr/endpoint.cc @@ -28,8 +28,8 @@ void grpc_endpoint_read(grpc_endpoint* ep, grpc_slice_buffer* slices, } void grpc_endpoint_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { - ep->vtable->write(ep, slices, cb); + grpc_closure* cb, void* arg) { + ep->vtable->write(ep, slices, cb, arg); } void grpc_endpoint_add_to_pollset(grpc_endpoint* ep, grpc_pollset* pollset) { diff --git a/src/core/lib/iomgr/endpoint.h b/src/core/lib/iomgr/endpoint.h index 15db1649fa..1f590a80ca 100644 --- a/src/core/lib/iomgr/endpoint.h +++ b/src/core/lib/iomgr/endpoint.h @@ -33,10 +33,12 @@ typedef struct grpc_endpoint grpc_endpoint; typedef struct grpc_endpoint_vtable grpc_endpoint_vtable; +class Timestamps; struct grpc_endpoint_vtable { void (*read)(grpc_endpoint* ep, grpc_slice_buffer* slices, grpc_closure* cb); - void (*write)(grpc_endpoint* ep, grpc_slice_buffer* slices, grpc_closure* cb); + void (*write)(grpc_endpoint* ep, grpc_slice_buffer* slices, grpc_closure* cb, + void* arg); void (*add_to_pollset)(grpc_endpoint* ep, grpc_pollset* pollset); void (*add_to_pollset_set)(grpc_endpoint* ep, grpc_pollset_set* pollset); void (*delete_from_pollset_set)(grpc_endpoint* ep, grpc_pollset_set* pollset); @@ -70,9 +72,11 @@ int grpc_endpoint_get_fd(grpc_endpoint* ep); \a slices may be mutated at will by the endpoint until cb is called. No guarantee is made to the content of slices after a write EXCEPT that it is a valid slice buffer. + \a arg is platform specific. It is currently only used by TCP on linux + platforms as an argument that would be forwarded to the timestamps callback. */ void grpc_endpoint_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb); + grpc_closure* cb, void* arg); /* Causes any pending and future read/write callbacks to run immediately with success==0 */ diff --git a/src/core/lib/iomgr/endpoint_cfstream.cc b/src/core/lib/iomgr/endpoint_cfstream.cc index c3bc0cc8fd..df2cf508c8 100644 --- a/src/core/lib/iomgr/endpoint_cfstream.cc +++ b/src/core/lib/iomgr/endpoint_cfstream.cc @@ -268,7 +268,7 @@ static void CFStreamRead(grpc_endpoint* ep, grpc_slice_buffer* slices, } static void CFStreamWrite(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { CFStreamEndpoint* ep_impl = reinterpret_cast<CFStreamEndpoint*>(ep); if (grpc_tcp_trace.enabled()) { gpr_log(GPR_DEBUG, "CFStream endpoint:%p write (%p, %p) length:%zu", diff --git a/src/core/lib/iomgr/endpoint_pair_posix.cc b/src/core/lib/iomgr/endpoint_pair_posix.cc index 5c5c246f99..3afbfd7254 100644 --- a/src/core/lib/iomgr/endpoint_pair_posix.cc +++ b/src/core/lib/iomgr/endpoint_pair_posix.cc @@ -59,11 +59,11 @@ grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char* name, grpc_core::ExecCtx exec_ctx; gpr_asprintf(&final_name, "%s:client", name); - p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name, false), args, + p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name, true), args, "socketpair-server"); gpr_free(final_name); gpr_asprintf(&final_name, "%s:server", name); - p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name, false), args, + p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name, true), args, "socketpair-client"); gpr_free(final_name); diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 0205363d5c..d4377e2d50 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -237,14 +237,19 @@ void grpc_event_engine_shutdown(void) { } bool grpc_event_engine_can_track_errors(void) { +/* Only track errors if platform supports errqueue. */ +#ifdef GRPC_LINUX_ERRQUEUE return g_event_engine->can_track_err; +#else + return false; +#endif /* GRPC_LINUX_ERRQUEUE */ } grpc_fd* grpc_fd_create(int fd, const char* name, bool track_err) { GRPC_POLLING_API_TRACE("fd_create(%d, %s, %d)", fd, name, track_err); GRPC_FD_TRACE("fd_create(%d, %s, %d)", fd, name, track_err); - GPR_DEBUG_ASSERT(!track_err || g_event_engine->can_track_err); - return g_event_engine->fd_create(fd, name, track_err); + return g_event_engine->fd_create(fd, name, + track_err && g_event_engine->can_track_err); } int grpc_fd_wrapped_fd(grpc_fd* fd) { diff --git a/src/core/lib/iomgr/internal_errqueue.cc b/src/core/lib/iomgr/internal_errqueue.cc new file mode 100644 index 0000000000..99c22e9055 --- /dev/null +++ b/src/core/lib/iomgr/internal_errqueue.cc @@ -0,0 +1,36 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <grpc/support/port_platform.h> + +#include "src/core/lib/iomgr/port.h" + +#include "src/core/lib/iomgr/internal_errqueue.h" + +#ifdef GRPC_POSIX_SOCKET_TCP + +bool kernel_supports_errqueue() { +#ifdef LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) + return true; +#endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(4, 0, 0) */ +#endif /* LINUX_VERSION_CODE */ + return false; +} + +#endif /* GRPC_POSIX_SOCKET_TCP */ diff --git a/src/core/lib/iomgr/internal_errqueue.h b/src/core/lib/iomgr/internal_errqueue.h new file mode 100644 index 0000000000..9d122808f9 --- /dev/null +++ b/src/core/lib/iomgr/internal_errqueue.h @@ -0,0 +1,83 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* This file contains constants defined in <linux/errqueue.h> and + * <linux/net_tstamp.h> so as to allow collecting network timestamps in the + * kernel. This file allows tcp_posix.cc to compile on platforms that do not + * have <linux/errqueue.h> and <linux/net_tstamp.h>. + */ + +#ifndef GRPC_CORE_LIB_IOMGR_INTERNAL_ERRQUEUE_H +#define GRPC_CORE_LIB_IOMGR_INTERNAL_ERRQUEUE_H + +#include <grpc/support/port_platform.h> + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_POSIX_SOCKET_TCP + +#include <sys/types.h> +#include <time.h> + +#ifdef GRPC_LINUX_ERRQUEUE +#include <linux/errqueue.h> +#include <linux/net_tstamp.h> +#include <sys/socket.h> +#endif /* GRPC_LINUX_ERRQUEUE */ + +namespace grpc_core { + +#ifdef GRPC_LINUX_ERRQUEUE + +/* Redefining scm_timestamping in the same way that <linux/errqueue.h> defines + * it, so that code compiles on systems that don't have it. */ +struct scm_timestamping { + struct timespec ts[3]; +}; +/* Also redefine timestamp types */ +/* The timestamp type for when the driver passed skb to NIC, or HW. */ +constexpr int SCM_TSTAMP_SND = 0; +/* The timestamp type for when data entered the packet scheduler. */ +constexpr int SCM_TSTAMP_SCHED = 1; +/* The timestamp type for when data acknowledged by peer. */ +constexpr int SCM_TSTAMP_ACK = 2; +/* Redefine required constants from <linux/net_tstamp.h> */ +constexpr uint32_t SOF_TIMESTAMPING_TX_SOFTWARE = 1u << 1; +constexpr uint32_t SOF_TIMESTAMPING_SOFTWARE = 1u << 4; +constexpr uint32_t SOF_TIMESTAMPING_OPT_ID = 1u << 7; +constexpr uint32_t SOF_TIMESTAMPING_TX_SCHED = 1u << 8; +constexpr uint32_t SOF_TIMESTAMPING_TX_ACK = 1u << 9; +constexpr uint32_t SOF_TIMESTAMPING_OPT_TSONLY = 1u << 11; + +constexpr uint32_t kTimestampingSocketOptions = SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_OPT_ID | + SOF_TIMESTAMPING_OPT_TSONLY; +constexpr uint32_t kTimestampingRecordingOptions = + SOF_TIMESTAMPING_TX_SCHED | SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_TX_ACK; +#endif /* GRPC_LINUX_ERRQUEUE */ + +/* Returns true if kernel is capable of supporting errqueue and timestamping. + * Currently allowing only linux kernels above 4.0.0 + */ +bool kernel_supports_errqueue(); +} // namespace grpc_core + +#endif /* GRPC_POSIX_SOCKET_TCP */ + +#endif /* GRPC_CORE_LIB_IOMGR_INTERNAL_ERRQUEUE_H */ diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index 066417b93c..abf96662f5 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -60,6 +60,11 @@ #define GRPC_HAVE_IP_PKTINFO 1 #define GRPC_HAVE_MSG_NOSIGNAL 1 #define GRPC_HAVE_UNIX_SOCKET 1 +#ifdef LINUX_VERSION_CODE +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) +#define GRPC_LINUX_ERRQUEUE 1 +#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) */ +#endif /* LINUX_VERSION_CODE */ #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1 #define GRPC_POSIX_FORK 1 #define GRPC_POSIX_HOST_NAME_MAX 1 diff --git a/src/core/lib/iomgr/tcp_client_posix.cc b/src/core/lib/iomgr/tcp_client_posix.cc index 296ee74311..9c989b7dfe 100644 --- a/src/core/lib/iomgr/tcp_client_posix.cc +++ b/src/core/lib/iomgr/tcp_client_posix.cc @@ -279,7 +279,7 @@ grpc_error* grpc_tcp_client_prepare_fd(const grpc_channel_args* channel_args, } addr_str = grpc_sockaddr_to_uri(mapped_addr); gpr_asprintf(&name, "tcp-client:%s", addr_str); - *fdobj = grpc_fd_create(fd, name, false); + *fdobj = grpc_fd_create(fd, name, true); gpr_free(name); gpr_free(addr_str); return GRPC_ERROR_NONE; diff --git a/src/core/lib/iomgr/tcp_custom.cc b/src/core/lib/iomgr/tcp_custom.cc index 990e8d632b..e02a1898f2 100644 --- a/src/core/lib/iomgr/tcp_custom.cc +++ b/src/core/lib/iomgr/tcp_custom.cc @@ -221,7 +221,7 @@ static void custom_write_callback(grpc_custom_socket* socket, } static void endpoint_write(grpc_endpoint* ep, grpc_slice_buffer* write_slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { custom_tcp_endpoint* tcp = (custom_tcp_endpoint*)ep; GRPC_CUSTOM_IOMGR_ASSERT_SAME_THREAD(); diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index b53ffbf01c..ac1e919acb 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -27,7 +27,9 @@ #include <errno.h> #include <limits.h> +#include <netinet/in.h> #include <stdbool.h> +#include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/socket.h> @@ -46,6 +48,7 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/iomgr/buffer_list.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/profiling/timers.h" @@ -97,17 +100,42 @@ struct grpc_tcp { grpc_closure read_done_closure; grpc_closure write_done_closure; + grpc_closure error_closure; char* peer_string; grpc_resource_user* resource_user; grpc_resource_user_slice_allocator slice_allocator; + + grpc_core::TracedBuffer* tb_head; /* List of traced buffers */ + gpr_mu tb_mu; /* Lock for access to list of traced buffers */ + + /* grpc_endpoint_write takes an argument which if non-null means that the + * transport layer wants the TCP layer to collect timestamps for this write. + * This arg is forwarded to the timestamps callback function when the ACK + * timestamp is received from the kernel. This arg is a (void *) which allows + * users of this API to pass in a pointer to any kind of structure. This + * structure could actually be a tag or any book-keeping object that the user + * can use to distinguish between different traced writes. The only + * requirement from the TCP endpoint layer is that this arg should be non-null + * if the user wants timestamps for the write. */ + void* outgoing_buffer_arg; + /* A counter which starts at 0. It is initialized the first time the socket + * options for collecting timestamps are set, and is incremented with each + * byte sent. */ + int bytes_counter; + bool socket_ts_enabled; /* True if timestamping options are set on the socket + */ + gpr_atm + stop_error_notification; /* Set to 1 if we do not want to be notified on + errors anymore */ }; struct backup_poller { gpr_mu* pollset_mu; grpc_closure run_poller; }; + } // namespace #define BACKUP_POLLER_POLLSET(b) ((grpc_pollset*)((b) + 1)) @@ -302,6 +330,7 @@ static void tcp_free(grpc_tcp* tcp) { grpc_slice_buffer_destroy_internal(&tcp->last_read_buffer); grpc_resource_user_unref(tcp->resource_user); gpr_free(tcp->peer_string); + gpr_mu_destroy(&tcp->tb_mu); gpr_free(tcp); } @@ -347,6 +376,10 @@ static void tcp_destroy(grpc_endpoint* ep) { grpc_network_status_unregister_endpoint(ep); grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep); grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer); + if (grpc_event_engine_can_track_errors()) { + gpr_atm_no_barrier_store(&tcp->stop_error_notification, true); + grpc_fd_set_error(tcp->em_fd); + } TCP_UNREF(tcp, "destroy"); } @@ -513,6 +546,235 @@ static void tcp_read(grpc_endpoint* ep, grpc_slice_buffer* incoming_buffer, } } +/* A wrapper around sendmsg. It sends \a msg over \a fd and returns the number + * of bytes sent. */ +ssize_t tcp_send(int fd, const struct msghdr* msg) { + GPR_TIMER_SCOPE("sendmsg", 1); + ssize_t sent_length; + do { + /* TODO(klempner): Cork if this is a partial write */ + GRPC_STATS_INC_SYSCALL_WRITE(); + sent_length = sendmsg(fd, msg, SENDMSG_FLAGS); + } while (sent_length < 0 && errno == EINTR); + return sent_length; +} + +/** This is to be called if outgoing_buffer_arg is not null. On linux platforms, + * this will call sendmsg with socket options set to collect timestamps inside + * the kernel. On return, sent_length is set to the return value of the sendmsg + * call. Returns false if setting the socket options failed. This is not + * implemented for non-linux platforms currently, and crashes out. + */ +static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg, + size_t sending_length, + ssize_t* sent_length, grpc_error** error); + +/** The callback function to be invoked when we get an error on the socket. */ +static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error); + +#ifdef GRPC_LINUX_ERRQUEUE +static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg, + size_t sending_length, + ssize_t* sent_length, + grpc_error** error) { + if (!tcp->socket_ts_enabled) { + uint32_t opt = grpc_core::kTimestampingSocketOptions; + if (setsockopt(tcp->fd, SOL_SOCKET, SO_TIMESTAMPING, + static_cast<void*>(&opt), sizeof(opt)) != 0) { + *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "setsockopt"), tcp); + grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer); + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_ERROR, "Failed to set timestamping options on the socket."); + } + return false; + } + tcp->bytes_counter = -1; + tcp->socket_ts_enabled = true; + } + /* Set control message to indicate that you want timestamps. */ + union { + char cmsg_buf[CMSG_SPACE(sizeof(uint32_t))]; + struct cmsghdr align; + } u; + cmsghdr* cmsg = reinterpret_cast<cmsghdr*>(u.cmsg_buf); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SO_TIMESTAMPING; + cmsg->cmsg_len = CMSG_LEN(sizeof(uint32_t)); + *reinterpret_cast<int*>(CMSG_DATA(cmsg)) = + grpc_core::kTimestampingRecordingOptions; + msg->msg_control = u.cmsg_buf; + msg->msg_controllen = CMSG_SPACE(sizeof(uint32_t)); + + /* If there was an error on sendmsg the logic in tcp_flush will handle it. */ + ssize_t length = tcp_send(tcp->fd, msg); + *sent_length = length; + /* Only save timestamps if all the bytes were taken by sendmsg. */ + if (sending_length == static_cast<size_t>(length)) { + gpr_mu_lock(&tcp->tb_mu); + grpc_core::TracedBuffer::AddNewEntry( + &tcp->tb_head, static_cast<int>(tcp->bytes_counter + length), + tcp->outgoing_buffer_arg); + gpr_mu_unlock(&tcp->tb_mu); + tcp->outgoing_buffer_arg = nullptr; + } + return true; +} + +/** Reads \a cmsg to derive timestamps from the control messages. If a valid + * timestamp is found, the traced buffer list is updated with this timestamp. + * The caller of this function should be looping on the control messages found + * in \a msg. \a cmsg should point to the control message that the caller wants + * processed. + * On return, a pointer to a control message is returned. On the next iteration, + * CMSG_NXTHDR(msg, ret_val) should be passed as \a cmsg. */ +struct cmsghdr* process_timestamp(grpc_tcp* tcp, msghdr* msg, + struct cmsghdr* cmsg) { + auto next_cmsg = CMSG_NXTHDR(msg, cmsg); + if (next_cmsg == nullptr) { + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_ERROR, "Received timestamp without extended error"); + } + return cmsg; + } + + if (!(next_cmsg->cmsg_level == SOL_IP || next_cmsg->cmsg_level == SOL_IPV6) || + !(next_cmsg->cmsg_type == IP_RECVERR || + next_cmsg->cmsg_type == IPV6_RECVERR)) { + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_ERROR, "Unexpected control message"); + } + return cmsg; + } + + auto tss = + reinterpret_cast<struct grpc_core::scm_timestamping*>(CMSG_DATA(cmsg)); + auto serr = reinterpret_cast<struct sock_extended_err*>(CMSG_DATA(next_cmsg)); + if (serr->ee_errno != ENOMSG || + serr->ee_origin != SO_EE_ORIGIN_TIMESTAMPING) { + gpr_log(GPR_ERROR, "Unexpected control message"); + return cmsg; + } + /* The error handling can potentially be done on another thread so we need + * to protect the traced buffer list. A lock free list might be better. Using + * a simple mutex for now. */ + gpr_mu_lock(&tcp->tb_mu); + grpc_core::TracedBuffer::ProcessTimestamp(&tcp->tb_head, serr, tss); + gpr_mu_unlock(&tcp->tb_mu); + return next_cmsg; +} + +/** For linux platforms, reads the socket's error queue and processes error + * messages from the queue. Returns true if all the errors processed were + * timestamps. Returns false if any of the errors were not timestamps. For + * non-linux platforms, error processing is not used/enabled currently. + */ +static bool process_errors(grpc_tcp* tcp) { + while (true) { + struct iovec iov; + iov.iov_base = nullptr; + iov.iov_len = 0; + struct msghdr msg; + msg.msg_name = nullptr; + msg.msg_namelen = 0; + msg.msg_iov = &iov; + msg.msg_iovlen = 0; + msg.msg_flags = 0; + + union { + char rbuf[1024 /*CMSG_SPACE(sizeof(scm_timestamping)) + + CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in))*/]; + struct cmsghdr align; + } aligned_buf; + memset(&aligned_buf, 0, sizeof(aligned_buf)); + + msg.msg_control = aligned_buf.rbuf; + msg.msg_controllen = sizeof(aligned_buf.rbuf); + + int r, saved_errno; + do { + r = recvmsg(tcp->fd, &msg, MSG_ERRQUEUE); + saved_errno = errno; + } while (r < 0 && saved_errno == EINTR); + + if (r == -1 && saved_errno == EAGAIN) { + return true; /* No more errors to process */ + } + if (r == -1) { + return false; + } + if (grpc_tcp_trace.enabled()) { + if ((msg.msg_flags & MSG_CTRUNC) == 1) { + gpr_log(GPR_INFO, "Error message was truncated."); + } + } + + if (msg.msg_controllen == 0) { + /* There was no control message found. It was probably spurious. */ + return true; + } + for (auto cmsg = CMSG_FIRSTHDR(&msg); cmsg && cmsg->cmsg_len; + cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (cmsg->cmsg_level != SOL_SOCKET || + cmsg->cmsg_type != SCM_TIMESTAMPING) { + /* Got a control message that is not a timestamp. Don't know how to + * handle this. */ + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_INFO, + "unknown control message cmsg_level:%d cmsg_type:%d", + cmsg->cmsg_level, cmsg->cmsg_type); + } + return false; + } + process_timestamp(tcp, &msg, cmsg); + } + } +} + +static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) { + grpc_tcp* tcp = static_cast<grpc_tcp*>(arg); + if (grpc_tcp_trace.enabled()) { + gpr_log(GPR_INFO, "TCP:%p got_error: %s", tcp, grpc_error_string(error)); + } + + if (error != GRPC_ERROR_NONE || + static_cast<bool>(gpr_atm_acq_load(&tcp->stop_error_notification))) { + /* We aren't going to register to hear on error anymore, so it is safe to + * unref. */ + grpc_core::TracedBuffer::Shutdown(&tcp->tb_head, GRPC_ERROR_REF(error)); + TCP_UNREF(tcp, "error-tracking"); + return; + } + + /* We are still interested in collecting timestamps, so let's try reading + * them. */ + if (!process_errors(tcp)) { + /* This was not a timestamps error. This was an actual error. Set the + * read and write closures to be ready. + */ + grpc_fd_set_readable(tcp->em_fd); + grpc_fd_set_writable(tcp->em_fd); + } + GRPC_CLOSURE_INIT(&tcp->error_closure, tcp_handle_error, tcp, + grpc_schedule_on_exec_ctx); + grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure); +} + +#else /* GRPC_LINUX_ERRQUEUE */ +static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg, + size_t sending_length, + ssize_t* sent_length, + grpc_error** error) { + gpr_log(GPR_ERROR, "Write with timestamps not supported for this platform"); + GPR_ASSERT(0); + return false; +} + +static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) { + gpr_log(GPR_ERROR, "Error handling is not supported for this platform"); + GPR_ASSERT(0); +} +#endif /* GRPC_LINUX_ERRQUEUE */ + /* returns true if done, false if pending; if returning true, *error is set */ #if defined(IOV_MAX) && IOV_MAX < 1000 #define MAX_WRITE_IOVEC IOV_MAX @@ -557,19 +819,20 @@ static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) { msg.msg_namelen = 0; msg.msg_iov = iov; msg.msg_iovlen = iov_size; - msg.msg_control = nullptr; - msg.msg_controllen = 0; msg.msg_flags = 0; + if (tcp->outgoing_buffer_arg != nullptr) { + if (!tcp_write_with_timestamps(tcp, &msg, sending_length, &sent_length, + error)) + return true; /* something went wrong with timestamps */ + } else { + msg.msg_control = nullptr; + msg.msg_controllen = 0; - GRPC_STATS_INC_TCP_WRITE_SIZE(sending_length); - GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(iov_size); + GRPC_STATS_INC_TCP_WRITE_SIZE(sending_length); + GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(iov_size); - GPR_TIMER_SCOPE("sendmsg", 1); - do { - /* TODO(klempner): Cork if this is a partial write */ - GRPC_STATS_INC_SYSCALL_WRITE(); - sent_length = sendmsg(tcp->fd, &msg, SENDMSG_FLAGS); - } while (sent_length < 0 && errno == EINTR); + sent_length = tcp_send(tcp->fd, &msg); + } if (sent_length < 0) { if (errno == EAGAIN) { @@ -593,6 +856,7 @@ static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) { } GPR_ASSERT(tcp->outgoing_byte_idx == 0); + tcp->bytes_counter += sent_length; trailing = sending_length - static_cast<size_t>(sent_length); while (trailing > 0) { size_t slice_length; @@ -607,7 +871,6 @@ static bool tcp_flush(grpc_tcp* tcp, grpc_error** error) { trailing -= slice_length; } } - if (outgoing_slice_idx == tcp->outgoing_buffer->count) { *error = GRPC_ERROR_NONE; grpc_slice_buffer_reset_and_unref_internal(tcp->outgoing_buffer); @@ -640,14 +903,13 @@ static void tcp_handle_write(void* arg /* grpc_tcp */, grpc_error* error) { const char* str = grpc_error_string(error); gpr_log(GPR_INFO, "write: %s", str); } - GRPC_CLOSURE_SCHED(cb, error); TCP_UNREF(tcp, "write"); } } static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { GPR_TIMER_SCOPE("tcp_write", 0); grpc_tcp* tcp = reinterpret_cast<grpc_tcp*>(ep); grpc_error* error = GRPC_ERROR_NONE; @@ -675,6 +937,10 @@ static void tcp_write(grpc_endpoint* ep, grpc_slice_buffer* buf, } tcp->outgoing_buffer = buf; tcp->outgoing_byte_idx = 0; + tcp->outgoing_buffer_arg = arg; + if (arg) { + GPR_ASSERT(grpc_event_engine_can_track_errors()); + } if (!tcp_flush(tcp, &error)) { TCP_REF(tcp, "write"); @@ -792,6 +1058,8 @@ grpc_endpoint* grpc_tcp_create(grpc_fd* em_fd, tcp->bytes_read_this_round = 0; /* Will be set to false by the very first endpoint read function */ tcp->is_first_read = true; + tcp->bytes_counter = -1; + tcp->socket_ts_enabled = false; /* paired with unref in grpc_tcp_destroy */ gpr_ref_init(&tcp->refcount, 1); gpr_atm_no_barrier_store(&tcp->shutdown_count, 0); @@ -803,6 +1071,19 @@ grpc_endpoint* grpc_tcp_create(grpc_fd* em_fd, /* Tell network status tracker about new endpoint */ grpc_network_status_register_endpoint(&tcp->base); grpc_resource_quota_unref_internal(resource_quota); + gpr_mu_init(&tcp->tb_mu); + tcp->tb_head = nullptr; + /* Start being notified on errors if event engine can track errors. */ + if (grpc_event_engine_can_track_errors()) { + /* Grab a ref to tcp so that we can safely access the tcp struct when + * processing errors. We unref when we no longer want to track errors + * separately. */ + TCP_REF(tcp, "error-tracking"); + gpr_atm_rel_store(&tcp->stop_error_notification, 0); + GRPC_CLOSURE_INIT(&tcp->error_closure, tcp_handle_error, tcp, + grpc_schedule_on_exec_ctx); + grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure); + } return &tcp->base; } @@ -821,6 +1102,11 @@ void grpc_tcp_destroy_and_release_fd(grpc_endpoint* ep, int* fd, tcp->release_fd = fd; tcp->release_fd_cb = done; grpc_slice_buffer_reset_and_unref_internal(&tcp->last_read_buffer); + if (grpc_event_engine_can_track_errors()) { + /* Stop errors notification. */ + gpr_atm_no_barrier_store(&tcp->stop_error_notification, true); + grpc_fd_set_error(tcp->em_fd); + } TCP_UNREF(tcp, "destroy"); } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index af89bd24db..eff825cb92 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -31,7 +31,10 @@ #include <grpc/support/port_platform.h> +#include "src/core/lib/iomgr/port.h" + #include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/buffer_list.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/ev_posix.h" diff --git a/src/core/lib/iomgr/tcp_server_posix.cc b/src/core/lib/iomgr/tcp_server_posix.cc index 8ddf684fea..824db07fbf 100644 --- a/src/core/lib/iomgr/tcp_server_posix.cc +++ b/src/core/lib/iomgr/tcp_server_posix.cc @@ -226,7 +226,7 @@ static void on_read(void* arg, grpc_error* err) { gpr_log(GPR_INFO, "SERVER_CONNECT: incoming connection: %s", addr_str); } - grpc_fd* fdobj = grpc_fd_create(fd, name, false); + grpc_fd* fdobj = grpc_fd_create(fd, name, true); read_notifier_pollset = sp->server->pollsets[static_cast<size_t>(gpr_atm_no_barrier_fetch_add( @@ -362,7 +362,7 @@ static grpc_error* clone_port(grpc_tcp_listener* listener, unsigned count) { listener->sibling = sp; sp->server = listener->server; sp->fd = fd; - sp->emfd = grpc_fd_create(fd, name, false); + sp->emfd = grpc_fd_create(fd, name, true); memcpy(&sp->addr, &listener->addr, sizeof(grpc_resolved_address)); sp->port = port; sp->port_index = listener->port_index; diff --git a/src/core/lib/iomgr/tcp_server_utils_posix_common.cc b/src/core/lib/iomgr/tcp_server_utils_posix_common.cc index b9f8145572..9595c028ce 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +++ b/src/core/lib/iomgr/tcp_server_utils_posix_common.cc @@ -105,7 +105,7 @@ static grpc_error* add_socket_to_server(grpc_tcp_server* s, int fd, s->tail = sp; sp->server = s; sp->fd = fd; - sp->emfd = grpc_fd_create(fd, name, false); + sp->emfd = grpc_fd_create(fd, name, true); memcpy(&sp->addr, addr, sizeof(grpc_resolved_address)); sp->port = port; sp->port_index = port_index; diff --git a/src/core/lib/iomgr/tcp_windows.cc b/src/core/lib/iomgr/tcp_windows.cc index b3cb442f18..64c4a56ae9 100644 --- a/src/core/lib/iomgr/tcp_windows.cc +++ b/src/core/lib/iomgr/tcp_windows.cc @@ -296,7 +296,7 @@ static void on_write(void* tcpp, grpc_error* error) { /* Initiates a write. */ static void win_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { grpc_tcp* tcp = (grpc_tcp*)ep; grpc_winsocket* socket = tcp->socket; grpc_winsocket_callback_info* info = &socket->write_info; diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc index 4294162af7..008d37119a 100644 --- a/src/core/lib/iomgr/timer_generic.cc +++ b/src/core/lib/iomgr/timer_generic.cc @@ -291,7 +291,7 @@ static void timer_list_init() { static void timer_list_shutdown() { size_t i; run_some_expired_timers( - GPR_ATM_MAX, nullptr, + GRPC_MILLIS_INF_FUTURE, nullptr, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Timer list shutdown")); for (i = 0; i < g_num_shards; i++) { timer_shard* shard = &g_shards[i]; @@ -714,9 +714,10 @@ static grpc_timer_check_result timer_check(grpc_millis* next) { #if GPR_ARCH_64 gpr_log(GPR_INFO, "TIMER CHECK BEGIN: now=%" PRId64 " next=%s tls_min=%" PRId64 - " glob_min=%" PRIdPTR, + " glob_min=%" PRId64, now, next_str, min_timer, - gpr_atm_no_barrier_load((gpr_atm*)(&g_shared_mutables.min_timer))); + static_cast<grpc_millis>(gpr_atm_no_barrier_load( + (gpr_atm*)(&g_shared_mutables.min_timer)))); #else gpr_log(GPR_INFO, "TIMER CHECK BEGIN: now=%" PRId64 " next=%s min=%" PRId64, now, next_str, min_timer); diff --git a/src/core/lib/iomgr/udp_server.cc b/src/core/lib/iomgr/udp_server.cc index bdb2d0e764..3dd7cab855 100644 --- a/src/core/lib/iomgr/udp_server.cc +++ b/src/core/lib/iomgr/udp_server.cc @@ -152,7 +152,7 @@ GrpcUdpListener::GrpcUdpListener(grpc_udp_server* server, int fd, grpc_sockaddr_to_string(&addr_str, addr, 1); gpr_asprintf(&name, "udp-server-listener:%s", addr_str); gpr_free(addr_str); - emfd_ = grpc_fd_create(fd, name, false); + emfd_ = grpc_fd_create(fd, name, true); memcpy(&addr_, addr, sizeof(grpc_resolved_address)); GPR_ASSERT(emfd_); gpr_free(name); diff --git a/src/core/lib/security/transport/secure_endpoint.cc b/src/core/lib/security/transport/secure_endpoint.cc index 840b2e73bc..f40f969bb7 100644 --- a/src/core/lib/security/transport/secure_endpoint.cc +++ b/src/core/lib/security/transport/secure_endpoint.cc @@ -254,7 +254,7 @@ static void flush_write_staging_buffer(secure_endpoint* ep, uint8_t** cur, } static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { GPR_TIMER_SCOPE("secure_endpoint.endpoint_write", 0); unsigned i; @@ -342,7 +342,7 @@ static void endpoint_write(grpc_endpoint* secure_ep, grpc_slice_buffer* slices, return; } - grpc_endpoint_write(ep->wrapped_ep, &ep->output_buffer, cb); + grpc_endpoint_write(ep->wrapped_ep, &ep->output_buffer, cb, arg); } static void endpoint_shutdown(grpc_endpoint* secure_ep, grpc_error* why) { diff --git a/src/core/lib/security/transport/security_handshaker.cc b/src/core/lib/security/transport/security_handshaker.cc index aff723ed04..d76d582638 100644 --- a/src/core/lib/security/transport/security_handshaker.cc +++ b/src/core/lib/security/transport/security_handshaker.cc @@ -259,7 +259,7 @@ static grpc_error* on_handshake_next_done_locked( grpc_slice_buffer_reset_and_unref_internal(&h->outgoing); grpc_slice_buffer_add(&h->outgoing, to_send); grpc_endpoint_write(h->args->endpoint, &h->outgoing, - &h->on_handshake_data_sent_to_peer); + &h->on_handshake_data_sent_to_peer, nullptr); } else if (handshaker_result == nullptr) { // There is nothing to send, but need to read from peer. grpc_endpoint_read(h->args->endpoint, h->args->read_buffer, diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index 478865015c..a44f9acdc3 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -23,6 +23,6 @@ #include <grpc/grpc.h> -const char* grpc_version_string(void) { return "6.0.0-pre1"; } +const char* grpc_version_string(void) { return "6.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "glider"; } +const char* grpc_g_stands_for(void) { return "gao"; } diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 577b6ffa91..cc797f1546 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include <grpcpp/grpcpp.h> namespace grpc { -grpc::string Version() { return "1.15.0-pre1"; } +grpc::string Version() { return "1.16.0-dev"; } } // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 0e3c47fe80..18515ea1e8 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ <!-- This file is generated --> <Project> <PropertyGroup> - <GrpcCsharpVersion>1.15.0-pre1</GrpcCsharpVersion> + <GrpcCsharpVersion>1.16.0-dev</GrpcCsharpVersion> <GoogleProtobufVersion>3.6.1</GoogleProtobufVersion> </PropertyGroup> </Project> diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index af37cfc971..55d09dda7a 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// <summary> /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies /// </summary> - public const string CurrentAssemblyFileVersion = "1.15.0.0"; + public const string CurrentAssemblyFileVersion = "1.16.0.0"; /// <summary> /// Current version of gRPC C# /// </summary> - public const string CurrentVersion = "1.15.0-pre1"; + public const string CurrentVersion = "1.16.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index d3c19bec38..24d016104c 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.15.0-pre1 +set VERSION=1.16.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index b2dc6e46e5..4e7ac4e414 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.15.0-pre1 +set VERSION=1.16.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index dfadcdc211..200cee5b7b 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.15.0-pre1' + v = '1.16.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 4cdd8653bb..38862e102a 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -22,4 +22,4 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.15.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.16.0-dev" diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 8bd1f6c525..82833102ad 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -22,5 +22,5 @@ // instead. This file can be regenerated from the template by running // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.15.0-pre1" -#define GRPC_C_VERSION_STRING @"6.0.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.16.0-dev" +#define GRPC_C_VERSION_STRING @"6.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index 91913d73d2..f31423f8c0 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.15.0", + "version": "1.16.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index c519809a36..469a48e782 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.15.0RC1" +#define PHP_GRPC_VERSION "1.16.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index e68a13c8e8..24e1557578 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.15.0rc1""" +__version__ = """1.16.0.dev0""" diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 6e6d756eec..0f68e823d7 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -82,6 +82,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/http/format_request.cc', 'src/core/lib/http/httpcli.cc', 'src/core/lib/http/parser.cc', + 'src/core/lib/iomgr/buffer_list.cc', 'src/core/lib/iomgr/call_combiner.cc', 'src/core/lib/iomgr/combiner.cc', 'src/core/lib/iomgr/endpoint.cc', @@ -102,6 +103,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/gethostname_fallback.cc', 'src/core/lib/iomgr/gethostname_host_name_max.cc', 'src/core/lib/iomgr/gethostname_sysconf.cc', + 'src/core/lib/iomgr/internal_errqueue.cc', 'src/core/lib/iomgr/iocp_windows.cc', 'src/core/lib/iomgr/iomgr.cc', 'src/core/lib/iomgr/iomgr_custom.cc', diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index dd45ba6640..6ffe1eb827 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 33a02cf270..e080bf2cbc 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 19b613c7ae..4b3b95fee9 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 5f117c6efa..c12aa153a4 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 5fd88b3ee1..f4b8a34a46 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 8f981eeebe..0c3e1ef734 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.15.0.pre1' + VERSION = '1.16.0.dev' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 862d975e5c..03d977c064 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.15.0.pre1' + VERSION = '1.16.0.dev' end end diff --git a/test/core/bad_client/bad_client.cc b/test/core/bad_client/bad_client.cc index c03ebcf409..ade23133c5 100644 --- a/test/core/bad_client/bad_client.cc +++ b/test/core/bad_client/bad_client.cc @@ -115,7 +115,7 @@ void grpc_run_client_side_validator(grpc_bad_client_arg* arg, uint32_t flags, grpc_schedule_on_exec_ctx); /* Write data */ - grpc_endpoint_write(sfd->client, &outgoing, &done_write_closure); + grpc_endpoint_write(sfd->client, &outgoing, &done_write_closure, nullptr); grpc_core::ExecCtx::Get()->Flush(); /* Await completion, unless the request is large and write may not finish diff --git a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc index b1f3a1c08a..1a7db40f59 100644 --- a/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc +++ b/test/core/client_channel/resolvers/dns_resolver_cooldown_test.cc @@ -28,12 +28,16 @@ #include "src/core/lib/iomgr/sockaddr_utils.h" #include "test/core/util/test_config.h" +constexpr int kMinResolutionPeriodMs = 1000; +// Provide some slack when checking intervals, to allow for test timing issues. +constexpr int kMinResolutionPeriodForCheckMs = 900; + extern grpc_address_resolver_vtable* grpc_resolve_address_impl; static grpc_address_resolver_vtable* default_resolve_address; static grpc_combiner* g_combiner; -grpc_ares_request* (*g_default_dns_lookup_ares_locked)( +static grpc_ares_request* (*g_default_dns_lookup_ares_locked)( const char* dns_server, const char* name, const char* default_port, grpc_pollset_set* interested_parties, grpc_closure* on_done, grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, @@ -43,7 +47,7 @@ grpc_ares_request* (*g_default_dns_lookup_ares_locked)( // times a system-level resolution has happened. static int g_resolution_count; -struct iomgr_args { +static struct iomgr_args { gpr_event ev; gpr_atm done_atm; gpr_mu* mu; @@ -61,6 +65,16 @@ static void test_resolve_address_impl(const char* name, default_resolve_address->resolve_address( name, default_port, g_iomgr_args.pollset_set, on_done, addrs); ++g_resolution_count; + static grpc_millis last_resolution_time = 0; + if (last_resolution_time == 0) { + last_resolution_time = + grpc_timespec_to_millis_round_up(gpr_now(GPR_CLOCK_MONOTONIC)); + } else { + grpc_millis now = + grpc_timespec_to_millis_round_up(gpr_now(GPR_CLOCK_MONOTONIC)); + GPR_ASSERT(now - last_resolution_time >= kMinResolutionPeriodForCheckMs); + last_resolution_time = now; + } } static grpc_error* test_blocking_resolve_address_impl( @@ -73,7 +87,7 @@ static grpc_error* test_blocking_resolve_address_impl( static grpc_address_resolver_vtable test_resolver = { test_resolve_address_impl, test_blocking_resolve_address_impl}; -grpc_ares_request* test_dns_lookup_ares_locked( +static grpc_ares_request* test_dns_lookup_ares_locked( const char* dns_server, const char* name, const char* default_port, grpc_pollset_set* interested_parties, grpc_closure* on_done, grpc_lb_addresses** addrs, bool check_grpclb, char** service_config_json, @@ -82,6 +96,16 @@ grpc_ares_request* test_dns_lookup_ares_locked( dns_server, name, default_port, g_iomgr_args.pollset_set, on_done, addrs, check_grpclb, service_config_json, combiner); ++g_resolution_count; + static grpc_millis last_resolution_time = 0; + if (last_resolution_time == 0) { + last_resolution_time = + grpc_timespec_to_millis_round_up(gpr_now(GPR_CLOCK_MONOTONIC)); + } else { + grpc_millis now = + grpc_timespec_to_millis_round_up(gpr_now(GPR_CLOCK_MONOTONIC)); + GPR_ASSERT(now - last_resolution_time >= kMinResolutionPeriodForCheckMs); + last_resolution_time = now; + } return result; } @@ -91,7 +115,7 @@ static gpr_timespec test_deadline(void) { static void do_nothing(void* arg, grpc_error* error) {} -void iomgr_args_init(iomgr_args* args) { +static void iomgr_args_init(iomgr_args* args) { gpr_event_init(&args->ev); args->pollset = static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); grpc_pollset_init(args->pollset, &args->mu); @@ -100,7 +124,7 @@ void iomgr_args_init(iomgr_args* args) { gpr_atm_rel_store(&args->done_atm, 0); } -void iomgr_args_finish(iomgr_args* args) { +static void iomgr_args_finish(iomgr_args* args) { GPR_ASSERT(gpr_event_wait(&args->ev, test_deadline())); grpc_pollset_set_del_pollset(args->pollset_set, args->pollset); grpc_pollset_set_destroy(args->pollset_set); @@ -146,29 +170,19 @@ struct OnResolutionCallbackArg { const char* uri_str = nullptr; grpc_core::OrphanablePtr<grpc_core::Resolver> resolver; grpc_channel_args* result = nullptr; - grpc_millis delay_before_second_resolution = 0; }; -// Counter for the number of times a resolution notification callback has been -// invoked. -static int g_on_resolution_invocations_count; - // Set to true by the last callback in the resolution chain. -bool g_all_callbacks_invoked; +static bool g_all_callbacks_invoked; -void on_fourth_resolution(void* arg, grpc_error* error) { +static void on_second_resolution(void* arg, grpc_error* error) { OnResolutionCallbackArg* cb_arg = static_cast<OnResolutionCallbackArg*>(arg); grpc_channel_args_destroy(cb_arg->result); GPR_ASSERT(error == GRPC_ERROR_NONE); - ++g_on_resolution_invocations_count; - gpr_log(GPR_INFO, - "4th: g_on_resolution_invocations_count: %d, g_resolution_count: %d", - g_on_resolution_invocations_count, g_resolution_count); - // In this case we expect to have incurred in another system-level resolution - // because on_third_resolution slept for longer than the min resolution - // period. - GPR_ASSERT(g_on_resolution_invocations_count == 4); - GPR_ASSERT(g_resolution_count == 3); + gpr_log(GPR_INFO, "2nd: g_resolution_count: %d", g_resolution_count); + // The resolution callback was not invoked until new data was + // available, which was delayed until after the cooldown period. + GPR_ASSERT(g_resolution_count == 2); cb_arg->resolver.reset(); gpr_atm_rel_store(&g_iomgr_args.done_atm, 1); gpr_mu_lock(g_iomgr_args.mu); @@ -179,67 +193,13 @@ void on_fourth_resolution(void* arg, grpc_error* error) { g_all_callbacks_invoked = true; } -void on_third_resolution(void* arg, grpc_error* error) { - OnResolutionCallbackArg* cb_arg = static_cast<OnResolutionCallbackArg*>(arg); - grpc_channel_args_destroy(cb_arg->result); - GPR_ASSERT(error == GRPC_ERROR_NONE); - ++g_on_resolution_invocations_count; - gpr_log(GPR_INFO, - "3rd: g_on_resolution_invocations_count: %d, g_resolution_count: %d", - g_on_resolution_invocations_count, g_resolution_count); - // The timer set because of the previous re-resolution request fires, so a new - // system-level resolution happened. - GPR_ASSERT(g_on_resolution_invocations_count == 3); - GPR_ASSERT(g_resolution_count == 2); - grpc_core::ExecCtx::Get()->TestOnlySetNow( - cb_arg->delay_before_second_resolution * 2); - cb_arg->resolver->NextLocked( - &cb_arg->result, - GRPC_CLOSURE_CREATE(on_fourth_resolution, arg, - grpc_combiner_scheduler(g_combiner))); - cb_arg->resolver->RequestReresolutionLocked(); - gpr_mu_lock(g_iomgr_args.mu); - GRPC_LOG_IF_ERROR("pollset_kick", - grpc_pollset_kick(g_iomgr_args.pollset, nullptr)); - gpr_mu_unlock(g_iomgr_args.mu); -} - -void on_second_resolution(void* arg, grpc_error* error) { - OnResolutionCallbackArg* cb_arg = static_cast<OnResolutionCallbackArg*>(arg); - grpc_channel_args_destroy(cb_arg->result); - GPR_ASSERT(error == GRPC_ERROR_NONE); - ++g_on_resolution_invocations_count; - gpr_log(GPR_INFO, - "2nd: g_on_resolution_invocations_count: %d, g_resolution_count: %d", - g_on_resolution_invocations_count, g_resolution_count); - // The resolution request for which this function is the callback happened - // before the min resolution period. Therefore, no new system-level - // resolutions happened, as indicated by g_resolution_count. But a resolution - // timer was set to fire when the cooldown finishes. - GPR_ASSERT(g_on_resolution_invocations_count == 2); - GPR_ASSERT(g_resolution_count == 1); - // Register a new callback to capture the timer firing. - cb_arg->resolver->NextLocked( - &cb_arg->result, - GRPC_CLOSURE_CREATE(on_third_resolution, arg, - grpc_combiner_scheduler(g_combiner))); - gpr_mu_lock(g_iomgr_args.mu); - GRPC_LOG_IF_ERROR("pollset_kick", - grpc_pollset_kick(g_iomgr_args.pollset, nullptr)); - gpr_mu_unlock(g_iomgr_args.mu); -} - -void on_first_resolution(void* arg, grpc_error* error) { +static void on_first_resolution(void* arg, grpc_error* error) { OnResolutionCallbackArg* cb_arg = static_cast<OnResolutionCallbackArg*>(arg); grpc_channel_args_destroy(cb_arg->result); GPR_ASSERT(error == GRPC_ERROR_NONE); - ++g_on_resolution_invocations_count; - gpr_log(GPR_INFO, - "1st: g_on_resolution_invocations_count: %d, g_resolution_count: %d", - g_on_resolution_invocations_count, g_resolution_count); + gpr_log(GPR_INFO, "1st: g_resolution_count: %d", g_resolution_count); // There's one initial system-level resolution and one invocation of a // notification callback (the current function). - GPR_ASSERT(g_on_resolution_invocations_count == 1); GPR_ASSERT(g_resolution_count == 1); cb_arg->resolver->NextLocked( &cb_arg->result, @@ -265,9 +225,7 @@ static void start_test_under_combiner(void* arg, grpc_error* error) { grpc_core::ResolverArgs args; args.uri = uri; args.combiner = g_combiner; - g_on_resolution_invocations_count = 0; g_resolution_count = 0; - constexpr int kMinResolutionPeriodMs = 1000; grpc_arg cooldown_arg; cooldown_arg.key = @@ -280,7 +238,6 @@ static void start_test_under_combiner(void* arg, grpc_error* error) { res_cb_arg->resolver = factory->CreateResolver(args); grpc_channel_args_destroy(cooldown_channel_args); GPR_ASSERT(res_cb_arg->resolver != nullptr); - res_cb_arg->delay_before_second_resolution = kMinResolutionPeriodMs; // First resolution, would incur in system-level resolution. res_cb_arg->resolver->NextLocked( &res_cb_arg->result, diff --git a/test/core/client_channel/resolvers/fake_resolver_test.cc b/test/core/client_channel/resolvers/fake_resolver_test.cc index 14caa3ea5d..f6696bf127 100644 --- a/test/core/client_channel/resolvers/fake_resolver_test.cc +++ b/test/core/client_channel/resolvers/fake_resolver_test.cc @@ -124,8 +124,8 @@ static void test_fake_resolver() { build_fake_resolver(combiner, response_generator.get()); GPR_ASSERT(resolver.get() != nullptr); // Test 1: normal resolution. - // next_results != NULL, reresolution_results == NULL, last_used_results == - // NULL. Expected response is next_results. + // next_results != NULL, reresolution_results == NULL. + // Expected response is next_results. grpc_channel_args* results = create_new_resolver_result(); on_resolution_arg on_res_arg = create_on_resolution_arg(results); grpc_closure* on_resolution = GRPC_CLOSURE_CREATE( @@ -137,10 +137,9 @@ static void test_fake_resolver() { GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, grpc_timeout_seconds_to_deadline(5)) != nullptr); // Test 2: update resolution. - // next_results != NULL, reresolution_results == NULL, last_used_results != - // NULL. Expected response is next_results. + // next_results != NULL, reresolution_results == NULL. + // Expected response is next_results. results = create_new_resolver_result(); - grpc_channel_args* last_used_results = grpc_channel_args_copy(results); on_res_arg = create_on_resolution_arg(results); on_resolution = GRPC_CLOSURE_CREATE(on_resolution_cb, &on_res_arg, grpc_combiner_scheduler(combiner)); @@ -150,21 +149,9 @@ static void test_fake_resolver() { grpc_core::ExecCtx::Get()->Flush(); GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 3: fallback re-resolution. - // next_results == NULL, reresolution_results == NULL, last_used_results != - // NULL. Expected response is last_used_results. - on_res_arg = create_on_resolution_arg(last_used_results); - on_resolution = GRPC_CLOSURE_CREATE(on_resolution_cb, &on_res_arg, - grpc_combiner_scheduler(combiner)); - resolver->NextLocked(&on_res_arg.resolver_result, on_resolution); - // Trigger a re-resolution. - resolver->RequestReresolutionLocked(); - grpc_core::ExecCtx::Get()->Flush(); - GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, - grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 4: normal re-resolution. - // next_results == NULL, reresolution_results != NULL, last_used_results != - // NULL. Expected response is reresolution_results. + // Test 3: normal re-resolution. + // next_results == NULL, reresolution_results != NULL. + // Expected response is reresolution_results. grpc_channel_args* reresolution_results = create_new_resolver_result(); on_res_arg = create_on_resolution_arg(grpc_channel_args_copy(reresolution_results)); @@ -180,9 +167,9 @@ static void test_fake_resolver() { grpc_core::ExecCtx::Get()->Flush(); GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 5: repeat re-resolution. - // next_results == NULL, reresolution_results != NULL, last_used_results != - // NULL. Expected response is reresolution_results. + // Test 4: repeat re-resolution. + // next_results == NULL, reresolution_results != NULL. + // Expected response is reresolution_results. on_res_arg = create_on_resolution_arg(reresolution_results); on_resolution = GRPC_CLOSURE_CREATE(on_resolution_cb, &on_res_arg, grpc_combiner_scheduler(combiner)); @@ -192,11 +179,10 @@ static void test_fake_resolver() { grpc_core::ExecCtx::Get()->Flush(); GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 6: normal resolution. - // next_results != NULL, reresolution_results != NULL, last_used_results != - // NULL. Expected response is next_results. + // Test 5: normal resolution. + // next_results != NULL, reresolution_results != NULL. + // Expected response is next_results. results = create_new_resolver_result(); - last_used_results = grpc_channel_args_copy(results); on_res_arg = create_on_resolution_arg(results); on_resolution = GRPC_CLOSURE_CREATE(on_resolution_cb, &on_res_arg, grpc_combiner_scheduler(combiner)); @@ -206,23 +192,7 @@ static void test_fake_resolver() { grpc_core::ExecCtx::Get()->Flush(); GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 7: fallback re-resolution. - // next_results == NULL, reresolution_results == NULL, last_used_results != - // NULL. Expected response is last_used_results. - on_res_arg = create_on_resolution_arg(last_used_results); - on_resolution = GRPC_CLOSURE_CREATE(on_resolution_cb, &on_res_arg, - grpc_combiner_scheduler(combiner)); - resolver->NextLocked(&on_res_arg.resolver_result, on_resolution); - // Reset reresolution_results. - response_generator->SetReresolutionResponse(nullptr); - // Flush here to guarantee that reresolution_results has been reset. - grpc_core::ExecCtx::Get()->Flush(); - // Trigger a re-resolution. - resolver->RequestReresolutionLocked(); - grpc_core::ExecCtx::Get()->Flush(); - GPR_ASSERT(gpr_event_wait(&on_res_arg.ev, - grpc_timeout_seconds_to_deadline(5)) != nullptr); - // Test 8: no-op. + // Test 6: no-op. // Requesting a new resolution without setting the response shouldn't trigger // the resolution callback. memset(&on_res_arg, 0, sizeof(on_res_arg)); diff --git a/test/core/end2end/bad_server_response_test.cc b/test/core/end2end/bad_server_response_test.cc index 3d133cfc18..f7396a1684 100644 --- a/test/core/end2end/bad_server_response_test.cc +++ b/test/core/end2end/bad_server_response_test.cc @@ -104,7 +104,7 @@ static void handle_write() { grpc_slice_buffer_reset_and_unref(&state.outgoing_buffer); grpc_slice_buffer_add(&state.outgoing_buffer, slice); - grpc_endpoint_write(state.tcp, &state.outgoing_buffer, &on_write); + grpc_endpoint_write(state.tcp, &state.outgoing_buffer, &on_write, nullptr); } static void handle_read(void* arg, grpc_error* error) { diff --git a/test/core/end2end/end2end_test.sh b/test/core/end2end/end2end_test.sh index 5bfb253090..6b23d848be 100755 --- a/test/core/end2end/end2end_test.sh +++ b/test/core/end2end/end2end_test.sh @@ -15,7 +15,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -if [ -z "$3" ] +if [ -n "$3" ] then export GRPC_POLL_STRATEGY=$3 fi diff --git a/test/core/end2end/fixtures/http_proxy_fixture.cc b/test/core/end2end/fixtures/http_proxy_fixture.cc index f02fa9d998..ea9c000efb 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.cc +++ b/test/core/end2end/fixtures/http_proxy_fixture.cc @@ -201,7 +201,7 @@ static void on_client_write_done(void* arg, grpc_error* error) { &conn->client_write_buffer); conn->client_is_writing = true; grpc_endpoint_write(conn->client_endpoint, &conn->client_write_buffer, - &conn->on_client_write_done); + &conn->on_client_write_done, nullptr); } else { // No more writes. Unref the connection. proxy_connection_unref(conn, "write_done"); @@ -226,7 +226,7 @@ static void on_server_write_done(void* arg, grpc_error* error) { &conn->server_write_buffer); conn->server_is_writing = true; grpc_endpoint_write(conn->server_endpoint, &conn->server_write_buffer, - &conn->on_server_write_done); + &conn->on_server_write_done, nullptr); } else { // No more writes. Unref the connection. proxy_connection_unref(conn, "server_write"); @@ -257,7 +257,7 @@ static void on_client_read_done(void* arg, grpc_error* error) { proxy_connection_ref(conn, "client_read"); conn->server_is_writing = true; grpc_endpoint_write(conn->server_endpoint, &conn->server_write_buffer, - &conn->on_server_write_done); + &conn->on_server_write_done, nullptr); } // Read more data. grpc_endpoint_read(conn->client_endpoint, &conn->client_read_buffer, @@ -288,7 +288,7 @@ static void on_server_read_done(void* arg, grpc_error* error) { proxy_connection_ref(conn, "server_read"); conn->client_is_writing = true; grpc_endpoint_write(conn->client_endpoint, &conn->client_write_buffer, - &conn->on_client_write_done); + &conn->on_client_write_done, nullptr); } // Read more data. grpc_endpoint_read(conn->server_endpoint, &conn->server_read_buffer, @@ -340,7 +340,7 @@ static void on_server_connect_done(void* arg, grpc_error* error) { grpc_slice_buffer_add(&conn->client_write_buffer, slice); conn->client_is_writing = true; grpc_endpoint_write(conn->client_endpoint, &conn->client_write_buffer, - &conn->on_write_response_done); + &conn->on_write_response_done, nullptr); } /** diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index 002671a5fa..675d9e6278 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -247,6 +247,19 @@ grpc_cc_test( ) grpc_cc_test( + name = "buffer_list_test", + srcs = ["buffer_list_test.cc"], + language = "C++", + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + + +grpc_cc_test( name = "tcp_server_posix_test", srcs = ["tcp_server_posix_test.cc"], language = "C++", diff --git a/test/core/iomgr/buffer_list_test.cc b/test/core/iomgr/buffer_list_test.cc new file mode 100644 index 0000000000..c7f30fa092 --- /dev/null +++ b/test/core/iomgr/buffer_list_test.cc @@ -0,0 +1,111 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#include "src/core/lib/iomgr/buffer_list.h" + +#include <grpc/grpc.h> + +#include "test/core/util/test_config.h" + +#ifdef GRPC_LINUX_ERRQUEUE + +static void TestShutdownFlushesListVerifier(void* arg, + grpc_core::Timestamps* ts, + grpc_error* error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + GPR_ASSERT(arg != nullptr); + gpr_atm* done = reinterpret_cast<gpr_atm*>(arg); + gpr_atm_rel_store(done, static_cast<gpr_atm>(1)); +} + +/** Tests that all TracedBuffer elements in the list are flushed out on + * shutdown. + * Also tests that arg is passed correctly. + */ +static void TestShutdownFlushesList() { + grpc_core::grpc_tcp_set_write_timestamps_callback( + TestShutdownFlushesListVerifier); + grpc_core::TracedBuffer* list = nullptr; +#define NUM_ELEM 5 + gpr_atm verifier_called[NUM_ELEM]; + for (auto i = 0; i < NUM_ELEM; i++) { + gpr_atm_rel_store(&verifier_called[i], static_cast<gpr_atm>(0)); + grpc_core::TracedBuffer::AddNewEntry( + &list, i, static_cast<void*>(&verifier_called[i])); + } + grpc_core::TracedBuffer::Shutdown(&list, GRPC_ERROR_NONE); + GPR_ASSERT(list == nullptr); + for (auto i = 0; i < NUM_ELEM; i++) { + GPR_ASSERT(gpr_atm_acq_load(&verifier_called[i]) == + static_cast<gpr_atm>(1)); + } +} + +static void TestVerifierCalledOnAckVerifier(void* arg, + grpc_core::Timestamps* ts, + grpc_error* error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + GPR_ASSERT(arg != nullptr); + GPR_ASSERT(ts->acked_time.clock_type == GPR_CLOCK_REALTIME); + GPR_ASSERT(ts->acked_time.tv_sec == 123); + GPR_ASSERT(ts->acked_time.tv_nsec == 456); + gpr_atm* done = reinterpret_cast<gpr_atm*>(arg); + gpr_atm_rel_store(done, static_cast<gpr_atm>(1)); +} + +/** Tests that the timestamp verifier is called on an ACK timestamp. + */ +static void TestVerifierCalledOnAck() { + struct sock_extended_err serr; + serr.ee_data = 213; + serr.ee_info = grpc_core::SCM_TSTAMP_ACK; + struct grpc_core::scm_timestamping tss; + tss.ts[0].tv_sec = 123; + tss.ts[0].tv_nsec = 456; + grpc_core::grpc_tcp_set_write_timestamps_callback( + TestVerifierCalledOnAckVerifier); + grpc_core::TracedBuffer* list = nullptr; + gpr_atm verifier_called; + gpr_atm_rel_store(&verifier_called, static_cast<gpr_atm>(0)); + grpc_core::TracedBuffer::AddNewEntry(&list, 213, &verifier_called); + grpc_core::TracedBuffer::ProcessTimestamp(&list, &serr, &tss); + GPR_ASSERT(gpr_atm_acq_load(&verifier_called) == static_cast<gpr_atm>(1)); + GPR_ASSERT(list == nullptr); + grpc_core::TracedBuffer::Shutdown(&list, GRPC_ERROR_NONE); +} + +static void TestTcpBufferList() { + TestVerifierCalledOnAck(); + TestShutdownFlushesList(); +} + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + grpc_init(); + TestTcpBufferList(); + grpc_shutdown(); + return 0; +} + +#else /* GRPC_LINUX_ERRQUEUE */ + +int main(int argc, char** argv) { return 0; } + +#endif /* GRPC_LINUX_ERRQUEUE */ diff --git a/test/core/iomgr/endpoint_tests.cc b/test/core/iomgr/endpoint_tests.cc index 8db8ac5ed6..a9e8ba86c5 100644 --- a/test/core/iomgr/endpoint_tests.cc +++ b/test/core/iomgr/endpoint_tests.cc @@ -150,8 +150,8 @@ static void read_and_write_test_write_handler(void* data, grpc_error* error) { &state->current_write_data); grpc_slice_buffer_reset_and_unref(&state->outgoing); grpc_slice_buffer_addn(&state->outgoing, slices, nslices); - grpc_endpoint_write(state->write_ep, &state->outgoing, - &state->done_write); + grpc_endpoint_write(state->write_ep, &state->outgoing, &state->done_write, + nullptr); gpr_free(slices); return; } @@ -294,7 +294,8 @@ static void multiple_shutdown_test(grpc_endpoint_test_config config) { grpc_slice_buffer_add(&slice_buffer, grpc_slice_from_copied_string("a")); grpc_endpoint_write(f.client_ep, &slice_buffer, GRPC_CLOSURE_CREATE(inc_on_failure, &fail_count, - grpc_schedule_on_exec_ctx)); + grpc_schedule_on_exec_ctx), + nullptr); wait_for_fail_count(&fail_count, 3); grpc_endpoint_shutdown(f.client_ep, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Test Shutdown")); diff --git a/test/core/iomgr/tcp_posix_test.cc b/test/core/iomgr/tcp_posix_test.cc index 3e87831e44..6447cc234d 100644 --- a/test/core/iomgr/tcp_posix_test.cc +++ b/test/core/iomgr/tcp_posix_test.cc @@ -36,6 +36,9 @@ #include <grpc/support/time.h> #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/iomgr/buffer_list.h" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/sockaddr_posix.h" #include "src/core/lib/slice/slice_internal.h" #include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" @@ -68,6 +71,43 @@ static void create_sockets(int sv[2]) { GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0); } +static void create_inet_sockets(int sv[2]) { + /* Prepare listening socket */ + struct sockaddr_in addr; + memset(&addr, 0, sizeof(struct sockaddr_in)); + addr.sin_family = AF_INET; + int sock = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sock); + GPR_ASSERT(bind(sock, (sockaddr*)&addr, sizeof(sockaddr_in)) == 0); + listen(sock, 1); + + /* Prepare client socket and connect to server */ + socklen_t len = sizeof(sockaddr_in); + GPR_ASSERT(getsockname(sock, (sockaddr*)&addr, &len) == 0); + + int client = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(client); + int ret; + do { + ret = connect(client, (sockaddr*)&addr, sizeof(sockaddr_in)); + } while (ret == -1 && errno == EINTR); + + /* Accept client connection */ + len = sizeof(socklen_t); + int server; + do { + server = accept(sock, (sockaddr*)&addr, (socklen_t*)&len); + } while (server == -1 && errno == EINTR); + GPR_ASSERT(server != -1); + + sv[0] = server; + sv[1] = client; + int flags = fcntl(sv[0], F_GETFL, 0); + GPR_ASSERT(fcntl(sv[0], F_SETFL, flags | O_NONBLOCK) == 0); + flags = fcntl(sv[1], F_GETFL, 0); + GPR_ASSERT(fcntl(sv[1], F_SETFL, flags | O_NONBLOCK) == 0); +} + static ssize_t fill_socket(int fd) { ssize_t write_bytes; ssize_t total_bytes = 0; @@ -289,11 +329,10 @@ static grpc_slice* allocate_blocks(size_t num_bytes, size_t slice_size, static void write_done(void* user_data /* write_socket_state */, grpc_error* error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); struct write_socket_state* state = static_cast<struct write_socket_state*>(user_data); - gpr_log(GPR_INFO, "Write done callback called"); gpr_mu_lock(g_mu); - gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; GPR_ASSERT( GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, nullptr))); @@ -340,10 +379,24 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) { gpr_free(buf); } +/* Verifier for timestamps callback for write_test */ +void timestamps_verifier(void* arg, grpc_core::Timestamps* ts, + grpc_error* error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + GPR_ASSERT(arg != nullptr); + GPR_ASSERT(ts->sendmsg_time.clock_type == GPR_CLOCK_REALTIME); + GPR_ASSERT(ts->scheduled_time.clock_type == GPR_CLOCK_REALTIME); + GPR_ASSERT(ts->acked_time.clock_type == GPR_CLOCK_REALTIME); + gpr_atm* done_timestamps = (gpr_atm*)arg; + gpr_atm_rel_store(done_timestamps, static_cast<gpr_atm>(1)); +} + /* Write to a socket using the grpc_tcp API, then drain it directly. Note that if the write does not complete immediately we need to drain the - socket in parallel with the read. */ -static void write_test(size_t num_bytes, size_t slice_size) { + socket in parallel with the read. If collect_timestamps is true, it will + try to get timestamps for the write. */ +static void write_test(size_t num_bytes, size_t slice_size, + bool collect_timestamps) { int sv[2]; grpc_endpoint* ep; struct write_socket_state state; @@ -356,19 +409,27 @@ static void write_test(size_t num_bytes, size_t slice_size) { grpc_timespec_to_millis_round_up(grpc_timeout_seconds_to_deadline(20)); grpc_core::ExecCtx exec_ctx; + if (collect_timestamps && !grpc_event_engine_can_track_errors()) { + return; + } + gpr_log(GPR_INFO, "Start write test with %" PRIuPTR " bytes, slice size %" PRIuPTR, num_bytes, slice_size); - create_sockets(sv); + if (collect_timestamps) { + create_inet_sockets(sv); + } else { + create_sockets(sv); + } grpc_arg a[1]; a[0].key = const_cast<char*>(GRPC_ARG_TCP_READ_CHUNK_SIZE); a[0].type = GRPC_ARG_INTEGER, a[0].value.integer = static_cast<int>(slice_size); grpc_channel_args args = {GPR_ARRAY_SIZE(a), a}; - ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test", false), &args, - "test"); + ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test", collect_timestamps), + &args, "test"); grpc_endpoint_add_to_pollset(ep, g_pollset); state.ep = ep; @@ -381,18 +442,26 @@ static void write_test(size_t num_bytes, size_t slice_size) { GRPC_CLOSURE_INIT(&write_done_closure, write_done, &state, grpc_schedule_on_exec_ctx); - grpc_endpoint_write(ep, &outgoing, &write_done_closure); + gpr_atm done_timestamps; + gpr_atm_rel_store(&done_timestamps, static_cast<gpr_atm>(0)); + grpc_endpoint_write(ep, &outgoing, &write_done_closure, + grpc_event_engine_can_track_errors() && collect_timestamps + ? (void*)&done_timestamps + : nullptr); drain_socket_blocking(sv[0], num_bytes, num_bytes); + exec_ctx.Flush(); gpr_mu_lock(g_mu); for (;;) { grpc_pollset_worker* worker = nullptr; - if (state.write_done) { + if (state.write_done && + (!(grpc_event_engine_can_track_errors() && collect_timestamps) || + gpr_atm_acq_load(&done_timestamps) == static_cast<gpr_atm>(1))) { break; } GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_work", grpc_pollset_work(g_pollset, &worker, deadline))); gpr_mu_unlock(g_mu); - + exec_ctx.Flush(); gpr_mu_lock(g_mu); } gpr_mu_unlock(g_mu); @@ -497,14 +566,21 @@ void run_tests(void) { large_read_test(8192); large_read_test(1); - write_test(100, 8192); - write_test(100, 1); - write_test(100000, 8192); - write_test(100000, 1); - write_test(100000, 137); + write_test(100, 8192, false); + write_test(100, 1, false); + write_test(100000, 8192, false); + write_test(100000, 1, false); + write_test(100000, 137, false); + + write_test(100, 8192, true); + write_test(100, 1, true); + write_test(100000, 8192, true); + write_test(100000, 1, true); + write_test(100, 137, true); for (i = 1; i < 1000; i = GPR_MAX(i + 1, i * 5 / 4)) { - write_test(40320, i); + write_test(40320, i, false); + write_test(40320, i, true); } release_fd_test(100, 8192); @@ -549,6 +625,7 @@ int main(int argc, char** argv) { grpc_closure destroyed; grpc_test_init(argc, argv); grpc_init(); + grpc_core::grpc_tcp_set_write_timestamps_callback(timestamps_verifier); { grpc_core::ExecCtx exec_ctx; g_pollset = static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); diff --git a/test/core/iomgr/timer_list_test.cc b/test/core/iomgr/timer_list_test.cc index feedf3f149..fd65d1abf1 100644 --- a/test/core/iomgr/timer_list_test.cc +++ b/test/core/iomgr/timer_list_test.cc @@ -248,11 +248,7 @@ int main(int argc, char** argv) { grpc_determine_iomgr_platform(); grpc_iomgr_platform_init(); gpr_set_log_verbosity(GPR_LOG_SEVERITY_DEBUG); -#ifndef GPR_WINDOWS - /* Skip this test on Windows until we figure out why it fails */ - /* https://github.com/grpc/grpc/issues/16417 */ long_running_service_cleanup_test(); -#endif // GPR_WINDOWS add_test(); destruction_test(); grpc_iomgr_platform_shutdown(); diff --git a/test/core/util/mock_endpoint.cc b/test/core/util/mock_endpoint.cc index 1156cd5fc5..ef6fd62b51 100644 --- a/test/core/util/mock_endpoint.cc +++ b/test/core/util/mock_endpoint.cc @@ -55,7 +55,7 @@ static void me_read(grpc_endpoint* ep, grpc_slice_buffer* slices, } static void me_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { mock_endpoint* m = reinterpret_cast<mock_endpoint*>(ep); for (size_t i = 0; i < slices->count; i++) { m->on_write(slices->slices[i]); diff --git a/test/core/util/passthru_endpoint.cc b/test/core/util/passthru_endpoint.cc index 5958216747..3cc8ad6fe1 100644 --- a/test/core/util/passthru_endpoint.cc +++ b/test/core/util/passthru_endpoint.cc @@ -76,7 +76,7 @@ static half* other_half(half* h) { } static void me_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { half* m = other_half(reinterpret_cast<half*>(ep)); gpr_mu_lock(&m->parent->mu); grpc_error* error = GRPC_ERROR_NONE; diff --git a/test/core/util/trickle_endpoint.cc b/test/core/util/trickle_endpoint.cc index f2efb049b4..62ed72a629 100644 --- a/test/core/util/trickle_endpoint.cc +++ b/test/core/util/trickle_endpoint.cc @@ -62,7 +62,7 @@ static void maybe_call_write_cb_locked(trickle_endpoint* te) { } static void te_write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { trickle_endpoint* te = reinterpret_cast<trickle_endpoint*>(ep); gpr_mu_lock(&te->mu); GPR_ASSERT(te->write_cb == nullptr); @@ -186,7 +186,8 @@ size_t grpc_trickle_endpoint_trickle(grpc_endpoint* ep) { te->last_write = now; grpc_endpoint_write( te->wrapped, &te->writing_buffer, - GRPC_CLOSURE_CREATE(te_finish_write, te, grpc_schedule_on_exec_ctx)); + GRPC_CLOSURE_CREATE(te_finish_write, te, grpc_schedule_on_exec_ctx), + nullptr); maybe_call_write_cb_locked(te); } } diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 68219c16dc..e5d6132012 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -129,12 +129,23 @@ class ClientLbEnd2endTest : public ::testing::Test { } } - void StartServers(size_t num_servers, - std::vector<int> ports = std::vector<int>()) { + void CreateServers(size_t num_servers, + std::vector<int> ports = std::vector<int>()) { + servers_.clear(); for (size_t i = 0; i < num_servers; ++i) { int port = 0; if (ports.size() == num_servers) port = ports[i]; - servers_.emplace_back(new ServerData(server_host_, port)); + servers_.emplace_back(new ServerData(port)); + } + } + + void StartServer(size_t index) { servers_[index]->Start(server_host_); } + + void StartServers(size_t num_servers, + std::vector<int> ports = std::vector<int>()) { + CreateServers(num_servers, ports); + for (size_t i = 0; i < num_servers; ++i) { + StartServer(i); } } @@ -240,20 +251,23 @@ class ClientLbEnd2endTest : public ::testing::Test { std::unique_ptr<std::thread> thread_; bool server_ready_ = false; - explicit ServerData(const grpc::string& server_host, int port = 0) { + explicit ServerData(int port = 0) { port_ = port > 0 ? port : grpc_pick_unused_port_or_die(); + } + + void Start(const grpc::string& server_host) { gpr_log(GPR_INFO, "starting server on port %d", port_); std::mutex mu; std::unique_lock<std::mutex> lock(mu); std::condition_variable cond; thread_.reset(new std::thread( - std::bind(&ServerData::Start, this, server_host, &mu, &cond))); + std::bind(&ServerData::Serve, this, server_host, &mu, &cond))); cond.wait(lock, [this] { return server_ready_; }); server_ready_ = false; gpr_log(GPR_INFO, "server startup complete"); } - void Start(const grpc::string& server_host, std::mutex* mu, + void Serve(const grpc::string& server_host, std::mutex* mu, std::condition_variable* cond) { std::ostringstream server_address; server_address << server_host << ":" << port_; @@ -601,6 +615,41 @@ TEST_F(ClientLbEnd2endTest, PickFirstReresolutionNoSelected) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstReconnectWithoutNewResolverResult) { + std::vector<int> ports = {grpc_pick_unused_port_or_die()}; + StartServers(1, ports); + auto channel = BuildChannel("pick_first"); + auto stub = BuildStub(channel); + SetNextResolution(ports); + gpr_log(GPR_INFO, "****** INITIAL CONNECTION *******"); + WaitForServer(stub, 0, DEBUG_LOCATION); + gpr_log(GPR_INFO, "****** STOPPING SERVER ******"); + servers_[0]->Shutdown(); + EXPECT_TRUE(WaitForChannelNotReady(channel.get())); + gpr_log(GPR_INFO, "****** RESTARTING SERVER ******"); + StartServers(1, ports); + WaitForServer(stub, 0, DEBUG_LOCATION); +} + +TEST_F(ClientLbEnd2endTest, + PickFirstReconnectWithoutNewResolverResultStartsFromTopOfList) { + std::vector<int> ports = {grpc_pick_unused_port_or_die(), + grpc_pick_unused_port_or_die()}; + CreateServers(2, ports); + StartServer(1); + auto channel = BuildChannel("pick_first"); + auto stub = BuildStub(channel); + SetNextResolution(ports); + gpr_log(GPR_INFO, "****** INITIAL CONNECTION *******"); + WaitForServer(stub, 1, DEBUG_LOCATION); + gpr_log(GPR_INFO, "****** STOPPING SERVER ******"); + servers_[1]->Shutdown(); + EXPECT_TRUE(WaitForChannelNotReady(channel.get())); + gpr_log(GPR_INFO, "****** STARTING BOTH SERVERS ******"); + StartServers(2, ports); + WaitForServer(stub, 0, DEBUG_LOCATION); +} + TEST_F(ClientLbEnd2endTest, PickFirstCheckStateBeforeStartWatch) { std::vector<int> ports = {grpc_pick_unused_port_or_die()}; StartServers(1, ports); @@ -613,7 +662,6 @@ TEST_F(ClientLbEnd2endTest, PickFirstCheckStateBeforeStartWatch) { servers_[0]->Shutdown(); // Channel 1 will receive a re-resolution containing the same server. It will // create a new subchannel and hold a ref to it. - servers_.clear(); StartServers(1, ports); gpr_log(GPR_INFO, "****** SERVER RESTARTED *******"); auto channel_2 = BuildChannel("pick_first"); @@ -632,7 +680,6 @@ TEST_F(ClientLbEnd2endTest, PickFirstCheckStateBeforeStartWatch) { EXPECT_TRUE(WaitForChannelNotReady(channel_2.get())); // Channel 2 will also receive a re-resolution containing the same server. // Both channels will ref the same subchannel that failed. - servers_.clear(); StartServers(1, ports); gpr_log(GPR_INFO, "****** SERVER RESTARTED AGAIN *******"); gpr_log(GPR_INFO, "****** CHANNEL 2 STARTING A CALL *******"); @@ -860,7 +907,7 @@ TEST_F(ClientLbEnd2endTest, RoundRobinReresolve) { // Kill all servers gpr_log(GPR_INFO, "****** ABOUT TO KILL SERVERS *******"); for (size_t i = 0; i < servers_.size(); ++i) { - servers_[i]->Shutdown(false); + servers_[i]->Shutdown(true); } gpr_log(GPR_INFO, "****** SERVERS KILLED *******"); gpr_log(GPR_INFO, "****** SENDING DOOMED REQUESTS *******"); @@ -921,7 +968,8 @@ TEST_F(ClientLbEnd2endTest, RoundRobinSingleReconnect) { // No requests have gone to the deceased server. EXPECT_EQ(pre_death, post_death); // Bring the first server back up. - servers_[0].reset(new ServerData(server_host_, ports[0])); + servers_[0].reset(new ServerData(ports[0])); + StartServer(0); // Requests should start arriving at the first server either right away (if // the server managed to start before the RR policy retried the subchannel) or // after the subchannel retry delay otherwise (RR's subchannel retried before diff --git a/test/cpp/microbenchmarks/bm_chttp2_transport.cc b/test/cpp/microbenchmarks/bm_chttp2_transport.cc index 1e9bd273aa..189923a841 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_transport.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_transport.cc @@ -96,7 +96,7 @@ class DummyEndpoint : public grpc_endpoint { } static void write(grpc_endpoint* ep, grpc_slice_buffer* slices, - grpc_closure* cb) { + grpc_closure* cb, void* arg) { GRPC_CLOSURE_SCHED(cb, GRPC_ERROR_NONE); } diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index b958c75fc7..483b29b1b3 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -15,6 +15,7 @@ licenses(["notice"]) # Apache v2 load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary", "grpc_package") +load("//test/cpp/qps:qps_benchmark_script.bzl", "qps_json_driver_batch", "json_run_localhost_batch") grpc_package(name = "test/cpp/qps") @@ -22,8 +23,8 @@ grpc_cc_library( name = "parse_json", srcs = ["parse_json.cc"], hdrs = ["parse_json.h"], - deps = ["//:grpc++"], external_deps = ["protobuf"], + deps = ["//:grpc++"], ) grpc_cc_library( @@ -31,16 +32,19 @@ grpc_cc_library( srcs = [ "client_async.cc", "client_sync.cc", + "qps_server_builder.cc", "qps_worker.cc", "server_async.cc", "server_sync.cc", - "qps_server_builder.cc", ], hdrs = [ "client.h", + "qps_server_builder.h", "qps_worker.h", "server.h", - "qps_server_builder.h", + ], + external_deps = [ + "gflags", ], deps = [ ":histogram", @@ -56,11 +60,8 @@ grpc_cc_library( "//test/core/end2end:ssl_test_data", "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", - "//test/cpp/util:test_util", "//test/cpp/util:test_config", - ], - external_deps = [ - "gflags", + "//test/cpp/util:test_util", ], ) @@ -97,15 +98,15 @@ grpc_cc_library( hdrs = [ "benchmark_config.h", ], + external_deps = [ + "gflags", + ], deps = [ ":driver_impl", ":histogram", "//:grpc++", "//src/proto/grpc/testing:control_proto", ], - external_deps = [ - "gflags", - ], ) grpc_cc_library( @@ -117,6 +118,21 @@ grpc_cc_library( deps = ["//test/core/util:grpc_test_util"], ) +grpc_cc_binary( + name = "qps_json_driver", + srcs = ["qps_json_driver.cc"], + external_deps = [ + "gflags", + ], + deps = [ + ":benchmark_config", + ":driver_impl", + "//:grpc++", + "//test/cpp/util:test_config", + "//test/cpp/util:test_util", + ], +) + grpc_cc_test( name = "inproc_sync_unary_ping_pong_test", srcs = ["inproc_sync_unary_ping_pong_test.cc"], @@ -135,17 +151,9 @@ grpc_cc_library( deps = ["//:grpc++"], ) -grpc_cc_binary( - name = "json_run_localhost", - srcs = ["json_run_localhost.cc"], - deps = [ - "//:gpr", - "//test/core/util:gpr_test_util", - "//test/core/util:grpc_test_util", - "//test/cpp/util:test_config", - "//test/cpp/util:test_util", - ], -) +qps_json_driver_batch() + +json_run_localhost_batch() grpc_cc_test( name = "qps_interarrival_test", @@ -157,24 +165,10 @@ grpc_cc_test( ], ) -grpc_cc_binary( - name = "qps_json_driver", - srcs = ["qps_json_driver.cc"], - deps = [ - ":benchmark_config", - ":driver_impl", - "//:grpc++", - "//test/cpp/util:test_config", - "//test/cpp/util:test_util", - ], - external_deps = [ - "gflags", - ], -) - grpc_cc_test( name = "qps_openloop_test", srcs = ["qps_openloop_test.cc"], + data = ["//third_party/toolchains:RBE_USE_MACHINE_TYPE_LARGE"], deps = [ ":benchmark_config", ":driver_impl", @@ -182,7 +176,6 @@ grpc_cc_test( "//test/cpp/util:test_config", "//test/cpp/util:test_util", ], - data = ["//third_party/toolchains:RBE_USE_MACHINE_TYPE_LARGE"], ) grpc_cc_test( diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 776283c25a..fb2caf5486 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function import json import pipes import shutil @@ -68,62 +69,66 @@ def maybe_exclude_gcov(scenario_json): return ['gcov'] return [] -print yaml.dump({ - 'tests': [ - { - 'name': 'json_run_localhost', - 'shortname': 'json_run_localhost:%s' % scenario_json['name'], - 'args': ['--scenarios_json', _scenario_json_string(scenario_json, False)], - 'ci_platforms': ['linux'], - 'platforms': ['linux'], - 'flaky': False, - 'language': 'c++', - 'boringssl': True, - 'defaults': 'boringssl', - 'cpu_cost': guess_cpu(scenario_json, False), - 'exclude_configs': ['tsan', 'asan'] + maybe_exclude_gcov(scenario_json), - 'timeout_seconds': 2*60, - 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), - 'auto_timeout_scaling': False - } - for scenario_json in scenario_config.CXXLanguage().scenarios() - if 'scalable' in scenario_json.get('CATEGORIES', []) - ] + [ - { - 'name': 'qps_json_driver', - 'shortname': 'qps_json_driver:inproc_%s' % scenario_json['name'], - 'args': ['--run_inproc', '--scenarios_json', _scenario_json_string(scenario_json, False)], - 'ci_platforms': ['linux'], - 'platforms': ['linux'], - 'flaky': False, - 'language': 'c++', - 'boringssl': True, - 'defaults': 'boringssl', - 'cpu_cost': guess_cpu(scenario_json, False), - 'exclude_configs': ['tsan', 'asan'], - 'timeout_seconds': 6*60, - 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []) - } - for scenario_json in scenario_config.CXXLanguage().scenarios() - if 'inproc' in scenario_json.get('CATEGORIES', []) - ] + [ - { - 'name': 'json_run_localhost', - 'shortname': 'json_run_localhost:%s_low_thread_count' % scenario_json['name'], - 'args': ['--scenarios_json', _scenario_json_string(scenario_json, True)], - 'ci_platforms': ['linux'], - 'platforms': ['linux'], - 'flaky': False, - 'language': 'c++', - 'boringssl': True, - 'defaults': 'boringssl', - 'cpu_cost': guess_cpu(scenario_json, True), - 'exclude_configs': sorted(c for c in configs_from_yaml if c not in ('tsan', 'asan')), - 'timeout_seconds': 10*60, - 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), - 'auto_timeout_scaling': False - } - for scenario_json in scenario_config.CXXLanguage().scenarios() - if 'scalable' in scenario_json.get('CATEGORIES', []) - ] -}) +def generate_yaml(): + return { + 'tests': [ + { + 'name': 'json_run_localhost', + 'shortname': 'json_run_localhost:%s' % scenario_json['name'], + 'args': ['--scenarios_json', _scenario_json_string(scenario_json, False)], + 'ci_platforms': ['linux'], + 'platforms': ['linux'], + 'flaky': False, + 'language': 'c++', + 'boringssl': True, + 'defaults': 'boringssl', + 'cpu_cost': guess_cpu(scenario_json, False), + 'exclude_configs': ['tsan', 'asan'] + maybe_exclude_gcov(scenario_json), + 'timeout_seconds': 2*60, + 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), + 'auto_timeout_scaling': False + } + for scenario_json in scenario_config.CXXLanguage().scenarios() + if 'scalable' in scenario_json.get('CATEGORIES', []) + ] + [ + { + 'name': 'qps_json_driver', + 'shortname': 'qps_json_driver:inproc_%s' % scenario_json['name'], + 'args': ['--run_inproc', '--scenarios_json', _scenario_json_string(scenario_json, False)], + 'ci_platforms': ['linux'], + 'platforms': ['linux'], + 'flaky': False, + 'language': 'c++', + 'boringssl': True, + 'defaults': 'boringssl', + 'cpu_cost': guess_cpu(scenario_json, False), + 'exclude_configs': ['tsan', 'asan'], + 'timeout_seconds': 6*60, + 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []) + } + for scenario_json in scenario_config.CXXLanguage().scenarios() + if 'inproc' in scenario_json.get('CATEGORIES', []) + ] + [ + { + 'name': 'json_run_localhost', + 'shortname': 'json_run_localhost:%s_low_thread_count' % scenario_json['name'], + 'args': ['--scenarios_json', _scenario_json_string(scenario_json, True)], + 'ci_platforms': ['linux'], + 'platforms': ['linux'], + 'flaky': False, + 'language': 'c++', + 'boringssl': True, + 'defaults': 'boringssl', + 'cpu_cost': guess_cpu(scenario_json, True), + 'exclude_configs': sorted(c for c in configs_from_yaml if c not in ('tsan', 'asan')), + 'timeout_seconds': 10*60, + 'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', []), + 'auto_timeout_scaling': False + } + for scenario_json in scenario_config.CXXLanguage().scenarios() + if 'scalable' in scenario_json.get('CATEGORIES', []) + ] + } + + +print(yaml.dump(generate_yaml()))
\ No newline at end of file diff --git a/test/cpp/qps/json_run_localhost_scenario_gen.py b/test/cpp/qps/json_run_localhost_scenario_gen.py new file mode 100755 index 0000000000..7fe12234d4 --- /dev/null +++ b/test/cpp/qps/json_run_localhost_scenario_gen.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python2.7 + +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gen_build_yaml as gen +import json + +def generate_args(): + all_scenario_set = gen.generate_yaml() + all_scenario_set = all_scenario_set['tests'] + json_run_localhost_scenarios = \ + [item for item in all_scenario_set if item['name'] == 'qps_json_driver'] + json_run_localhost_arg_set = \ + [item['args'][2] for item in json_run_localhost_scenarios \ + if 'args' in item and len(item['args']) > 2] + deserialized_scenarios = [json.loads(item)['scenarios'][0] \ + for item in json_run_localhost_arg_set] + all_scenarios = {scenario['name'].encode('ascii', 'ignore'): \ + '\'{\'scenarios\' : [' + json.dumps(scenario) + ']}\'' \ + for scenario in deserialized_scenarios} + + serialized_scenarios_str = str(all_scenarios).encode('ascii', 'ignore') + with open('json_run_localhost_scenarios.bzl', 'wb') as f: + f.write('"""Scenarios run on localhost."""\n\n') + f.write('JSON_RUN_LOCALHOST_SCENARIOS = ' + serialized_scenarios_str + '\n') + +generate_args() diff --git a/test/cpp/qps/json_run_localhost_scenarios.bzl b/test/cpp/qps/json_run_localhost_scenarios.bzl new file mode 100644 index 0000000000..33e49e9214 --- /dev/null +++ b/test/cpp/qps/json_run_localhost_scenarios.bzl @@ -0,0 +1,3 @@ +"""Scenarios run on localhost.""" + +JSON_RUN_LOCALHOST_SCENARIOS = {'cpp_protobuf_sync_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_ping_pong_insecure_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_ping_pong_insecure_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 2, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 1000000, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 2, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 1000000}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 65536, "req_size": 65536}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 1000000, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING_FROM_SERVER", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_client_1channel_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_client_1channel_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_async_unary_1channel_100rpcs_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_1channel_100rpcs_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 1048576, "req_size": 1048576}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 8388608, "req_size": 128}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 2}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING_FROM_SERVER", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_ping_pong_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_ping_pong_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\''} diff --git a/test/cpp/qps/qps_benchmark_script.bzl b/test/cpp/qps/qps_benchmark_script.bzl new file mode 100644 index 0000000000..0904ceb036 --- /dev/null +++ b/test/cpp/qps/qps_benchmark_script.bzl @@ -0,0 +1,74 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This is for the gRPC build system. This isn't intended to be used outsite of +# the BUILD file for gRPC. It contains the mapping for the template system we +# use to generate other platform's build system files. +# +# Please consider that there should be a high bar for additions and changes to +# this file. +# Each rule listed must be re-written for Google's internal build system, and +# each change must be ported from one to the other. +# + +"""Script to run qps benchmark.""" + +load("//bazel:grpc_build_system.bzl", "grpc_cc_test") +load("//test/cpp/qps:qps_json_driver_scenarios.bzl", "QPS_JSON_DRIVER_SCENARIOS") +load("//test/cpp/qps:json_run_localhost_scenarios.bzl", "JSON_RUN_LOCALHOST_SCENARIOS") + +def qps_json_driver_batch(): + for scenario in QPS_JSON_DRIVER_SCENARIOS: + grpc_cc_test( + name = "qps_json_driver_test_%s" % scenario, + srcs = ["qps_json_driver.cc"], + args = [ + "--run_inproc", + "--scenarios_json", + QPS_JSON_DRIVER_SCENARIOS[scenario], + ], + external_deps = [ + "gflags", + ], + deps = [ + ":benchmark_config", + ":driver_impl", + "//:grpc++", + "//test/cpp/util:test_config", + "//test/cpp/util:test_util", + ], + ) + +def json_run_localhost_batch(): + for scenario in JSON_RUN_LOCALHOST_SCENARIOS: + grpc_cc_test( + name = "json_run_localhost_%s" % scenario, + srcs = ["json_run_localhost.cc"], + args = [ + "--scenarios_json", + JSON_RUN_LOCALHOST_SCENARIOS[scenario], + ], + data = [ + "//test/cpp/qps:qps_json_driver", + "//test/cpp/qps:qps_worker", + ], + deps = [ + "//:gpr", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + "//test/cpp/util:test_config", + "//test/cpp/util:test_util", + ], + ) diff --git a/test/cpp/qps/qps_json_driver_scenario_gen.py b/test/cpp/qps/qps_json_driver_scenario_gen.py new file mode 100755 index 0000000000..2b66c69d8a --- /dev/null +++ b/test/cpp/qps/qps_json_driver_scenario_gen.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python2.7 + +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gen_build_yaml as gen +import json + +def generate_args(): + all_scenario_set = gen.generate_yaml() + all_scenario_set = all_scenario_set['tests'] + qps_json_driver_scenario_set = \ + [item for item in all_scenario_set if item['name'] == 'qps_json_driver'] + qps_json_driver_arg_set = \ + [item['args'][2] for item in qps_json_driver_scenario_set \ + if 'args' in item and len(item['args']) > 2] + deserialized_scenarios = [json.loads(item)['scenarios'][0] \ + for item in qps_json_driver_arg_set] + all_scenarios = {scenario['name'].encode('ascii', 'ignore'): \ + '\'{\'scenarios\' : [' + json.dumps(scenario) + ']}\'' \ + for scenario in deserialized_scenarios} + + serialized_scenarios_str = str(all_scenarios).encode('ascii', 'ignore') + with open('qps_json_driver_scenarios.bzl', 'w') as f: + f.write('"""Scenarios of qps driver."""\n\n') + f.write('QPS_JSON_DRIVER_SCENARIOS = ' + serialized_scenarios_str + '\n') + +generate_args() diff --git a/test/cpp/qps/qps_json_driver_scenarios.bzl b/test/cpp/qps/qps_json_driver_scenarios.bzl new file mode 100644 index 0000000000..95b4911325 --- /dev/null +++ b/test/cpp/qps/qps_json_driver_scenarios.bzl @@ -0,0 +1,3 @@ +"""Scenarios of qps driver.""" + +QPS_JSON_DRIVER_SCENARIOS = {'cpp_protobuf_sync_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_ping_pong_insecure_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_ping_pong_insecure_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 2, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 1000000, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 2, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 1000000}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 65536, "req_size": 65536}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_1cq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 1000000, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 13, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 1000000}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING_FROM_SERVER", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_client_1channel_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_client_1channel_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_async_unary_1channel_100rpcs_1MB': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_1channel_100rpcs_1MB", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 1048576, "req_size": 1048576}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 8388608, "req_size": 128}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING_FROM_CLIENT", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 2}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 2}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "SYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 16, "rpc_type": "STREAMING_FROM_SERVER", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "SYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_protobuf_async_unary_qps_unconstrained_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_unary_qps_unconstrained_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 3, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 3}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_qps_unconstrained_10mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_qps_unconstrained_10mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 0, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 10, "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\'', 'cpp_generic_async_streaming_ping_pong_insecure': '\'{\'scenarios\' : [{"name": "cpp_generic_async_streaming_ping_pong_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_GENERIC_SERVER", "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}], "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "threads_per_cq": 0}, "client_config": {"security_params": null, "channel_args": [{"str_value": "latency", "name": "grpc.optimization_target"}], "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "payload_config": {"bytebuf_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 1, "threads_per_cq": 0}, "num_clients": 1}]}\'', 'cpp_protobuf_async_streaming_qps_unconstrained_1mps_insecure': '\'{\'scenarios\' : [{"name": "cpp_protobuf_async_streaming_qps_unconstrained_1mps_insecure", "warmup_seconds": 0, "benchmark_seconds": 1, "num_servers": 1, "server_config": {"security_params": null, "server_type": "ASYNC_SERVER", "async_server_threads": 0, "threads_per_cq": 0, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}]}, "client_config": {"security_params": null, "channel_args": [{"str_value": "throughput", "name": "grpc.optimization_target"}, {"int_value": 1, "name": "grpc.minimal_stack"}], "async_client_threads": 0, "outstanding_rpcs_per_channel": 100, "rpc_type": "STREAMING", "load_params": {"closed_loop": {}}, "histogram_params": {"resolution": 0.01, "max_possible": 60000000000.0}, "client_type": "ASYNC_CLIENT", "messages_per_stream": 1, "payload_config": {"simple_params": {"resp_size": 0, "req_size": 0}}, "client_channels": 64, "threads_per_cq": 0}, "num_clients": 0}]}\''} diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 693c02fdb2..693c02fdb2 100644..100755 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 2642fe4393..581dab3b4e 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.15.0rc1' +VERSION = '1.16.0.dev0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index b13cf937e2..9a97ee84f2 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.15.0-pre1 +PROJECT_NUMBER = 1.16.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 1578679f40..0cd4cfd647 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.15.0-pre1 +PROJECT_NUMBER = 1.16.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -1067,6 +1067,7 @@ src/core/lib/http/format_request.h \ src/core/lib/http/httpcli.h \ src/core/lib/http/parser.h \ src/core/lib/iomgr/block_annotate.h \ +src/core/lib/iomgr/buffer_list.h \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/combiner.h \ @@ -1082,6 +1083,7 @@ src/core/lib/iomgr/ev_posix.h \ src/core/lib/iomgr/exec_ctx.h \ src/core/lib/iomgr/executor.h \ src/core/lib/iomgr/gethostname.h \ +src/core/lib/iomgr/internal_errqueue.h \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.h \ src/core/lib/iomgr/iomgr_custom.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 26e7a7ccc1..aa75bc6828 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre1 +PROJECT_NUMBER = 6.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 094df63db7..7cd1dc7bf3 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-pre1 +PROJECT_NUMBER = 6.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -1159,6 +1159,8 @@ src/core/lib/http/parser.cc \ src/core/lib/http/parser.h \ src/core/lib/iomgr/README.md \ src/core/lib/iomgr/block_annotate.h \ +src/core/lib/iomgr/buffer_list.cc \ +src/core/lib/iomgr/buffer_list.h \ src/core/lib/iomgr/call_combiner.cc \ src/core/lib/iomgr/call_combiner.h \ src/core/lib/iomgr/closure.h \ @@ -1194,6 +1196,8 @@ src/core/lib/iomgr/gethostname.h \ src/core/lib/iomgr/gethostname_fallback.cc \ src/core/lib/iomgr/gethostname_host_name_max.cc \ src/core/lib/iomgr/gethostname_sysconf.cc \ +src/core/lib/iomgr/internal_errqueue.cc \ +src/core/lib/iomgr/internal_errqueue.h \ src/core/lib/iomgr/iocp_windows.cc \ src/core/lib/iomgr/iocp_windows.h \ src/core/lib/iomgr/iomgr.cc \ diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 6419030bbb..3d4d90317c 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -56,6 +56,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ $1 \ -- //test/... || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh index 7d11719ea4..19cfc1739a 100644 --- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh @@ -65,6 +65,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index 1eda166620..edb2069fb2 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -62,6 +62,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ --cache_test_results=no \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh index ffe644a48b..f8c9314236 100644 --- a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh @@ -61,6 +61,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ + --remote_instance_name=grpc-testing/instances/default_instance \ + --test_env=GRPC_VERBOSITY=debug \ -- //test/... || FAILED="true" if [ "$FAILED" != "" ] diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index 9d442346a7..6cd6f43167 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -235,7 +235,7 @@ def run_tests_for_lang(lang, runtime, images): maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( - resultset, args.bq_result_table, args) + resultset, args.bq_result_table) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 5014aea9cd..8ea5126fde 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -173,6 +173,23 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "buffer_list_test", + "src": [ + "test/core/iomgr/buffer_list_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", "name": "channel_create_test", "src": [ "test/core/surface/channel_create_test.cc" @@ -9488,6 +9505,7 @@ "src/core/lib/http/format_request.cc", "src/core/lib/http/httpcli.cc", "src/core/lib/http/parser.cc", + "src/core/lib/iomgr/buffer_list.cc", "src/core/lib/iomgr/call_combiner.cc", "src/core/lib/iomgr/combiner.cc", "src/core/lib/iomgr/endpoint.cc", @@ -9508,6 +9526,7 @@ "src/core/lib/iomgr/gethostname_fallback.cc", "src/core/lib/iomgr/gethostname_host_name_max.cc", "src/core/lib/iomgr/gethostname_sysconf.cc", + "src/core/lib/iomgr/internal_errqueue.cc", "src/core/lib/iomgr/iocp_windows.cc", "src/core/lib/iomgr/iomgr.cc", "src/core/lib/iomgr/iomgr_custom.cc", @@ -9667,6 +9686,7 @@ "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", "src/core/lib/iomgr/block_annotate.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", @@ -9682,6 +9702,7 @@ "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", "src/core/lib/iomgr/gethostname.h", + "src/core/lib/iomgr/internal_errqueue.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_custom.h", @@ -9817,6 +9838,7 @@ "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", "src/core/lib/iomgr/block_annotate.h", + "src/core/lib/iomgr/buffer_list.h", "src/core/lib/iomgr/call_combiner.h", "src/core/lib/iomgr/closure.h", "src/core/lib/iomgr/combiner.h", @@ -9832,6 +9854,7 @@ "src/core/lib/iomgr/exec_ctx.h", "src/core/lib/iomgr/executor.h", "src/core/lib/iomgr/gethostname.h", + "src/core/lib/iomgr/internal_errqueue.h", "src/core/lib/iomgr/iocp_windows.h", "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_custom.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 51c7b57d8a..fba76d69d1 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -199,6 +199,26 @@ "args": [], "benchmark": false, "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "buffer_list_test", + "platforms": [ + "linux" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ "linux", "mac", "posix", diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index 9d99703725..0ca23f56cf 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -104,14 +104,13 @@ def _insert_rows_with_retries(bq, bq_table, bq_rows): sys.exit(1) -def upload_results_to_bq(resultset, bq_table, args, platform): +def upload_results_to_bq(resultset, bq_table, extra_fields): """Upload test results to a BQ table. Args: resultset: dictionary generated by jobset.run bq_table: string name of table to create/upload results to in BQ - args: args in run_tests.py, generated by argparse - platform: string name of platform tests were run on + extra_fields: dict with extra values that will be uploaded along with the results """ bq = big_query_utils.create_big_query() big_query_utils.create_partitioned_table( @@ -129,32 +128,26 @@ def upload_results_to_bq(resultset, bq_table, args, platform): for result in results: test_results = {} _get_build_metadata(test_results) - test_results['compiler'] = args.compiler - test_results['config'] = args.config test_results['cpu_estimated'] = result.cpu_estimated test_results['cpu_measured'] = result.cpu_measured test_results['elapsed_time'] = '%.2f' % result.elapsed_time - test_results['iomgr_platform'] = args.iomgr_platform - # args.language is a list, but will always have one element in the contexts - # this function is used. - test_results['language'] = args.language[0] - test_results['platform'] = platform test_results['result'] = result.state test_results['return_code'] = result.returncode test_results['test_name'] = shortname test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') + for field_name, field_value in six.iteritems(extra_fields): + test_results[field_name] = field_value row = big_query_utils.make_row(str(uuid.uuid4()), test_results) bq_rows.append(row) _insert_rows_with_retries(bq, bq_table, bq_rows) -def upload_interop_results_to_bq(resultset, bq_table, args): +def upload_interop_results_to_bq(resultset, bq_table): """Upload interop test results to a BQ table. Args: resultset: dictionary generated by jobset.run bq_table: string name of table to create/upload results to in BQ - args: args in run_interop_tests.py, generated by argparse """ bq = big_query_utils.create_big_query() big_query_utils.create_partitioned_table( diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 741e3c2b32..9bbc2e3e0e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1494,7 +1494,7 @@ try: maxjobs=args.jobs, skip_jobs=args.manual_run) if args.bq_result_table and resultset: - upload_interop_results_to_bq(resultset, args.bq_result_table, args) + upload_interop_results_to_bq(resultset, args.bq_result_table) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e04b13b24c..ecb5e1d899 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1821,8 +1821,16 @@ def _build_and_run(check_cancelled, for antagonist in antagonists: antagonist.kill() if args.bq_result_table and resultset: - upload_results_to_bq(resultset, args.bq_result_table, args, - platform_string()) + upload_extra_fields = { + 'compiler': args.compiler, + 'config': args.config, + 'iomgr_platform': args.iomgr_platform, + 'language': args.language[ + 0], # args.language is a list but will always have one element when uploading to BQ is enabled. + 'platform': platform_string() + } + upload_results_to_bq(resultset, args.bq_result_table, + upload_extra_fields) if xml_report and resultset: report_utils.render_junit_xml_report( resultset, xml_report, suite_name=args.report_suite_name) diff --git a/tools/run_tests/sanity/check_qps_scenario_changes.py b/tools/run_tests/sanity/check_qps_scenario_changes.py new file mode 100755 index 0000000000..635fee6a60 --- /dev/null +++ b/tools/run_tests/sanity/check_qps_scenario_changes.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import subprocess + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../../test/cpp/qps')) +subprocess.call(['./json_run_localhost_scenario_gen.py']) +subprocess.call(['./qps_json_driver_scenario_gen.py']) + +output = subprocess.check_output(['git', 'status', '--porcelain']) +qps_json_driver_bzl = 'test/cpp/qps/qps_json_driver_scenarios.bzl' +json_run_localhost_bzl = 'test/cpp/qps/json_run_localhost_scenarios.bzl' + +if qps_json_driver_bzl in output or json_run_localhost_bzl in output: + print('qps benchmark scenarios have been updated, please commit ' + 'test/cpp/qps/qps_json_driver_scenarios.bzl and/or ' + 'test/cpp/qps/json_run_localhost_scenarios.bzl') + sys.exit(1) diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index fd9b34a198..1913edd425 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -3,6 +3,7 @@ - script: tools/run_tests/sanity/check_bazel_workspace.py - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_owners.sh +- script: tools/run_tests/sanity/check_qps_scenario_changes.py - script: tools/run_tests/sanity/check_shellcheck.sh - script: tools/run_tests/sanity/check_submodules.sh - script: tools/run_tests/sanity/check_test_filtering.py |