diff options
155 files changed, 4669 insertions, 1123 deletions
diff --git a/.yardopts b/.yardopts new file mode 100644 index 0000000000..3bc779d421 --- /dev/null +++ b/.yardopts @@ -0,0 +1 @@ +src/ruby/**/*.rb
\ No newline at end of file @@ -195,6 +195,7 @@ cc_library( "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -284,6 +285,7 @@ cc_library( "src/core/transport/transport.h", "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "third_party/nanopb/pb.h", "third_party/nanopb/pb_common.h", @@ -338,6 +340,7 @@ cc_library( "src/core/client_config/resolvers/sockaddr_resolver.c", "src/core/client_config/subchannel.c", "src/core/client_config/subchannel_factory.c", + "src/core/client_config/subchannel_index.c", "src/core/client_config/uri_parser.c", "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", @@ -441,6 +444,7 @@ cc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", @@ -508,6 +512,7 @@ cc_library( "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -597,6 +602,7 @@ cc_library( "src/core/transport/transport.h", "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "third_party/nanopb/pb.h", "third_party/nanopb/pb_common.h", @@ -631,6 +637,7 @@ cc_library( "src/core/client_config/resolvers/sockaddr_resolver.c", "src/core/client_config/subchannel.c", "src/core/client_config/subchannel_factory.c", + "src/core/client_config/subchannel_index.c", "src/core/client_config/uri_parser.c", "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", @@ -734,6 +741,7 @@ cc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", @@ -797,7 +805,6 @@ cc_library( "src/cpp/client/create_channel_internal.h", "src/cpp/common/create_auth_context.h", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/client/secure_credentials.cc", "src/cpp/common/auth_property_iterator.cc", @@ -821,7 +828,6 @@ cc_library( "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", - "src/cpp/server/fixed_size_thread_pool.cc", "src/cpp/server/insecure_server_credentials.cc", "src/cpp/server/server.cc", "src/cpp/server/server_builder.cc", @@ -928,7 +934,6 @@ cc_library( "src/cpp/client/create_channel_internal.h", "src/cpp/common/create_auth_context.h", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/common/insecure_create_auth_context.cc", "src/cpp/client/channel.cc", @@ -947,7 +952,6 @@ cc_library( "src/cpp/server/async_generic_service.cc", "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", - "src/cpp/server/fixed_size_thread_pool.cc", "src/cpp/server/insecure_server_credentials.cc", "src/cpp/server/server.cc", "src/cpp/server/server_builder.cc", @@ -1316,6 +1320,7 @@ objc_library( "src/core/client_config/resolvers/sockaddr_resolver.c", "src/core/client_config/subchannel.c", "src/core/client_config/subchannel_factory.c", + "src/core/client_config/subchannel_index.c", "src/core/client_config/uri_parser.c", "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", @@ -1419,6 +1424,7 @@ objc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", @@ -1481,6 +1487,7 @@ objc_library( "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -1570,6 +1577,7 @@ objc_library( "src/core/transport/transport.h", "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "third_party/nanopb/pb.h", "third_party/nanopb/pb_common.h", @@ -826,6 +826,7 @@ alloc_test: $(BINDIR)/$(CONFIG)/alloc_test alpn_test: $(BINDIR)/$(CONFIG)/alpn_test bin_encoder_test: $(BINDIR)/$(CONFIG)/bin_encoder_test census_context_test: $(BINDIR)/$(CONFIG)/census_context_test +census_log_test: $(BINDIR)/$(CONFIG)/census_log_test channel_create_test: $(BINDIR)/$(CONFIG)/channel_create_test chttp2_hpack_encoder_test: $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test @@ -1135,6 +1136,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/alpn_test \ $(BINDIR)/$(CONFIG)/bin_encoder_test \ $(BINDIR)/$(CONFIG)/census_context_test \ + $(BINDIR)/$(CONFIG)/census_log_test \ $(BINDIR)/$(CONFIG)/channel_create_test \ $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test \ $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test \ @@ -1371,6 +1373,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/bin_encoder_test || ( echo test bin_encoder_test failed ; exit 1 ) $(E) "[RUN] Testing census_context_test" $(Q) $(BINDIR)/$(CONFIG)/census_context_test || ( echo test census_context_test failed ; exit 1 ) + $(E) "[RUN] Testing census_log_test" + $(Q) $(BINDIR)/$(CONFIG)/census_log_test || ( echo test census_log_test failed ; exit 1 ) $(E) "[RUN] Testing channel_create_test" $(Q) $(BINDIR)/$(CONFIG)/channel_create_test || ( echo test channel_create_test failed ; exit 1 ) $(E) "[RUN] Testing chttp2_hpack_encoder_test" @@ -1601,6 +1605,8 @@ test_cxx: test_zookeeper buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/interop_test || ( echo test interop_test failed ; exit 1 ) $(E) "[RUN] Testing mock_test" $(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 ) + $(E) "[RUN] Testing qps_openloop_test" + $(Q) $(BINDIR)/$(CONFIG)/qps_openloop_test || ( echo test qps_openloop_test failed ; exit 1 ) $(E) "[RUN] Testing qps_test" $(Q) $(BINDIR)/$(CONFIG)/qps_test || ( echo test qps_test failed ; exit 1 ) $(E) "[RUN] Testing secure_auth_context_test" @@ -2380,6 +2386,7 @@ LIBGRPC_SRC = \ src/core/client_config/resolvers/sockaddr_resolver.c \ src/core/client_config/subchannel.c \ src/core/client_config/subchannel_factory.c \ + src/core/client_config/subchannel_index.c \ src/core/client_config/uri_parser.c \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ @@ -2483,6 +2490,7 @@ LIBGRPC_SRC = \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ + src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ @@ -2669,6 +2677,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/client_config/resolvers/sockaddr_resolver.c \ src/core/client_config/subchannel.c \ src/core/client_config/subchannel_factory.c \ + src/core/client_config/subchannel_index.c \ src/core/client_config/uri_parser.c \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ @@ -2772,6 +2781,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ + src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ @@ -2972,7 +2982,6 @@ LIBGRPC++_SRC = \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ - src/cpp/server/fixed_size_thread_pool.cc \ src/cpp/server/insecure_server_credentials.cc \ src/cpp/server/server.cc \ src/cpp/server/server_builder.cc \ @@ -3252,7 +3261,6 @@ LIBGRPC++_UNSECURE_SRC = \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ - src/cpp/server/fixed_size_thread_pool.cc \ src/cpp/server/insecure_server_credentials.cc \ src/cpp/server/server.cc \ src/cpp/server/server_builder.cc \ @@ -5858,6 +5866,38 @@ endif endif +CENSUS_LOG_TEST_SRC = \ + test/core/census/log_test.c \ + +CENSUS_LOG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CENSUS_LOG_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/census_log_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/census_log_test: $(CENSUS_LOG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CENSUS_LOG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/census_log_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/census/log_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_census_log_test: $(CENSUS_LOG_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CENSUS_LOG_TEST_OBJS:.o=.dep) +endif +endif + + CHANNEL_CREATE_TEST_SRC = \ test/core/surface/channel_create_test.c \ diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 3c46d61170..072089ac51 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,6 +7,7 @@ graft third_party/zlib include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py include src/python/grpcio/grpc_core_dependencies.py +include src/python/grpcio/precompiled.py include src/python/grpcio/support.py include src/python/grpcio/README.rst include requirements.txt @@ -3,6 +3,8 @@ [gRPC - An RPC library and framework](http://github.com/grpc/grpc) =================================== +[](https://gitter.im/grpc/grpc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + Copyright 2015-2016 Google Inc. #Documentation diff --git a/binding.gyp b/binding.gyp index f7a21e56cd..07a06f7d66 100644 --- a/binding.gyp +++ b/binding.gyp @@ -55,7 +55,8 @@ 'UNICODE', '_UNICODE', 'NOMINMAX', - 'OPENSSL_NO_ASM' + 'OPENSSL_NO_ASM', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], "msvs_settings": { 'VCCLCompilerTool': { @@ -78,7 +79,8 @@ # supports ALPN. The target is "[major].[minor].[patch]". We split by # periods and take the first field to get the major version. 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)' + 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], 'include_dirs': [ '<(node_root_dir)/deps/openssl/openssl/include', @@ -604,6 +606,7 @@ 'src/core/client_config/resolvers/sockaddr_resolver.c', 'src/core/client_config/subchannel.c', 'src/core/client_config/subchannel_factory.c', + 'src/core/client_config/subchannel_index.c', 'src/core/client_config/uri_parser.c', 'src/core/compression/algorithm.c', 'src/core/compression/message_compress.c', @@ -707,6 +710,7 @@ 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tracing.c', diff --git a/build.yaml b/build.yaml index ffa16675d8..dacb066141 100644 --- a/build.yaml +++ b/build.yaml @@ -14,10 +14,12 @@ filegroups: - include/grpc/census.h headers: - src/core/census/aggregation.h + - src/core/census/log.h - src/core/census/rpc_metric_id.h src: - src/core/census/context.c - src/core/census/initialize.c + - src/core/census/log.c - src/core/census/operation.c - src/core/census/placeholders.c - src/core/census/tracing.c @@ -172,7 +174,6 @@ filegroups: - src/cpp/client/create_channel_internal.h - src/cpp/common/create_auth_context.h - src/cpp/server/dynamic_thread_pool.h - - src/cpp/server/fixed_size_thread_pool.h - src/cpp/server/thread_pool_interface.h src: - src/cpp/client/channel.cc @@ -191,7 +192,6 @@ filegroups: - src/cpp/server/async_generic_service.cc - src/cpp/server/create_default_thread_pool.cc - src/cpp/server/dynamic_thread_pool.cc - - src/cpp/server/fixed_size_thread_pool.cc - src/cpp/server/insecure_server_credentials.cc - src/cpp/server/server.cc - src/cpp/server/server_builder.cc @@ -271,6 +271,7 @@ filegroups: - src/core/client_config/resolvers/sockaddr_resolver.h - src/core/client_config/subchannel.h - src/core/client_config/subchannel_factory.h + - src/core/client_config/subchannel_index.h - src/core/client_config/uri_parser.h - src/core/compression/algorithm_metadata.h - src/core/compression/message_compress.h @@ -388,6 +389,7 @@ filegroups: - src/core/client_config/resolvers/sockaddr_resolver.c - src/core/client_config/subchannel.c - src/core/client_config/subchannel_factory.c + - src/core/client_config/subchannel_index.c - src/core/client_config/uri_parser.c - src/core/compression/algorithm.c - src/core/compression/message_compress.c @@ -969,6 +971,16 @@ targets: - grpc - gpr_test_util - gpr +- name: census_log_test + build: test + language: c + src: + - test/core/census/log_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: channel_create_test build: test language: c @@ -2326,7 +2338,6 @@ targets: - posix - name: qps_openloop_test build: test - run: false language: c++ src: - test/cpp/qps/qps_openloop_test.cc @@ -2645,8 +2656,8 @@ configs: LDXX: clang++ compile_the_world: true test_environ: - ASAN_OPTIONS: suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always - LSAN_OPTIONS: suppressions=tools/asan_suppressions.txt:report_objects=1 + ASAN_OPTIONS: detect_leaks=1:color=always + LSAN_OPTIONS: suppressions=tools/lsan_suppressions.txt:report_objects=1 timeout_multiplier: 1.5 asan-noleaks: CC: clang diff --git a/examples/node/README.md b/examples/node/README.md index 09c56f7fa6..7a2bc9794f 100644 --- a/examples/node/README.md +++ b/examples/node/README.md @@ -4,14 +4,10 @@ gRPC in 3 minutes (Node.js) PREREQUISITES ------------- -- `node`: This requires Node 0.10.x or greater. -- [homebrew][] on Mac OS X. This simplifies the installation of the gRPC C core. +- `node`: This requires Node 0.12.x or greater. INSTALL ------- - - [Install gRPC Node][] - - - Install this package's dependencies ```sh $ cd examples/node @@ -35,15 +31,9 @@ TRY IT! $ node ./greeter_client.js ``` -NOTE ----- -This directory has a copy of `helloworld.proto` because it currently depends on -some Protocol Buffer 2.0 syntax that is deprecated in Protocol Buffer 3.0. - TUTORIAL -------- You can find a more detailed tutorial in [gRPC Basics: Node.js][] -[homebrew]:http://brew.sh [Install gRPC Node]:../../src/node [gRPC Basics: Node.js]:http://www.grpc.io/docs/tutorials/basic/node.html diff --git a/examples/node/greeter_client.js b/examples/node/greeter_client.js index 9b4b0a7782..ca5781514d 100644 --- a/examples/node/greeter_client.js +++ b/examples/node/greeter_client.js @@ -31,7 +31,7 @@ * */ -var PROTO_PATH = __dirname + '/helloworld.proto'; +var PROTO_PATH = __dirname + '/../protos/helloworld.proto'; var grpc = require('grpc'); var hello_proto = grpc.load(PROTO_PATH).helloworld; diff --git a/examples/node/greeter_server.js b/examples/node/greeter_server.js index 2712b3dd3a..47d9892816 100644 --- a/examples/node/greeter_server.js +++ b/examples/node/greeter_server.js @@ -31,7 +31,7 @@ * */ -var PROTO_PATH = __dirname + '/helloworld.proto'; +var PROTO_PATH = __dirname + '/../protos/helloworld.proto'; var grpc = require('grpc'); var hello_proto = grpc.load(PROTO_PATH).helloworld; diff --git a/examples/node/helloworld.proto b/examples/node/helloworld.proto deleted file mode 100644 index a52c947f89..0000000000 --- a/examples/node/helloworld.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option java_package = "ex.grpc"; - -package helloworld; - -// The greeting service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - optional string name = 1; -} - -// The response message containing the greetings -message HelloReply { - optional string message = 1; -} diff --git a/examples/node/package.json b/examples/node/package.json index 65c5789ed7..00ba428d96 100644 --- a/examples/node/package.json +++ b/examples/node/package.json @@ -2,6 +2,6 @@ "name": "grpc-examples", "version": "0.1.0", "dependencies": { - "grpc": "0.12.0" + "grpc": "0.13.0" } } diff --git a/examples/node/route_guide/route_guide.proto b/examples/node/route_guide/route_guide.proto deleted file mode 100644 index 38daa933cd..0000000000 --- a/examples/node/route_guide/route_guide.proto +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option java_package = "io.grpc.routeguide"; - -package routeguide; - -// Interface exported by the server. -service RouteGuide { - // A simple RPC. - // - // Obtains the feature at a given position. - rpc GetFeature(Point) returns (Feature) {} - - // A server-to-client streaming RPC. - // - // Obtains the Features available within the given Rectangle. Results are - // streamed rather than returned at once (e.g. in a response message with a - // repeated field), as the rectangle may cover a large area and contain a - // huge number of features. - rpc ListFeatures(Rectangle) returns (stream Feature) {} - - // A client-to-server streaming RPC. - // - // Accepts a stream of Points on a route being traversed, returning a - // RouteSummary when traversal is completed. - rpc RecordRoute(stream Point) returns (RouteSummary) {} - - // A Bidirectional streaming RPC. - // - // Accepts a stream of RouteNotes sent while a route is being traversed, - // while receiving other RouteNotes (e.g. from other users). - rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} -} - -// Points are represented as latitude-longitude pairs in the E7 representation -// (degrees multiplied by 10**7 and rounded to the nearest integer). -// Latitudes should be in the range +/- 90 degrees and longitude should be in -// the range +/- 180 degrees (inclusive). -message Point { - optional int32 latitude = 1; - optional int32 longitude = 2; -} - -// A latitude-longitude rectangle, represented as two diagonally opposite -// points "lo" and "hi". -message Rectangle { - // One corner of the rectangle. - optional Point lo = 1; - - // The other corner of the rectangle. - optional Point hi = 2; -} - -// A feature names something at a given point. -// -// If a feature could not be named, the name is empty. -message Feature { - // The name of the feature. - optional string name = 1; - - // The point where the feature is detected. - optional Point location = 2; -} - -// A RouteNote is a message sent while at a given point. -message RouteNote { - // The location from which the message is sent. - optional Point location = 1; - - // The message to be sent. - optional string message = 2; -} - -// A RouteSummary is received in response to a RecordRoute rpc. -// -// It contains the number of individual points received, the number of -// detected features, and the total distance covered as the cumulative sum of -// the distance between each point. -message RouteSummary { - // The number of points received. - optional int32 point_count = 1; - - // The number of known features passed while traversing the route. - optional int32 feature_count = 2; - - // The distance covered in metres. - optional int32 distance = 3; - - // The duration of the traversal in seconds. - optional int32 elapsed_time = 4; -} diff --git a/examples/node/route_guide/route_guide_client.js b/examples/node/route_guide/route_guide_client.js index e38a21f422..6ff0279184 100644 --- a/examples/node/route_guide/route_guide_client.js +++ b/examples/node/route_guide/route_guide_client.js @@ -31,15 +31,17 @@ * */ +var PROTO_PATH = __dirname + '/../../protos/route_guide.proto'; + var async = require('async'); var fs = require('fs'); var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(__dirname + '/route_guide.proto').routeguide; +var routeguide = grpc.load(PROTO_PATH).routeguide; var client = new routeguide.RouteGuide('localhost:50051', - grpc.Credentials.createInsecure()); + grpc.credentials.createInsecure()); var COORD_FACTOR = 1e7; diff --git a/examples/node/route_guide/route_guide_server.js b/examples/node/route_guide/route_guide_server.js index 06cf2925e1..9fa9827991 100644 --- a/examples/node/route_guide/route_guide_server.js +++ b/examples/node/route_guide/route_guide_server.js @@ -31,12 +31,14 @@ * */ +var PROTO_PATH = __dirname + '/../../protos/route_guide.proto'; + var fs = require('fs'); var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(__dirname + '/route_guide.proto').routeguide; +var routeguide = grpc.load(PROTO_PATH).routeguide; var COORD_FACTOR = 1e7; diff --git a/gRPC.podspec b/gRPC.podspec index 05e3802b77..8e4fb0facb 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -199,6 +199,7 @@ Pod::Spec.new do |s| 'src/core/client_config/resolvers/sockaddr_resolver.h', 'src/core/client_config/subchannel.h', 'src/core/client_config/subchannel_factory.h', + 'src/core/client_config/subchannel_index.h', 'src/core/client_config/uri_parser.h', 'src/core/compression/algorithm_metadata.h', 'src/core/compression/message_compress.h', @@ -288,6 +289,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport.h', 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', + 'src/core/census/log.h', 'src/core/census/rpc_metric_id.h', 'third_party/nanopb/pb.h', 'third_party/nanopb/pb_common.h', @@ -355,6 +357,7 @@ Pod::Spec.new do |s| 'src/core/client_config/resolvers/sockaddr_resolver.c', 'src/core/client_config/subchannel.c', 'src/core/client_config/subchannel_factory.c', + 'src/core/client_config/subchannel_index.c', 'src/core/client_config/uri_parser.c', 'src/core/compression/algorithm.c', 'src/core/compression/message_compress.c', @@ -458,6 +461,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tracing.c', @@ -516,6 +520,7 @@ Pod::Spec.new do |s| 'src/core/client_config/resolvers/sockaddr_resolver.h', 'src/core/client_config/subchannel.h', 'src/core/client_config/subchannel_factory.h', + 'src/core/client_config/subchannel_index.h', 'src/core/client_config/uri_parser.h', 'src/core/compression/algorithm_metadata.h', 'src/core/compression/message_compress.h', @@ -605,6 +610,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport.h', 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', + 'src/core/census/log.h', 'src/core/census/rpc_metric_id.h', 'third_party/nanopb/pb.h', 'third_party/nanopb/pb_common.h', @@ -99,6 +99,7 @@ EXPORTS grpc_auth_context_set_peer_identity_property_name grpc_channel_credentials_release grpc_google_default_credentials_create + grpc_set_ssl_roots_override_callback grpc_ssl_credentials_create grpc_call_credentials_release grpc_composite_channel_credentials_create diff --git a/grpc.gemspec b/grpc.gemspec index 1ef26761b7..a7c864054a 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 2.0.0' - s.files = %w( Makefile ) + s.files = %w( Makefile .yardopts ) s.files += %w( etc/roots.pem ) s.files += Dir.glob('src/ruby/bin/**/*') s.files += Dir.glob('src/ruby/ext/**/*') @@ -31,7 +31,7 @@ Gem::Specification.new do |s| s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.2' + s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' @@ -195,6 +195,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/client_config/resolvers/sockaddr_resolver.h ) s.files += %w( src/core/client_config/subchannel.h ) s.files += %w( src/core/client_config/subchannel_factory.h ) + s.files += %w( src/core/client_config/subchannel_index.h ) s.files += %w( src/core/client_config/uri_parser.h ) s.files += %w( src/core/compression/algorithm_metadata.h ) s.files += %w( src/core/compression/message_compress.h ) @@ -284,6 +285,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/transport/transport.h ) s.files += %w( src/core/transport/transport_impl.h ) s.files += %w( src/core/census/aggregation.h ) + s.files += %w( src/core/census/log.h ) s.files += %w( src/core/census/rpc_metric_id.h ) s.files += %w( third_party/nanopb/pb.h ) s.files += %w( third_party/nanopb/pb_common.h ) @@ -338,6 +340,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/client_config/resolvers/sockaddr_resolver.c ) s.files += %w( src/core/client_config/subchannel.c ) s.files += %w( src/core/client_config/subchannel_factory.c ) + s.files += %w( src/core/client_config/subchannel_index.c ) s.files += %w( src/core/client_config/uri_parser.c ) s.files += %w( src/core/compression/algorithm.c ) s.files += %w( src/core/compression/message_compress.c ) @@ -441,6 +444,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/transport/transport_op_string.c ) s.files += %w( src/core/census/context.c ) s.files += %w( src/core/census/initialize.c ) + s.files += %w( src/core/census/log.c ) s.files += %w( src/core/census/operation.c ) s.files += %w( src/core/census/placeholders.c ) s.files += %w( src/core/census/tracing.c ) diff --git a/include/grpc++/create_channel.h b/include/grpc++/create_channel.h index aff9c291c2..80eed067b7 100644 --- a/include/grpc++/create_channel.h +++ b/include/grpc++/create_channel.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,6 +36,7 @@ #include <memory> +#include <grpc++/channel.h> #include <grpc++/security/credentials.h> #include <grpc++/support/channel_arguments.h> #include <grpc++/support/config.h> diff --git a/include/grpc++/impl/codegen/sync_stream.h b/include/grpc++/impl/codegen/sync_stream.h index 33d25e837c..9ae48bd23d 100644 --- a/include/grpc++/impl/codegen/sync_stream.h +++ b/include/grpc++/impl/codegen/sync_stream.h @@ -193,6 +193,15 @@ class ClientWriter : public ClientWriterInterface<W> { cq_.Pluck(&ops); } + void WaitForInitialMetadata() { + GPR_ASSERT(!context_->initial_metadata_received_); + + CallOpSet<CallOpRecvInitialMetadata> ops; + ops.RecvInitialMetadata(context_); + call_.PerformOps(&ops); + cq_.Pluck(&ops); // status ignored + } + using WriterInterface<W>::Write; bool Write(const W& msg, const WriteOptions& options) GRPC_OVERRIDE { CallOpSet<CallOpSendMessage> ops; @@ -213,6 +222,9 @@ class ClientWriter : public ClientWriterInterface<W> { /// Read the final response and wait for the final status. Status Finish() GRPC_OVERRIDE { Status status; + if (!context_->initial_metadata_received_) { + finish_ops_.RecvInitialMetadata(context_); + } finish_ops_.ClientRecvStatus(context_, &status); call_.PerformOps(&finish_ops_); GPR_ASSERT(cq_.Pluck(&finish_ops_)); @@ -221,7 +233,8 @@ class ClientWriter : public ClientWriterInterface<W> { private: ClientContext* context_; - CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_; + CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage, + CallOpClientRecvStatus> finish_ops_; CompletionQueue cq_; Call call_; }; @@ -292,7 +305,10 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> { } Status Finish() GRPC_OVERRIDE { - CallOpSet<CallOpClientRecvStatus> ops; + CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> ops; + if (!context_->initial_metadata_received_) { + ops.RecvInitialMetadata(context_); + } Status status; ops.ClientRecvStatus(context_, &status); call_.PerformOps(&ops); diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index a2960a7ecc..a9ede35f90 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -51,7 +51,7 @@ class ChannelArgumentsTest; /// concrete setters are provided. class ChannelArguments { public: - ChannelArguments() {} + ChannelArguments(); ~ChannelArguments() {} ChannelArguments(const ChannelArguments& other); @@ -62,8 +62,8 @@ class ChannelArguments { void Swap(ChannelArguments& other); - /// Populates this instance with the arguments from \a channel_args. Does not - /// take ownership of \a channel_args. + /// Dump arguments in this instance to \a channel_args. Does not take + /// ownership of \a channel_args. /// /// Note that the underlying arguments are shared. Changes made to either \a /// channel_args or this instance would be reflected on both. @@ -77,6 +77,9 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); + /// The given string will be sent at the front of the user agent string. + void SetUserAgentPrefix(const grpc::string& user_agent_prefix); + // Generic channel argument setters. Only for advanced use cases. /// Set an integer argument \a value under \a key. void SetInt(const grpc::string& key, int value); @@ -92,6 +95,17 @@ class ChannelArguments { friend class SecureChannelCredentials; friend class testing::ChannelArgumentsTest; + /// Default pointer argument operations. + struct PointerVtableMembers { + static void* Copy(void* in) { return in; } + static void Destroy(void* in) {} + static int Compare(void* a, void* b) { + if (a < b) return -1; + if (a > b) return 1; + return 0; + } + }; + // Returns empty string when it is not set. grpc::string GetSslTargetNameOverride() const; diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 3de5cae8be..ef7205ded8 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -167,7 +167,8 @@ typedef grpc_ssl_roots_override_result (*grpc_ssl_roots_override_callback)( before any ssl credentials are created to have the desired side effect. If GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment is set to a valid path, the callback will not be called. */ -void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb); +GRPCAPI void grpc_set_ssl_roots_override_callback( + grpc_ssl_roots_override_callback cb); /* Object that holds a private key / certificate chain pair in PEM format. */ typedef struct { diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index ea1e96cf1d..b11f6ffec4 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -68,6 +68,12 @@ typedef enum { GRPC_ARG_POINTER } grpc_arg_type; +typedef struct grpc_arg_pointer_vtable { + void *(*copy)(void *p); + void (*destroy)(void *p); + int (*cmp)(void *p, void *q); +} grpc_arg_pointer_vtable; + /** A single argument... each argument has a key and a value A note on naming keys: @@ -88,8 +94,7 @@ typedef struct { int integer; struct { void *p; - void *(*copy)(void *p); - void (*destroy)(void *p); + const grpc_arg_pointer_vtable *vtable; } pointer; } value; } grpc_arg; diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 258063fd54..92569043fc 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -34,6 +34,14 @@ #ifndef GRPC_IMPL_CODEGEN_PORT_PLATFORM_H #define GRPC_IMPL_CODEGEN_PORT_PLATFORM_H +/* + * Define GPR_BACKWARDS_COMPATIBILITY_MODE to try harder to be ABI + * compatible with older platforms (currently only on Linux) + * Causes: + * - some libc calls to be gotten via dlsym + * - some syscalls to be made directly + */ + /* Get windows.h included everywhere (we need it) */ #if defined(_WIN64) || defined(WIN64) || defined(_WIN32) || defined(WIN32) #ifndef WIN32_LEAN_AND_MEAN diff --git a/include/grpc/support/avl.h b/include/grpc/support/avl.h index 28eb5b175e..2fe2d0b9d0 100644 --- a/include/grpc/support/avl.h +++ b/include/grpc/support/avl.h @@ -81,11 +81,12 @@ GPRAPI void gpr_avl_unref(gpr_avl avl); if key exists in avl, the new tree's key entry updated (i.e. a duplicate is not created) */ GPRAPI gpr_avl gpr_avl_add(gpr_avl avl, void *key, void *value); -/** return a new tree with key deleted */ +/** return a new tree with key deleted + implicitly unrefs avl to allow easy chaining. */ GPRAPI gpr_avl gpr_avl_remove(gpr_avl avl, void *key); /** lookup key, and return the associated value. does not mutate avl. returns NULL if key is not found. */ GPRAPI void *gpr_avl_get(gpr_avl avl, void *key); -#endif +#endif /* GRPC_SUPPORT_AVL_H */ diff --git a/include/grpc/support/useful.h b/include/grpc/support/useful.h index 9f08d788c0..a0e76da29e 100644 --- a/include/grpc/support/useful.h +++ b/include/grpc/support/useful.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -72,4 +72,6 @@ 0x0f0f0f0f) % \ 255) +#define GPR_ICMP(a, b) ((a) < (b) ? -1 : ((a) > (b) ? 1 : 0)) + #endif /* GRPC_SUPPORT_USEFUL_H */ diff --git a/package.json b/package.json index b8f8c0a256..fc1d06ce00 100644 --- a/package.json +++ b/package.json @@ -23,13 +23,12 @@ "test": "./node_modules/.bin/mocha src/node/test && npm run-script lint", "gen_docs": "./node_modules/.bin/jsdoc -c src/node/jsdoc_conf.json", "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", - "preinstall": "npm install node-pre-gyp", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, + "bundledDependencies": ["node-pre-gyp"], "dependencies": { "lodash": "^3.9.3", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.19", "protobufjs": "^4.0.0" }, "devDependencies": { @@ -141,6 +140,7 @@ "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -230,6 +230,7 @@ "src/core/transport/transport.h", "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "third_party/nanopb/pb.h", "third_party/nanopb/pb_common.h", @@ -284,6 +285,7 @@ "src/core/client_config/resolvers/sockaddr_resolver.c", "src/core/client_config/subchannel.c", "src/core/client_config/subchannel_factory.c", + "src/core/client_config/subchannel_index.c", "src/core/client_config/uri_parser.c", "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", @@ -387,6 +389,7 @@ "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tracing.c", diff --git a/requirements.txt b/requirements.txt index a1cc88cfe7..e3208e6355 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ futures>=2.2.0 cython>=0.23 coverage>=4.0 six>=1.10 +wheel>=0.29 @@ -53,6 +53,7 @@ sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. import commands +import precompiled import grpc_core_dependencies import grpc_version @@ -94,10 +95,10 @@ if "linux" in sys.platform: if not "win32" in sys.platform: EXTENSION_LIBRARIES += ('m',) -DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600)) +DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) -CFLAGS = () LDFLAGS = () +CFLAGS = () if "linux" in sys.platform: LDFLAGS += ('-Wl,-wrap,memcpy',) if "linux" in sys.platform or "darwin" in sys.platform: @@ -156,15 +157,14 @@ SETUP_REQUIRES = ( ) + INSTALL_REQUIRES COMMAND_CLASS = { - 'install': commands.Install, 'doc': commands.SphinxDocumentation, 'build_proto_modules': commands.BuildProtoModules, 'build_project_metadata': commands.BuildProjectMetadata, 'build_py': commands.BuildPy, 'build_ext': commands.BuildExt, + 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, - 'bdist_egg_grpc_custom': commands.BdistEggCustomName, } # Ensure that package data is copied over before any commands have been run: @@ -202,10 +202,13 @@ TEST_LOADER = 'tests:Loader' TEST_RUNNER = 'tests:Runner' PACKAGE_DATA = { + # Binaries that may or may not be present in the final installation, but are + # mentioned here for completeness. 'grpc._cython': [ '_credentials/roots.pem', '_windows/grpc_c.32.python', '_windows/grpc_c.64.python', + 'cygrpc.so', ], } if INSTALL_TESTS: @@ -215,19 +218,23 @@ else: PACKAGES = setuptools.find_packages( PYTHON_STEM, exclude=['tests', 'tests.*']) -setuptools.setup( - name='grpcio', - version=grpc_version.VERSION, - license=LICENSE, - ext_modules=CYTHON_EXTENSION_MODULES, - packages=list(PACKAGES), - package_dir=PACKAGE_DIRECTORIES, - package_data=PACKAGE_DATA, - install_requires=INSTALL_REQUIRES, - setup_requires=SETUP_REQUIRES, - cmdclass=COMMAND_CLASS, - tests_require=TESTS_REQUIRE, - test_suite=TEST_SUITE, - test_loader=TEST_LOADER, - test_runner=TEST_RUNNER, -) +setup_arguments = { + 'name': 'grpcio', + 'version': grpc_version.VERSION, + 'license': LICENSE, + 'ext_modules': CYTHON_EXTENSION_MODULES, + 'packages': list(PACKAGES), + 'package_dir': PACKAGE_DIRECTORIES, + 'package_data': PACKAGE_DATA, + 'install_requires': INSTALL_REQUIRES, + 'setup_requires': SETUP_REQUIRES, + 'cmdclass': COMMAND_CLASS, + 'tests_require': TESTS_REQUIRE, + 'test_suite': TEST_SUITE, + 'test_loader': TEST_LOADER, + 'test_runner': TEST_RUNNER, +} + +precompiled.update_setup_arguments(setup_arguments) + +setuptools.setup(**setup_arguments) diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c index a8db32b9d5..c8aaf31e2d 100644 --- a/src/core/census/grpc_filter.c +++ b/src/core/census/grpc_filter.c @@ -107,8 +107,8 @@ static void server_mutate_op(grpc_call_element *elem, if (op->recv_initial_metadata) { /* substitute our callback for the op callback */ calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->on_complete; - op->on_complete = &calld->finish_recv; + calld->on_done_recv = op->recv_initial_metadata_ready; + op->recv_initial_metadata_ready = &calld->finish_recv; } } diff --git a/src/core/census/log.c b/src/core/census/log.c new file mode 100644 index 0000000000..91b26941b8 --- /dev/null +++ b/src/core/census/log.c @@ -0,0 +1,600 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +// Implements an efficient in-memory log, optimized for multiple writers and +// a single reader. Available log space is divided up in blocks of +// CENSUS_LOG_2_MAX_RECORD_SIZE bytes. A block can be in one of the following +// three data structures: +// - Free blocks (free_block_list) +// - Blocks with unread data (dirty_block_list) +// - Blocks currently attached to cores (core_local_blocks[]) +// +// census_log_start_write() moves a block from core_local_blocks[] to the end of +// dirty_block_list when block: +// - is out-of-space OR +// - has an incomplete record (an incomplete record occurs when a thread calls +// census_log_start_write() and is context-switched before calling +// census_log_end_write() +// So, blocks in dirty_block_list are ordered, from oldest to newest, by the +// time when block is detached from the core. +// +// census_log_read_next() first iterates over dirty_block_list and then +// core_local_blocks[]. It moves completely read blocks from dirty_block_list +// to free_block_list. Blocks in core_local_blocks[] are not freed, even when +// completely read. +// +// If the log is configured to discard old records and free_block_list is empty, +// census_log_start_write() iterates over dirty_block_list to allocate a +// new block. It moves the oldest available block (no pending read/write) to +// core_local_blocks[]. +// +// core_local_block_struct is used to implement a map from core id to the block +// associated with that core. This mapping is advisory. It is possible that the +// block returned by this mapping is no longer associated with that core. This +// mapping is updated, lazily, by census_log_start_write(). +// +// Locking in block struct: +// +// Exclusive g_log.lock must be held before calling any functions operating on +// block structs except census_log_start_write() and census_log_end_write(). +// +// Writes to a block are serialized via writer_lock. census_log_start_write() +// acquires this lock and census_log_end_write() releases it. On failure to +// acquire the lock, writer allocates a new block for the current core and +// updates core_local_block accordingly. +// +// Simultaneous read and write access is allowed. Readers can safely read up to +// committed bytes (bytes_committed). +// +// reader_lock protects the block, currently being read, from getting recycled. +// start_read() acquires reader_lock and end_read() releases the lock. +// +// Read/write access to a block is disabled via try_disable_access(). It returns +// with both writer_lock and reader_lock held. These locks are subsequently +// released by enable_access() to enable access to the block. +// +// A note on naming: Most function/struct names are prepended by cl_ +// (shorthand for census_log). Further, functions that manipulate structures +// include the name of the structure, which will be passed as the first +// argument. E.g. cl_block_initialize() will initialize a cl_block. + +#include "src/core/census/log.h" +#include <grpc/support/alloc.h> +#include <grpc/support/atm.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/sync.h> +#include <grpc/support/useful.h> +#include <stdbool.h> +#include <string.h> + +// End of platform specific code + +typedef struct census_log_block_list_struct { + struct census_log_block_list_struct* next; + struct census_log_block_list_struct* prev; + struct census_log_block* block; +} cl_block_list_struct; + +typedef struct census_log_block { + // Pointer to underlying buffer. + char* buffer; + gpr_atm writer_lock; + gpr_atm reader_lock; + // Keeps completely written bytes. Declared atomic because accessed + // simultaneously by reader and writer. + gpr_atm bytes_committed; + // Bytes already read. + size_t bytes_read; + // Links for list. + cl_block_list_struct link; +// We want this structure to be cacheline aligned. We assume the following +// sizes for the various parts on 32/64bit systems: +// type 32b size 64b size +// char* 4 8 +// 3x gpr_atm 12 24 +// size_t 4 8 +// cl_block_list_struct 12 24 +// TOTAL 32 64 +// +// Depending on the size of our cacheline and the architecture, we +// selectively add char buffering to this structure. The size is checked +// via assert in census_log_initialize(). +#if defined(GPR_ARCH_64) +#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 64) +#else +#if defined(GPR_ARCH_32) +#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 32) +#else +#error "Unknown architecture" +#endif +#endif +#if CL_BLOCK_PAD_SIZE > 0 + char padding[CL_BLOCK_PAD_SIZE]; +#endif +} cl_block; + +// A list of cl_blocks, doubly-linked through cl_block::link. +typedef struct census_log_block_list { + int32_t count; // Number of items in list. + cl_block_list_struct ht; // head/tail of linked list. +} cl_block_list; + +// Cacheline aligned block pointers to avoid false sharing. Block pointer must +// be initialized via set_block(), before calling other functions +typedef struct census_log_core_local_block { + gpr_atm block; +// Ensure cachline alignment: we assume sizeof(gpr_atm) == 4 or 8 +#if defined(GPR_ARCH_64) +#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 8) +#else +#if defined(GPR_ARCH_32) +#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 4) +#else +#error "Unknown architecture" +#endif +#endif +#if CL_CORE_LOCAL_BLOCK_PAD_SIZE > 0 + char padding[CL_CORE_LOCAL_BLOCK_PAD_SIZE]; +#endif +} cl_core_local_block; + +struct census_log { + int discard_old_records; + // Number of cores (aka hardware-contexts) + unsigned num_cores; + // number of CENSUS_LOG_2_MAX_RECORD_SIZE blocks in log + uint32_t num_blocks; + cl_block* blocks; // Block metadata. + cl_core_local_block* core_local_blocks; // Keeps core to block mappings. + gpr_mu lock; + int initialized; // has log been initialized? + // Keeps the state of the reader iterator. A value of 0 indicates that + // iterator has reached the end. census_log_init_reader() resets the value + // to num_core to restart iteration. + uint32_t read_iterator_state; + // Points to the block being read. If non-NULL, the block is locked for + // reading(block_being_read_->reader_lock is held). + cl_block* block_being_read; + char* buffer; + cl_block_list free_block_list; + cl_block_list dirty_block_list; + gpr_atm out_of_space_count; +}; + +// Single internal log. +static struct census_log g_log; + +// Functions that operate on an atomic memory location used as a lock. + +// Returns non-zero if lock is acquired. +static int cl_try_lock(gpr_atm* lock) { return gpr_atm_acq_cas(lock, 0, 1); } + +static void cl_unlock(gpr_atm* lock) { gpr_atm_rel_store(lock, 0); } + +// Functions that operate on cl_core_local_block's. + +static void cl_core_local_block_set_block(cl_core_local_block* clb, + cl_block* block) { + gpr_atm_rel_store(&clb->block, (gpr_atm)block); +} + +static cl_block* cl_core_local_block_get_block(cl_core_local_block* clb) { + return (cl_block*)gpr_atm_acq_load(&clb->block); +} + +// Functions that operate on cl_block_list_struct's. + +static void cl_block_list_struct_initialize(cl_block_list_struct* bls, + cl_block* block) { + bls->next = bls->prev = bls; + bls->block = block; +} + +// Functions that operate on cl_block_list's. + +static void cl_block_list_initialize(cl_block_list* list) { + list->count = 0; + cl_block_list_struct_initialize(&list->ht, NULL); +} + +// Returns head of *this, or NULL if empty. +static cl_block* cl_block_list_head(cl_block_list* list) { + return list->ht.next->block; +} + +// Insert element *e after *pos. +static void cl_block_list_insert(cl_block_list* list, cl_block_list_struct* pos, + cl_block_list_struct* e) { + list->count++; + e->next = pos->next; + e->prev = pos; + e->next->prev = e; + e->prev->next = e; +} + +// Insert block at the head of the list +static void cl_block_list_insert_at_head(cl_block_list* list, cl_block* block) { + cl_block_list_insert(list, &list->ht, &block->link); +} + +// Insert block at the tail of the list. +static void cl_block_list_insert_at_tail(cl_block_list* list, cl_block* block) { + cl_block_list_insert(list, list->ht.prev, &block->link); +} + +// Removes block *b. Requires *b be in the list. +static void cl_block_list_remove(cl_block_list* list, cl_block* b) { + list->count--; + b->link.next->prev = b->link.prev; + b->link.prev->next = b->link.next; +} + +// Functions that operate on cl_block's + +static void cl_block_initialize(cl_block* block, char* buffer) { + block->buffer = buffer; + gpr_atm_rel_store(&block->writer_lock, 0); + gpr_atm_rel_store(&block->reader_lock, 0); + gpr_atm_rel_store(&block->bytes_committed, 0); + block->bytes_read = 0; + cl_block_list_struct_initialize(&block->link, block); +} + +// Guards against exposing partially written buffer to the reader. +static void cl_block_set_bytes_committed(cl_block* block, + size_t bytes_committed) { + gpr_atm_rel_store(&block->bytes_committed, (gpr_atm)bytes_committed); +} + +static size_t cl_block_get_bytes_committed(cl_block* block) { + return (size_t)gpr_atm_acq_load(&block->bytes_committed); +} + +// Tries to disable future read/write access to this block. Succeeds if: +// - no in-progress write AND +// - no in-progress read AND +// - 'discard_data' set to true OR no unread data +// On success, clears the block state and returns with writer_lock_ and +// reader_lock_ held. These locks are released by a subsequent +// cl_block_access_enable() call. +static bool cl_block_try_disable_access(cl_block* block, int discard_data) { + if (!cl_try_lock(&block->writer_lock)) { + return false; + } + if (!cl_try_lock(&block->reader_lock)) { + cl_unlock(&block->writer_lock); + return false; + } + if (!discard_data && + (block->bytes_read != cl_block_get_bytes_committed(block))) { + cl_unlock(&block->reader_lock); + cl_unlock(&block->writer_lock); + return false; + } + cl_block_set_bytes_committed(block, 0); + block->bytes_read = 0; + return true; +} + +static void cl_block_enable_access(cl_block* block) { + cl_unlock(&block->reader_lock); + cl_unlock(&block->writer_lock); +} + +// Returns with writer_lock held. +static void* cl_block_start_write(cl_block* block, size_t size) { + if (!cl_try_lock(&block->writer_lock)) { + return NULL; + } + size_t bytes_committed = cl_block_get_bytes_committed(block); + if (bytes_committed + size > CENSUS_LOG_MAX_RECORD_SIZE) { + cl_unlock(&block->writer_lock); + return NULL; + } + return block->buffer + bytes_committed; +} + +// Releases writer_lock and increments committed bytes by 'bytes_written'. +// 'bytes_written' must be <= 'size' specified in the corresponding +// StartWrite() call. This function is thread-safe. +static void cl_block_end_write(cl_block* block, size_t bytes_written) { + cl_block_set_bytes_committed( + block, cl_block_get_bytes_committed(block) + bytes_written); + cl_unlock(&block->writer_lock); +} + +// Returns a pointer to the first unread byte in buffer. The number of bytes +// available are returned in 'bytes_available'. Acquires reader lock that is +// released by a subsequent cl_block_end_read() call. Returns NULL if: +// - read in progress +// - no data available +static void* cl_block_start_read(cl_block* block, size_t* bytes_available) { + if (!cl_try_lock(&block->reader_lock)) { + return NULL; + } + // bytes_committed may change from under us. Use bytes_available to update + // bytes_read below. + size_t bytes_committed = cl_block_get_bytes_committed(block); + GPR_ASSERT(bytes_committed >= block->bytes_read); + *bytes_available = bytes_committed - block->bytes_read; + if (*bytes_available == 0) { + cl_unlock(&block->reader_lock); + return NULL; + } + void* record = block->buffer + block->bytes_read; + block->bytes_read += *bytes_available; + return record; +} + +static void cl_block_end_read(cl_block* block) { + cl_unlock(&block->reader_lock); +} + +// Internal functions operating on g_log + +// Allocates a new free block (or recycles an available dirty block if log is +// configured to discard old records). Returns NULL if out-of-space. +static cl_block* cl_allocate_block(void) { + cl_block* block = cl_block_list_head(&g_log.free_block_list); + if (block != NULL) { + cl_block_list_remove(&g_log.free_block_list, block); + return block; + } + if (!g_log.discard_old_records) { + // No free block and log is configured to keep old records. + return NULL; + } + // Recycle dirty block. Start from the oldest. + for (block = cl_block_list_head(&g_log.dirty_block_list); block != NULL; + block = block->link.next->block) { + if (cl_block_try_disable_access(block, 1 /* discard data */)) { + cl_block_list_remove(&g_log.dirty_block_list, block); + return block; + } + } + return NULL; +} + +// Allocates a new block and updates core id => block mapping. 'old_block' +// points to the block that the caller thinks is attached to +// 'core_id'. 'old_block' may be NULL. Returns true if: +// - allocated a new block OR +// - 'core_id' => 'old_block' mapping changed (another thread allocated a +// block before lock was acquired). +static bool cl_allocate_core_local_block(uint32_t core_id, + cl_block* old_block) { + // Now that we have the lock, check if core-local mapping has changed. + cl_core_local_block* core_local_block = &g_log.core_local_blocks[core_id]; + cl_block* block = cl_core_local_block_get_block(core_local_block); + if ((block != NULL) && (block != old_block)) { + return true; + } + if (block != NULL) { + cl_core_local_block_set_block(core_local_block, NULL); + cl_block_list_insert_at_tail(&g_log.dirty_block_list, block); + } + block = cl_allocate_block(); + if (block == NULL) { + return false; + } + cl_core_local_block_set_block(core_local_block, block); + cl_block_enable_access(block); + return true; +} + +static cl_block* cl_get_block(void* record) { + uintptr_t p = (uintptr_t)((char*)record - g_log.buffer); + uintptr_t index = p >> CENSUS_LOG_2_MAX_RECORD_SIZE; + return &g_log.blocks[index]; +} + +// Gets the next block to read and tries to free 'prev' block (if not NULL). +// Returns NULL if reached the end. +static cl_block* cl_next_block_to_read(cl_block* prev) { + cl_block* block = NULL; + if (g_log.read_iterator_state == g_log.num_cores) { + // We are traversing dirty list; find the next dirty block. + if (prev != NULL) { + // Try to free the previous block if there is no unread data. This + // block + // may have unread data if previously incomplete record completed + // between + // read_next() calls. + block = prev->link.next->block; + if (cl_block_try_disable_access(prev, 0 /* do not discard data */)) { + cl_block_list_remove(&g_log.dirty_block_list, prev); + cl_block_list_insert_at_head(&g_log.free_block_list, prev); + } + } else { + block = cl_block_list_head(&g_log.dirty_block_list); + } + if (block != NULL) { + return block; + } + // We are done with the dirty list; moving on to core-local blocks. + } + while (g_log.read_iterator_state > 0) { + g_log.read_iterator_state--; + block = cl_core_local_block_get_block( + &g_log.core_local_blocks[g_log.read_iterator_state]); + if (block != NULL) { + return block; + } + } + return NULL; +} + +#define CL_LOG_2_MB 20 // 2^20 = 1MB + +// External functions: primary stats_log interface +void census_log_initialize(size_t size_in_mb, int discard_old_records) { + // Check cacheline alignment. + GPR_ASSERT(sizeof(cl_block) % GPR_CACHELINE_SIZE == 0); + GPR_ASSERT(sizeof(cl_core_local_block) % GPR_CACHELINE_SIZE == 0); + GPR_ASSERT(!g_log.initialized); + g_log.discard_old_records = discard_old_records; + g_log.num_cores = gpr_cpu_num_cores(); + // Ensure that we will not get any overflow in calaculating num_blocks + GPR_ASSERT(CL_LOG_2_MB >= CENSUS_LOG_2_MAX_RECORD_SIZE); + GPR_ASSERT(size_in_mb < 1000); + // Ensure at least 2x as many blocks as there are cores. + g_log.num_blocks = + (uint32_t)GPR_MAX(2 * g_log.num_cores, (size_in_mb << CL_LOG_2_MB) >> + CENSUS_LOG_2_MAX_RECORD_SIZE); + gpr_mu_init(&g_log.lock); + g_log.read_iterator_state = 0; + g_log.block_being_read = NULL; + g_log.core_local_blocks = (cl_core_local_block*)gpr_malloc_aligned( + g_log.num_cores * sizeof(cl_core_local_block), GPR_CACHELINE_SIZE_LOG); + memset(g_log.core_local_blocks, 0, + g_log.num_cores * sizeof(cl_core_local_block)); + g_log.blocks = (cl_block*)gpr_malloc_aligned( + g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE_LOG); + memset(g_log.blocks, 0, g_log.num_blocks * sizeof(cl_block)); + g_log.buffer = gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE); + memset(g_log.buffer, 0, g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE); + cl_block_list_initialize(&g_log.free_block_list); + cl_block_list_initialize(&g_log.dirty_block_list); + for (uint32_t i = 0; i < g_log.num_blocks; ++i) { + cl_block* block = g_log.blocks + i; + cl_block_initialize(block, g_log.buffer + (CENSUS_LOG_MAX_RECORD_SIZE * i)); + cl_block_try_disable_access(block, 1 /* discard data */); + cl_block_list_insert_at_tail(&g_log.free_block_list, block); + } + gpr_atm_rel_store(&g_log.out_of_space_count, 0); + g_log.initialized = 1; +} + +void census_log_shutdown(void) { + GPR_ASSERT(g_log.initialized); + gpr_mu_destroy(&g_log.lock); + gpr_free_aligned(g_log.core_local_blocks); + g_log.core_local_blocks = NULL; + gpr_free_aligned(g_log.blocks); + g_log.blocks = NULL; + gpr_free(g_log.buffer); + g_log.buffer = NULL; + g_log.initialized = 0; +} + +void* census_log_start_write(size_t size) { + // Used to bound number of times block allocation is attempted. + GPR_ASSERT(size > 0); + GPR_ASSERT(g_log.initialized); + if (size > CENSUS_LOG_MAX_RECORD_SIZE) { + return NULL; + } + uint32_t attempts_remaining = g_log.num_blocks; + uint32_t core_id = gpr_cpu_current_cpu(); + do { + void* record = NULL; + cl_block* block = + cl_core_local_block_get_block(&g_log.core_local_blocks[core_id]); + if (block && (record = cl_block_start_write(block, size))) { + return record; + } + // Need to allocate a new block. We are here if: + // - No block associated with the core OR + // - Write in-progress on the block OR + // - block is out of space + gpr_mu_lock(&g_log.lock); + bool allocated = cl_allocate_core_local_block(core_id, block); + gpr_mu_unlock(&g_log.lock); + if (!allocated) { + gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1); + return NULL; + } + } while (attempts_remaining--); + // Give up. + gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1); + return NULL; +} + +void census_log_end_write(void* record, size_t bytes_written) { + GPR_ASSERT(g_log.initialized); + cl_block_end_write(cl_get_block(record), bytes_written); +} + +void census_log_init_reader(void) { + GPR_ASSERT(g_log.initialized); + gpr_mu_lock(&g_log.lock); + // If a block is locked for reading unlock it. + if (g_log.block_being_read != NULL) { + cl_block_end_read(g_log.block_being_read); + g_log.block_being_read = NULL; + } + g_log.read_iterator_state = g_log.num_cores; + gpr_mu_unlock(&g_log.lock); +} + +const void* census_log_read_next(size_t* bytes_available) { + GPR_ASSERT(g_log.initialized); + gpr_mu_lock(&g_log.lock); + if (g_log.block_being_read != NULL) { + cl_block_end_read(g_log.block_being_read); + } + do { + g_log.block_being_read = cl_next_block_to_read(g_log.block_being_read); + if (g_log.block_being_read != NULL) { + void* record = + cl_block_start_read(g_log.block_being_read, bytes_available); + if (record != NULL) { + gpr_mu_unlock(&g_log.lock); + return record; + } + } + } while (g_log.block_being_read != NULL); + gpr_mu_unlock(&g_log.lock); + return NULL; +} + +size_t census_log_remaining_space(void) { + GPR_ASSERT(g_log.initialized); + size_t space = 0; + gpr_mu_lock(&g_log.lock); + if (g_log.discard_old_records) { + // Remaining space is not meaningful; just return the entire log space. + space = g_log.num_blocks << CENSUS_LOG_2_MAX_RECORD_SIZE; + } else { + GPR_ASSERT(g_log.free_block_list.count >= 0); + space = (size_t)g_log.free_block_list.count * CENSUS_LOG_MAX_RECORD_SIZE; + } + gpr_mu_unlock(&g_log.lock); + return space; +} + +int64_t census_log_out_of_space_count(void) { + GPR_ASSERT(g_log.initialized); + return gpr_atm_acq_load(&g_log.out_of_space_count); +} diff --git a/src/core/census/log.h b/src/core/census/log.h new file mode 100644 index 0000000000..05daea066f --- /dev/null +++ b/src/core/census/log.h @@ -0,0 +1,93 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CENSUS_LOG_H +#define GRPC_INTERNAL_CORE_CENSUS_LOG_H + +#include <grpc/support/port_platform.h> +#include <stddef.h> + +/* Maximum record size, in bytes. */ +#define CENSUS_LOG_2_MAX_RECORD_SIZE 14 /* 2^14 = 16KB */ +#define CENSUS_LOG_MAX_RECORD_SIZE (1 << CENSUS_LOG_2_MAX_RECORD_SIZE) + +/* Initialize the statistics logging subsystem with the given log size. A log + size of 0 will result in the smallest possible log for the platform + (approximately CENSUS_LOG_MAX_RECORD_SIZE * gpr_cpu_num_cores()). If + discard_old_records is non-zero, then new records will displace older ones + when the log is full. This function must be called before any other + census_log functions. +*/ +void census_log_initialize(size_t size_in_mb, int discard_old_records); + +/* Shutdown the logging subsystem. Caller must ensure that: + - no in progress or future call to any census_log functions + - no incomplete records +*/ +void census_log_shutdown(void); + +/* Allocates and returns a 'size' bytes record and marks it in use. A + subsequent census_log_end_write() marks the record complete. The + 'bytes_written' census_log_end_write() argument must be <= + 'size'. Returns NULL if out-of-space AND: + - log is configured to keep old records OR + - all blocks are pinned by incomplete records. +*/ +void* census_log_start_write(size_t size); + +void census_log_end_write(void* record, size_t bytes_written); + +void census_log_init_reader(void); + +/* census_log_read_next() iterates over blocks with data and for each block + returns a pointer to the first unread byte. The number of bytes that can be + read are returned in 'bytes_available'. Reader is expected to read all + available data. Reading the data consumes it i.e. it cannot be read again. + census_log_read_next() returns NULL if the end is reached i.e last block + is read. census_log_init_reader() starts the iteration or aborts the + current iteration. +*/ +const void* census_log_read_next(size_t* bytes_available); + +/* Returns estimated remaining space across all blocks, in bytes. If log is + configured to discard old records, returns total log space. Otherwise, + returns space available in empty blocks (partially filled blocks are + treated as full). +*/ +size_t census_log_remaining_space(void); + +/* Returns the number of times gprc_stats_log_start_write() failed due to + out-of-space. */ +int64_t census_log_out_of_space_count(void); + +#endif /* GRPC_INTERNAL_CORE_CENSUS_LOG_H */ diff --git a/src/core/channel/channel_args.c b/src/core/channel/channel_args.c index 0427ce0b8d..bae7a90a01 100644 --- a/src/core/channel/channel_args.c +++ b/src/core/channel/channel_args.c @@ -37,6 +37,7 @@ #include <grpc/census.h> #include <grpc/support/alloc.h> +#include <grpc/support/log.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -55,9 +56,8 @@ static grpc_arg copy_arg(const grpc_arg *src) { break; case GRPC_ARG_POINTER: dst.value.pointer = src->value.pointer; - dst.value.pointer.p = src->value.pointer.copy - ? src->value.pointer.copy(src->value.pointer.p) - : src->value.pointer.p; + dst.value.pointer.p = + src->value.pointer.vtable->copy(src->value.pointer.p); break; } return dst; @@ -94,6 +94,58 @@ grpc_channel_args *grpc_channel_args_merge(const grpc_channel_args *a, return grpc_channel_args_copy_and_add(a, b->args, b->num_args); } +static int cmp_arg(const grpc_arg *a, const grpc_arg *b) { + int c = GPR_ICMP(a->type, b->type); + if (c != 0) return c; + c = strcmp(a->key, b->key); + if (c != 0) return c; + switch (a->type) { + case GRPC_ARG_STRING: + return strcmp(a->value.string, b->value.string); + case GRPC_ARG_INTEGER: + return GPR_ICMP(a->value.integer, b->value.integer); + case GRPC_ARG_POINTER: + c = GPR_ICMP(a->value.pointer.p, b->value.pointer.p); + if (c != 0) { + c = GPR_ICMP(a->value.pointer.vtable, b->value.pointer.vtable); + if (c == 0) { + c = a->value.pointer.vtable->cmp(a->value.pointer.p, + b->value.pointer.p); + } + } + return c; + } + GPR_UNREACHABLE_CODE(return 0); +} + +/* stabilizing comparison function: since channel_args ordering matters for + * keys with the same name, we need to preserve that ordering */ +static int cmp_key_stable(const void *ap, const void *bp) { + const grpc_arg *const *a = ap; + const grpc_arg *const *b = bp; + int c = strcmp((*a)->key, (*b)->key); + if (c == 0) c = GPR_ICMP(*a, *b); + return c; +} + +grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) { + grpc_arg **args = gpr_malloc(sizeof(grpc_arg *) * a->num_args); + for (size_t i = 0; i < a->num_args; i++) { + args[i] = &a->args[i]; + } + qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable); + + grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args)); + b->num_args = a->num_args; + b->args = gpr_malloc(sizeof(grpc_arg) * b->num_args); + for (size_t i = 0; i < a->num_args; i++) { + b->args[i] = copy_arg(args[i]); + } + + gpr_free(args); + return b; +} + void grpc_channel_args_destroy(grpc_channel_args *a) { size_t i; for (i = 0; i < a->num_args; i++) { @@ -104,9 +156,7 @@ void grpc_channel_args_destroy(grpc_channel_args *a) { case GRPC_ARG_INTEGER: break; case GRPC_ARG_POINTER: - if (a->args[i].value.pointer.destroy) { - a->args[i].value.pointer.destroy(a->args[i].value.pointer.p); - } + a->args[i].value.pointer.vtable->destroy(a->args[i].value.pointer.p); break; } gpr_free(a->args[i].key); @@ -208,3 +258,14 @@ int grpc_channel_args_compression_algorithm_get_states( return (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; /* All algs. enabled */ } } + +int grpc_channel_args_compare(const grpc_channel_args *a, + const grpc_channel_args *b) { + int c = GPR_ICMP(a->num_args, b->num_args); + if (c != 0) return c; + for (size_t i = 0; i < a->num_args; i++) { + c = cmp_arg(&a->args[i], &b->args[i]); + if (c != 0) return c; + } + return 0; +} diff --git a/src/core/channel/channel_args.h b/src/core/channel/channel_args.h index 480cc9aec2..b3a7c9f434 100644 --- a/src/core/channel/channel_args.h +++ b/src/core/channel/channel_args.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -40,6 +40,9 @@ /* Copy some arguments */ grpc_channel_args *grpc_channel_args_copy(const grpc_channel_args *src); +/* Copy some arguments, stably sorting keys */ +grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a); + /** Copy some arguments and add the to_add parameter in the end. If to_add is NULL, it is equivalent to call grpc_channel_args_copy. */ grpc_channel_args *grpc_channel_args_copy_and_add(const grpc_channel_args *src, @@ -85,4 +88,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( int grpc_channel_args_compression_algorithm_get_states( const grpc_channel_args *a); +int grpc_channel_args_compare(const grpc_channel_args *a, + const grpc_channel_args *b); + #endif /* GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_ARGS_H */ diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c index 43eee046b8..1aa27208c2 100644 --- a/src/core/channel/http_client_filter.c +++ b/src/core/channel/http_client_filter.c @@ -127,8 +127,8 @@ static void hc_mutate_op(grpc_call_element *elem, if (op->recv_initial_metadata != NULL) { /* substitute our callback for the higher callback */ calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->on_complete; - op->on_complete = &calld->hc_on_recv; + calld->on_done_recv = op->recv_initial_metadata_ready; + op->recv_initial_metadata_ready = &calld->hc_on_recv; } } diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c index bb75323933..370f8dbe42 100644 --- a/src/core/channel/http_server_filter.c +++ b/src/core/channel/http_server_filter.c @@ -186,8 +186,8 @@ static void hs_mutate_op(grpc_call_element *elem, if (op->recv_initial_metadata) { /* substitute our callback for the higher callback */ calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->on_complete; - op->on_complete = &calld->hs_on_recv; + calld->on_done_recv = op->recv_initial_metadata_ready; + op->recv_initial_metadata_ready = &calld->hs_on_recv; } } diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c index 3ad9fd9efb..81297c8d44 100644 --- a/src/core/channel/subchannel_call_holder.c +++ b/src/core/channel/subchannel_call_holder.c @@ -241,10 +241,8 @@ static void fail_locked(grpc_exec_ctx *exec_ctx, grpc_subchannel_call_holder *holder) { size_t i; for (i = 0; i < holder->waiting_ops_count; i++) { - grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].on_complete, false, - NULL); - grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].recv_message_ready, - false, NULL); + grpc_transport_stream_op_finish_with_failure(exec_ctx, + &holder->waiting_ops[i]); } holder->waiting_ops_count = 0; } diff --git a/src/core/client_config/connector.c b/src/core/client_config/connector.c index 1603ffb8be..aa34aa7fab 100644 --- a/src/core/client_config/connector.c +++ b/src/core/client_config/connector.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -33,8 +33,9 @@ #include "src/core/client_config/connector.h" -void grpc_connector_ref(grpc_connector* connector) { +grpc_connector* grpc_connector_ref(grpc_connector* connector) { connector->vtable->ref(connector); + return connector; } void grpc_connector_unref(grpc_exec_ctx* exec_ctx, grpc_connector* connector) { diff --git a/src/core/client_config/connector.h b/src/core/client_config/connector.h index b4482fa2ee..b91eb512c3 100644 --- a/src/core/client_config/connector.h +++ b/src/core/client_config/connector.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -81,7 +81,7 @@ struct grpc_connector_vtable { grpc_connect_out_args *out_args, grpc_closure *notify); }; -void grpc_connector_ref(grpc_connector *connector); +grpc_connector *grpc_connector_ref(grpc_connector *connector); void grpc_connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *connector); /** Connect using the connector: max one outstanding call at a time */ void grpc_connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *connector, diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 0a996a1e8b..6599c75dba 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -36,16 +36,17 @@ #include <string.h> #include <grpc/support/alloc.h> +#include <grpc/support/avl.h> #include "src/core/channel/channel_args.h" #include "src/core/channel/client_channel.h" #include "src/core/channel/connected_channel.h" #include "src/core/client_config/initial_connect_string.h" +#include "src/core/client_config/subchannel_index.h" #include "src/core/iomgr/timer.h" #include "src/core/profiling/timers.h" #include "src/core/surface/channel.h" #include "src/core/transport/connectivity_state.h" -#include "src/core/transport/connectivity_state.h" #define INTERNAL_REF_BITS 16 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1)) @@ -94,6 +95,8 @@ struct grpc_subchannel { struct sockaddr *addr; size_t addr_len; + grpc_subchannel_key *key; + /** initial string to send to peer */ gpr_slice initial_connect_string; @@ -207,6 +210,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_connectivity_state_destroy(exec_ctx, &c->state_tracker); grpc_connector_unref(exec_ctx, c->connector); grpc_pollset_set_destroy(&c->pollset_set); + grpc_subchannel_key_destroy(exec_ctx, c->key); gpr_free(c); } @@ -222,22 +226,42 @@ static gpr_atm ref_mutate(grpc_subchannel *c, gpr_atm delta, return old_val; } -void grpc_subchannel_ref(grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, (1 << INTERNAL_REF_BITS), 0 REF_MUTATE_PURPOSE("STRONG_REF")); GPR_ASSERT((old_refs & STRONG_REF_MASK) != 0); + return c; } -void grpc_subchannel_weak_ref(grpc_subchannel *c - GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *c + GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { gpr_atm old_refs; old_refs = ref_mutate(c, 1, 0 REF_MUTATE_PURPOSE("WEAK_REF")); GPR_ASSERT(old_refs != 0); + return c; +} + +grpc_subchannel *grpc_subchannel_ref_from_weak_ref( + grpc_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) { + if (!c) return NULL; + for (;;) { + gpr_atm old_refs = gpr_atm_acq_load(&c->ref_pair); + if (old_refs >= (1 << INTERNAL_REF_BITS)) { + gpr_atm new_refs = old_refs + (1 << INTERNAL_REF_BITS); + if (gpr_atm_rel_cas(&c->ref_pair, old_refs, new_refs)) { + return c; + } + } else { + return NULL; + } + } } static void disconnect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { grpc_connected_subchannel *con; + grpc_subchannel_index_unregister(exec_ctx, c->key, c); gpr_mu_lock(&c->mu); GPR_ASSERT(!c->disconnected); c->disconnected = 1; @@ -276,10 +300,19 @@ static uint32_t random_seed() { return (uint32_t)(gpr_time_to_millis(gpr_now(GPR_CLOCK_MONOTONIC))); } -grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, +grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, + grpc_connector *connector, grpc_subchannel_args *args) { - grpc_subchannel *c = gpr_malloc(sizeof(*c)); + grpc_subchannel_key *key = grpc_subchannel_key_create(connector, args); + grpc_subchannel *c = grpc_subchannel_index_find(exec_ctx, key); + if (c) { + grpc_subchannel_key_destroy(exec_ctx, key); + return c; + } + + c = gpr_malloc(sizeof(*c)); memset(c, 0, sizeof(*c)); + c->key = key; gpr_atm_no_barrier_store(&c->ref_pair, 1 << INTERNAL_REF_BITS); c->connector = connector; grpc_connector_ref(c->connector); @@ -305,7 +338,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE, "subchannel"); gpr_mu_init(&c->mu); - return c; + + return grpc_subchannel_index_register(exec_ctx, key, c); } static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h index 57c7c9dc67..313e63c75c 100644 --- a/src/core/client_config/subchannel.h +++ b/src/core/client_config/subchannel.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -48,6 +48,8 @@ typedef struct grpc_subchannel_args grpc_subchannel_args; #ifdef GRPC_STREAM_REFCOUNT_DEBUG #define GRPC_SUBCHANNEL_REF(p, r) \ grpc_subchannel_ref((p), __FILE__, __LINE__, (r)) +#define GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(p, r) \ + grpc_subchannel_ref_from_weak_ref((p), __FILE__, __LINE__, (r)) #define GRPC_SUBCHANNEL_UNREF(cl, p, r) \ grpc_subchannel_unref((cl), (p), __FILE__, __LINE__, (r)) #define GRPC_SUBCHANNEL_WEAK_REF(p, r) \ @@ -66,6 +68,8 @@ typedef struct grpc_subchannel_args grpc_subchannel_args; , const char *file, int line, const char *reason #else #define GRPC_SUBCHANNEL_REF(p, r) grpc_subchannel_ref((p)) +#define GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(p, r) \ + grpc_subchannel_ref_from_weak_ref((p)) #define GRPC_SUBCHANNEL_UNREF(cl, p, r) grpc_subchannel_unref((cl), (p)) #define GRPC_SUBCHANNEL_WEAK_REF(p, r) grpc_subchannel_weak_ref((p)) #define GRPC_SUBCHANNEL_WEAK_UNREF(cl, p, r) \ @@ -79,13 +83,15 @@ typedef struct grpc_subchannel_args grpc_subchannel_args; #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS #endif -void grpc_subchannel_ref(grpc_subchannel *channel - GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +grpc_subchannel *grpc_subchannel_ref(grpc_subchannel *channel + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +grpc_subchannel *grpc_subchannel_ref_from_weak_ref( + grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx, grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -void grpc_subchannel_weak_ref(grpc_subchannel *channel - GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +grpc_subchannel *grpc_subchannel_weak_ref(grpc_subchannel *channel + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx, grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); @@ -146,6 +152,8 @@ grpc_call_stack *grpc_subchannel_call_get_call_stack( grpc_subchannel_call *subchannel_call); struct grpc_subchannel_args { + /* When updating this struct, also update subchannel_index.c */ + /** Channel filters for this channel - wrapped factories will likely want to mutate this */ const grpc_channel_filter **filters; @@ -159,7 +167,8 @@ struct grpc_subchannel_args { }; /** create a subchannel given a connector */ -grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, +grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, + grpc_connector *connector, grpc_subchannel_args *args); #endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_H */ diff --git a/src/core/client_config/subchannel_index.c b/src/core/client_config/subchannel_index.c new file mode 100644 index 0000000000..f78a7fd588 --- /dev/null +++ b/src/core/client_config/subchannel_index.c @@ -0,0 +1,259 @@ +// +// +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// + +#include "src/core/client_config/subchannel_index.h" + +#include <stdbool.h> +#include <string.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/avl.h> +#include <grpc/support/tls.h> + +#include "src/core/channel/channel_args.h" + +// a map of subchannel_key --> subchannel, used for detecting connections +// to the same destination in order to share them +static gpr_avl g_subchannel_index; + +static gpr_mu g_mu; + +struct grpc_subchannel_key { + grpc_connector *connector; + grpc_subchannel_args args; +}; + +GPR_TLS_DECL(subchannel_index_exec_ctx); + +static void enter_ctx(grpc_exec_ctx *exec_ctx) { + GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == 0); + gpr_tls_set(&subchannel_index_exec_ctx, (intptr_t)exec_ctx); +} + +static void leave_ctx(grpc_exec_ctx *exec_ctx) { + GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == (intptr_t)exec_ctx); + gpr_tls_set(&subchannel_index_exec_ctx, 0); +} + +static grpc_exec_ctx *current_ctx() { + grpc_exec_ctx *c = (grpc_exec_ctx *)gpr_tls_get(&subchannel_index_exec_ctx); + GPR_ASSERT(c != NULL); + return c; +} + +static grpc_subchannel_key *create_key( + grpc_connector *connector, grpc_subchannel_args *args, + grpc_channel_args *(*copy_channel_args)(const grpc_channel_args *args)) { + grpc_subchannel_key *k = gpr_malloc(sizeof(*k)); + k->connector = grpc_connector_ref(connector); + k->args.filter_count = args->filter_count; + k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count); + memcpy((grpc_channel_filter *)k->args.filters, args->filters, + sizeof(*k->args.filters) * k->args.filter_count); + k->args.addr_len = args->addr_len; + k->args.addr = gpr_malloc(args->addr_len); + memcpy(k->args.addr, args->addr, k->args.addr_len); + k->args.args = copy_channel_args(args->args); + return k; +} + +grpc_subchannel_key *grpc_subchannel_key_create(grpc_connector *connector, + grpc_subchannel_args *args) { + return create_key(connector, args, grpc_channel_args_normalize); +} + +static grpc_subchannel_key *subchannel_key_copy(grpc_subchannel_key *k) { + return create_key(k->connector, &k->args, grpc_channel_args_copy); +} + +static int subchannel_key_compare(grpc_subchannel_key *a, + grpc_subchannel_key *b) { + int c = GPR_ICMP(a->connector, b->connector); + if (c != 0) return c; + c = GPR_ICMP(a->args.addr_len, b->args.addr_len); + if (c != 0) return c; + c = GPR_ICMP(a->args.filter_count, b->args.filter_count); + if (c != 0) return c; + c = memcmp(a->args.addr, b->args.addr, a->args.addr_len); + if (c != 0) return c; + c = memcmp(a->args.filters, b->args.filters, + a->args.filter_count * sizeof(*a->args.filters)); + return grpc_channel_args_compare(a->args.args, b->args.args); +} + +void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *k) { + grpc_connector_unref(exec_ctx, k->connector); + gpr_free(k->args.addr); + gpr_free((grpc_channel_args *)k->args.filters); + grpc_channel_args_destroy((grpc_channel_args *)k->args.args); + gpr_free(k); +} + +static void sck_avl_destroy(void *p) { + grpc_subchannel_key_destroy(current_ctx(), p); +} + +static void *sck_avl_copy(void *p) { return subchannel_key_copy(p); } + +static long sck_avl_compare(void *a, void *b) { + return subchannel_key_compare(a, b); +} + +static void scv_avl_destroy(void *p) { + GRPC_SUBCHANNEL_WEAK_UNREF(current_ctx(), p, "subchannel_index"); +} + +static void *scv_avl_copy(void *p) { + GRPC_SUBCHANNEL_WEAK_REF(p, "subchannel_index"); + return p; +} + +static const gpr_avl_vtable subchannel_avl_vtable = { + .destroy_key = sck_avl_destroy, + .copy_key = sck_avl_copy, + .compare_keys = sck_avl_compare, + .destroy_value = scv_avl_destroy, + .copy_value = scv_avl_copy}; + +void grpc_subchannel_index_init(void) { + g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable); + gpr_mu_init(&g_mu); +} + +void grpc_subchannel_index_shutdown(void) { + gpr_mu_destroy(&g_mu); + gpr_avl_unref(g_subchannel_index); +} + +grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key) { + enter_ctx(exec_ctx); + + // Lock, and take a reference to the subchannel index. + // We don't need to do the search under a lock as avl's are immutable. + gpr_mu_lock(&g_mu); + gpr_avl index = gpr_avl_ref(g_subchannel_index); + gpr_mu_unlock(&g_mu); + + grpc_subchannel *c = + GRPC_SUBCHANNEL_REF_FROM_WEAK_REF(gpr_avl_get(index, key), "index_find"); + gpr_avl_unref(index); + + leave_ctx(exec_ctx); + return c; +} + +grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key, + grpc_subchannel *constructed) { + enter_ctx(exec_ctx); + + grpc_subchannel *c = NULL; + + while (c == NULL) { + // Compare and swap loop: + // - take a reference to the current index + gpr_mu_lock(&g_mu); + gpr_avl index = gpr_avl_ref(g_subchannel_index); + gpr_mu_unlock(&g_mu); + + // - Check to see if a subchannel already exists + c = gpr_avl_get(index, key); + if (c != NULL) { + // yes -> we're done + GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, constructed, "index_register"); + } else { + // no -> update the avl and compare/swap + gpr_avl updated = + gpr_avl_add(gpr_avl_ref(index), subchannel_key_copy(key), + GRPC_SUBCHANNEL_WEAK_REF(constructed, "index_register")); + + // it may happen (but it's expected to be unlikely) + // that some other thread has changed the index: + // compare/swap here to check that, and retry as necessary + gpr_mu_lock(&g_mu); + if (index.root == g_subchannel_index.root) { + GPR_SWAP(gpr_avl, updated, g_subchannel_index); + c = constructed; + } + gpr_mu_unlock(&g_mu); + + gpr_avl_unref(updated); + } + gpr_avl_unref(index); + } + + leave_ctx(exec_ctx); + + return c; +} + +void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key, + grpc_subchannel *constructed) { + enter_ctx(exec_ctx); + + bool done = false; + while (!done) { + // Compare and swap loop: + // - take a reference to the current index + gpr_mu_lock(&g_mu); + gpr_avl index = gpr_avl_ref(g_subchannel_index); + gpr_mu_unlock(&g_mu); + + // Check to see if this key still refers to the previously + // registered subchannel + grpc_subchannel *c = gpr_avl_get(index, key); + if (c != constructed) { + gpr_avl_unref(index); + break; + } + + // compare and swap the update (some other thread may have + // mutated the index behind us) + gpr_avl updated = gpr_avl_remove(gpr_avl_ref(index), key); + + gpr_mu_lock(&g_mu); + if (index.root == g_subchannel_index.root) { + GPR_SWAP(gpr_avl, updated, g_subchannel_index); + done = true; + } + gpr_mu_unlock(&g_mu); + + gpr_avl_unref(updated); + gpr_avl_unref(index); + } + + leave_ctx(exec_ctx); +} diff --git a/src/core/client_config/subchannel_index.h b/src/core/client_config/subchannel_index.h new file mode 100644 index 0000000000..095ef17819 --- /dev/null +++ b/src/core/client_config/subchannel_index.h @@ -0,0 +1,77 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H +#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H + +#include "src/core/client_config/connector.h" +#include "src/core/client_config/subchannel.h" + +/** \file Provides an index of active subchannels so that they can be + shared amongst channels */ + +typedef struct grpc_subchannel_key grpc_subchannel_key; + +/** Create a key that can be used to uniquely identify a subchannel */ +grpc_subchannel_key *grpc_subchannel_key_create(grpc_connector *con, + grpc_subchannel_args *args); + +/** Destroy a subchannel key */ +void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key); + +/** Given a subchannel key, find the subchannel registered for it. + Returns NULL if no such channel exists. + Thread-safe. */ +grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key); + +/** Register a subchannel against a key. + Takes ownership of \a constructed. + Returns the registered subchannel. This may be different from + \a constructed in the case of a registration race. */ +grpc_subchannel *grpc_subchannel_index_register(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key, + grpc_subchannel *constructed); + +/** Remove \a constructed as the registered subchannel for \a key. */ +void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx, + grpc_subchannel_key *key, + grpc_subchannel *constructed); + +/** Initialize the subchannel index (global) */ +void grpc_subchannel_index_init(void); +/** Shutdown the subchannel index (global) */ +void grpc_subchannel_index_shutdown(void); + +#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_SUBCHANNEL_INDEX_H */ diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c index fe006c603c..ef548cfe4d 100644 --- a/src/core/iomgr/udp_server.c +++ b/src/core/iomgr/udp_server.c @@ -137,7 +137,7 @@ grpc_udp_server *grpc_udp_server_create(void) { } static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { - grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1); + grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1, NULL); gpr_mu_destroy(&s->mu); gpr_cv_destroy(&s->cv); @@ -146,7 +146,8 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { gpr_free(s); } -static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) { +static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, + bool success) { grpc_udp_server *s = server; gpr_mu_lock(&s->mu); s->destroyed_ports++; @@ -263,10 +264,10 @@ error: } /* event manager callback when reads are ready */ -static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { +static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) { server_port *sp = arg; - if (success == 0) { + if (!success) { gpr_mu_lock(&sp->server->mu); if (0 == --sp->server->active_ports) { gpr_mu_unlock(&sp->server->mu); diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c index afba0079f5..c58574bd6d 100644 --- a/src/core/security/credentials.c +++ b/src/core/security/credentials.c @@ -196,14 +196,21 @@ static void *server_credentials_pointer_arg_copy(void *p) { return grpc_server_credentials_ref(p); } +static int server_credentials_pointer_cmp(void *a, void *b) { + return GPR_ICMP(a, b); +} + +static const grpc_arg_pointer_vtable cred_ptr_vtable = { + server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy, + server_credentials_pointer_cmp}; + grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) { grpc_arg arg; memset(&arg, 0, sizeof(grpc_arg)); arg.type = GRPC_ARG_POINTER; arg.key = GRPC_SERVER_CREDENTIALS_ARG; arg.value.pointer.p = p; - arg.value.pointer.copy = server_credentials_pointer_arg_copy; - arg.value.pointer.destroy = server_credentials_pointer_arg_destroy; + arg.value.pointer.vtable = &cred_ptr_vtable; return arg; } diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index bdccbabfea..b46205323b 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -202,12 +202,17 @@ static void *connector_pointer_arg_copy(void *p) { return GRPC_SECURITY_CONNECTOR_REF(p, "connector_pointer_arg"); } +static int connector_pointer_cmp(void *a, void *b) { return GPR_ICMP(a, b); } + +static const grpc_arg_pointer_vtable connector_pointer_vtable = { + connector_pointer_arg_copy, connector_pointer_arg_destroy, + connector_pointer_cmp}; + grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc) { grpc_arg result; result.type = GRPC_ARG_POINTER; result.key = GRPC_SECURITY_CONNECTOR_ARG; - result.value.pointer.destroy = connector_pointer_arg_destroy; - result.value.pointer.copy = connector_pointer_arg_copy; + result.value.pointer.vtable = &connector_pointer_vtable; result.value.pointer.p = sc; return result; } diff --git a/src/core/security/security_context.c b/src/core/security/security_context.c index 2068c97d78..a71b3bc915 100644 --- a/src/core/security/security_context.c +++ b/src/core/security/security_context.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -309,14 +309,19 @@ static void *auth_context_pointer_arg_copy(void *p) { return GRPC_AUTH_CONTEXT_REF(p, "auth_context_pointer_arg"); } +static int auth_context_pointer_cmp(void *a, void *b) { return GPR_ICMP(a, b); } + +static const grpc_arg_pointer_vtable auth_context_pointer_vtable = { + auth_context_pointer_arg_copy, auth_context_pointer_arg_destroy, + auth_context_pointer_cmp}; + grpc_arg grpc_auth_context_to_arg(grpc_auth_context *p) { grpc_arg arg; memset(&arg, 0, sizeof(grpc_arg)); arg.type = GRPC_ARG_POINTER; arg.key = GRPC_AUTH_CONTEXT_ARG; arg.value.pointer.p = p; - arg.value.pointer.copy = auth_context_pointer_arg_copy; - arg.value.pointer.destroy = auth_context_pointer_arg_destroy; + arg.value.pointer.vtable = &auth_context_pointer_vtable; return arg; } diff --git a/src/core/security/server_auth_filter.c b/src/core/security/server_auth_filter.c index 4c78711387..3d8e5e8d35 100644 --- a/src/core/security/server_auth_filter.c +++ b/src/core/security/server_auth_filter.c @@ -176,8 +176,8 @@ static void set_recv_ops_md_callbacks(grpc_call_element *elem, if (op->recv_initial_metadata != NULL) { /* substitute our callback for the higher callback */ calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv = op->on_complete; - op->on_complete = &calld->auth_on_recv; + calld->on_done_recv = op->recv_initial_metadata_ready; + op->recv_initial_metadata_ready = &calld->auth_on_recv; calld->transport_op = *op; } } diff --git a/src/core/support/avl.c b/src/core/support/avl.c index 9734c9987f..f378b3ee17 100644 --- a/src/core/support/avl.c +++ b/src/core/support/avl.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -167,7 +167,7 @@ static gpr_avl_node *rotate_right_left(const gpr_avl_vtable *vtable, void *key, vtable->copy_key(right->left->key), vtable->copy_value(right->left->value), new_node(key, value, left, ref_node(right->left->left)), - new_node(vtable->copy_key(right->key), vtable->copy_key(right->value), + new_node(vtable->copy_key(right->key), vtable->copy_value(right->value), ref_node(right->left->right), ref_node(right->right))); unref_node(vtable, right); return n; diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c index 442cd8298e..1ca6fa1aff 100644 --- a/src/core/support/env_linux.c +++ b/src/core/support/env_linux.c @@ -52,6 +52,7 @@ #include "src/core/support/string.h" char *gpr_getenv(const char *name) { +#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE) typedef char *(*getenv_type)(const char *); static getenv_type getenv_func = NULL; /* Check to see which getenv variant is supported (go from most @@ -62,6 +63,10 @@ char *gpr_getenv(const char *name) { } char *result = getenv_func(name); return result == NULL ? result : gpr_strdup(result); +#else + char *result = secure_getenv(name); + return result == NULL ? result : gpr_strdup(result); +#endif } void gpr_setenv(const char *name, const char *value) { diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c index 1f92d7f090..36d75e7da2 100644 --- a/src/core/support/time_posix.c +++ b/src/core/support/time_posix.c @@ -86,7 +86,7 @@ gpr_timespec gpr_now(gpr_clock_type clock_type) { gpr_precise_clock_now(&ret); return ret; } else { -#if defined(__linux__) && !defined(GPR_NO_DIRECT_SYSCALLS) +#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE) && defined(__linux__) /* avoid ABI problems by invoking syscalls directly */ syscall(SYS_clock_gettime, clockid_for_gpr_clock[clock_type], &now); #else diff --git a/src/core/surface/alarm.c b/src/core/surface/alarm.c index d753023ca9..fb496f6c47 100644 --- a/src/core/surface/alarm.c +++ b/src/core/surface/alarm.c @@ -63,9 +63,9 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline, alarm->cq = cq; alarm->tag = tag; + grpc_cq_begin_op(cq, tag); grpc_timer_init(&exec_ctx, &alarm->alarm, deadline, alarm_cb, alarm, gpr_now(GPR_CLOCK_MONOTONIC)); - grpc_cq_begin_op(cq, tag); grpc_exec_ctx_finish(&exec_ctx); return alarm; } diff --git a/src/core/surface/call.c b/src/core/surface/call.c index 9495e748b5..1b117aa6b8 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -159,6 +159,9 @@ struct grpc_call { uint8_t receiving_message; uint8_t received_final_op; + /* have we received initial metadata */ + bool has_initial_md_been_received; + batch_control active_batches[MAX_CONCURRENT_BATCHES]; /* first idx: is_receiving, second idx: is_trailing */ @@ -200,6 +203,7 @@ struct grpc_call { gpr_slice receiving_slice; grpc_closure receiving_slice_ready; grpc_closure receiving_stream_ready; + grpc_closure receiving_initial_metadata_ready; uint32_t test_only_last_message_flags; union { @@ -212,6 +216,11 @@ struct grpc_call { int *cancelled; } server; } final_op; + + struct { + void *bctlp; + bool success; + } saved_receiving_stream_ready_ctx; }; #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) @@ -993,6 +1002,94 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } } +static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl, + bool success) { + grpc_call *call = bctl->call; + if (call->receiving_stream == NULL) { + *call->receiving_buffer = NULL; + call->receiving_message = 0; + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } + } else if (call->receiving_stream->length > + grpc_channel_get_max_message_length(call->channel)) { + cancel_with_status(exec_ctx, call, GRPC_STATUS_INTERNAL, + "Max message size exceeded"); + grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); + call->receiving_stream = NULL; + *call->receiving_buffer = NULL; + call->receiving_message = 0; + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } + } else { + call->test_only_last_message_flags = call->receiving_stream->flags; + if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) && + (call->compression_algorithm > GRPC_COMPRESS_NONE)) { + *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create( + NULL, 0, call->compression_algorithm); + } else { + *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0); + } + grpc_closure_init(&call->receiving_slice_ready, receiving_slice_ready, + bctl); + continue_receiving_slices(exec_ctx, bctl); + /* early out */ + return; + } +} + +static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, + bool success) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + + gpr_mu_lock(&bctl->call->mu); + if (bctl->call->has_initial_md_been_received) { + gpr_mu_unlock(&bctl->call->mu); + process_data_after_md(exec_ctx, bctlp, success); + } else { + call->saved_receiving_stream_ready_ctx.bctlp = bctlp; + call->saved_receiving_stream_ready_ctx.success = success; + gpr_mu_unlock(&bctl->call->mu); + } +} + +static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx, + void *bctlp, bool success) { + batch_control *bctl = bctlp; + grpc_call *call = bctl->call; + + gpr_mu_lock(&call->mu); + + grpc_metadata_batch *md = + &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; + grpc_metadata_batch_filter(md, recv_initial_filter, call); + call->has_initial_md_been_received = true; + + if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) != + 0 && + !call->is_client) { + GPR_TIMER_BEGIN("set_deadline_alarm", 0); + set_deadline_alarm(exec_ctx, call, md->deadline); + GPR_TIMER_END("set_deadline_alarm", 0); + } + + if (call->saved_receiving_stream_ready_ctx.bctlp != NULL) { + grpc_closure *saved_rsr_closure = grpc_closure_create( + receiving_stream_ready, call->saved_receiving_stream_ready_ctx.bctlp); + grpc_exec_ctx_enqueue(exec_ctx, saved_rsr_closure, + call->saved_receiving_stream_ready_ctx.success, NULL); + call->saved_receiving_stream_ready_ctx.bctlp = NULL; + } + + gpr_mu_unlock(&call->mu); + + if (gpr_unref(&bctl->steps_to_complete)) { + post_batch_completion(exec_ctx, bctl); + } +} + static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) { batch_control *bctl = bctlp; grpc_call *call = bctl->call; @@ -1011,19 +1108,6 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) { grpc_metadata_batch_destroy( &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]); } - if (bctl->recv_initial_metadata) { - grpc_metadata_batch *md = - &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; - grpc_metadata_batch_filter(md, recv_initial_filter, call); - - if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) != - 0 && - !call->is_client) { - GPR_TIMER_BEGIN("set_deadline_alarm", 0); - set_deadline_alarm(exec_ctx, call, md->deadline); - GPR_TIMER_END("set_deadline_alarm", 0); - } - } if (bctl->recv_final_op) { grpc_metadata_batch *md = &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */]; @@ -1065,45 +1149,6 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) { } } -static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, - bool success) { - batch_control *bctl = bctlp; - grpc_call *call = bctl->call; - - if (call->receiving_stream == NULL) { - *call->receiving_buffer = NULL; - call->receiving_message = 0; - if (gpr_unref(&bctl->steps_to_complete)) { - post_batch_completion(exec_ctx, bctl); - } - } else if (call->receiving_stream->length > - grpc_channel_get_max_message_length(call->channel)) { - cancel_with_status(exec_ctx, call, GRPC_STATUS_INTERNAL, - "Max message size exceeded"); - grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); - call->receiving_stream = NULL; - *call->receiving_buffer = NULL; - call->receiving_message = 0; - if (gpr_unref(&bctl->steps_to_complete)) { - post_batch_completion(exec_ctx, bctl); - } - } else { - call->test_only_last_message_flags = call->receiving_stream->flags; - if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) && - (call->compression_algorithm > GRPC_COMPRESS_NONE)) { - *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create( - NULL, 0, call->compression_algorithm); - } else { - *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0); - } - grpc_closure_init(&call->receiving_slice_ready, receiving_slice_ready, - bctl); - continue_receiving_slices(exec_ctx, bctl); - /* early out */ - return; - } -} - static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_call *call, const grpc_op *ops, size_t nops, void *notify_tag, @@ -1273,9 +1318,14 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } call->received_initial_metadata = 1; call->buffered_metadata[0] = op->data.recv_initial_metadata; + grpc_closure_init(&call->receiving_initial_metadata_ready, + receiving_initial_metadata_ready, bctl); bctl->recv_initial_metadata = 1; stream_op.recv_initial_metadata = &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */]; + stream_op.recv_initial_metadata_ready = + &call->receiving_initial_metadata_ready; + num_completion_callbacks_needed++; break; case GRPC_OP_RECV_MESSAGE: /* Flag validation: currently allow no flags */ diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 4d4337d288..fd7e20e9cc 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -172,7 +172,7 @@ static grpc_subchannel *subchannel_factory_create_subchannel( c->base.vtable = &connector_vtable; gpr_ref_init(&c->refs, 1); args->args = final_args; - s = grpc_subchannel_create(&c->base, args); + s = grpc_subchannel_create(exec_ctx, &c->base, args); grpc_connector_unref(exec_ctx, &c->base); grpc_channel_args_destroy(final_args); return s; diff --git a/src/core/surface/init.c b/src/core/surface/init.c index e3ab70dba7..a4a53d3ec1 100644 --- a/src/core/surface/init.c +++ b/src/core/surface/init.c @@ -46,6 +46,8 @@ #include "src/core/client_config/resolver_registry.h" #include "src/core/client_config/resolvers/dns_resolver.h" #include "src/core/client_config/resolvers/sockaddr_resolver.h" +#include "src/core/client_config/subchannel.h" +#include "src/core/client_config/subchannel_index.h" #include "src/core/debug/trace.h" #include "src/core/iomgr/executor.h" #include "src/core/iomgr/iomgr.h" @@ -127,6 +129,7 @@ void grpc_init(void) { } gpr_timers_global_init(); grpc_cq_global_init(); + grpc_subchannel_index_init(); for (i = 0; i < g_number_of_plugins; i++) { if (g_all_of_the_plugins[i].init != NULL) { g_all_of_the_plugins[i].init(); @@ -145,6 +148,7 @@ void grpc_shutdown(void) { grpc_executor_shutdown(); grpc_cq_global_shutdown(); grpc_iomgr_shutdown(); + grpc_subchannel_index_shutdown(); census_shutdown(); gpr_timers_global_destroy(); grpc_tracer_shutdown(); diff --git a/src/core/surface/lame_client.c b/src/core/surface/lame_client.c index 705996cad3..537069e984 100644 --- a/src/core/surface/lame_client.c +++ b/src/core/surface/lame_client.c @@ -78,8 +78,7 @@ static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx, } else if (op->recv_trailing_metadata != NULL) { fill_metadata(elem, op->recv_trailing_metadata); } - grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, false, NULL); - grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, false, NULL); + grpc_transport_stream_op_finish_with_failure(exec_ctx, op); } static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c index dd1441ad67..9c04426d87 100644 --- a/src/core/surface/secure_channel_create.c +++ b/src/core/surface/secure_channel_create.c @@ -238,7 +238,7 @@ static grpc_subchannel *subchannel_factory_create_subchannel( gpr_mu_init(&c->mu); gpr_ref_init(&c->refs, 1); args->args = final_args; - s = grpc_subchannel_create(&c->base, args); + s = grpc_subchannel_create(exec_ctx, &c->base, args); grpc_connector_unref(exec_ctx, &c->base); grpc_channel_args_destroy(final_args); return s; diff --git a/src/core/surface/server.c b/src/core/surface/server.c index 42cffccb4c..fb5e0d4b9e 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -596,8 +596,8 @@ static void server_mutate_op(grpc_call_element *elem, if (op->recv_initial_metadata != NULL) { calld->recv_initial_metadata = op->recv_initial_metadata; - calld->on_done_recv_initial_metadata = op->on_complete; - op->on_complete = &calld->server_on_recv_initial_metadata; + calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready; + op->recv_initial_metadata_ready = &calld->server_on_recv_initial_metadata; } } diff --git a/src/core/surface/validate_metadata.c b/src/core/surface/validate_metadata.c index df2e80b4b7..bf4126867f 100644 --- a/src/core/surface/validate_metadata.c +++ b/src/core/surface/validate_metadata.c @@ -50,7 +50,7 @@ static int conforms_to(const char *s, size_t len, const uint8_t *legal_bits) { int grpc_header_key_is_legal(const char *key, size_t length) { static const uint8_t legal_header_bits[256 / 8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xff, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0x00, 0x00, 0x00, 0x80, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; if (length == 0) { diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index c611496e7e..0e1e2c4265 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -385,7 +385,7 @@ typedef struct { grpc_closure *send_trailing_metadata_finished; grpc_metadata_batch *recv_initial_metadata; - grpc_closure *recv_initial_metadata_finished; + grpc_closure *recv_initial_metadata_ready; grpc_byte_stream **recv_message; grpc_closure *recv_message_ready; grpc_metadata_batch *recv_trailing_metadata; diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 9298573c7f..617d98875c 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -544,7 +544,7 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, GPR_ASSERT(s->global.send_initial_metadata_finished == NULL); GPR_ASSERT(s->global.send_message_finished == NULL); GPR_ASSERT(s->global.send_trailing_metadata_finished == NULL); - GPR_ASSERT(s->global.recv_initial_metadata_finished == NULL); + GPR_ASSERT(s->global.recv_initial_metadata_ready == NULL); GPR_ASSERT(s->global.recv_message_ready == NULL); GPR_ASSERT(s->global.recv_trailing_metadata_finished == NULL); grpc_chttp2_data_parser_destroy(exec_ctx, &s->parsing.data_parser); @@ -863,9 +863,9 @@ static void perform_stream_op_locked( } if (op->recv_initial_metadata != NULL) { - GPR_ASSERT(stream_global->recv_initial_metadata_finished == NULL); - stream_global->recv_initial_metadata_finished = - add_closure_barrier(on_complete); + GPR_ASSERT(stream_global->recv_initial_metadata_ready == NULL); + stream_global->recv_initial_metadata_ready = + op->recv_initial_metadata_ready; stream_global->recv_initial_metadata = op->recv_initial_metadata; grpc_chttp2_list_add_check_read_ops(transport_global, stream_global); } @@ -1009,13 +1009,14 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx, grpc_byte_stream *bs; while ( grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) { - if (stream_global->recv_initial_metadata_finished != NULL && + if (stream_global->recv_initial_metadata_ready != NULL && stream_global->published_initial_metadata) { grpc_chttp2_incoming_metadata_buffer_publish( &stream_global->received_initial_metadata, stream_global->recv_initial_metadata); - grpc_chttp2_complete_closure_step( - exec_ctx, &stream_global->recv_initial_metadata_finished, 1); + grpc_exec_ctx_enqueue( + exec_ctx, stream_global->recv_initial_metadata_ready, true, NULL); + stream_global->recv_initial_metadata_ready = NULL; } if (stream_global->recv_message_ready != NULL) { if (stream_global->incoming_frames.head != NULL) { diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c index 08d685668c..6e154b629a 100644 --- a/src/core/transport/transport.c +++ b/src/core/transport/transport.c @@ -126,6 +126,7 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx, void grpc_transport_stream_op_finish_with_failure( grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op) { grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, false, NULL); + grpc_exec_ctx_enqueue(exec_ctx, op->recv_initial_metadata_ready, false, NULL); grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, false, NULL); } diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h index f5cac77adc..8902c5d2f6 100644 --- a/src/core/transport/transport.h +++ b/src/core/transport/transport.h @@ -92,6 +92,8 @@ typedef struct grpc_transport_stream_op { /** Receive initial metadata from the stream, into provided metadata batch. */ grpc_metadata_batch *recv_initial_metadata; + /** Should be enqueued when initial metadata is ready to be processed. */ + grpc_closure *recv_initial_metadata_ready; /** Receive message data from the stream, into provided byte stream. */ grpc_byte_stream **recv_message; @@ -103,7 +105,8 @@ typedef struct grpc_transport_stream_op { grpc_metadata_batch *recv_trailing_metadata; /** Should be enqueued when all requested operations (excluding recv_message - which has its own closure) in a given batch have been completed. */ + and recv_initial_metadata which have their own closures) in a given batch + have been completed. */ grpc_closure *on_complete; /** If != GRPC_STATUS_OK, cancel this stream */ diff --git a/src/cpp/client/create_channel.cc b/src/cpp/client/create_channel.cc index fdaa28ffef..76a1b31e2f 100644 --- a/src/cpp/client/create_channel.cc +++ b/src/cpp/client/create_channel.cc @@ -32,7 +32,6 @@ */ #include <memory> -#include <sstream> #include <grpc++/channel.h> #include <grpc++/create_channel.h> @@ -56,13 +55,8 @@ std::shared_ptr<Channel> CreateCustomChannel( const ChannelArguments& args) { internal::GrpcLibrary init_lib; // We need to call init in case of a bad creds. - ChannelArguments cp_args = args; - std::ostringstream user_agent_prefix; - user_agent_prefix << "grpc-c++/" << grpc_version_string(); - cp_args.SetString(GRPC_ARG_PRIMARY_USER_AGENT_STRING, - user_agent_prefix.str()); return creds - ? creds->CreateChannel(target, cp_args) + ? creds->CreateChannel(target, args) : CreateChannelInternal("", grpc_lame_client_channel_create( NULL, GRPC_STATUS_INVALID_ARGUMENT, "Invalid credentials.")); diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index 90cd5136af..d7faa5e173 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,14 +30,23 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - #include <grpc++/support/channel_arguments.h> +#include <sstream> + +#include <grpc/impl/codegen/grpc_types.h> #include <grpc/support/log.h> #include "src/core/channel/channel_args.h" namespace grpc { +ChannelArguments::ChannelArguments() { + std::ostringstream user_agent_prefix; + user_agent_prefix << "grpc-c++/" << grpc_version_string(); + // This will be ignored if used on the server side. + SetString(GRPC_ARG_PRIMARY_USER_AGENT_STRING, user_agent_prefix.str()); +} + ChannelArguments::ChannelArguments(const ChannelArguments& other) : strings_(other.strings_) { args_.reserve(other.args_.size()); @@ -62,9 +71,7 @@ ChannelArguments::ChannelArguments(const ChannelArguments& other) break; case GRPC_ARG_POINTER: ap.value.pointer = a->value.pointer; - ap.value.pointer.p = a->value.pointer.copy - ? a->value.pointer.copy(ap.value.pointer.p) - : ap.value.pointer.p; + ap.value.pointer.p = a->value.pointer.vtable->copy(ap.value.pointer.p); break; } args_.push_back(ap); @@ -81,6 +88,31 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_ALGORITHM_ARG, algorithm); } +// Note: a second call to this will add in front the result of the first call. +// An example is calling this on a copy of ChannelArguments which already has a +// prefix. The user can build up a prefix string by calling this multiple times, +// each with more significant identifier. +void ChannelArguments::SetUserAgentPrefix( + const grpc::string& user_agent_prefix) { + if (user_agent_prefix.empty()) { + return; + } + bool replaced = false; + for (auto it = args_.begin(); it != args_.end(); ++it) { + const grpc_arg& arg = *it; + if (arg.type == GRPC_ARG_STRING && + grpc::string(arg.key) == GRPC_ARG_PRIMARY_USER_AGENT_STRING) { + strings_.push_back(user_agent_prefix + " " + arg.value.string); + it->value.string = const_cast<char*>(strings_.back().c_str()); + replaced = true; + break; + } + } + if (!replaced) { + SetString(GRPC_ARG_PRIMARY_USER_AGENT_STRING, user_agent_prefix); + } +} + void ChannelArguments::SetInt(const grpc::string& key, int value) { grpc_arg arg; arg.type = GRPC_ARG_INTEGER; @@ -92,13 +124,15 @@ void ChannelArguments::SetInt(const grpc::string& key, int value) { } void ChannelArguments::SetPointer(const grpc::string& key, void* value) { + static const grpc_arg_pointer_vtable vtable = { + &PointerVtableMembers::Copy, &PointerVtableMembers::Destroy, + &PointerVtableMembers::Compare}; grpc_arg arg; arg.type = GRPC_ARG_POINTER; strings_.push_back(key); arg.key = const_cast<char*>(strings_.back().c_str()); arg.value.pointer.p = value; - arg.value.pointer.copy = nullptr; - arg.value.pointer.destroy = nullptr; + arg.value.pointer.vtable = &vtable; args_.push_back(arg); } diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index a8c188e5a5..c54cf6474f 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -38,7 +38,6 @@ #include <grpc++/impl/service_type.h> #include <grpc++/server.h> #include "src/cpp/server/thread_pool_interface.h" -#include "src/cpp/server/fixed_size_thread_pool.h" namespace grpc { diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 9587503e4b..8d7d2cae0d 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -59,6 +59,7 @@ <Compile Include="IServerStreamWriter.cs" /> <Compile Include="IAsyncStreamWriter.cs" /> <Compile Include="IAsyncStreamReader.cs" /> + <Compile Include="Logging\NullLogger.cs" /> <Compile Include="ServerPort.cs" /> <Compile Include="Version.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> diff --git a/src/csharp/Grpc.Core/Logging/NullLogger.cs b/src/csharp/Grpc.Core/Logging/NullLogger.cs new file mode 100644 index 0000000000..58679a0ff9 --- /dev/null +++ b/src/csharp/Grpc.Core/Logging/NullLogger.cs @@ -0,0 +1,122 @@ +#region Copyright notice and license + +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; + +namespace Grpc.Core.Logging +{ + /// <summary> + /// Logger which doesn't log any information anywhere. + /// </summary> + public sealed class NullLogger : ILogger + { + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Debug(string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Debug(string format, params object[] formatArgs) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Error(string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Error(Exception exception, string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Error(string format, params object[] formatArgs) + { + } + + /// <summary> + /// Returns a reference to the instance on which the method is called, as + /// instances aren't associated with specific types. + /// </summary> + public ILogger ForType<T>() + { + return this; + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Info(string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Info(string format, params object[] formatArgs) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Warning(string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Warning(Exception exception, string message) + { + } + + /// <summary> + /// As with all logging calls on this logger, this method is a no-op. + /// </summary> + public void Warning(string format, params object[] formatArgs) + { + } + } +} diff --git a/src/node/performance/worker_server.js b/src/node/performance/worker.js index 7c8ab00026..7c8ab00026 100644 --- a/src/node/performance/worker_server.js +++ b/src/node/performance/worker.js diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c index 7ba14a38d8..4f48d6f2e2 100644 --- a/src/php/ext/grpc/call.c +++ b/src/php/ext/grpc/call.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -129,9 +129,9 @@ zval *grpc_parse_metadata_array(grpc_metadata_array *metadata_array) { zend_throw_exception(zend_exception_get_default(), "Metadata hash somehow contains wrong types.", 1 TSRMLS_CC); - efree(str_key); - efree(str_val); - return NULL; + efree(str_key); + efree(str_val); + return NULL; } add_next_index_stringl(*data, str_val, elem->value_length, false); } else { diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index 60c94412dc..f0bc7340ba 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -141,44 +141,40 @@ PHP_METHOD(Channel, __construct) { HashTable *array_hash; zval **creds_obj = NULL; wrapped_grpc_channel_credentials *creds = NULL; - /* "s|a" == 1 string, 1 optional array */ - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|a", &target, + /* "sa" == 1 string, 1 array */ + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sa", &target, &target_length, &args_array) == FAILURE) { zend_throw_exception(spl_ce_InvalidArgumentException, "Channel expects a string and an array", 1 TSRMLS_CC); return; } - if (args_array == NULL) { - channel->wrapped = grpc_insecure_channel_create(target, NULL, NULL); - } else { - array_hash = Z_ARRVAL_P(args_array); - if (zend_hash_find(array_hash, "credentials", sizeof("credentials"), - (void **)&creds_obj) == SUCCESS) { - if (Z_TYPE_P(*creds_obj) == IS_NULL) { - creds = NULL; - zend_hash_del(array_hash, "credentials", 12); - } else if (zend_get_class_entry(*creds_obj TSRMLS_CC) != - grpc_ce_channel_credentials) { - zend_throw_exception(spl_ce_InvalidArgumentException, - "credentials must be a ChannelCredentials object", - 1 TSRMLS_CC); - return; - } else { - creds = (wrapped_grpc_channel_credentials *)zend_object_store_get_object( - *creds_obj TSRMLS_CC); - zend_hash_del(array_hash, "credentials", 12); - } - } - php_grpc_read_args_array(args_array, &args); - if (creds == NULL) { - channel->wrapped = grpc_insecure_channel_create(target, &args, NULL); + array_hash = Z_ARRVAL_P(args_array); + if (zend_hash_find(array_hash, "credentials", sizeof("credentials"), + (void **)&creds_obj) == SUCCESS) { + if (Z_TYPE_P(*creds_obj) == IS_NULL) { + creds = NULL; + zend_hash_del(array_hash, "credentials", 12); + } else if (zend_get_class_entry(*creds_obj TSRMLS_CC) != + grpc_ce_channel_credentials) { + zend_throw_exception(spl_ce_InvalidArgumentException, + "credentials must be a ChannelCredentials object", + 1 TSRMLS_CC); + return; } else { - gpr_log(GPR_DEBUG, "Initialized secure channel"); - channel->wrapped = - grpc_secure_channel_create(creds->wrapped, target, &args, NULL); + creds = (wrapped_grpc_channel_credentials *)zend_object_store_get_object( + *creds_obj TSRMLS_CC); + zend_hash_del(array_hash, "credentials", 12); } - efree(args.args); } + php_grpc_read_args_array(args_array, &args); + if (creds == NULL) { + channel->wrapped = grpc_insecure_channel_create(target, &args, NULL); + } else { + gpr_log(GPR_DEBUG, "Initialized secure channel"); + channel->wrapped = + grpc_secure_channel_create(creds->wrapped, target, &args, NULL); + } + efree(args.args); } /** diff --git a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php index 1fe81b9d54..f70525ef15 100644 --- a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php +++ b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php @@ -1,7 +1,7 @@ <?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -106,6 +106,34 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase $this->assertSame(\Grpc\STATUS_CANCELLED, $status->code); } + public function testCallCredentialsCallback() + { + $div_arg = new math\DivArgs(); + $call = self::$client->Div($div_arg, array(), array( + 'call_credentials_callback' => function ($context) { + return array(); + }, + )); + $call->cancel(); + list($response, $status) = $call->wait(); + $this->assertSame(\Grpc\STATUS_CANCELLED, $status->code); + } + + public function testCallCredentialsCallback2() + { + $div_arg = new math\DivArgs(); + $call = self::$client->Div($div_arg); + $call_credentials = Grpc\CallCredentials::createFromPlugin( + function ($context) { + return array(); + } + ); + $call->setCallCredentials($call_credentials); + $call->cancel(); + list($response, $status) = $call->wait(); + $this->assertSame(\Grpc\STATUS_CANCELLED, $status->code); + } + /** * @expectedException InvalidArgumentException */ @@ -118,6 +146,23 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase $invalid_client->InvalidUnaryCall($div_arg); } + /** + * @expectedException Exception + */ + public function testMissingCredentials() + { + $invalid_client = new DummyInvalidClient('host', [ + ]); + } + + public function testPrimaryUserAgentString() + { + $invalid_client = new DummyInvalidClient('host', [ + 'credentials' => Grpc\ChannelCredentials::createInsecure(), + 'grpc.primary_user_agent' => 'testUserAgent', + ]); + } + public function testWriteFlags() { $div_arg = new math\DivArgs(); diff --git a/src/php/tests/unit_tests/CallCredentials2Test.php b/src/php/tests/unit_tests/CallCredentials2Test.php new file mode 100644 index 0000000000..1282db6eed --- /dev/null +++ b/src/php/tests/unit_tests/CallCredentials2Test.php @@ -0,0 +1,135 @@ +<?php +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +class CallCredentials2Test extends PHPUnit_Framework_TestCase +{ + public function setUp() + { + $credentials = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + $server_credentials = Grpc\ServerCredentials::createSsl( + null, + file_get_contents(dirname(__FILE__).'/../data/server1.key'), + file_get_contents(dirname(__FILE__).'/../data/server1.pem')); + $this->server = new Grpc\Server(); + $this->port = $this->server->addSecureHttp2Port('0.0.0.0:0', + $server_credentials); + $this->server->start(); + $this->host_override = 'foo.test.google.fr'; + $this->channel = new Grpc\Channel( + 'localhost:'.$this->port, + [ + 'grpc.ssl_target_name_override' => $this->host_override, + 'grpc.default_authority' => $this->host_override, + 'credentials' => $credentials, + ] + ); + } + + public function tearDown() + { + unset($this->channel); + unset($this->server); + } + + public function callbackFunc($context) + { + $this->assertTrue(is_string($context->service_url)); + $this->assertTrue(is_string($context->method_name)); + + return ['k1' => ['v1'], 'k2' => ['v2']]; + } + + public function testCreateFromPlugin() + { + $deadline = Grpc\Timeval::infFuture(); + $status_text = 'xyz'; + $call = new Grpc\Call($this->channel, + '/abc/dummy_method', + $deadline, + $this->host_override); + + $call_credentials = Grpc\CallCredentials::createFromPlugin( + array($this, 'callbackFunc')); + $call->setCredentials($call_credentials); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + + $event = $this->server->requestCall(); + + $this->assertTrue(is_array($event->metadata)); + $metadata = $event->metadata; + $this->assertTrue(array_key_exists('k1', $metadata)); + $this->assertTrue(array_key_exists('k2', $metadata)); + $this->assertSame($metadata['k1'], ['v1']); + $this->assertSame($metadata['k2'], ['v2']); + + $this->assertSame('/abc/dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => Grpc\STATUS_OK, + 'details' => $status_text, + ], + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_status); + $this->assertFalse($event->cancelled); + + $event = $call->startBatch([ + Grpc\OP_RECV_INITIAL_METADATA => true, + Grpc\OP_RECV_STATUS_ON_CLIENT => true, + ]); + + $this->assertSame([], $event->metadata); + $status = $event->status; + $this->assertSame([], $status->metadata); + $this->assertSame(Grpc\STATUS_OK, $status->code); + $this->assertSame($status_text, $status->details); + + unset($call); + unset($server_call); + } +} diff --git a/src/php/tests/unit_tests/CallCredentials3Test.php b/src/php/tests/unit_tests/CallCredentials3Test.php new file mode 100644 index 0000000000..a458f1d322 --- /dev/null +++ b/src/php/tests/unit_tests/CallCredentials3Test.php @@ -0,0 +1,136 @@ +<?php +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +class CallCredentials3Test extends PHPUnit_Framework_TestCase +{ + public function setUp() + { + $this->credentials = Grpc\ChannelCredentials::createSsl( + file_get_contents(dirname(__FILE__).'/../data/ca.pem')); + $server_credentials = Grpc\ServerCredentials::createSsl( + null, + file_get_contents(dirname(__FILE__).'/../data/server1.key'), + file_get_contents(dirname(__FILE__).'/../data/server1.pem')); + $this->server = new Grpc\Server(); + $this->port = $this->server->addSecureHttp2Port('0.0.0.0:0', + $server_credentials); + $this->server->start(); + $this->host_override = 'foo.test.google.fr'; + $this->channel = new Grpc\Channel( + 'localhost:'.$this->port, + [ + 'grpc.ssl_target_name_override' => $this->host_override, + 'grpc.default_authority' => $this->host_override, + 'credentials' => $this->credentials, + ] + ); + } + + public function tearDown() + { + unset($this->channel); + unset($this->server); + } + + public function callbackFunc($context) + { + $this->assertTrue(is_string($context->service_url)); + $this->assertTrue(is_string($context->method_name)); + + return ['k1' => ['v1'], 'k2' => ['v2']]; + } + + public function testCreateFromPlugin() + { + $deadline = Grpc\Timeval::infFuture(); + $status_text = 'xyz'; + $call = new Grpc\Call($this->channel, + '/abc/dummy_method', + $deadline, + $this->host_override); + + $call_credentials = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc']); + $call->setCredentials($call_credentials); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + + $event = $this->server->requestCall(); + + $this->assertTrue(is_array($event->metadata)); + $metadata = $event->metadata; + $this->assertTrue(array_key_exists('k1', $metadata)); + $this->assertTrue(array_key_exists('k2', $metadata)); + $this->assertSame($metadata['k1'], ['v1']); + $this->assertSame($metadata['k2'], ['v2']); + + $this->assertSame('/abc/dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => Grpc\STATUS_OK, + 'details' => $status_text, + ], + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_status); + $this->assertFalse($event->cancelled); + + $event = $call->startBatch([ + Grpc\OP_RECV_INITIAL_METADATA => true, + Grpc\OP_RECV_STATUS_ON_CLIENT => true, + ]); + + $this->assertSame([], $event->metadata); + $status = $event->status; + $this->assertSame([], $status->metadata); + $this->assertSame(Grpc\STATUS_OK, $status->code); + $this->assertSame($status_text, $status->details); + + unset($call); + unset($server_call); + } + +} diff --git a/src/php/tests/unit_tests/CallCredentialsTest.php b/src/php/tests/unit_tests/CallCredentialsTest.php index 0918412781..287024839d 100644 --- a/src/php/tests/unit_tests/CallCredentialsTest.php +++ b/src/php/tests/unit_tests/CallCredentialsTest.php @@ -1,7 +1,7 @@ <?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,13 +36,13 @@ class CallCredentialsTest extends PHPUnit_Framework_TestCase { public function setUp() { - $credentials = Grpc\ChannelCredentials::createSsl( + $this->credentials = Grpc\ChannelCredentials::createSsl( file_get_contents(dirname(__FILE__).'/../data/ca.pem')); - $call_credentials = Grpc\CallCredentials::createFromPlugin( - array($this, 'callbackFunc')); - $credentials = Grpc\ChannelCredentials::createComposite( - $credentials, - $call_credentials + $this->call_credentials = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc']); + $this->credentials = Grpc\ChannelCredentials::createComposite( + $this->credentials, + $this->call_credentials ); $server_credentials = Grpc\ServerCredentials::createSsl( null, @@ -58,7 +58,7 @@ class CallCredentialsTest extends PHPUnit_Framework_TestCase [ 'grpc.ssl_target_name_override' => $this->host_override, 'grpc.default_authority' => $this->host_override, - 'credentials' => $credentials, + 'credentials' => $this->credentials, ] ); } @@ -134,4 +134,41 @@ class CallCredentialsTest extends PHPUnit_Framework_TestCase unset($call); unset($server_call); } + + public function callbackFunc2($context) + { + return []; + } + + public function testCreateComposite() + { + $call_credentials2 = Grpc\CallCredentials::createFromPlugin( + [$this, 'callbackFunc2']); + $call_credentials3 = Grpc\CallCredentials::createComposite( + $this->call_credentials, + $call_credentials2 + ); + $this->assertSame('Grpc\CallCredentials', get_class($call_credentials3)); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testCreateFromPluginInvalidParam() + { + $call_credentials = Grpc\CallCredentials::createFromPlugin( + 'callbackFunc' + ); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testCreateCompositeInvalidParam() + { + $call_credentials3 = Grpc\CallCredentials::createComposite( + $this->call_credentials, + $this->credentials + ); + } } diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php index 3b697b50c3..a2522fb120 100755 --- a/src/php/tests/unit_tests/CallTest.php +++ b/src/php/tests/unit_tests/CallTest.php @@ -1,7 +1,7 @@ <?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -91,4 +91,32 @@ class CallTest extends PHPUnit_Framework_TestCase { $this->assertTrue(is_string($this->call->getPeer())); } + + public function testCancel() + { + $this->assertNull($this->call->cancel()); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidMetadataKey() + { + $batch = [ + 'invalid' => ['key1' => 'value1'], + ]; + $result = $this->call->startBatch($batch); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidMetadataInnerValue() + { + $batch = [ + Grpc\OP_SEND_INITIAL_METADATA => ['key1' => 'value1'], + ]; + $result = $this->call->startBatch($batch); + } + } diff --git a/src/cpp/server/fixed_size_thread_pool.cc b/src/php/tests/unit_tests/ChannelCredentialsTest.php index 2bdc44be2e..6d472dc876 100644 --- a/src/cpp/server/fixed_size_thread_pool.cc +++ b/src/php/tests/unit_tests/ChannelCredentialsTest.php @@ -1,6 +1,7 @@ +<?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,55 +32,42 @@ * */ -#include <grpc++/impl/sync.h> -#include <grpc++/impl/thd.h> -#include "src/cpp/server/fixed_size_thread_pool.h" - -namespace grpc { - -void FixedSizeThreadPool::ThreadFunc() { - for (;;) { - // Wait until work is available or we are shutting down. - grpc::unique_lock<grpc::mutex> lock(mu_); - if (!shutdown_ && callbacks_.empty()) { - cv_.wait(lock); +class ChanellCredentialsTest extends PHPUnit_Framework_TestCase +{ + public function setUp() + { } - // Drain callbacks before considering shutdown to ensure all work - // gets completed. - if (!callbacks_.empty()) { - auto cb = callbacks_.front(); - callbacks_.pop(); - lock.unlock(); - cb(); - } else if (shutdown_) { - return; + + public function tearDown() + { } - } -} -FixedSizeThreadPool::FixedSizeThreadPool(int num_threads) : shutdown_(false) { - for (int i = 0; i < num_threads; i++) { - threads_.push_back( - new grpc::thread(&FixedSizeThreadPool::ThreadFunc, this)); - } -} + public function testCreateDefault() + { + $channel_credentials = Grpc\ChannelCredentials::createDefault(); + $this->assertSame('Grpc\ChannelCredentials', get_class($channel_credentials)); + } -FixedSizeThreadPool::~FixedSizeThreadPool() { - { - grpc::lock_guard<grpc::mutex> lock(mu_); - shutdown_ = true; - cv_.notify_all(); - } - for (auto t = threads_.begin(); t != threads_.end(); t++) { - (*t)->join(); - delete *t; - } -} + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidCreateSsl() + { + $channel_credentials = Grpc\ChannelCredentials::createSsl([]); + } -void FixedSizeThreadPool::Add(const std::function<void()>& callback) { - grpc::lock_guard<grpc::mutex> lock(mu_); - callbacks_.push(callback); - cv_.notify_one(); -} + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidCreateComposite() + { + $channel_credentials = Grpc\ChannelCredentials::createComposite( + 'something', 'something'); + } -} // namespace grpc + public function testCreateInsecure() + { + $channel_credentials = Grpc\ChannelCredentials::createInsecure(); + $this->assertNull($channel_credentials); + } +}
\ No newline at end of file diff --git a/src/php/tests/unit_tests/ChannelTest.php b/src/php/tests/unit_tests/ChannelTest.php new file mode 100644 index 0000000000..acb8a0a70d --- /dev/null +++ b/src/php/tests/unit_tests/ChannelTest.php @@ -0,0 +1,82 @@ +<?php +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +class ChannelTest extends PHPUnit_Framework_TestCase +{ + public function setUp() + { + } + + public function tearDown() + { + } + + public function testInsecureCredentials() + { + $this->channel = new Grpc\Channel( + 'localhost:0', + [ + 'credentials' => Grpc\ChannelCredentials::createInsecure(), + ] + ); + $this->assertSame('Grpc\Channel', get_class($this->channel)); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidCredentials() + { + $this->channel = new Grpc\Channel( + 'localhost:0', + [ + 'credentials' => new Grpc\Timeval(100), + ] + ); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidOptionsArray() + { + $this->channel = new Grpc\Channel( + 'localhost:0', + [ + 'abc' => [], + ] + ); + } + +}
\ No newline at end of file diff --git a/src/php/tests/unit_tests/EndToEndTest.php b/src/php/tests/unit_tests/EndToEndTest.php index 5a38262451..45f6708b1f 100755 --- a/src/php/tests/unit_tests/EndToEndTest.php +++ b/src/php/tests/unit_tests/EndToEndTest.php @@ -1,7 +1,7 @@ <?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -201,6 +201,318 @@ class EndToEndTest extends PHPUnit_Framework_TestCase unset($server_call); } + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidClientMessageArray() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => 'invalid', + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidClientMessageString() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => 0], + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidClientMessageFlags() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => 'abc', + 'flags' => 'invalid'], + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidServerStatusMetadata() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + $this->assertTrue($event->send_message); + + $event = $this->server->requestCall(); + $this->assertSame('dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_MESSAGE => ['message' => $reply_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => 'invalid', + 'code' => Grpc\STATUS_OK, + 'details' => $status_text, + ], + Grpc\OP_RECV_MESSAGE => true, + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidServerStatusCode() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + $this->assertTrue($event->send_message); + + $event = $this->server->requestCall(); + $this->assertSame('dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_MESSAGE => ['message' => $reply_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => 'invalid', + 'details' => $status_text, + ], + Grpc\OP_RECV_MESSAGE => true, + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testMissingServerStatusCode() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + $this->assertTrue($event->send_message); + + $event = $this->server->requestCall(); + $this->assertSame('dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_MESSAGE => ['message' => $reply_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'details' => $status_text, + ], + Grpc\OP_RECV_MESSAGE => true, + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidServerStatusDetails() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + $this->assertTrue($event->send_message); + + $event = $this->server->requestCall(); + $this->assertSame('dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_MESSAGE => ['message' => $reply_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => Grpc\STATUS_OK, + 'details' => 0, + ], + Grpc\OP_RECV_MESSAGE => true, + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testMissingServerStatusDetails() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + ]); + + $this->assertTrue($event->send_metadata); + $this->assertTrue($event->send_close); + $this->assertTrue($event->send_message); + + $event = $this->server->requestCall(); + $this->assertSame('dummy_method', $event->method); + $server_call = $event->call; + + $event = $server_call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_MESSAGE => ['message' => $reply_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => Grpc\STATUS_OK, + ], + Grpc\OP_RECV_MESSAGE => true, + Grpc\OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidStartBatchKey() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + 9999999 => [], + ]); + } + + /** + * @expectedException LogicException + */ + public function testInvalidStartBatch() + { + $deadline = Grpc\Timeval::infFuture(); + $req_text = 'client_server_full_request_response'; + $reply_text = 'reply:client_server_full_request_response'; + $status_text = 'status:client_server_full_response_text'; + + $call = new Grpc\Call($this->channel, + 'dummy_method', + $deadline); + + $event = $call->startBatch([ + Grpc\OP_SEND_INITIAL_METADATA => [], + Grpc\OP_SEND_CLOSE_FROM_CLIENT => true, + Grpc\OP_SEND_MESSAGE => ['message' => $req_text], + Grpc\OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => Grpc\STATUS_OK, + 'details' => 'abc', + ], + ]); + } + public function testGetTarget() { $this->assertTrue(is_string($this->channel->getTarget())); @@ -255,4 +567,36 @@ class EndToEndTest extends PHPUnit_Framework_TestCase $new_state = $this->channel->getConnectivityState(); $this->assertTrue($new_state == Grpc\CHANNEL_IDLE); } + + /** + * @expectedException InvalidArgumentException + */ + public function testGetConnectivityStateInvalidParam() + { + $this->assertTrue($this->channel->getConnectivityState( + new Grpc\Timeval)); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testWatchConnectivityStateInvalidParam() + { + $this->assertTrue($this->channel->watchConnectivityState( + 0, 1000)); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testChannelConstructorInvalidParam() + { + $this->channel = new Grpc\Channel('localhost:'.$this->port, NULL); + } + + public function testClose() + { + $this->assertNull($this->channel->close()); + } + } diff --git a/src/cpp/server/fixed_size_thread_pool.h b/src/php/tests/unit_tests/ServerTest.php index 394ae5821e..cde6a9a8f7 100644 --- a/src/cpp/server/fixed_size_thread_pool.h +++ b/src/php/tests/unit_tests/ServerTest.php @@ -1,6 +1,7 @@ +<?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,37 +32,40 @@ * */ -#ifndef GRPC_INTERNAL_CPP_FIXED_SIZE_THREAD_POOL_H -#define GRPC_INTERNAL_CPP_FIXED_SIZE_THREAD_POOL_H +class ServerTest extends PHPUnit_Framework_TestCase +{ + public function setUp() + { + } -#include <queue> -#include <vector> + public function tearDown() + { + } -#include <grpc++/impl/sync.h> -#include <grpc++/impl/thd.h> -#include <grpc++/support/config.h> + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidConstructor() + { + $server = new Grpc\Server('invalid_host'); + } -#include "src/cpp/server/thread_pool_interface.h" + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidAddHttp2Port() + { + $this->server = new Grpc\Server([]); + $this->port = $this->server->addHttp2Port(['0.0.0.0:0']); + } -namespace grpc { + /** + * @expectedException InvalidArgumentException + */ + public function testInvalidAddSecureHttp2Port() + { + $this->server = new Grpc\Server([]); + $this->port = $this->server->addSecureHttp2Port(['0.0.0.0:0']); + } -class FixedSizeThreadPool GRPC_FINAL : public ThreadPoolInterface { - public: - explicit FixedSizeThreadPool(int num_threads); - ~FixedSizeThreadPool(); - - void Add(const std::function<void()>& callback) GRPC_OVERRIDE; - - private: - grpc::mutex mu_; - grpc::condition_variable cv_; - bool shutdown_; - std::queue<std::function<void()>> callbacks_; - std::vector<grpc::thread*> threads_; - - void ThreadFunc(); -}; - -} // namespace grpc - -#endif // GRPC_INTERNAL_CPP_FIXED_SIZE_THREAD_POOL_H +}
\ No newline at end of file diff --git a/src/php/tests/unit_tests/TimevalTest.php b/src/php/tests/unit_tests/TimevalTest.php index 1d2a8d303e..9e4bc294da 100755 --- a/src/php/tests/unit_tests/TimevalTest.php +++ b/src/php/tests/unit_tests/TimevalTest.php @@ -1,7 +1,7 @@ <?php /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -91,4 +91,69 @@ class TimevalTest extends PHPUnit_Framework_TestCase $back_to_now = $deadline->subtract($delta); $this->assertSame(0, Grpc\Timeval::compare($back_to_now, $now)); } + + public function testSimilar() + { + $a = Grpc\Timeval::now(); + $delta = new Grpc\Timeval(1000); + $b = $a->add($delta); + $thresh = new Grpc\Timeval(1100); + $this->assertTrue(Grpc\Timeval::similar($a, $b, $thresh)); + $thresh = new Grpc\Timeval(900); + $this->assertFalse(Grpc\Timeval::similar($a, $b, $thresh)); + } + + public function testSleepUntil() + { + $curr_microtime = microtime(true); + $now = Grpc\Timeval::now(); + $delta = new Grpc\Timeval(1000); + $deadline = $now->add($delta); + $deadline->sleepUntil(); + $done_microtime = microtime(true); + $this->assertTrue(($done_microtime - $curr_microtime) > 0.0009); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testConstructorInvalidParam() + { + $delta = new Grpc\Timeval('abc'); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testAddInvalidParam() + { + $a = Grpc\Timeval::now(); + $a->add(1000); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testSubtractInvalidParam() + { + $a = Grpc\Timeval::now(); + $a->subtract(1000); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testCompareInvalidParam() + { + $a = Grpc\Timeval::compare(1000, 1100); + } + + /** + * @expectedException InvalidArgumentException + */ + public function testSimilarInvalidParam() + { + $a = Grpc\Timeval::similar(1000, 1100, 1200); + } + } diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index c7b5a3bde4..698c760ebe 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -1,22 +1,40 @@ gRPC Python =========== -Package for GRPC Python. +Package for gRPC Python. -Dependencies +Installation ------------ -Ensure you have installed the gRPC core. On Mac OS X, install homebrew_. -Run the following command to install gRPC Python. +gRPC Python is available for Linux and Mac OS X running Python 2.7. + +From PyPI +~~~~~~~~~ + +If you are installing locally... :: - $ curl -fsSL https://goo.gl/getgrpc | bash -s python + $ pip install grpcio + +Else system wide (on Ubuntu)... + +:: -This will download and run the [gRPC install script][] to install grpc core. The script then uses pip to install this package. It also installs the Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin for python. + $ sudo pip install grpcio + +From Source +~~~~~~~~~~~ + +Building from source requires that you have the Python headers (usually a +package named `python-dev`). + +:: -Otherwise, `install from source`_ + $ export REPO_ROOT=grpc + $ git clone https://github.com/grpc/grpc.git $REPO_ROOT + $ cd $REPO_ROOT + $ pip install . -.. _`install from source`: https://github.com/grpc/grpc/blob/master/src/python/README.md#building-from-source -.. _homebrew: http://brew.sh -.. _`gRPC install script`: https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install +Note that `$REPO_ROOT` can be assigned to whatever directory name floats your +fancy. diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 774e7ad6a1..aa29c728f2 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -41,7 +41,6 @@ import sys import traceback import setuptools -from setuptools.command import bdist_egg from setuptools.command import build_ext from setuptools.command import build_py from setuptools.command import easy_install @@ -52,13 +51,6 @@ import support PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) -BINARIES_REPOSITORY = os.environ.get( - 'GRPC_PYTHON_BINARIES_REPOSITORY', - 'https://storage.googleapis.com/grpc-precompiled-binaries/python') - -USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get( - 'GRPC_PYTHON_USE_CUSTOM_BDIST', '1'))) - CONF_PY_ADDENDUM = """ extensions.append('sphinx.ext.napoleon') napoleon_google_docstring = True @@ -74,126 +66,39 @@ class CommandError(Exception): # TODO(atash): Remove this once PyPI has better Linux bdist support. See # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported -def _get_grpc_custom_bdist_egg(decorated_basename, target_egg_basename): - """Returns a string path to a .egg file for Linux to install. +def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): + """Returns a string path to a bdist file for Linux to install. - If we can retrieve a pre-compiled egg from online, uses it. Else, emits a + If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a warning and builds from source. """ + # TODO(atash): somehow the name that's returned from `wheel` is different + # between different versions of 'wheel' (but from a compatibility standpoint, + # the names are compatible); we should have some way of determining name + # compatibility in the same way `wheel` does to avoid having to rename all of + # the custom wheels that we build/upload to GCS. + # Break import style to ensure that setup.py has had a chance to install the - # relevant package eggs. + # relevant package. from six.moves.urllib import request - decorated_path = decorated_basename + '.egg' + decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT try: url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path) - egg_data = request.urlopen(url).read() + bdist_data = request.urlopen(url).read() except IOError as error: raise CommandError( - '{}\n\nCould not find the bdist egg {}: {}' + '{}\n\nCould not find the bdist {}: {}' .format(traceback.format_exc(), decorated_path, error.message)) - # Our chosen local egg path. - egg_path = target_egg_basename + '.egg' + # Our chosen local bdist path. + bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT try: - with open(egg_path, 'w') as egg_file: - egg_file.write(egg_data) + with open(bdist_path, 'w') as bdist_file: + bdist_file.write(bdist_data) except IOError as error: raise CommandError( - '{}\n\nCould not write grpcio egg: {}' + '{}\n\nCould not write grpcio bdist: {}' .format(traceback.format_exc(), error.message)) - return egg_path - - -class EggNameMixin(object): - """Mixin for setuptools.Command classes to enable acquiring the egg name.""" - - def egg_name(self, with_custom): - """ - Args: - with_custom: Boolean describing whether or not to decorate the egg name - with custom gRPC-specific target information. - """ - egg_command = self.get_finalized_command('bdist_egg') - base = os.path.splitext(os.path.basename(egg_command.egg_output))[0] - if with_custom: - flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' - return '{base}-{flavor}'.format(base=base, flavor=flavor) - else: - return base - - -class Install(install.install, EggNameMixin): - """Custom Install command for gRPC Python. - - This is for bdist shims and whatever else we might need a custom install - command for. - """ - - user_options = install.install.user_options + [ - # TODO(atash): remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - ('use-grpc-custom-bdist', None, - 'Whether to retrieve a binary from the gRPC binary repository instead ' - 'of building from source.'), - ] - - def initialize_options(self): - install.install.initialize_options(self) - self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST - - def finalize_options(self): - install.install.finalize_options(self) - - def run(self): - if self.use_grpc_custom_bdist: - try: - try: - egg_path = _get_grpc_custom_bdist_egg(self.egg_name(True), - self.egg_name(False)) - except CommandError as error: - sys.stderr.write( - '\nWARNING: Failed to acquire grpcio prebuilt binary:\n' - '{}.\n\n'.format(error.message)) - raise - try: - self._run_bdist_retrieval_install(egg_path) - except Exception as error: - # if anything else happens (and given how there's no way to really know - # what's happening in setuptools here, I mean *anything*), warn the user - # and fall back to building from source. - sys.stderr.write( - '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n' - .format(traceback.format_exc())) - raise - except Exception: - install.install.run(self) - else: - install.install.run(self) - - # TODO(atash): Remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - def _run_bdist_retrieval_install(self, bdist_egg): - easy_install = self.distribution.get_command_class('easy_install') - easy_install_command = easy_install( - self.distribution, args='x', root=self.root, record=self.record, - ) - easy_install_command.ensure_finalized() - easy_install_command.always_copy_from = '.' - easy_install_command.package_index.scan(glob.glob('*.egg')) - arguments = [bdist_egg] - if setuptools.bootstrap_install_from: - args.insert(0, setuptools.bootstrap_install_from) - easy_install_command.args = arguments - easy_install_command.run() - setuptools.bootstrap_install_from = None - - -class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin): - """Thin wrapper around the bdist_egg command to build with our custom name.""" - - def run(self): - bdist_egg.bdist_egg.run(self) - target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True))) - shutil.move(self.get_outputs()[0], target) + return bdist_path class SphinxDocumentation(setuptools.Command): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index bbeed9ad40..800d0ea2f6 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -36,7 +36,7 @@ cdef extern from "grpc/_cython/loader.h": ctypedef unsigned uint32_t ctypedef long int64_t - int pygrpc_load_core(const char*) + int pygrpc_load_core(char*) void *gpr_malloc(size_t size) void gpr_free(void *ptr) diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c index 817303c8a4..4b1860ce8c 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.c +++ b/src/python/grpcio/grpc/_cython/imports.generated.c @@ -137,6 +137,7 @@ grpc_auth_context_add_cstring_property_type grpc_auth_context_add_cstring_proper grpc_auth_context_set_peer_identity_property_name_type grpc_auth_context_set_peer_identity_property_name_import; grpc_channel_credentials_release_type grpc_channel_credentials_release_import; grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import; +grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import; grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import; grpc_call_credentials_release_type grpc_call_credentials_release_import; grpc_composite_channel_credentials_create_type grpc_composite_channel_credentials_create_import; @@ -296,6 +297,10 @@ gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import; gpr_thd_currentid_type gpr_thd_currentid_import; gpr_thd_join_type gpr_thd_join_import; +#ifdef __cplusplus +extern "C" { +#endif /* __cpluslus */ + void pygrpc_load_imports(HMODULE library) { census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); @@ -397,6 +402,7 @@ void pygrpc_load_imports(HMODULE library) { grpc_auth_context_set_peer_identity_property_name_import = (grpc_auth_context_set_peer_identity_property_name_type) GetProcAddress(library, "grpc_auth_context_set_peer_identity_property_name"); grpc_channel_credentials_release_import = (grpc_channel_credentials_release_type) GetProcAddress(library, "grpc_channel_credentials_release"); grpc_google_default_credentials_create_import = (grpc_google_default_credentials_create_type) GetProcAddress(library, "grpc_google_default_credentials_create"); + grpc_set_ssl_roots_override_callback_import = (grpc_set_ssl_roots_override_callback_type) GetProcAddress(library, "grpc_set_ssl_roots_override_callback"); grpc_ssl_credentials_create_import = (grpc_ssl_credentials_create_type) GetProcAddress(library, "grpc_ssl_credentials_create"); grpc_call_credentials_release_import = (grpc_call_credentials_release_type) GetProcAddress(library, "grpc_call_credentials_release"); grpc_composite_channel_credentials_create_import = (grpc_composite_channel_credentials_create_type) GetProcAddress(library, "grpc_composite_channel_credentials_create"); @@ -557,4 +563,8 @@ void pygrpc_load_imports(HMODULE library) { gpr_thd_join_import = (gpr_thd_join_type) GetProcAddress(library, "gpr_thd_join"); } +#ifdef __cplusplus +} +#endif /* __cpluslus */ + #endif /* !GPR_WIN32 */ diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h index 6d0a6e06c0..ca30742abc 100644 --- a/src/python/grpcio/grpc/_cython/imports.generated.h +++ b/src/python/grpcio/grpc/_cython/imports.generated.h @@ -361,6 +361,9 @@ extern grpc_channel_credentials_release_type grpc_channel_credentials_release_im typedef grpc_channel_credentials *(*grpc_google_default_credentials_create_type)(void); extern grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import; #define grpc_google_default_credentials_create grpc_google_default_credentials_create_import +typedef void(*grpc_set_ssl_roots_override_callback_type)(grpc_ssl_roots_override_callback cb); +extern grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import; +#define grpc_set_ssl_roots_override_callback grpc_set_ssl_roots_override_callback_import typedef grpc_channel_credentials *(*grpc_ssl_credentials_create_type)(const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, void *reserved); extern grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import; #define grpc_ssl_credentials_create grpc_ssl_credentials_create_import @@ -836,8 +839,16 @@ typedef void(*gpr_thd_join_type)(gpr_thd_id t); extern gpr_thd_join_type gpr_thd_join_import; #define gpr_thd_join gpr_thd_join_import +#ifdef __cplusplus +extern "C" { +#endif /* __cpluslus */ + void pygrpc_load_imports(HMODULE library); +#ifdef __cplusplus +} +#endif /* __cpluslus */ + #else /* !GPR_WIN32 */ #include <grpc/support/alloc.h> diff --git a/src/python/grpcio/grpc/_cython/loader.c b/src/python/grpcio/grpc/_cython/loader.c index cdd47deed3..3b72806ea1 100644 --- a/src/python/grpcio/grpc/_cython/loader.c +++ b/src/python/grpcio/grpc/_cython/loader.c @@ -33,6 +33,10 @@ #include "loader.h" +#ifdef __cplusplus +extern "C" { +#endif /* __cpluslus */ + #if GPR_WIN32 int pygrpc_load_core(char *path) { @@ -56,4 +60,9 @@ int pygrpc_load_core(char *path) { int pygrpc_load_core(char *path) { return 1; } -#endif +#endif /* !GPR_WIN32 */ + +#ifdef __cplusplus +} +#endif /* __cpluslus */ + diff --git a/src/python/grpcio/grpc/_cython/loader.h b/src/python/grpcio/grpc/_cython/loader.h index dd31e1561b..3b8796d39f 100644 --- a/src/python/grpcio/grpc/_cython/loader.h +++ b/src/python/grpcio/grpc/_cython/loader.h @@ -39,7 +39,16 @@ /* Additional inclusions not covered by "imports.generated.h" */ #include <grpc/byte_buffer_reader.h> +#ifdef __cplusplus +extern "C" { +#endif /* __cpluslus */ + /* Attempts to load the core if necessary, and return non-zero upon succes. */ int pygrpc_load_core(char *path); +#ifdef __cplusplus +} +#endif /* __cpluslus */ + #endif /* GRPC_RB_BYTE_BUFFER_H_ */ + diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 60af862305..5d7e31ba01 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -122,6 +122,7 @@ CORE_SOURCE_FILES = [ 'src/core/client_config/resolvers/sockaddr_resolver.c', 'src/core/client_config/subchannel.c', 'src/core/client_config/subchannel_factory.c', + 'src/core/client_config/subchannel_index.c', 'src/core/client_config/uri_parser.c', 'src/core/compression/algorithm.c', 'src/core/compression/message_compress.c', @@ -225,6 +226,7 @@ CORE_SOURCE_FILES = [ 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tracing.c', diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py new file mode 100644 index 0000000000..05c651b506 --- /dev/null +++ b/src/python/grpcio/precompiled.py @@ -0,0 +1,102 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import platform +import shutil +import sys + +import setuptools + +import commands +import grpc_version + +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen + +PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) +BINARIES_REPOSITORY = os.environ.get( + 'GRPC_PYTHON_BINARIES_REPOSITORY', + 'https://storage.googleapis.com/grpc-precompiled-binaries/python') +USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( + 'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1'))) + +def _tagged_ext_name(base): + uname = platform.uname() + tags = '-'.join((grpc_version.VERSION, uname[0], uname[4])) + flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' + return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor) + + +class BuildTaggedExt(setuptools.Command): + + description = 'build the gRPC tagged extensions' + user_options = [] + + def initialize_options(self): + # distutils requires this override. + pass + + def finalize_options(self): + # distutils requires this override. + pass + + def run(self): + if 'linux' in sys.platform: + self.run_command('build_ext') + try: + os.makedirs('dist/') + except OSError: + pass + shutil.copyfile( + os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'), + 'dist/{}.so'.format(_tagged_ext_name('cygrpc'))) + else: + sys.stderr.write('nothing to do for build_tagged_ext\n') + + +def update_setup_arguments(setup_arguments): + url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc')) + target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so') + try: + extension = urlopen(url).read() + except: + sys.stderr.write( + 'could not download precompiled extension: {}\n'.format(url)) + return + try: + with open(target_path, 'w') as target: + target.write(extension) + setup_arguments['ext_modules'] = [] + except: + sys.stderr.write( + 'could not write precompiled extension to directory: {} -> {}\n' + .format(url, target_path)) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index b7c6cb3d7e..6b7001a489 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -79,6 +79,7 @@ unless File.exist?(File.join(grpc_lib_dir, 'libgrpc.a')) or windows ENV['EMBED_ZLIB'] = 'true' ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG'] ENV['ARCH_FLAGS'] = '-arch i386 -arch x86_64' if RUBY_PLATFORM =~ /darwin/ + ENV['CFLAGS'] = '-DGPR_BACKWARDS_COMPATIBILITY_MODE' output_dir = File.expand_path(RbConfig::CONFIG['topdir']) grpc_lib_dir = File.join(output_dir, 'libs', grpc_config) diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index d4ddb734c0..1af34d97fb 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -137,6 +137,7 @@ grpc_auth_context_add_cstring_property_type grpc_auth_context_add_cstring_proper grpc_auth_context_set_peer_identity_property_name_type grpc_auth_context_set_peer_identity_property_name_import; grpc_channel_credentials_release_type grpc_channel_credentials_release_import; grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import; +grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import; grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import; grpc_call_credentials_release_type grpc_call_credentials_release_import; grpc_composite_channel_credentials_create_type grpc_composite_channel_credentials_create_import; @@ -397,6 +398,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_auth_context_set_peer_identity_property_name_import = (grpc_auth_context_set_peer_identity_property_name_type) GetProcAddress(library, "grpc_auth_context_set_peer_identity_property_name"); grpc_channel_credentials_release_import = (grpc_channel_credentials_release_type) GetProcAddress(library, "grpc_channel_credentials_release"); grpc_google_default_credentials_create_import = (grpc_google_default_credentials_create_type) GetProcAddress(library, "grpc_google_default_credentials_create"); + grpc_set_ssl_roots_override_callback_import = (grpc_set_ssl_roots_override_callback_type) GetProcAddress(library, "grpc_set_ssl_roots_override_callback"); grpc_ssl_credentials_create_import = (grpc_ssl_credentials_create_type) GetProcAddress(library, "grpc_ssl_credentials_create"); grpc_call_credentials_release_import = (grpc_call_credentials_release_type) GetProcAddress(library, "grpc_call_credentials_release"); grpc_composite_channel_credentials_create_import = (grpc_composite_channel_credentials_create_type) GetProcAddress(library, "grpc_composite_channel_credentials_create"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 618ae5e7fc..b61c5282b6 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -361,6 +361,9 @@ extern grpc_channel_credentials_release_type grpc_channel_credentials_release_im typedef grpc_channel_credentials *(*grpc_google_default_credentials_create_type)(void); extern grpc_google_default_credentials_create_type grpc_google_default_credentials_create_import; #define grpc_google_default_credentials_create grpc_google_default_credentials_create_import +typedef void(*grpc_set_ssl_roots_override_callback_type)(grpc_ssl_roots_override_callback cb); +extern grpc_set_ssl_roots_override_callback_type grpc_set_ssl_roots_override_callback_import; +#define grpc_set_ssl_roots_override_callback grpc_set_ssl_roots_override_callback_import typedef grpc_channel_credentials *(*grpc_ssl_credentials_create_type)(const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, void *reserved); extern grpc_ssl_credentials_create_type grpc_ssl_credentials_create_import; #define grpc_ssl_credentials_create grpc_ssl_credentials_create_import diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb index 594fda1cd3..7ef534571f 100644 --- a/src/ruby/spec/client_server_spec.rb +++ b/src/ruby/spec/client_server_spec.rb @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -198,6 +198,7 @@ shared_examples 'basic GRPC message delivery is OK' do # confirm the client can receive the server response and status. client_ops = { CallOps::SEND_CLOSE_FROM_CLIENT => nil, + CallOps::RECV_INITIAL_METADATA => nil, CallOps::RECV_MESSAGE => nil, CallOps::RECV_STATUS_ON_CLIENT => nil } diff --git a/templates/binding.gyp.template b/templates/binding.gyp.template index 7cee562caa..3ffbc06643 100644 --- a/templates/binding.gyp.template +++ b/templates/binding.gyp.template @@ -57,7 +57,8 @@ 'UNICODE', '_UNICODE', 'NOMINMAX', - 'OPENSSL_NO_ASM' + 'OPENSSL_NO_ASM', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], "msvs_settings": { 'VCCLCompilerTool': { @@ -80,7 +81,8 @@ # supports ALPN. The target is "[major].[minor].[patch]". We split by # periods and take the first field to get the major version. 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)' + 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], 'include_dirs': [ '<(node_root_dir)/deps/openssl/openssl/include', diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index 6cc3e964a3..701e1c7485 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -17,7 +17,7 @@ s.required_ruby_version = '>= 2.0.0' - s.files = %w( Makefile ) + s.files = %w( Makefile .yardopts ) s.files += %w( etc/roots.pem ) s.files += Dir.glob('src/ruby/bin/**/*') s.files += Dir.glob('src/ruby/ext/**/*') @@ -33,7 +33,7 @@ s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.2' + s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' diff --git a/templates/package.json.template b/templates/package.json.template index d6279b996e..99e8287b35 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -25,13 +25,12 @@ "test": "./node_modules/.bin/mocha src/node/test && npm run-script lint", "gen_docs": "./node_modules/.bin/jsdoc -c src/node/jsdoc_conf.json", "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", - "preinstall": "npm install node-pre-gyp", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, + "bundledDependencies": ["node-pre-gyp"], "dependencies": { "lodash": "^3.9.3", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.19", "protobufjs": "^4.0.0" }, "devDependencies": { diff --git a/templates/src/python/grpcio/grpc/_cython/imports.generated.c.template b/templates/src/python/grpcio/grpc/_cython/imports.generated.c.template index be33280c0c..62fe0947d2 100644 --- a/templates/src/python/grpcio/grpc/_cython/imports.generated.c.template +++ b/templates/src/python/grpcio/grpc/_cython/imports.generated.c.template @@ -43,10 +43,19 @@ ${api.name}_type ${api.name}_import; %endfor + #ifdef __cplusplus + extern "C" { + #endif /* __cpluslus */ + void pygrpc_load_imports(HMODULE library) { %for api in c_apis: ${api.name}_import = (${api.name}_type) GetProcAddress(library, "${api.name}"); %endfor } + #ifdef __cplusplus + } + #endif /* __cpluslus */ + #endif /* !GPR_WIN32 */ + diff --git a/templates/src/python/grpcio/grpc/_cython/imports.generated.h.template b/templates/src/python/grpcio/grpc/_cython/imports.generated.h.template index 6866a61cae..8e7c183180 100644 --- a/templates/src/python/grpcio/grpc/_cython/imports.generated.h.template +++ b/templates/src/python/grpcio/grpc/_cython/imports.generated.h.template @@ -52,8 +52,16 @@ #define ${api.name} ${api.name}_import %endfor + #ifdef __cplusplus + extern "C" { + #endif /* __cpluslus */ + void pygrpc_load_imports(HMODULE library); + #ifdef __cplusplus + } + #endif /* __cpluslus */ + #else /* !GPR_WIN32 */ #include <grpc/support/alloc.h> diff --git a/test/core/census/log_test.c b/test/core/census/log_test.c new file mode 100644 index 0000000000..b68ca11504 --- /dev/null +++ b/test/core/census/log_test.c @@ -0,0 +1,589 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/census/log.h" +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/port_platform.h> +#include <grpc/support/sync.h> +#include <grpc/support/thd.h> +#include <grpc/support/time.h> +#include <grpc/support/useful.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "test/core/util/test_config.h" + +// Change this to non-zero if you want more output. +#define VERBOSE 0 + +// Log size to use for all tests. +#define LOG_SIZE_IN_MB 1 +#define LOG_SIZE_IN_BYTES (LOG_SIZE_IN_MB << 20) + +// Fills in 'record' of size 'size'. Each byte in record is filled in with the +// same value. The value is extracted from 'record' pointer. +static void write_record(char* record, size_t size) { + char data = (char)((uintptr_t)record % 255); + memset(record, data, size); +} + +// Reads fixed size records. Returns the number of records read in +// 'num_records'. +static void read_records(size_t record_size, const char* buffer, + size_t buffer_size, int* num_records) { + GPR_ASSERT(buffer_size >= record_size); + GPR_ASSERT(buffer_size % record_size == 0); + *num_records = (int)(buffer_size / record_size); + for (int i = 0; i < *num_records; ++i) { + const char* record = buffer + (record_size * (size_t)i); + char data = (char)((uintptr_t)record % 255); + for (size_t j = 0; j < record_size; ++j) { + GPR_ASSERT(data == record[j]); + } + } +} + +// Tries to write the specified number of records. Stops when the log gets +// full. Returns the number of records written. Spins for random +// number of times, up to 'max_spin_count', between writes. +static int write_records_to_log(int writer_id, size_t record_size, + int num_records, int max_spin_count) { + int counter = 0; + for (int i = 0; i < num_records; ++i) { + int spin_count = max_spin_count ? rand() % max_spin_count : 0; + if (VERBOSE && (counter++ == num_records / 10)) { + printf(" Writer %d: %d out of %d written\n", writer_id, i, num_records); + counter = 0; + } + char* record = (char*)(census_log_start_write(record_size)); + if (record == NULL) { + return i; + } + write_record(record, record_size); + census_log_end_write(record, record_size); + for (int j = 0; j < spin_count; ++j) { + GPR_ASSERT(j >= 0); + } + } + return num_records; +} + +// Performs a single read iteration. Returns the number of records read. +static int perform_read_iteration(size_t record_size) { + const void* read_buffer = NULL; + size_t bytes_available; + int records_read = 0; + census_log_init_reader(); + while ((read_buffer = census_log_read_next(&bytes_available))) { + int num_records = 0; + read_records(record_size, (const char*)read_buffer, bytes_available, + &num_records); + records_read += num_records; + } + return records_read; +} + +// Asserts that the log is empty. +static void assert_log_empty(void) { + census_log_init_reader(); + size_t bytes_available; + GPR_ASSERT(census_log_read_next(&bytes_available) == NULL); +} + +// Fills the log and verifies data. If 'no fragmentation' is true, records +// are sized such that CENSUS_LOG_2_MAX_RECORD_SIZE is a multiple of record +// size. If not a circular log, verifies that the number of records written +// match the number of records read. +static void fill_log(size_t log_size, int no_fragmentation, int circular_log) { + size_t size; + if (no_fragmentation) { + int log2size = rand() % (CENSUS_LOG_2_MAX_RECORD_SIZE + 1); + size = ((size_t)1 << log2size); + } else { + while (1) { + size = 1 + ((size_t)rand() % CENSUS_LOG_MAX_RECORD_SIZE); + if (CENSUS_LOG_MAX_RECORD_SIZE % size) { + break; + } + } + } + int records_written = + write_records_to_log(0 /* writer id */, size, + (int)((log_size / size) * 2), 0 /* spin count */); + int records_read = perform_read_iteration(size); + if (!circular_log) { + GPR_ASSERT(records_written == records_read); + } + assert_log_empty(); +} + +// Structure to pass args to writer_thread +typedef struct writer_thread_args { + // Index of this thread in the writers vector. + int index; + // Record size. + size_t record_size; + // Number of records to write. + int num_records; + // Used to signal when writer is complete + gpr_cv* done; + gpr_mu* mu; + int* count; +} writer_thread_args; + +// Writes the given number of records of random size (up to kMaxRecordSize) and +// random data to the specified log. +static void writer_thread(void* arg) { + writer_thread_args* args = (writer_thread_args*)arg; + // Maximum number of times to spin between writes. + static const int MAX_SPIN_COUNT = 50; + int records_written = 0; + if (VERBOSE) { + printf(" Writer %d starting\n", args->index); + } + while (records_written < args->num_records) { + records_written += write_records_to_log(args->index, args->record_size, + args->num_records - records_written, + MAX_SPIN_COUNT); + if (records_written < args->num_records) { + // Ran out of log space. Sleep for a bit and let the reader catch up. + // This should never happen for circular logs. + if (VERBOSE) { + printf( + " Writer %d stalled due to out-of-space: %d out of %d " + "written\n", + args->index, records_written, args->num_records); + } + gpr_sleep_until(GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); + } + } + // Done. Decrement count and signal. + gpr_mu_lock(args->mu); + (*args->count)--; + gpr_cv_signal(args->done); + if (VERBOSE) { + printf(" Writer %d done\n", args->index); + } + gpr_mu_unlock(args->mu); +} + +// struct to pass args to reader_thread +typedef struct reader_thread_args { + // Record size. + size_t record_size; + // Interval between read iterations. + int read_iteration_interval_in_msec; + // Total number of records. + int total_records; + // Signalled when reader should stop. + gpr_cv stop; + int stop_flag; + // Used to signal when reader has finished + gpr_cv* done; + gpr_mu* mu; + int running; +} reader_thread_args; + +// Reads and verifies the specified number of records. Reader can also be +// stopped via gpr_cv_signal(&args->stop). Sleeps for 'read_interval_in_msec' +// between read iterations. +static void reader_thread(void* arg) { + reader_thread_args* args = (reader_thread_args*)arg; + if (VERBOSE) { + printf(" Reader starting\n"); + } + gpr_timespec interval = gpr_time_from_micros( + args->read_iteration_interval_in_msec * 1000, GPR_TIMESPAN); + gpr_mu_lock(args->mu); + int records_read = 0; + int num_iterations = 0; + int counter = 0; + while (!args->stop_flag && records_read < args->total_records) { + gpr_cv_wait(&args->stop, args->mu, interval); + if (!args->stop_flag) { + records_read += perform_read_iteration(args->record_size); + GPR_ASSERT(records_read <= args->total_records); + if (VERBOSE && (counter++ == 100000)) { + printf(" Reader: %d out of %d read\n", records_read, + args->total_records); + counter = 0; + } + ++num_iterations; + } + } + // Done + args->running = 0; + gpr_cv_signal(args->done); + if (VERBOSE) { + printf(" Reader: records: %d, iterations: %d\n", records_read, + num_iterations); + } + gpr_mu_unlock(args->mu); +} + +// Creates NUM_WRITERS writers where each writer writes NUM_RECORDS_PER_WRITER +// records. Also, starts a reader that iterates over and reads blocks every +// READ_ITERATION_INTERVAL_IN_MSEC. +// Number of writers. +#define NUM_WRITERS 5 +static void multiple_writers_single_reader(int circular_log) { + // Sleep interval between read iterations. + static const int READ_ITERATION_INTERVAL_IN_MSEC = 10; + // Maximum record size. + static const size_t MAX_RECORD_SIZE = 20; + // Number of records written by each writer. This is sized such that we + // will write through the entire log ~10 times. + const int NUM_RECORDS_PER_WRITER = + (int)((10 * census_log_remaining_space()) / (MAX_RECORD_SIZE / 2)) / + NUM_WRITERS; + size_t record_size = ((size_t)rand() % MAX_RECORD_SIZE) + 1; + // Create and start writers. + writer_thread_args writers[NUM_WRITERS]; + int writers_count = NUM_WRITERS; + gpr_cv writers_done; + gpr_mu writers_mu; // protects writers_done and writers_count + gpr_cv_init(&writers_done); + gpr_mu_init(&writers_mu); + gpr_thd_id id; + for (int i = 0; i < NUM_WRITERS; ++i) { + writers[i].index = i; + writers[i].record_size = record_size; + writers[i].num_records = NUM_RECORDS_PER_WRITER; + writers[i].done = &writers_done; + writers[i].count = &writers_count; + writers[i].mu = &writers_mu; + gpr_thd_new(&id, &writer_thread, &writers[i], NULL); + } + // Start reader. + gpr_cv reader_done; + gpr_mu reader_mu; // protects reader_done and reader.running + reader_thread_args reader; + reader.record_size = record_size; + reader.read_iteration_interval_in_msec = READ_ITERATION_INTERVAL_IN_MSEC; + reader.total_records = NUM_WRITERS * NUM_RECORDS_PER_WRITER; + reader.stop_flag = 0; + gpr_cv_init(&reader.stop); + gpr_cv_init(&reader_done); + reader.done = &reader_done; + gpr_mu_init(&reader_mu); + reader.mu = &reader_mu; + reader.running = 1; + gpr_thd_new(&id, &reader_thread, &reader, NULL); + // Wait for writers to finish. + gpr_mu_lock(&writers_mu); + while (writers_count != 0) { + gpr_cv_wait(&writers_done, &writers_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + gpr_mu_unlock(&writers_mu); + gpr_mu_destroy(&writers_mu); + gpr_cv_destroy(&writers_done); + gpr_mu_lock(&reader_mu); + if (circular_log) { + // Stop reader. + reader.stop_flag = 1; + gpr_cv_signal(&reader.stop); + } + // wait for reader to finish + while (reader.running) { + gpr_cv_wait(&reader_done, &reader_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + if (circular_log) { + // Assert that there were no out-of-space errors. + GPR_ASSERT(0 == census_log_out_of_space_count()); + } + gpr_mu_unlock(&reader_mu); + gpr_mu_destroy(&reader_mu); + gpr_cv_destroy(&reader_done); + if (VERBOSE) { + printf(" Reader: finished\n"); + } +} + +static void setup_test(int circular_log) { + census_log_initialize(LOG_SIZE_IN_MB, circular_log); + GPR_ASSERT(census_log_remaining_space() == LOG_SIZE_IN_BYTES); +} + +// Attempts to create a record of invalid size (size > +// CENSUS_LOG_MAX_RECORD_SIZE). +void test_invalid_record_size(void) { + static const size_t INVALID_SIZE = CENSUS_LOG_MAX_RECORD_SIZE + 1; + static const size_t VALID_SIZE = 1; + printf("Starting test: invalid record size\n"); + setup_test(0); + void* record = census_log_start_write(INVALID_SIZE); + GPR_ASSERT(record == NULL); + // Now try writing a valid record. + record = census_log_start_write(VALID_SIZE); + GPR_ASSERT(record != NULL); + census_log_end_write(record, VALID_SIZE); + // Verifies that available space went down by one block. In theory, this + // check can fail if the thread is context switched to a new CPU during the + // start_write execution (multiple blocks get allocated), but this has not + // been observed in practice. + GPR_ASSERT(LOG_SIZE_IN_BYTES - CENSUS_LOG_MAX_RECORD_SIZE == + census_log_remaining_space()); + census_log_shutdown(); +} + +// Tests end_write() with a different size than what was specified in +// start_write(). +void test_end_write_with_different_size(void) { + static const size_t START_WRITE_SIZE = 10; + static const size_t END_WRITE_SIZE = 7; + printf("Starting test: end write with different size\n"); + setup_test(0); + void* record_written = census_log_start_write(START_WRITE_SIZE); + GPR_ASSERT(record_written != NULL); + census_log_end_write(record_written, END_WRITE_SIZE); + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_written == record_read); + GPR_ASSERT(END_WRITE_SIZE == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +// Verifies that pending records are not available via read_next(). +void test_read_pending_record(void) { + static const size_t PR_RECORD_SIZE = 1024; + printf("Starting test: read pending record\n"); + setup_test(0); + // Start a write. + void* record_written = census_log_start_write(PR_RECORD_SIZE); + GPR_ASSERT(record_written != NULL); + // As write is pending, read should fail. + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read == NULL); + // A read followed by end_write() should succeed. + census_log_end_write(record_written, PR_RECORD_SIZE); + census_log_init_reader(); + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_written == record_read); + GPR_ASSERT(PR_RECORD_SIZE == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +// Tries reading beyond pending write. +void test_read_beyond_pending_record(void) { + printf("Starting test: read beyond pending record\n"); + setup_test(0); + // Start a write. + const size_t incomplete_record_size = 10; + void* incomplete_record = census_log_start_write(incomplete_record_size); + GPR_ASSERT(incomplete_record != NULL); + const size_t complete_record_size = 20; + void* complete_record = census_log_start_write(complete_record_size); + GPR_ASSERT(complete_record != NULL); + GPR_ASSERT(complete_record != incomplete_record); + census_log_end_write(complete_record, complete_record_size); + // Now iterate over blocks to read completed records. + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(complete_record == record_read); + GPR_ASSERT(complete_record_size == bytes_available); + // Complete first record. + census_log_end_write(incomplete_record, incomplete_record_size); + // Have read past the incomplete record, so read_next() should return NULL. + // NB: this test also assumes our thread did not get switched to a different + // CPU between the two start_write calls + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read == NULL); + // Reset reader to get the newly completed record. + census_log_init_reader(); + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(incomplete_record == record_read); + GPR_ASSERT(incomplete_record_size == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +// Tests scenario where block being read is detached from a core and put on the +// dirty list. +void test_detached_while_reading(void) { + printf("Starting test: detached while reading\n"); + setup_test(0); + // Start a write. + static const size_t DWR_RECORD_SIZE = 10; + void* record_written = census_log_start_write(DWR_RECORD_SIZE); + GPR_ASSERT(record_written != NULL); + census_log_end_write(record_written, DWR_RECORD_SIZE); + // Read this record. + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read != NULL); + GPR_ASSERT(DWR_RECORD_SIZE == bytes_available); + // Now fill the log. This will move the block being read from core-local + // array to the dirty list. + while ((record_written = census_log_start_write(DWR_RECORD_SIZE))) { + census_log_end_write(record_written, DWR_RECORD_SIZE); + } + + // In this iteration, read_next() should only traverse blocks in the + // core-local array. Therefore, we expect at most gpr_cpu_num_cores() more + // blocks. As log is full, if read_next() is traversing the dirty list, we + // will get more than gpr_cpu_num_cores() blocks. + int block_read = 0; + while ((record_read = census_log_read_next(&bytes_available))) { + ++block_read; + GPR_ASSERT(block_read <= (int)gpr_cpu_num_cores()); + } + census_log_shutdown(); +} + +// Fills non-circular log with records sized such that size is a multiple of +// CENSUS_LOG_MAX_RECORD_SIZE (no per-block fragmentation). +void test_fill_log_no_fragmentation(void) { + printf("Starting test: fill log no fragmentation\n"); + const int circular = 0; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 1 /* no fragmentation */, circular); + census_log_shutdown(); +} + +// Fills circular log with records sized such that size is a multiple of +// CENSUS_LOG_MAX_RECORD_SIZE (no per-block fragmentation). +void test_fill_circular_log_no_fragmentation(void) { + printf("Starting test: fill circular log no fragmentation\n"); + const int circular = 1; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 1 /* no fragmentation */, circular); + census_log_shutdown(); +} + +// Fills non-circular log with records that may straddle end of a block. +void test_fill_log_with_straddling_records(void) { + printf("Starting test: fill log with straddling records\n"); + const int circular = 0; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 0 /* block straddling records */, circular); + census_log_shutdown(); +} + +// Fills circular log with records that may straddle end of a block. +void test_fill_circular_log_with_straddling_records(void) { + printf("Starting test: fill circular log with straddling records\n"); + const int circular = 1; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 0 /* block straddling records */, circular); + census_log_shutdown(); +} + +// Tests scenario where multiple writers and a single reader are using a log +// that is configured to discard old records. +void test_multiple_writers_circular_log(void) { + printf("Starting test: multiple writers circular log\n"); + const int circular = 1; + setup_test(circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +// Tests scenario where multiple writers and a single reader are using a log +// that is configured to discard old records. +void test_multiple_writers(void) { + printf("Starting test: multiple writers\n"); + const int circular = 0; + setup_test(circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +// Repeat the straddling records and multiple writers tests with a small log. +void test_small_log(void) { + printf("Starting test: small log\n"); + const int circular = 0; + census_log_initialize(0, circular); + size_t log_size = census_log_remaining_space(); + GPR_ASSERT(log_size > 0); + fill_log(log_size, 0, circular); + census_log_shutdown(); + census_log_initialize(0, circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +void test_performance(void) { + for (size_t write_size = 1; write_size < CENSUS_LOG_MAX_RECORD_SIZE; + write_size *= 2) { + setup_test(0); + gpr_timespec start_time = gpr_now(GPR_CLOCK_REALTIME); + int nrecords = 0; + while (1) { + void* record = census_log_start_write(write_size); + if (record == NULL) { + break; + } + census_log_end_write(record, write_size); + nrecords++; + } + gpr_timespec write_time = + gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start_time); + double write_time_micro = + (double)write_time.tv_sec * 1000000 + (double)write_time.tv_nsec / 1000; + census_log_shutdown(); + printf( + "Wrote %d %d byte records in %.3g microseconds: %g records/us " + "(%g ns/record), %g gigabytes/s\n", + nrecords, (int)write_size, write_time_micro, + nrecords / write_time_micro, 1000 * write_time_micro / nrecords, + (double)((int)write_size * nrecords) / write_time_micro / 1000); + } +} + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + gpr_time_init(); + srand((unsigned)gpr_now(GPR_CLOCK_REALTIME).tv_nsec); + test_invalid_record_size(); + test_end_write_with_different_size(); + test_read_pending_record(); + test_read_beyond_pending_record(); + test_detached_while_reading(); + test_fill_log_no_fragmentation(); + test_fill_circular_log_no_fragmentation(); + test_fill_log_with_straddling_records(); + test_fill_circular_log_with_straddling_records(); + test_small_log(); + test_multiple_writers(); + test_multiple_writers_circular_log(); + test_performance(); + return 0; +} diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c index 5ab64f9800..dbdd3524ed 100644 --- a/test/core/end2end/fixtures/h2_uchannel.c +++ b/test/core/end2end/fixtures/h2_uchannel.c @@ -159,7 +159,7 @@ static grpc_subchannel *subchannel_factory_create_subchannel( c->base.vtable = &connector_vtable; gpr_ref_init(&c->refs, 1); args->args = final_args; - s = grpc_subchannel_create(&c->base, args); + s = grpc_subchannel_create(exec_ctx, &c->base, args); grpc_connector_unref(exec_ctx, &c->base); grpc_channel_args_destroy(final_args); if (*f->sniffed_subchannel) { diff --git a/test/core/fling/client.c b/test/core/fling/client.c index 02db681cfd..b36aef3093 100644 --- a/test/core/fling/client.c +++ b/test/core/fling/client.c @@ -51,7 +51,7 @@ static grpc_channel *channel; static grpc_completion_queue *cq; static grpc_call *call; static grpc_op ops[6]; -static grpc_op stream_init_op; +static grpc_op stream_init_ops[2]; static grpc_op stream_step_ops[2]; static grpc_metadata_array initial_metadata_recv; static grpc_metadata_array trailing_metadata_recv; @@ -105,13 +105,17 @@ static void step_ping_pong_request(void) { } static void init_ping_pong_stream(void) { + grpc_metadata_array_init(&initial_metadata_recv); + grpc_call_error error; call = grpc_channel_create_call(channel, NULL, GRPC_PROPAGATE_DEFAULTS, cq, "/Reflector/reflectStream", "localhost", gpr_inf_future(GPR_CLOCK_REALTIME), NULL); - stream_init_op.op = GRPC_OP_SEND_INITIAL_METADATA; - stream_init_op.data.send_initial_metadata.count = 0; - error = grpc_call_start_batch(call, &stream_init_op, 1, (void *)1, NULL); + stream_init_ops[0].op = GRPC_OP_SEND_INITIAL_METADATA; + stream_init_ops[0].data.send_initial_metadata.count = 0; + stream_init_ops[1].op = GRPC_OP_RECV_INITIAL_METADATA; + stream_init_ops[1].data.recv_initial_metadata = &initial_metadata_recv; + error = grpc_call_start_batch(call, stream_init_ops, 2, (void *)1, NULL); GPR_ASSERT(GRPC_CALL_OK == error); grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME), NULL); diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 85e28732e4..2e253d8a8a 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -173,7 +173,7 @@ static void test_receive(int number_of_clients) { grpc_exec_ctx_finish(&exec_ctx); } -static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) { +static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) { grpc_pollset_destroy(p); } diff --git a/test/cpp/common/channel_arguments_test.cc b/test/cpp/common/channel_arguments_test.cc index e010d375cf..a4821b4d0b 100644 --- a/test/cpp/common/channel_arguments_test.cc +++ b/test/cpp/common/channel_arguments_test.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,94 +41,141 @@ namespace testing { class ChannelArgumentsTest : public ::testing::Test { protected: + ChannelArgumentsTest() + : pointer_vtable_({&ChannelArguments::PointerVtableMembers::Copy, + &ChannelArguments::PointerVtableMembers::Destroy, + &ChannelArguments::PointerVtableMembers::Compare}) {} + void SetChannelArgs(const ChannelArguments& channel_args, grpc_channel_args* args) { channel_args.SetChannelArgs(args); } + + grpc::string GetDefaultUserAgentPrefix() { + std::ostringstream user_agent_prefix; + user_agent_prefix << "grpc-c++/" << grpc_version_string(); + return user_agent_prefix.str(); + } + + void VerifyDefaultChannelArgs() { + grpc_channel_args args; + SetChannelArgs(channel_args_, &args); + EXPECT_EQ(static_cast<size_t>(1), args.num_args); + EXPECT_STREQ(GRPC_ARG_PRIMARY_USER_AGENT_STRING, args.args[0].key); + EXPECT_EQ(GetDefaultUserAgentPrefix(), + grpc::string(args.args[0].value.string)); + } + + bool HasArg(grpc_arg expected_arg) { + grpc_channel_args args; + SetChannelArgs(channel_args_, &args); + for (size_t i = 0; i < args.num_args; i++) { + const grpc_arg& arg = args.args[i]; + if (arg.type == expected_arg.type && + grpc::string(arg.key) == expected_arg.key) { + if (arg.type == GRPC_ARG_INTEGER) { + return arg.value.integer == expected_arg.value.integer; + } else if (arg.type == GRPC_ARG_STRING) { + return grpc::string(arg.value.string) == expected_arg.value.string; + } else if (arg.type == GRPC_ARG_POINTER) { + return arg.value.pointer.p == expected_arg.value.pointer.p && + arg.value.pointer.vtable->copy == + expected_arg.value.pointer.vtable->copy && + arg.value.pointer.vtable->destroy == + expected_arg.value.pointer.vtable->destroy; + } + } + } + return false; + } + grpc_arg_pointer_vtable pointer_vtable_; + ChannelArguments channel_args_; }; TEST_F(ChannelArgumentsTest, SetInt) { - grpc_channel_args args; - ChannelArguments channel_args; - // Empty arguments. - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(0), args.num_args); - - grpc::string key("key0"); - channel_args.SetInt(key, 0); + VerifyDefaultChannelArgs(); + grpc::string key0("key0"); + grpc_arg arg0; + arg0.type = GRPC_ARG_INTEGER; + arg0.key = const_cast<char*>(key0.c_str()); + arg0.value.integer = 0; + grpc::string key1("key1"); + grpc_arg arg1; + arg1.type = GRPC_ARG_INTEGER; + arg1.key = const_cast<char*>(key1.c_str()); + arg1.value.integer = 1; + + grpc::string arg_key0(key0); + channel_args_.SetInt(arg_key0, arg0.value.integer); // Clear key early to make sure channel_args takes a copy - key = ""; - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(1), args.num_args); - EXPECT_EQ(GRPC_ARG_INTEGER, args.args[0].type); - EXPECT_STREQ("key0", args.args[0].key); - EXPECT_EQ(0, args.args[0].value.integer); - - key = "key1"; - channel_args.SetInt(key, 1); - key = ""; - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(2), args.num_args); - // We do not enforce order on the arguments. - for (size_t i = 0; i < args.num_args; i++) { - EXPECT_EQ(GRPC_ARG_INTEGER, args.args[i].type); - if (grpc::string(args.args[i].key) == "key0") { - EXPECT_EQ(0, args.args[i].value.integer); - } else if (grpc::string(args.args[i].key) == "key1") { - EXPECT_EQ(1, args.args[i].value.integer); - } - } + arg_key0.clear(); + EXPECT_TRUE(HasArg(arg0)); + + grpc::string arg_key1(key1); + channel_args_.SetInt(arg_key1, arg1.value.integer); + arg_key1.clear(); + EXPECT_TRUE(HasArg(arg0)); + EXPECT_TRUE(HasArg(arg1)); } TEST_F(ChannelArgumentsTest, SetString) { - grpc_channel_args args; - ChannelArguments channel_args; - // Empty arguments. - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(0), args.num_args); - - grpc::string key("key0"); - grpc::string val("val0"); - channel_args.SetString(key, val); + VerifyDefaultChannelArgs(); + grpc::string key0("key0"); + grpc::string val0("val0"); + grpc_arg arg0; + arg0.type = GRPC_ARG_STRING; + arg0.key = const_cast<char*>(key0.c_str()); + arg0.value.string = const_cast<char*>(val0.c_str()); + grpc::string key1("key1"); + grpc::string val1("val1"); + grpc_arg arg1; + arg1.type = GRPC_ARG_STRING; + arg1.key = const_cast<char*>(key1.c_str()); + arg1.value.string = const_cast<char*>(val1.c_str()); + + grpc::string key(key0); + grpc::string val(val0); + channel_args_.SetString(key, val); // Clear key/val early to make sure channel_args takes a copy key = ""; val = ""; - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(1), args.num_args); - EXPECT_EQ(GRPC_ARG_STRING, args.args[0].type); - EXPECT_STREQ("key0", args.args[0].key); - EXPECT_STREQ("val0", args.args[0].value.string); - - key = "key1"; - val = "val1"; - channel_args.SetString(key, val); - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(2), args.num_args); - // We do not enforce order on the arguments. - for (size_t i = 0; i < args.num_args; i++) { - EXPECT_EQ(GRPC_ARG_STRING, args.args[i].type); - if (grpc::string(args.args[i].key) == "key0") { - EXPECT_STREQ("val0", args.args[i].value.string); - } else if (grpc::string(args.args[i].key) == "key1") { - EXPECT_STREQ("val1", args.args[i].value.string); - } - } + EXPECT_TRUE(HasArg(arg0)); + + key = key1; + val = val1; + channel_args_.SetString(key, val); + // Clear key/val early to make sure channel_args takes a copy + key = ""; + val = ""; + EXPECT_TRUE(HasArg(arg0)); + EXPECT_TRUE(HasArg(arg1)); } TEST_F(ChannelArgumentsTest, SetPointer) { - grpc_channel_args args; - ChannelArguments channel_args; - // Empty arguments. - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(0), args.num_args); - - grpc::string key("key0"); - channel_args.SetPointer(key, &key); - SetChannelArgs(channel_args, &args); - EXPECT_EQ(static_cast<size_t>(1), args.num_args); - EXPECT_EQ(GRPC_ARG_POINTER, args.args[0].type); - EXPECT_STREQ("key0", args.args[0].key); - EXPECT_EQ(&key, args.args[0].value.pointer.p); + VerifyDefaultChannelArgs(); + grpc::string key0("key0"); + grpc_arg arg0; + arg0.type = GRPC_ARG_POINTER; + arg0.key = const_cast<char*>(key0.c_str()); + arg0.value.pointer.p = &key0; + arg0.value.pointer.vtable = &pointer_vtable_; + + grpc::string key(key0); + channel_args_.SetPointer(key, arg0.value.pointer.p); + EXPECT_TRUE(HasArg(arg0)); +} + +TEST_F(ChannelArgumentsTest, SetUserAgentPrefix) { + VerifyDefaultChannelArgs(); + grpc::string prefix("prefix"); + grpc::string whole_prefix = prefix + " " + GetDefaultUserAgentPrefix(); + grpc_arg arg0; + arg0.type = GRPC_ARG_STRING; + arg0.key = const_cast<char*>(GRPC_ARG_PRIMARY_USER_AGENT_STRING); + arg0.value.string = const_cast<char*>(whole_prefix.c_str()); + + channel_args_.SetUserAgentPrefix(prefix); + EXPECT_TRUE(HasArg(arg0)); } } // namespace testing diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 252bda3798..a194c615cd 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -479,8 +479,10 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { send_request.set_message("Hello"); std::pair<grpc::string, grpc::string> meta1("key1", "val1"); std::pair<grpc::string, grpc::string> meta2("key2", "val2"); + std::pair<grpc::string, grpc::string> meta3("g.r.d-bin", "xyz"); cli_ctx.AddMetadata(meta1.first, meta1.second); cli_ctx.AddMetadata(meta2.first, meta2.second); + cli_ctx.AddMetadata(meta3.first, meta3.second); std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -494,6 +496,8 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { ToString(client_initial_metadata.find(meta1.first)->second)); EXPECT_EQ(meta2.second, ToString(client_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(meta3.second, + ToString(client_initial_metadata.find(meta3.first)->second)); EXPECT_GE(client_initial_metadata.size(), static_cast<size_t>(2)); send_response.set_message(recv_request.message()); diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 65da71b391..c8523847ab 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -252,6 +252,9 @@ class End2endTest : public ::testing::TestWithParam<TestScenario> { args.SetSslTargetNameOverride("foo.test.google.fr"); channel_creds = SslCredentials(ssl_opts); } + if (!user_agent_prefix_.empty()) { + args.SetUserAgentPrefix(user_agent_prefix_); + } args.SetString(GRPC_ARG_SECONDARY_USER_AGENT_STRING, "end2end_test"); channel_ = CreateCustomChannel(server_address_.str(), channel_creds, args); } @@ -285,6 +288,7 @@ class End2endTest : public ::testing::TestWithParam<TestScenario> { TestServiceImpl service_; TestServiceImpl special_service_; TestServiceImplDupPkg dup_pkg_service_; + grpc::string user_agent_prefix_; }; static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs, @@ -601,6 +605,25 @@ TEST_P(End2endServerTryCancelTest, BidiStreamServerCancelAfter) { TestBidiStreamServerCancel(CANCEL_AFTER_PROCESSING, 5); } +TEST_P(End2endTest, SimpleRpcWithCustomeUserAgentPrefix) { + user_agent_prefix_ = "custom_prefix"; + ResetStub(); + EchoRequest request; + EchoResponse response; + request.set_message("Hello hello hello hello"); + request.mutable_param()->set_echo_metadata(true); + + ClientContext context; + Status s = stub_->Echo(&context, request, &response); + EXPECT_EQ(response.message(), request.message()); + EXPECT_TRUE(s.ok()); + const auto& trailing_metadata = context.GetServerTrailingMetadata(); + auto iter = trailing_metadata.find("user-agent"); + EXPECT_TRUE(iter != trailing_metadata.end()); + grpc::string expected_prefix = user_agent_prefix_ + " grpc-c++/"; + EXPECT_TRUE(iter->second.starts_with(expected_prefix)); +} + TEST_P(End2endTest, MultipleRpcsWithVariedBinaryMetadataValue) { ResetStub(); std::vector<std::thread*> threads; diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 50b2bf2514..c94a523fa1 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -41,6 +41,7 @@ #include <grpc++/support/byte_buffer.h> #include <grpc++/support/slice.h> #include <grpc/support/log.h> +#include <grpc/support/time.h> #include "src/proto/grpc/testing/payloads.grpc.pb.h" #include "src/proto/grpc/testing/services.grpc.pb.h" @@ -52,27 +53,8 @@ #include "test/cpp/util/create_test_channel.h" namespace grpc { - -#if defined(__APPLE__) -// Specialize Timepoint for high res clock as we need that -template <> -class TimePoint<std::chrono::high_resolution_clock::time_point> { - public: - TimePoint(const std::chrono::high_resolution_clock::time_point& time) { - TimepointHR2Timespec(time, &time_); - } - gpr_timespec raw_time() const { return time_; } - - private: - gpr_timespec time_; -}; -#endif - namespace testing { -typedef std::chrono::high_resolution_clock grpc_time_source; -typedef std::chrono::time_point<grpc_time_source> grpc_time; - template <class RequestType> class ClientRequestCreator { public: @@ -184,7 +166,7 @@ class Client { // Set up the load distribution based on the number of threads const auto& load = config.load_params(); - std::unique_ptr<RandomDist> random_dist; + std::unique_ptr<RandomDistInterface> random_dist; switch (load.load_case()) { case LoadParams::kClosedLoop: // Closed-loop doesn't use random dist at all @@ -218,25 +200,26 @@ class Client { closed_loop_ = false; // set up interarrival timer according to random dist interarrival_timer_.init(*random_dist, num_threads); + const auto now = gpr_now(GPR_CLOCK_MONOTONIC); for (size_t i = 0; i < num_threads; i++) { - next_time_.push_back( - grpc_time_source::now() + - std::chrono::duration_cast<grpc_time_source::duration>( - interarrival_timer_(i))); + next_time_.push_back(gpr_time_add( + now, + gpr_time_from_nanos(interarrival_timer_.next(i), GPR_TIMESPAN))); } } } - bool NextIssueTime(int thread_idx, grpc_time* time_delay) { - if (closed_loop_) { - return false; - } else { - *time_delay = next_time_[thread_idx]; - next_time_[thread_idx] += - std::chrono::duration_cast<grpc_time_source::duration>( - interarrival_timer_(thread_idx)); - return true; - } + gpr_timespec NextIssueTime(int thread_idx) { + const gpr_timespec result = next_time_[thread_idx]; + next_time_[thread_idx] = + gpr_time_add(next_time_[thread_idx], + gpr_time_from_nanos(interarrival_timer_.next(thread_idx), + GPR_TIMESPAN)); + return result; + } + std::function<gpr_timespec()> NextIssuer(int thread_idx) { + return closed_loop_ ? std::function<gpr_timespec()>() + : std::bind(&Client::NextIssueTime, this, thread_idx); } private: @@ -306,7 +289,7 @@ class Client { Histogram* new_stats_; Histogram histogram_; Client* client_; - size_t idx_; + const size_t idx_; std::thread impl_; }; @@ -314,7 +297,7 @@ class Client { std::unique_ptr<Timer> timer_; InterarrivalTimer interarrival_timer_; - std::vector<grpc_time> next_time_; + std::vector<gpr_timespec> next_time_; }; template <class StubType, class RequestType> @@ -323,9 +306,9 @@ class ClientImpl : public Client { ClientImpl(const ClientConfig& config, std::function<std::unique_ptr<StubType>(std::shared_ptr<Channel>)> create_stub) - : channels_(config.client_channels()), create_stub_(create_stub) { - cores_ = LimitCores(config.core_list().data(), config.core_list_size()); - + : cores_(LimitCores(config.core_list().data(), config.core_list_size())), + channels_(config.client_channels()), + create_stub_(create_stub) { for (int i = 0; i < config.client_channels(); i++) { channels_[i].init(config.server_targets(i % config.server_targets_size()), config, create_stub_); @@ -337,7 +320,7 @@ class ClientImpl : public Client { virtual ~ClientImpl() {} protected: - int cores_; + const int cores_; RequestType request_; class ClientChannelInfo { diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index f3f8f37051..9e8767d103 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -43,9 +43,9 @@ #include <vector> #include <gflags/gflags.h> +#include <grpc++/alarm.h> #include <grpc++/channel.h> #include <grpc++/client_context.h> -#include <grpc++/client_context.h> #include <grpc++/generic/generic_stub.h> #include <grpc/grpc.h> #include <grpc/support/cpu.h> @@ -60,11 +60,9 @@ namespace grpc { namespace testing { -typedef std::list<grpc_time> deadline_list; - class ClientRpcContext { public: - explicit ClientRpcContext(int ch) : channel_id_(ch) {} + ClientRpcContext() {} virtual ~ClientRpcContext() {} // next state, return false if done. Collect stats when appropriate virtual bool RunNextState(bool, Histogram* hist) = 0; @@ -74,72 +72,73 @@ class ClientRpcContext { return reinterpret_cast<ClientRpcContext*>(t); } - deadline_list::iterator deadline_posn() const { return deadline_posn_; } - void set_deadline_posn(const deadline_list::iterator& it) { - deadline_posn_ = it; - } virtual void Start(CompletionQueue* cq) = 0; - int channel_id() const { return channel_id_; } - - protected: - int channel_id_; - - private: - deadline_list::iterator deadline_posn_; }; template <class RequestType, class ResponseType> class ClientRpcContextUnaryImpl : public ClientRpcContext { public: ClientRpcContextUnaryImpl( - int channel_id, BenchmarkService::Stub* stub, const RequestType& req, + BenchmarkService::Stub* stub, const RequestType& req, + std::function<gpr_timespec()> next_issue, std::function< std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> start_req, std::function<void(grpc::Status, ResponseType*)> on_done) - : ClientRpcContext(channel_id), - context_(), + : context_(), stub_(stub), + cq_(nullptr), req_(req), response_(), - next_state_(&ClientRpcContextUnaryImpl::RespDone), + next_state_(State::READY), callback_(on_done), + next_issue_(next_issue), start_req_(start_req) {} + ~ClientRpcContextUnaryImpl() GRPC_OVERRIDE {} void Start(CompletionQueue* cq) GRPC_OVERRIDE { - start_ = Timer::Now(); - response_reader_ = start_req_(stub_, &context_, req_, cq); - response_reader_->Finish(&response_, &status_, ClientRpcContext::tag(this)); + cq_ = cq; + if (!next_issue_) { // ready to issue + RunNextState(true, nullptr); + } else { // wait for the issue time + alarm_.reset(new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this))); + } } - ~ClientRpcContextUnaryImpl() GRPC_OVERRIDE {} bool RunNextState(bool ok, Histogram* hist) GRPC_OVERRIDE { - bool ret = (this->*next_state_)(ok); - if (!ret) { - hist->Add((Timer::Now() - start_) * 1e9); + switch (next_state_) { + case State::READY: + start_ = Timer::Now(); + response_reader_ = start_req_(stub_, &context_, req_, cq_); + response_reader_->Finish(&response_, &status_, + ClientRpcContext::tag(this)); + next_state_ = State::RESP_DONE; + return true; + case State::RESP_DONE: + hist->Add((Timer::Now() - start_) * 1e9); + callback_(status_, &response_); + next_state_ = State::INVALID; + return false; + default: + GPR_ASSERT(false); + return false; } - return ret; } - ClientRpcContext* StartNewClone() GRPC_OVERRIDE { - return new ClientRpcContextUnaryImpl(channel_id_, stub_, req_, start_req_, + return new ClientRpcContextUnaryImpl(stub_, req_, next_issue_, start_req_, callback_); } private: - bool RespDone(bool) { - next_state_ = &ClientRpcContextUnaryImpl::DoCallBack; - return false; - } - bool DoCallBack(bool) { - callback_(status_, &response_); - return true; // we're done, this'll be ignored - } grpc::ClientContext context_; BenchmarkService::Stub* stub_; + CompletionQueue* cq_; + std::unique_ptr<Alarm> alarm_; RequestType req_; ResponseType response_; - bool (ClientRpcContextUnaryImpl::*next_state_)(bool); + enum State { INVALID, READY, RESP_DONE }; + State next_state_; std::function<void(grpc::Status, ResponseType*)> callback_; + std::function<gpr_timespec()> next_issue_; std::function<std::unique_ptr<grpc::ClientAsyncResponseReader<ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> start_req_; @@ -157,49 +156,35 @@ class AsyncClient : public ClientImpl<StubType, RequestType> { // member name resolution until the template types are fully resolved public: using Client::SetupLoadTest; - using Client::NextIssueTime; using Client::closed_loop_; + using Client::NextIssuer; using ClientImpl<StubType, RequestType>::cores_; using ClientImpl<StubType, RequestType>::channels_; using ClientImpl<StubType, RequestType>::request_; AsyncClient(const ClientConfig& config, - std::function<ClientRpcContext*(int, StubType*, - const RequestType&)> setup_ctx, + std::function<ClientRpcContext*( + StubType*, std::function<gpr_timespec()> next_issue, + const RequestType&)> setup_ctx, std::function<std::unique_ptr<StubType>(std::shared_ptr<Channel>)> create_stub) : ClientImpl<StubType, RequestType>(config, create_stub), - num_async_threads_(NumThreads(config)), - channel_lock_(new std::mutex[config.client_channels()]), - contexts_(config.client_channels()), - max_outstanding_per_channel_(config.outstanding_rpcs_per_channel()), - channel_count_(config.client_channels()), - pref_channel_inc_(num_async_threads_) { + num_async_threads_(NumThreads(config)) { SetupLoadTest(config, num_async_threads_); for (int i = 0; i < num_async_threads_; i++) { cli_cqs_.emplace_back(new CompletionQueue); - if (!closed_loop_) { - rpc_deadlines_.emplace_back(); - next_channel_.push_back(i % channel_count_); - issue_allowed_.emplace_back(true); - - grpc_time next_issue; - NextIssueTime(i, &next_issue); - next_issue_.push_back(next_issue); - } + next_issuers_.emplace_back(NextIssuer(i)); } + using namespace std::placeholders; int t = 0; for (int i = 0; i < config.outstanding_rpcs_per_channel(); i++) { - for (int ch = 0; ch < channel_count_; ch++) { + for (int ch = 0; ch < config.client_channels(); ch++) { auto* cq = cli_cqs_[t].get(); + auto ctx = + setup_ctx(channels_[ch].get_stub(), next_issuers_[t], request_); + ctx->Start(cq); t = (t + 1) % cli_cqs_.size(); - auto ctx = setup_ctx(ch, channels_[ch].get_stub(), request_); - if (closed_loop_) { - ctx->Start(cq); - } else { - contexts_[ch].push_front(ctx); - } } } } @@ -212,140 +197,34 @@ class AsyncClient : public ClientImpl<StubType, RequestType> { delete ClientRpcContext::detag(got_tag); } } - // Now clear out all the pre-allocated idle contexts - for (int ch = 0; ch < channel_count_; ch++) { - while (!contexts_[ch].empty()) { - // Get an idle context from the front of the list - auto* ctx = *(contexts_[ch].begin()); - contexts_[ch].pop_front(); - delete ctx; - } - } - delete[] channel_lock_; } bool ThreadFunc(Histogram* histogram, size_t thread_idx) GRPC_OVERRIDE GRPC_FINAL { void* got_tag; bool ok; - grpc_time deadline, short_deadline; - if (closed_loop_) { - deadline = grpc_time_source::now() + std::chrono::seconds(1); - short_deadline = deadline; - } else { - if (rpc_deadlines_[thread_idx].empty()) { - deadline = grpc_time_source::now() + std::chrono::seconds(1); - } else { - deadline = *(rpc_deadlines_[thread_idx].begin()); - } - short_deadline = - issue_allowed_[thread_idx] ? next_issue_[thread_idx] : deadline; - } - - bool got_event; - switch (cli_cqs_[thread_idx]->AsyncNext(&got_tag, &ok, short_deadline)) { - case CompletionQueue::SHUTDOWN: - return false; - case CompletionQueue::TIMEOUT: - got_event = false; - break; - case CompletionQueue::GOT_EVENT: - got_event = true; - break; - default: - GPR_ASSERT(false); - break; - } - if (got_event) { + if (cli_cqs_[thread_idx]->Next(&got_tag, &ok)) { + // Got a regular event, so process it ClientRpcContext* ctx = ClientRpcContext::detag(got_tag); - if (ctx->RunNextState(ok, histogram) == false) { - // call the callback and then clone the ctx - ctx->RunNextState(ok, histogram); - ClientRpcContext* clone_ctx = ctx->StartNewClone(); - if (closed_loop_) { - clone_ctx->Start(cli_cqs_[thread_idx].get()); - } else { - // Remove the entry from the rpc deadlines list - rpc_deadlines_[thread_idx].erase(ctx->deadline_posn()); - // Put the clone_ctx in the list of idle contexts for this channel - // Under lock - int ch = clone_ctx->channel_id(); - std::lock_guard<std::mutex> g(channel_lock_[ch]); - contexts_[ch].push_front(clone_ctx); - } + if (!ctx->RunNextState(ok, histogram)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + auto clone = ctx->StartNewClone(); + clone->Start(cli_cqs_[thread_idx].get()); // delete the old version delete ctx; } - if (!closed_loop_) - issue_allowed_[thread_idx] = - true; // may be ok now even if it hadn't been + return true; + } else { // queue is shutting down + return false; } - if (!closed_loop_ && issue_allowed_[thread_idx] && - grpc_time_source::now() >= next_issue_[thread_idx]) { - // Attempt to issue - bool issued = false; - for (int num_attempts = 0, channel_attempt = next_channel_[thread_idx]; - num_attempts < channel_count_ && !issued; num_attempts++) { - bool can_issue = false; - ClientRpcContext* ctx = nullptr; - { - std::lock_guard<std::mutex> g(channel_lock_[channel_attempt]); - if (!contexts_[channel_attempt].empty()) { - // Get an idle context from the front of the list - ctx = *(contexts_[channel_attempt].begin()); - contexts_[channel_attempt].pop_front(); - can_issue = true; - } - } - if (can_issue) { - // do the work to issue - rpc_deadlines_[thread_idx].emplace_back(grpc_time_source::now() + - std::chrono::seconds(1)); - auto it = rpc_deadlines_[thread_idx].end(); - --it; - ctx->set_deadline_posn(it); - ctx->Start(cli_cqs_[thread_idx].get()); - issued = true; - // If we did issue, then next time, try our thread's next - // preferred channel - next_channel_[thread_idx] += pref_channel_inc_; - if (next_channel_[thread_idx] >= channel_count_) - next_channel_[thread_idx] = (thread_idx % channel_count_); - } else { - // Do a modular increment of channel attempt if we couldn't issue - channel_attempt = (channel_attempt + 1) % channel_count_; - } - } - if (issued) { - // We issued one; see when we can issue the next - grpc_time next_issue; - NextIssueTime(thread_idx, &next_issue); - next_issue_[thread_idx] = next_issue; - } else { - issue_allowed_[thread_idx] = false; - } - } - return true; } protected: - int num_async_threads_; + const int num_async_threads_; private: - class boolean { // exists only to avoid data-race on vector<bool> - public: - boolean() : val_(false) {} - boolean(bool b) : val_(b) {} - operator bool() const { return val_; } - boolean& operator=(bool b) { - val_ = b; - return *this; - } - - private: - bool val_; - }; int NumThreads(const ClientConfig& config) { int num_threads = config.async_client_threads(); if (num_threads <= 0) { // Use dynamic sizing @@ -356,18 +235,7 @@ class AsyncClient : public ClientImpl<StubType, RequestType> { } std::vector<std::unique_ptr<CompletionQueue>> cli_cqs_; - - std::vector<deadline_list> rpc_deadlines_; // per thread deadlines - std::vector<int> next_channel_; // per thread round-robin channel ctr - std::vector<boolean> issue_allowed_; // may this thread attempt to issue - std::vector<grpc_time> next_issue_; // when should it issue? - - std::mutex* - channel_lock_; // a vector, but avoid std::vector for old compilers - std::vector<context_list> contexts_; // per-channel list of idle contexts - int max_outstanding_per_channel_; - int channel_count_; - int pref_channel_inc_; + std::vector<std::function<gpr_timespec()>> next_issuers_; }; static std::unique_ptr<BenchmarkService::Stub> BenchmarkStubCreator( @@ -391,11 +259,11 @@ class AsyncUnaryClient GRPC_FINAL const SimpleRequest& request, CompletionQueue* cq) { return stub->AsyncUnaryCall(ctx, request, cq); }; - static ClientRpcContext* SetupCtx(int channel_id, - BenchmarkService::Stub* stub, + static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, + std::function<gpr_timespec()> next_issue, const SimpleRequest& req) { return new ClientRpcContextUnaryImpl<SimpleRequest, SimpleResponse>( - channel_id, stub, req, AsyncUnaryClient::StartReq, + stub, req, next_issue, AsyncUnaryClient::StartReq, AsyncUnaryClient::CheckDone); } }; @@ -404,62 +272,94 @@ template <class RequestType, class ResponseType> class ClientRpcContextStreamingImpl : public ClientRpcContext { public: ClientRpcContextStreamingImpl( - int channel_id, BenchmarkService::Stub* stub, const RequestType& req, + BenchmarkService::Stub* stub, const RequestType& req, + std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr< grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> start_req, std::function<void(grpc::Status, ResponseType*)> on_done) - : ClientRpcContext(channel_id), - context_(), + : context_(), stub_(stub), + cq_(nullptr), req_(req), response_(), - next_state_(&ClientRpcContextStreamingImpl::ReqSent), + next_state_(State::INVALID), callback_(on_done), + next_issue_(next_issue), start_req_(start_req), start_(Timer::Now()) {} ~ClientRpcContextStreamingImpl() GRPC_OVERRIDE {} + void Start(CompletionQueue* cq) GRPC_OVERRIDE { + cq_ = cq; + stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this)); + next_state_ = State::STREAM_IDLE; + } bool RunNextState(bool ok, Histogram* hist) GRPC_OVERRIDE { - return (this->*next_state_)(ok, hist); + while (true) { + switch (next_state_) { + case State::STREAM_IDLE: + if (!next_issue_) { // ready to issue + next_state_ = State::READY_TO_WRITE; + } else { + next_state_ = State::WAIT; + } + break; // loop around, don't return + case State::WAIT: + alarm_.reset( + new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this))); + next_state_ = State::READY_TO_WRITE; + return true; + case State::READY_TO_WRITE: + if (!ok) { + return false; + } + start_ = Timer::Now(); + next_state_ = State::WRITE_DONE; + stream_->Write(req_, ClientRpcContext::tag(this)); + return true; + case State::WRITE_DONE: + if (!ok) { + return false; + } + next_state_ = State::READ_DONE; + stream_->Read(&response_, ClientRpcContext::tag(this)); + return true; + break; + case State::READ_DONE: + hist->Add((Timer::Now() - start_) * 1e9); + callback_(status_, &response_); + next_state_ = State::STREAM_IDLE; + break; // loop around + default: + GPR_ASSERT(false); + return false; + } + } } ClientRpcContext* StartNewClone() GRPC_OVERRIDE { - return new ClientRpcContextStreamingImpl(channel_id_, stub_, req_, + return new ClientRpcContextStreamingImpl(stub_, req_, next_issue_, start_req_, callback_); } - void Start(CompletionQueue* cq) GRPC_OVERRIDE { - stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this)); - } private: - bool ReqSent(bool ok, Histogram*) { return StartWrite(ok); } - bool StartWrite(bool ok) { - if (!ok) { - return (false); - } - start_ = Timer::Now(); - next_state_ = &ClientRpcContextStreamingImpl::WriteDone; - stream_->Write(req_, ClientRpcContext::tag(this)); - return true; - } - bool WriteDone(bool ok, Histogram*) { - if (!ok) { - return (false); - } - next_state_ = &ClientRpcContextStreamingImpl::ReadDone; - stream_->Read(&response_, ClientRpcContext::tag(this)); - return true; - } - bool ReadDone(bool ok, Histogram* hist) { - hist->Add((Timer::Now() - start_) * 1e9); - return StartWrite(ok); - } grpc::ClientContext context_; BenchmarkService::Stub* stub_; + CompletionQueue* cq_; + std::unique_ptr<Alarm> alarm_; RequestType req_; ResponseType response_; - bool (ClientRpcContextStreamingImpl::*next_state_)(bool, Histogram*); + enum State { + INVALID, + STREAM_IDLE, + WAIT, + READY_TO_WRITE, + WRITE_DONE, + READ_DONE + }; + State next_state_; std::function<void(grpc::Status, ResponseType*)> callback_; + std::function<gpr_timespec()> next_issue_; std::function< std::unique_ptr<grpc::ClientAsyncReaderWriter<RequestType, ResponseType>>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, @@ -475,9 +375,6 @@ class AsyncStreamingClient GRPC_FINAL public: explicit AsyncStreamingClient(const ClientConfig& config) : AsyncClient(config, SetupCtx, BenchmarkStubCreator) { - // async streaming currently only supports closed loop - GPR_ASSERT(closed_loop_); - StartThreads(num_async_threads_); } @@ -492,11 +389,11 @@ class AsyncStreamingClient GRPC_FINAL auto stream = stub->AsyncStreamingCall(ctx, cq, tag); return stream; }; - static ClientRpcContext* SetupCtx(int channel_id, - BenchmarkService::Stub* stub, + static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, + std::function<gpr_timespec()> next_issue, const SimpleRequest& req) { return new ClientRpcContextStreamingImpl<SimpleRequest, SimpleResponse>( - channel_id, stub, req, AsyncStreamingClient::StartReq, + stub, req, next_issue, AsyncStreamingClient::StartReq, AsyncStreamingClient::CheckDone); } }; @@ -504,64 +401,96 @@ class AsyncStreamingClient GRPC_FINAL class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { public: ClientRpcContextGenericStreamingImpl( - int channel_id, grpc::GenericStub* stub, const ByteBuffer& req, + grpc::GenericStub* stub, const ByteBuffer& req, + std::function<gpr_timespec()> next_issue, std::function<std::unique_ptr<grpc::GenericClientAsyncReaderWriter>( grpc::GenericStub*, grpc::ClientContext*, const grpc::string& method_name, CompletionQueue*, void*)> start_req, std::function<void(grpc::Status, ByteBuffer*)> on_done) - : ClientRpcContext(channel_id), - context_(), + : context_(), stub_(stub), + cq_(nullptr), req_(req), response_(), - next_state_(&ClientRpcContextGenericStreamingImpl::ReqSent), + next_state_(State::INVALID), callback_(on_done), + next_issue_(next_issue), start_req_(start_req), start_(Timer::Now()) {} ~ClientRpcContextGenericStreamingImpl() GRPC_OVERRIDE {} - bool RunNextState(bool ok, Histogram* hist) GRPC_OVERRIDE { - return (this->*next_state_)(ok, hist); - } - ClientRpcContext* StartNewClone() GRPC_OVERRIDE { - return new ClientRpcContextGenericStreamingImpl(channel_id_, stub_, req_, - start_req_, callback_); - } void Start(CompletionQueue* cq) GRPC_OVERRIDE { + cq_ = cq; const grpc::string kMethodName( "/grpc.testing.BenchmarkService/StreamingCall"); stream_ = start_req_(stub_, &context_, kMethodName, cq, ClientRpcContext::tag(this)); + next_state_ = State::STREAM_IDLE; } - - private: - bool ReqSent(bool ok, Histogram*) { return StartWrite(ok); } - bool StartWrite(bool ok) { - if (!ok) { - return (false); - } - start_ = Timer::Now(); - next_state_ = &ClientRpcContextGenericStreamingImpl::WriteDone; - stream_->Write(req_, ClientRpcContext::tag(this)); - return true; - } - bool WriteDone(bool ok, Histogram*) { - if (!ok) { - return (false); + bool RunNextState(bool ok, Histogram* hist) GRPC_OVERRIDE { + while (true) { + switch (next_state_) { + case State::STREAM_IDLE: + if (!next_issue_) { // ready to issue + next_state_ = State::READY_TO_WRITE; + } else { + next_state_ = State::WAIT; + } + break; // loop around, don't return + case State::WAIT: + alarm_.reset( + new Alarm(cq_, next_issue_(), ClientRpcContext::tag(this))); + next_state_ = State::READY_TO_WRITE; + return true; + case State::READY_TO_WRITE: + if (!ok) { + return false; + } + start_ = Timer::Now(); + next_state_ = State::WRITE_DONE; + stream_->Write(req_, ClientRpcContext::tag(this)); + return true; + case State::WRITE_DONE: + if (!ok) { + return false; + } + next_state_ = State::READ_DONE; + stream_->Read(&response_, ClientRpcContext::tag(this)); + return true; + break; + case State::READ_DONE: + hist->Add((Timer::Now() - start_) * 1e9); + callback_(status_, &response_); + next_state_ = State::STREAM_IDLE; + break; // loop around + default: + GPR_ASSERT(false); + return false; + } } - next_state_ = &ClientRpcContextGenericStreamingImpl::ReadDone; - stream_->Read(&response_, ClientRpcContext::tag(this)); - return true; } - bool ReadDone(bool ok, Histogram* hist) { - hist->Add((Timer::Now() - start_) * 1e9); - return StartWrite(ok); + ClientRpcContext* StartNewClone() GRPC_OVERRIDE { + return new ClientRpcContextGenericStreamingImpl(stub_, req_, next_issue_, + start_req_, callback_); } + + private: grpc::ClientContext context_; grpc::GenericStub* stub_; + CompletionQueue* cq_; + std::unique_ptr<Alarm> alarm_; ByteBuffer req_; ByteBuffer response_; - bool (ClientRpcContextGenericStreamingImpl::*next_state_)(bool, Histogram*); + enum State { + INVALID, + STREAM_IDLE, + WAIT, + READY_TO_WRITE, + WRITE_DONE, + READ_DONE + }; + State next_state_; std::function<void(grpc::Status, ByteBuffer*)> callback_; + std::function<gpr_timespec()> next_issue_; std::function<std::unique_ptr<grpc::GenericClientAsyncReaderWriter>( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, CompletionQueue*, void*)> start_req_; @@ -580,9 +509,6 @@ class GenericAsyncStreamingClient GRPC_FINAL public: explicit GenericAsyncStreamingClient(const ClientConfig& config) : AsyncClient(config, SetupCtx, GenericStubCreator) { - // async streaming currently only supports closed loop - GPR_ASSERT(closed_loop_); - StartThreads(num_async_threads_); } @@ -596,10 +522,11 @@ class GenericAsyncStreamingClient GRPC_FINAL auto stream = stub->Call(ctx, method_name, cq, tag); return stream; }; - static ClientRpcContext* SetupCtx(int channel_id, grpc::GenericStub* stub, + static ClientRpcContext* SetupCtx(grpc::GenericStub* stub, + std::function<gpr_timespec()> next_issue, const ByteBuffer& req) { return new ClientRpcContextGenericStreamingImpl( - channel_id, stub, req, GenericAsyncStreamingClient::StartReq, + stub, req, next_issue, GenericAsyncStreamingClient::StartReq, GenericAsyncStreamingClient::CheckDone); } }; diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc index d93537b279..edfc246a25 100644 --- a/test/cpp/qps/client_sync.cc +++ b/test/cpp/qps/client_sync.cc @@ -84,11 +84,8 @@ class SynchronousClient protected: void WaitToIssue(int thread_idx) { - grpc_time next_time; - if (NextIssueTime(thread_idx, &next_time)) { - gpr_timespec next_timespec; - TimepointHR2Timespec(next_time, &next_timespec); - gpr_sleep_until(next_timespec); + if (!closed_loop_) { + gpr_sleep_until(NextIssueTime(thread_idx)); } } diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 80f6ada409..1c7fdf8796 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -197,9 +197,7 @@ std::unique_ptr<ScenarioResult> RunScenario( workers.resize(num_clients + num_servers); gpr_timespec deadline = - gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_seconds( - warmup_seconds + benchmark_seconds + 20, GPR_TIMESPAN)); + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(warmup_seconds + benchmark_seconds + 20); // Start servers using runsc::ServerData; diff --git a/test/cpp/qps/interarrival.h b/test/cpp/qps/interarrival.h index 841619e3ff..b6fd67b77c 100644 --- a/test/cpp/qps/interarrival.h +++ b/test/cpp/qps/interarrival.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -51,15 +51,15 @@ namespace testing { // stacks. Thus, this code only uses a uniform distribution of doubles [0,1) // and then provides the distribution functions itself. -class RandomDist { +class RandomDistInterface { public: - RandomDist() {} - virtual ~RandomDist() = 0; - // Argument to operator() is a uniform double in the range [0,1) - virtual double operator()(double uni) const = 0; + RandomDistInterface() {} + virtual ~RandomDistInterface() = 0; + // Argument to transform is a uniform double in the range [0,1) + virtual double transform(double uni) const = 0; }; -inline RandomDist::~RandomDist() {} +inline RandomDistInterface::~RandomDistInterface() {} // ExpDist implements an exponential distribution, which is the // interarrival distribution for a Poisson process. The parameter @@ -69,11 +69,11 @@ inline RandomDist::~RandomDist() {} // independent identical stationary sources. For more information, // see http://en.wikipedia.org/wiki/Exponential_distribution -class ExpDist GRPC_FINAL : public RandomDist { +class ExpDist GRPC_FINAL : public RandomDistInterface { public: explicit ExpDist(double lambda) : lambda_recip_(1.0 / lambda) {} ~ExpDist() GRPC_OVERRIDE {} - double operator()(double uni) const GRPC_OVERRIDE { + double transform(double uni) const GRPC_OVERRIDE { // Note: Use 1.0-uni above to avoid NaN if uni is 0 return lambda_recip_ * (-log(1.0 - uni)); } @@ -87,11 +87,11 @@ class ExpDist GRPC_FINAL : public RandomDist { // mean interarrival time is (lo+hi)/2. For more information, // see http://en.wikipedia.org/wiki/Uniform_distribution_%28continuous%29 -class UniformDist GRPC_FINAL : public RandomDist { +class UniformDist GRPC_FINAL : public RandomDistInterface { public: UniformDist(double lo, double hi) : lo_(lo), range_(hi - lo) {} ~UniformDist() GRPC_OVERRIDE {} - double operator()(double uni) const GRPC_OVERRIDE { + double transform(double uni) const GRPC_OVERRIDE { return uni * range_ + lo_; } @@ -106,11 +106,11 @@ class UniformDist GRPC_FINAL : public RandomDist { // clients) will not preserve any deterministic interarrival gap across // requests. -class DetDist GRPC_FINAL : public RandomDist { +class DetDist GRPC_FINAL : public RandomDistInterface { public: explicit DetDist(double val) : val_(val) {} ~DetDist() GRPC_OVERRIDE {} - double operator()(double uni) const GRPC_OVERRIDE { return val_; } + double transform(double uni) const GRPC_OVERRIDE { return val_; } private: double val_; @@ -123,12 +123,12 @@ class DetDist GRPC_FINAL : public RandomDist { // good representation of the response times of data center jobs. See // http://en.wikipedia.org/wiki/Pareto_distribution -class ParetoDist GRPC_FINAL : public RandomDist { +class ParetoDist GRPC_FINAL : public RandomDistInterface { public: ParetoDist(double base, double alpha) : base_(base), alpha_recip_(1.0 / alpha) {} ~ParetoDist() GRPC_OVERRIDE {} - double operator()(double uni) const GRPC_OVERRIDE { + double transform(double uni) const GRPC_OVERRIDE { // Note: Use 1.0-uni above to avoid div by zero if uni is 0 return base_ / pow(1.0 - uni, alpha_recip_); } @@ -145,13 +145,14 @@ class ParetoDist GRPC_FINAL : public RandomDist { class InterarrivalTimer { public: InterarrivalTimer() {} - void init(const RandomDist& r, int threads, int entries = 1000000) { + void init(const RandomDistInterface& r, int threads, int entries = 1000000) { for (int i = 0; i < entries; i++) { // rand is the only choice that is portable across POSIX and Windows // and that supports new and old compilers - const double uniform_0_1 = rand() / RAND_MAX; + const double uniform_0_1 = + static_cast<double>(rand()) / static_cast<double>(RAND_MAX); random_table_.push_back( - std::chrono::nanoseconds(static_cast<int64_t>(1e9 * r(uniform_0_1)))); + static_cast<int64_t>(1e9 * r.transform(uniform_0_1))); } // Now set up the thread positions for (int i = 0; i < threads; i++) { @@ -160,7 +161,7 @@ class InterarrivalTimer { } virtual ~InterarrivalTimer(){}; - std::chrono::nanoseconds operator()(int thread_num) { + int64_t next(int thread_num) { auto ret = *(thread_posns_[thread_num]++); if (thread_posns_[thread_num] == random_table_.end()) thread_posns_[thread_num] = random_table_.begin(); @@ -168,7 +169,7 @@ class InterarrivalTimer { } private: - typedef std::vector<std::chrono::nanoseconds> time_table; + typedef std::vector<int64_t> time_table; std::vector<time_table::const_iterator> thread_posns_; time_table random_table_; }; diff --git a/test/cpp/qps/qps-sweep.sh b/test/cpp/qps/qps-sweep.sh index 539da1d893..7a35788849 100755 --- a/test/cpp/qps/qps-sweep.sh +++ b/test/cpp/qps/qps-sweep.sh @@ -37,9 +37,26 @@ fi bins=`find . .. ../.. ../../.. -name bins | head -1` +# Print out each command that gets executed set -x +# +# Specify parameters used in some of the tests +# + +# big is the size in bytes of large messages (0 is the size otherwise) big=65536 + +# wide is the number of client channels in multi-channel tests (1 otherwise) +wide=64 + +# deep is the number of RPCs outstanding on a channel in non-ping-pong tests +# (the value used is 1 otherwise) +deep=100 + +# half is half the count of worker processes, used in the crossbar scenario +# that uses equal clients and servers. The other scenarios use only 1 server +# and either 1 client or N-1 clients as appropriate half=`echo $QPS_WORKERS | awk -F, '{print int(NF/2)}'` for secure in true false; do @@ -52,30 +69,40 @@ for secure in true false; do # Scenario 2: generic async streaming "unconstrained" (QPS) "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ - --client_channels=64 --bbuf_req_size=0 --bbuf_resp_size=0 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ + --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ - --num_servers=1 --num_clients=0 + --num_servers=1 --num_clients=0 |& tee /tmp/qps-test.$$ # Scenario 2b: QPS with a single server core "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ - --client_channels=64 --bbuf_req_size=0 --bbuf_resp_size=0 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ + --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 --server_core_limit=1 # Scenario 2c: protobuf-based QPS "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=100 \ - --client_channels=64 --simple_req_size=0 --simple_resp_size=0 \ + --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ + --client_channels=$wide --simple_req_size=0 --simple_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=0 - # Scenario 3: Latency at near-peak load (TBD) + # Scenario 3: Latency at sub-peak load (all clients equally loaded) + for loadfactor in 0.2 0.5 0.7; do + "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ + --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ + --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ + --num_servers=1 --num_clients=0 --poisson_load=`awk -v lf=$loadfactor \ + '$5 == "QPS:" {print int(lf * $6); exit}' /tmp/qps-test.$$` + done + + rm /tmp/qps-test.$$ # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM). "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 @@ -108,35 +135,35 @@ for secure in true false; do # Scenario 9: Crossbar QPS test "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ - --client_channels=64 --bbuf_req_size=0 --bbuf_resp_size=0 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ + --client_channels=$wide --bbuf_req_size=0 --bbuf_resp_size=0 \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=$half --num_clients=0 # Scenario 10: Multi-channel bidir throughput test "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ - --client_channels=64 --bbuf_req_size=$big --bbuf_resp_size=$big \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \ + --client_channels=$wide --bbuf_req_size=$big --bbuf_resp_size=$big \ --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \ --num_servers=1 --num_clients=1 # Scenario 11: Single-channel request throughput test "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=0 \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 # Scenario 12: Single-channel response throughput test "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \ + --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 # Scenario 13: Single-channel bidirectional protobuf throughput test "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \ - --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=100 \ + --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=$deep \ --client_channels=1 --simple_req_size=$big --simple_resp_size=$big \ --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \ --num_servers=1 --num_clients=1 diff --git a/test/cpp/qps/qps_interarrival_test.cc b/test/cpp/qps/qps_interarrival_test.cc index ccda28f09a..77e81fb84b 100644 --- a/test/cpp/qps/qps_interarrival_test.cc +++ b/test/cpp/qps/qps_interarrival_test.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,17 +39,17 @@ #include "test/cpp/qps/interarrival.h" -using grpc::testing::RandomDist; +using grpc::testing::RandomDistInterface; using grpc::testing::InterarrivalTimer; -static void RunTest(RandomDist &&r, int threads, std::string title) { +static void RunTest(RandomDistInterface &&r, int threads, std::string title) { InterarrivalTimer timer; timer.init(r, threads); gpr_histogram *h(gpr_histogram_create(0.01, 60e9)); for (int i = 0; i < 10000000; i++) { for (int j = 0; j < threads; j++) { - gpr_histogram_add(h, timer(j).count()); + gpr_histogram_add(h, timer.next(j)); } } @@ -70,7 +70,7 @@ using grpc::testing::ParetoDist; int main(int argc, char **argv) { RunTest(ExpDist(10.0), 5, std::string("Exponential(10)")); RunTest(DetDist(5.0), 5, std::string("Det(5)")); - RunTest(UniformDist(0.0, 10.0), 5, std::string("Uniform(1,10)")); + RunTest(UniformDist(0.0, 10.0), 5, std::string("Uniform(0,10)")); RunTest(ParetoDist(1.0, 1.0), 5, std::string("Pareto(1,1)")); return 0; } diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc index fe5f685b6e..0ac41d9f96 100644 --- a/test/cpp/qps/qps_openloop_test.cc +++ b/test/cpp/qps/qps_openloop_test.cc @@ -53,7 +53,7 @@ static void RunQPS() { client_config.set_outstanding_rpcs_per_channel(1000); client_config.set_client_channels(8); client_config.set_async_client_threads(8); - client_config.set_rpc_type(UNARY); + client_config.set_rpc_type(STREAMING); client_config.mutable_load_params()->mutable_poisson()->set_offered_load( 1000.0); diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc index 15054db892..27aaf137f6 100644 --- a/test/cpp/qps/qps_test.cc +++ b/test/cpp/qps/qps_test.cc @@ -53,7 +53,7 @@ static void RunQPS() { client_config.set_outstanding_rpcs_per_channel(1000); client_config.set_client_channels(8); client_config.set_async_client_threads(8); - client_config.set_rpc_type(UNARY); + client_config.set_rpc_type(STREAMING); client_config.mutable_load_params()->mutable_closed_loop(); ServerConfig server_config; diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 1302d718f0..2024e0bfef 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -51,6 +51,7 @@ #include <gtest/gtest.h> #include "src/proto/grpc/testing/services.grpc.pb.h" +#include "test/core/util/test_config.h" #include "test/cpp/qps/server.h" namespace grpc { @@ -129,7 +130,7 @@ class AsyncQpsServerTest : public Server { } } ~AsyncQpsServerTest() { - auto deadline = std::chrono::system_clock::now() + std::chrono::seconds(10); + auto deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10); server_->Shutdown(deadline); for (auto ss = shutdown_state_.begin(); ss != shutdown_state_.end(); ++ss) { (*ss)->set_shutdown(); diff --git a/test/distrib/node/run_distrib_test.sh b/test/distrib/node/run_distrib_test.sh index 99a51f01f7..9b8f15771b 100755 --- a/test/distrib/node/run_distrib_test.sh +++ b/test/distrib/node/run_distrib_test.sh @@ -38,6 +38,9 @@ nvm install $NODE_VERSION npm install -g node-static +# Kill off existing static servers +kill -9 $(ps aux | grep '[n]ode .*static' | awk '{print $2}') || true + STATIC_SERVER=127.0.0.1 STATIC_PORT=8080 diff --git a/test/distrib/python/distribtest.py b/test/distrib/python/distribtest.py index 66c1f88796..dc20688140 100644 --- a/test/distrib/python/distribtest.py +++ b/test/distrib/python/distribtest.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/tools/README.md b/tools/README.md index eb6633a866..a0c41eb79f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,6 +8,8 @@ dockerfile: Docker files to test gRPC. doxygen: gRPC C/C++ documentation generation via Doxygen. +gce: scripts to help setup testing infrastructure on GCE. + jenkins: support for running tests on Jenkins. profile_analyzer: pretty printer for gRPC profiling data. diff --git a/tools/codegen/core/gen_legal_metadata_characters.c b/tools/codegen/core/gen_legal_metadata_characters.c index 3c9e1c7619..10605d52bf 100644 --- a/tools/codegen/core/gen_legal_metadata_characters.c +++ b/tools/codegen/core/gen_legal_metadata_characters.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -52,7 +52,7 @@ static void legal(int x) { static void dump(void) { int i; - printf("static const gpr_uint8 legal_header_bits[256/8] = "); + printf("static const uint8_t legal_header_bits[256/8] = "); for (i = 0; i < 256 / 8; i++) printf("%c 0x%02x", i ? ',' : '{', legal_bits[i]); printf(" };\n"); diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index a7efdc85cc..99771f9632 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -57,6 +57,9 @@ argp.add_argument('-a', '--ancient', argp.add_argument('-f', '--fix', default=False, action='store_true'); +argp.add_argument('--precommit', + default=False, + action='store_true') args = argp.parse_args() # open the license text @@ -68,9 +71,9 @@ with open('LICENSE') as f: # that given a line of license text, returns what should # be in the file LICENSE_PREFIX = { - '.c': r'\s*\*\s*', - '.cc': r'\s*\*\s*', - '.h': r'\s*\*\s*', + '.c': r'\s*(?://|\*)\s*', + '.cc': r'\s*(?://|\*)\s*', + '.h': r'\s*(?://|\*)\s*', '.m': r'\s*\*\s*', '.php': r'\s*\*\s*', '.js': r'\s*\*\s*', @@ -101,6 +104,10 @@ RE_LICENSE = dict( for line in LICENSE)) for k, v in LICENSE_PREFIX.iteritems()) +if args.precommit: + FILE_LIST_COMMAND = 'git diff --name-only HEAD | grep -v ^third_party/' +else: + FILE_LIST_COMMAND = 'git ls-tree -r --name-only -r HEAD | grep -v ^third_party/' def load(name): with open(name) as f: @@ -124,8 +131,14 @@ def log(cond, why, filename): # scan files, validate the text ok = True -for filename in subprocess.check_output('git ls-tree -r --name-only -r HEAD | grep -v ^third_party/', - shell=True).splitlines(): +filename_list = [] +try: + filename_list = subprocess.check_output(FILE_LIST_COMMAND, + shell=True).splitlines() +except subprocess.CalledProcessError: + sys.exit(0) + +for filename in filename_list: if filename in KNOWN_BAD: continue ext = os.path.splitext(filename)[1] base = os.path.basename(filename) diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh index 6bfa278cae..d904a841d4 100755 --- a/tools/distrib/clang_format_code.sh +++ b/tools/distrib/clang_format_code.sh @@ -37,4 +37,4 @@ cd $(dirname $0)/../.. docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format # run clang-format against the checked out codebase -docker run -e TEST=$TEST --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh +docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh diff --git a/tools/distrib/sanitize.sh b/tools/distrib/sanitize.sh new file mode 100755 index 0000000000..3b7ca6fd88 --- /dev/null +++ b/tools/distrib/sanitize.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. + +DIFF_COMMAND="git diff --name-only HEAD | grep -v ^third_party/" + +if [ "x$1" == 'x--pre-commit' ]; then + if eval $DIFF_COMMAND | grep '^build.yaml$'; then + ./tools/buildgen/generate_projects.sh + else + templates=$(eval $DIFF_COMMAND | grep '\.template$' || true) + if [ -n "$templates" ]; then + ./tools/buildgen/generate_projects.sh --templates $templates + fi + fi + CHANGED_FILES=$(eval $DIFF_COMMAND) ./tools/distrib/clang_format_code.sh + ./tools/distrib/check_copyright.py --fix --precommit + ./tools/distrib/check_trailing_newlines.sh +else + ./tools/buildgen/generate_projects.sh + ./tools/distrib/clang_format_code.sh + ./tools/distrib/check_copyright.py --fix + ./tools/distrib/check_trailing_newlines.sh +fi diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile index b44fcff7e3..1eb4c1e775 100644 --- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile @@ -29,4 +29,9 @@ FROM fedora:21 +# Make yum work properly under docker when using overlay storage driver. +# https://bugzilla.redhat.com/show_bug.cgi?id=1213602#c9 +# https://github.com/docker/docker/issues/10180 +RUN yum install -y yum-plugin-ovl + RUN yum clean all && yum update -y && yum install -y python python-pip diff --git a/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile index 598dac5a11..b567c5b109 100644 --- a/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile @@ -29,6 +29,11 @@ FROM fedora:21 +# Make yum work properly under docker when using overlay storage driver. +# https://bugzilla.redhat.com/show_bug.cgi?id=1213602#c9 +# https://github.com/docker/docker/issues/10180 +RUN yum install -y yum-plugin-ovl + RUN yum clean all && yum update -y && yum install -y ruby RUN gem install bundler diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index af0b22a07f..d56bc01831 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -48,6 +48,12 @@ do done done +# The CHANGED_FILES variable is used to restrict the set of files to check. +# Here we set files to the intersection of files and CHANGED_FILES +if [ -n "$CHANGED_FILES" ]; then + files=$(comm -12 <(echo $files | tr ' ' '\n' | sort -u) <(echo $CHANGED_FILES | tr ' ' '\n' | sort -u)) +fi + if [ "x$TEST" = "x" ] then echo $files | xargs $CLANG_FORMAT -i diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e0650d74f9..d5e5df86f6 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -840,7 +840,6 @@ src/cpp/server/secure_server_credentials.h \ src/cpp/client/create_channel_internal.h \ src/cpp/common/create_auth_context.h \ src/cpp/server/dynamic_thread_pool.h \ -src/cpp/server/fixed_size_thread_pool.h \ src/cpp/server/thread_pool_interface.h \ src/cpp/client/secure_credentials.cc \ src/cpp/common/auth_property_iterator.cc \ @@ -864,7 +863,6 @@ src/cpp/proto/proto_utils.cc \ src/cpp/server/async_generic_service.cc \ src/cpp/server/create_default_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.cc \ -src/cpp/server/fixed_size_thread_pool.cc \ src/cpp/server/insecure_server_credentials.cc \ src/cpp/server/server.cc \ src/cpp/server/server_builder.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index c6f1b6ae7c..fd02b98614 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -814,6 +814,7 @@ src/core/client_config/resolvers/dns_resolver.h \ src/core/client_config/resolvers/sockaddr_resolver.h \ src/core/client_config/subchannel.h \ src/core/client_config/subchannel_factory.h \ +src/core/client_config/subchannel_index.h \ src/core/client_config/uri_parser.h \ src/core/compression/algorithm_metadata.h \ src/core/compression/message_compress.h \ @@ -903,6 +904,7 @@ src/core/transport/static_metadata.h \ src/core/transport/transport.h \ src/core/transport/transport_impl.h \ src/core/census/aggregation.h \ +src/core/census/log.h \ src/core/census/rpc_metric_id.h \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.h \ @@ -957,6 +959,7 @@ src/core/client_config/resolvers/dns_resolver.c \ src/core/client_config/resolvers/sockaddr_resolver.c \ src/core/client_config/subchannel.c \ src/core/client_config/subchannel_factory.c \ +src/core/client_config/subchannel_index.c \ src/core/client_config/uri_parser.c \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ @@ -1060,6 +1063,7 @@ src/core/transport/transport.c \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ +src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tracing.c \ diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh new file mode 100755 index 0000000000..2a9e77ab17 --- /dev/null +++ b/tools/gce/create_linux_worker.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Creates a standard jenkins worker on GCE. + +set -ex + +cd $(dirname $0) + +CLOUD_PROJECT=grpc-testing +ZONE=us-central1-a + +INSTANCE_NAME=grpc-jenkins-worker1 + +gcloud compute instances create $INSTANCE_NAME \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + --machine-type n1-standard-8 \ + --image ubuntu-14-04 \ + --boot-disk-size 1000 + +echo 'Created GCE instance, waiting 60 seconds for it to come online.' +sleep 60 + +gcloud compute copy-files \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + jenkins_master.pub linux_worker_init.sh ${INSTANCE_NAME}:~ + +gcloud compute ssh \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + $INSTANCE_NAME --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub new file mode 100644 index 0000000000..e9853224e1 --- /dev/null +++ b/tools/gce/jenkins_master.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh new file mode 100755 index 0000000000..f56cac0ce2 --- /dev/null +++ b/tools/gce/linux_worker_init.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Initializes a fresh GCE VM to become a jenkins linux worker. +# You shouldn't run this script on your own, use create_linux_worker.sh +# instead. + +set -ex + +sudo apt-get update + +# Install JRE +sudo apt-get install -y openjdk-7-jre +sudo apt-get install -y unzip lsof + +# Install Docker +curl -sSL https://get.docker.com/ | sh + +# Setup jenkins user (or the user will already exist bcuz magic) +sudo adduser jenkins --disabled-password || true + +# Enable jenkins to use docker without sudo: +sudo usermod -aG docker jenkins + +# Use "overlay" storage driver for docker +# see https://github.com/grpc/grpc/issues/4988 +echo 'DOCKER_OPTS="${DOCKER_OPTS} --storage-driver=overlay"' | sudo tee --append /etc/default/docker + +# Install RVM +# TODO(jtattermusch): why is RVM needed? +gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +curl -sSL https://get.rvm.io | bash -s stable --ruby + +# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ +# This needs to happen as the last step to prevent Jenkins master from connecting +# to a machine that hasn't been properly setup yet. +cat jenkins_master.pub | sudo tee --append ~jenkins/.ssh/authorized_keys + +# Restart for docker to pickup the config changes. +echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' +sleep 10 + +sudo reboot diff --git a/tools/asan_suppressions.txt b/tools/lsan_suppressions.txt index d672cb6cae..d672cb6cae 100644 --- a/tools/asan_suppressions.txt +++ b/tools/lsan_suppressions.txt diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 74d2a4859c..9cd02c5e43 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -175,7 +175,8 @@ class CSharpExtArtifact: else: environ = {'CONFIG': 'opt', 'EMBED_OPENSSL': 'true', - 'EMBED_ZLIB': 'true'} + 'EMBED_ZLIB': 'true', + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'} if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh index 9a3b9bd1ba..6aa4824538 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/build_artifact_node.sh @@ -36,7 +36,7 @@ nvm use 4 cd $(dirname $0)/../.. -rm -rf build +rm -rf build || true mkdir -p artifacts @@ -46,6 +46,6 @@ node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 ) for version in ${node_versions[@]} do - node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH + ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH cp -r build/stage/* artifacts/ done diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 835fad83e1..6e7ab911d5 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -39,12 +39,20 @@ then pip install -rrequirements.txt fi +# The bdist_wheel_grpc_custom command is finicky about command output ordering +# and thus ought to be run in a shell command separate of others. Further, it +# trashes the actual bdist_wheel output, so it should be run first so that +# bdist_wheel may be run unmolested. +GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ +GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ +${SETARCH_CMD} python setup.py \ + build_tagged_ext + GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ bdist_wheel \ - sdist \ - bdist_egg_grpc_custom + sdist mkdir -p artifacts diff --git a/tools/run_tests/build_node.bat b/tools/run_tests/build_node.bat new file mode 100644 index 0000000000..6896bc1d1b --- /dev/null +++ b/tools/run_tests/build_node.bat @@ -0,0 +1,30 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +npm install --build-from-source
\ No newline at end of file diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index a8b9448973..aca90a3750 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -38,6 +38,7 @@ cd $(dirname $0)/../.. mkdir -p artifacts/ cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* artifacts/ || true +npm update npm pack cp grpc-*.tgz artifacts/grpc.tgz diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index d508c6394c..9d7b8a3c72 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -45,8 +45,8 @@ { "config": "asan", "environ": { - "ASAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:detect_leaks=1:color=always", - "LSAN_OPTIONS": "suppressions=tools/asan_suppressions.txt:report_objects=1" + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" }, "timeout_multiplier": 1.5 }, diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index b26a870778..261f44bc6d 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -122,11 +122,15 @@ class NodeDistribTest(object): def build_jobspec(self): if self.platform == 'linux': + linux32 = '' + if self.arch == 'x86': + linux32 = 'linux32' return create_docker_jobspec(self.name, 'tools/dockerfile/distribtest/node_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/node/run_distrib_test.sh %s' % ( + '%s test/distrib/node/run_distrib_test.sh %s' % ( + linux32, self.node_version)) elif self.platform == 'macos': return create_jobspec(self.name, @@ -236,11 +240,7 @@ def targets(): RubyDistribTest('linux', 'x64', 'ubuntu1504'), RubyDistribTest('linux', 'x64', 'ubuntu1510'), RubyDistribTest('linux', 'x64', 'ubuntu1604'), - NodeDistribTest('macos', 'x64', None, '0.10'), - NodeDistribTest('macos', 'x64', None, '0.12'), - NodeDistribTest('macos', 'x64', None, '3'), NodeDistribTest('macos', 'x64', None, '4'), - NodeDistribTest('macos', 'x64', None, '5'), NodeDistribTest('linux', 'x86', 'jessie', '4') ] + [ NodeDistribTest('linux', 'x64', os, version) diff --git a/tools/run_tests/pre_build_node.bat b/tools/run_tests/pre_build_node.bat new file mode 100644 index 0000000000..6e7cbe5d42 --- /dev/null +++ b/tools/run_tests/pre_build_node.bat @@ -0,0 +1,39 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +@rem Expire cache after 1 week +npm update --cache-min 604800 + +npm install node-gyp-install +.\node_modules\.bin\node-gyp-install.cmd + +@rem delete the redundant openssl headers +for /f "delims=v" %%v in ('node --version') do ( + rmdir "%HOMEDRIVE%%HOMEPATH%\.node-gyp\%%v\include\node\openssl" /S /Q +)
\ No newline at end of file diff --git a/tools/run_tests/run_node.bat b/tools/run_tests/run_node.bat new file mode 100644 index 0000000000..f5cf01f095 --- /dev/null +++ b/tools/run_tests/run_node.bat @@ -0,0 +1,32 @@ +@rem Copyright 2016, Google Inc. +@rem All rights reserved. +@rem +@rem Redistribution and use in source and binary forms, with or without +@rem modification, are permitted provided that the following conditions are +@rem met: +@rem +@rem * Redistributions of source code must retain the above copyright +@rem notice, this list of conditions and the following disclaimer. +@rem * Redistributions in binary form must reproduce the above +@rem copyright notice, this list of conditions and the following disclaimer +@rem in the documentation and/or other materials provided with the +@rem distribution. +@rem * Neither the name of Google Inc. nor the names of its +@rem contributors may be used to endorse or promote products derived from +@rem this software without specific prior written permission. +@rem +@rem THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +@rem "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +@rem LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +@rem A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +@rem OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +@rem SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +@rem LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +@rem DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +@rem THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +@rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +@rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set JUNIT_REPORT_PATH=src\node\reports.xml +set JUNIT_REPORT_STACK=1 +.\node_modules\.bin\mocha.cmd --reporter mocha-jenkins-reporter src\node\test
\ No newline at end of file diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index f405866442..de3716bc88 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -195,16 +195,22 @@ class CLanguage(object): class NodeLanguage(object): def __init__(self): + self.platform = platform_string() self.node_version = '0.12' def test_specs(self, config, args): - return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version], - None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + if self.platform == 'windows': + return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)] + else: + return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version], + None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): - # Default to 1 week cache expiration - return [['tools/run_tests/pre_build_node.sh', self.node_version]] + if self.platform == 'windows': + return [['tools\\run_tests\\pre_build_node.bat']] + else: + return [['tools/run_tests/pre_build_node.sh', self.node_version]] def make_targets(self, test_regex): return [] @@ -213,7 +219,10 @@ class NodeLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_node.sh', self.node_version]] + if self.platform == 'windows': + return [['tools\\run_tests\\build_node.bat']] + else: + return [['tools/run_tests/build_node.sh', self.node_version]] def post_tests_steps(self): return [] diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 76fa3d0776..82afb5715c 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -90,6 +90,20 @@ ], "headers": [], "language": "c", + "name": "census_log_test", + "src": [ + "test/core/census/log_test.c" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "channel_create_test", "src": [ "test/core/surface/channel_create_test.c" @@ -2986,6 +3000,7 @@ "include/grpc/status.h", "src/core/census/aggregation.h", "src/core/census/grpc_filter.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", @@ -3013,6 +3028,7 @@ "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -3142,6 +3158,8 @@ "src/core/census/grpc_filter.c", "src/core/census/grpc_filter.h", "src/core/census/initialize.c", + "src/core/census/log.c", + "src/core/census/log.h", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/rpc_metric_id.h", @@ -3198,6 +3216,8 @@ "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.c", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.c", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.c", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm.c", @@ -3521,6 +3541,7 @@ "include/grpc/status.h", "src/core/census/aggregation.h", "src/core/census/grpc_filter.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", @@ -3548,6 +3569,7 @@ "src/core/client_config/resolvers/sockaddr_resolver.h", "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm_metadata.h", "src/core/compression/message_compress.h", @@ -3662,6 +3684,8 @@ "src/core/census/grpc_filter.c", "src/core/census/grpc_filter.h", "src/core/census/initialize.c", + "src/core/census/log.c", + "src/core/census/log.h", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/rpc_metric_id.h", @@ -3718,6 +3742,8 @@ "src/core/client_config/subchannel.h", "src/core/client_config/subchannel_factory.c", "src/core/client_config/subchannel_factory.h", + "src/core/client_config/subchannel_index.c", + "src/core/client_config/subchannel_index.h", "src/core/client_config/uri_parser.c", "src/core/client_config/uri_parser.h", "src/core/compression/algorithm.c", @@ -4046,7 +4072,6 @@ "src/cpp/common/create_auth_context.h", "src/cpp/common/secure_auth_context.h", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/secure_server_credentials.h", "src/cpp/server/thread_pool_interface.h" ], @@ -4154,8 +4179,6 @@ "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.cc", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/insecure_server_credentials.cc", "src/cpp/server/secure_server_credentials.cc", "src/cpp/server/secure_server_credentials.h", @@ -4302,7 +4325,6 @@ "src/cpp/client/create_channel_internal.h", "src/cpp/common/create_auth_context.h", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/thread_pool_interface.h" ], "language": "c++", @@ -4403,8 +4425,6 @@ "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.h", - "src/cpp/server/fixed_size_thread_pool.cc", - "src/cpp/server/fixed_size_thread_pool.h", "src/cpp/server/insecure_server_credentials.cc", "src/cpp/server/server.cc", "src/cpp/server/server_builder.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 3b9bce1ec3..480af97368 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -133,6 +133,26 @@ "exclude_configs": [], "flaky": false, "language": "c", + "name": "census_log_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", "name": "channel_create_test", "platforms": [ "linux", @@ -2104,6 +2124,24 @@ "mac", "posix" ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "qps_openloop_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], "cpu_cost": 10, "exclude_configs": [ "tsan" diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index ea8e50456a..b30941ff73 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -176,6 +176,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "census_context_test", "vcxp {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "census_log_test", "vcxproj\test\census_log_test\census_log_test.vcxproj", "{C27CEE16-2BEC-5572-3956-677E9F6F8BED}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_create_test", "vcxproj\test\channel_create_test\channel_create_test.vcxproj", "{AFC88484-3A2E-32BC-25B2-23DF741D4F3D}" ProjectSection(myProperties) = preProject lib = "False" @@ -1604,6 +1615,22 @@ Global {5C1CFC2D-AF3C-D7CB-BA74-D267E91CBC73}.Release-DLL|Win32.Build.0 = Release|Win32 {5C1CFC2D-AF3C-D7CB-BA74-D267E91CBC73}.Release-DLL|x64.ActiveCfg = Release|x64 {5C1CFC2D-AF3C-D7CB-BA74-D267E91CBC73}.Release-DLL|x64.Build.0 = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|Win32.ActiveCfg = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|x64.ActiveCfg = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|Win32.ActiveCfg = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|x64.ActiveCfg = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|Win32.Build.0 = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|x64.Build.0 = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|Win32.Build.0 = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|x64.Build.0 = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|x64.Build.0 = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|Win32.Build.0 = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|x64.ActiveCfg = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|x64.Build.0 = Release|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Debug|Win32.ActiveCfg = Debug|Win32 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Debug|x64.ActiveCfg = Debug|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index bb2d0e782b..c62faf33e6 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -340,7 +340,6 @@ <ClInclude Include="$(SolutionDir)\..\src\cpp\client\create_channel_internal.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\common\create_auth_context.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.h" /> - <ClInclude Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\thread_pool_interface.h" /> </ItemGroup> <ItemGroup> @@ -388,8 +387,6 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.cc"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.cc"> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\insecure_server_credentials.cc"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\server.cc"> diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 72ecfe26fc..5f9350e76a 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -67,9 +67,6 @@ <ClCompile Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.cc"> <Filter>src\cpp\server</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.cc"> - <Filter>src\cpp\server</Filter> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\insecure_server_credentials.cc"> <Filter>src\cpp\server</Filter> </ClCompile> @@ -347,9 +344,6 @@ <ClInclude Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.h"> <Filter>src\cpp\server</Filter> </ClInclude> - <ClInclude Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.h"> - <Filter>src\cpp\server</Filter> - </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\thread_pool_interface.h"> <Filter>src\cpp\server</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 8ff8d8b800..fb4246580f 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -337,7 +337,6 @@ <ClInclude Include="$(SolutionDir)\..\src\cpp\client\create_channel_internal.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\common\create_auth_context.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.h" /> - <ClInclude Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.h" /> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\thread_pool_interface.h" /> </ItemGroup> <ItemGroup> @@ -375,8 +374,6 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.cc"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.cc"> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\insecure_server_credentials.cc"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\server.cc"> diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 316fdd7caa..eeff7d3697 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -52,9 +52,6 @@ <ClCompile Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.cc"> <Filter>src\cpp\server</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.cc"> - <Filter>src\cpp\server</Filter> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\cpp\server\insecure_server_credentials.cc"> <Filter>src\cpp\server</Filter> </ClCompile> @@ -323,9 +320,6 @@ <ClInclude Include="$(SolutionDir)\..\src\cpp\server\dynamic_thread_pool.h"> <Filter>src\cpp\server</Filter> </ClInclude> - <ClInclude Include="$(SolutionDir)\..\src\cpp\server\fixed_size_thread_pool.h"> - <Filter>src\cpp\server</Filter> - </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\cpp\server\thread_pool_interface.h"> <Filter>src\cpp\server</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 4e1155f84b..6f523f7b63 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -323,6 +323,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\client_config\resolvers\sockaddr_resolver.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\uri_parser.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\compression\algorithm_metadata.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\compression\message_compress.h" /> @@ -412,6 +413,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport_impl.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h" /> <ClInclude Include="$(SolutionDir)\..\third_party\nanopb\pb.h" /> <ClInclude Include="$(SolutionDir)\..\third_party\nanopb\pb_common.h" /> @@ -517,6 +519,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\uri_parser.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\compression\algorithm.c"> @@ -723,6 +727,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\placeholders.c"> diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index e0cfd8295b..1453bffa15 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -148,6 +148,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.c"> <Filter>src\core\client_config</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.c"> + <Filter>src\core\client_config</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\uri_parser.c"> <Filter>src\core\client_config</Filter> </ClCompile> @@ -457,6 +460,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> <Filter>src\core\census</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + <Filter>src\core\census</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> <Filter>src\core\census</Filter> </ClCompile> @@ -641,6 +647,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.h"> <Filter>src\core\client_config</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.h"> + <Filter>src\core\client_config</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\uri_parser.h"> <Filter>src\core\client_config</Filter> </ClInclude> @@ -908,6 +917,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h"> <Filter>src\core\census</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h"> + <Filter>src\core\census</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h"> <Filter>src\core\census</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 3820d89968..da7c03bf8f 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -299,6 +299,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\client_config\resolvers\sockaddr_resolver.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\uri_parser.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\compression\algorithm_metadata.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\compression\message_compress.h" /> @@ -388,6 +389,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport_impl.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h" /> <ClInclude Include="$(SolutionDir)\..\third_party\nanopb\pb.h" /> <ClInclude Include="$(SolutionDir)\..\third_party\nanopb\pb_common.h" /> @@ -453,6 +455,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\uri_parser.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\compression\algorithm.c"> @@ -659,6 +663,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\placeholders.c"> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index c4c957d36b..85e2c76580 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -88,6 +88,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.c"> <Filter>src\core\client_config</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.c"> + <Filter>src\core\client_config</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\client_config\uri_parser.c"> <Filter>src\core\client_config</Filter> </ClCompile> @@ -397,6 +400,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> <Filter>src\core\census</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + <Filter>src\core\census</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> <Filter>src\core\census</Filter> </ClCompile> @@ -536,6 +542,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_factory.h"> <Filter>src\core\client_config</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\client_config\subchannel_index.h"> + <Filter>src\core\client_config</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\client_config\uri_parser.h"> <Filter>src\core\client_config</Filter> </ClInclude> @@ -803,6 +812,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h"> <Filter>src\core\census</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h"> + <Filter>src\core\census</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h"> <Filter>src\core\census</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj new file mode 100644 index 0000000000..851086d663 --- /dev/null +++ b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\1.0.204.1.props')" /> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{C27CEE16-2BEC-5572-3956-677E9F6F8BED}</ProjectGuid> + <IgnoreWarnIntDirInTempDetected>true</IgnoreWarnIntDirInTempDetected> + <IntDir>$(SolutionDir)IntDir\$(MSBuildProjectName)\</IntDir> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '10.0'" Label="Configuration"> + <PlatformToolset>v100</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '11.0'" Label="Configuration"> + <PlatformToolset>v110</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '12.0'" Label="Configuration"> + <PlatformToolset>v120</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '14.0'" Label="Configuration"> + <PlatformToolset>v140</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <WholeProgramOptimization>true</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="$(SolutionDir)\..\vsprojects\global.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\openssl.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\winsock.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\zlib.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)'=='Debug'"> + <TargetName>census_log_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Debug</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Debug</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'"> + <TargetName>census_log_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Release</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Release</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\census\log_test.c"> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc_test_util\grpc_test_util.vcxproj"> + <Project>{17BCAFC0-5FDC-4C94-AEB9-95F3E220614B}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc\grpc.vcxproj"> + <Project>{29D16885-7228-4C31-81ED-5F9187C7F2A9}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr_test_util\gpr_test_util.vcxproj"> + <Project>{EAB0A629-17A9-44DB-B5FF-E91A721FE037}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj"> + <Project>{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}</Project> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <None Include="packages.config" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + </ImportGroup> + <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> + <PropertyGroup> + <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> + </PropertyGroup> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" /> + </Target> +</Project> + diff --git a/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters new file mode 100644 index 0000000000..135c77847f --- /dev/null +++ b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\census\log_test.c"> + <Filter>test\core\census</Filter> + </ClCompile> + </ItemGroup> + + <ItemGroup> + <Filter Include="test"> + <UniqueIdentifier>{4d0aae38-6975-cafb-30a6-a7c2c87d22ff}</UniqueIdentifier> + </Filter> + <Filter Include="test\core"> + <UniqueIdentifier>{fb85321f-d3b5-ef2f-c5aa-34660a5e0c7b}</UniqueIdentifier> + </Filter> + <Filter Include="test\core\census"> + <UniqueIdentifier>{f23141da-cbe2-70fa-8207-858af868eb18}</UniqueIdentifier> + </Filter> + </ItemGroup> +</Project> + |