From 7225408dc7fd7af82348f4e123b0906ef411caa6 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 21 Aug 2017 17:05:44 -0700 Subject: Add Z_FINISH flush at end of stream --- src/core/ext/transport/chttp2/transport/writing.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 80eb51ff0d..304f5f199e 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -325,6 +325,19 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( is_last_frame = is_last_data_frame && s->send_trailing_metadata != NULL && grpc_metadata_batch_is_empty(s->send_trailing_metadata); + if (is_last_frame && s->stream_compression_ctx) { + GPR_ASSERT(grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(s->stream_compression_ctx); + s->stream_compression_ctx = NULL; + /* After finish, bytes in s->compressed_data_buffer may be + * more than max_outgoing. Start another round of the current + * while loop so that send_bytes, is_last_data_frame and + * is_last_frame are recalculated. */ + continue; + } grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); @@ -345,6 +358,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); + GPR_ASSERT(s->compressed_data_buffer->length > 0); } } } else { -- cgit v1.2.3 From 1e95aa1e3b79d3094197eb3af8e5613442a18241 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 21 Aug 2017 18:06:02 -0700 Subject: clang-format --- src/core/ext/transport/chttp2/transport/writing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 304f5f199e..50f0d8588c 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -330,7 +330,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); - grpc_stream_compression_context_destroy(s->stream_compression_ctx); + grpc_stream_compression_context_destroy( + s->stream_compression_ctx); s->stream_compression_ctx = NULL; /* After finish, bytes in s->compressed_data_buffer may be * more than max_outgoing. Start another round of the current -- cgit v1.2.3 From b02f1bf6fc3dd1943ab2e4f9e7ed7d9278a0a113 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 10:03:16 -0700 Subject: Minor bug fix --- src/core/ext/transport/chttp2/transport/writing.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 50f0d8588c..e338caf39f 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,10 +322,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - if (is_last_frame && s->stream_compression_ctx) { + if (is_last_data_frame && s->stream_compression_ctx) { GPR_ASSERT(grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, @@ -335,10 +332,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->stream_compression_ctx = NULL; /* After finish, bytes in s->compressed_data_buffer may be * more than max_outgoing. Start another round of the current - * while loop so that send_bytes, is_last_data_frame and - * is_last_frame are recalculated. */ + * while loop so that send_bytes and is_last_data_frame are + * recalculated. */ continue; } + is_last_frame = + is_last_data_frame && s->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty(s->send_trailing_metadata); grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); -- cgit v1.2.3 From 2e0e4e91ed46c5fe39a327ab6484f4531d29788f Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 22 Aug 2017 11:33:15 -0700 Subject: Increase Python mac target build time 30m -> 60m --- tools/run_tests/artifacts/artifact_targets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 841bbdf912..41ab8b9339 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -157,6 +157,7 @@ class PythonArtifact: return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_python.sh'], environ=environ, + timeout_seconds=60*60, use_workspace=True) def __str__(self): -- cgit v1.2.3 From f85caa63829211614a3a252bc993ea10c53c8d2e Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 12:31:54 -0700 Subject: Advance dependency version of Protobuf on ObjC --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler.podspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index b7387c897a..7fc065c60c 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index c3f95f9f42..25c437911f 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.3.0' + v = '3.4.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC -- cgit v1.2.3 From 9d3f3a2a39db7907a29e2e0d86b7ea242dc7d7cc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 17:15:58 -0700 Subject: fix the template --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7fc065c60c..b7387c897a 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.4.0' + s.dependency '!ProtoCompiler', '3.3.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' -- cgit v1.2.3 From c457b47fcf4c5e63a545a60096b9783650586773 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 17:19:21 -0700 Subject: fix the template --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index b7387c897a..7fc065c60c 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index b8223417b7..196c405468 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.3.0' + s.dependency '!ProtoCompiler', '3.4.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' -- cgit v1.2.3 From 2746d84282136cea8e101b392ba53a25ade863cc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 22 Aug 2017 18:19:14 -0700 Subject: Add missing files to podspec --- gRPC-Core.podspec | 1 + templates/gRPC-Core.podspec.template | 1 + 2 files changed, 2 insertions(+) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 77ddc6a33e..083305762a 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -987,6 +987,7 @@ Pod::Spec.new do |s| 'test/core/end2end/end2end_tests.{c,h}', 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', + 'test/core/end2end/fixtures/*.h', 'test/core/end2end/data/*.{c,h}', 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 616f2cbbff..6077f8098d 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -171,6 +171,7 @@ 'test/core/end2end/end2end_tests.{c,h}', 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', + 'test/core/end2end/fixtures/*.h', 'test/core/end2end/data/*.{c,h}', 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', -- cgit v1.2.3 From 103fc3e3f45941ab471de19ff17c7a51a373b80b Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 23 Aug 2017 11:24:47 -0700 Subject: nit fix --- src/core/ext/transport/chttp2/transport/writing.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index e338caf39f..0ae10ee883 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,7 +322,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - if (is_last_data_frame && s->stream_compression_ctx) { + if (is_last_data_frame && s->stream_compression_ctx != NULL) { GPR_ASSERT(grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, -- cgit v1.2.3 From 66999d219274b278c02ffcd3e40c0902e904c6c7 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Wed, 23 Aug 2017 13:51:42 -0700 Subject: fix build.yaml to expand nanopb headers file group --- build.yaml | 2 +- grpc.gemspec | 4 ++++ package.xml | 4 ++++ tools/doxygen/Doxyfile.core.internal | 6 +++++- vsprojects/vcxproj/grpc/grpc.vcxproj | 4 ++++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 ++++++++++++ vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 4 ++++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters | 12 ++++++++++++ 8 files changed, 46 insertions(+), 2 deletions(-) diff --git a/build.yaml b/build.yaml index dba7da1245..905d0e4cd9 100644 --- a/build.yaml +++ b/build.yaml @@ -911,7 +911,7 @@ filegroups: - third_party/nanopb/pb_common.c - third_party/nanopb/pb_decode.c - third_party/nanopb/pb_encode.c - filegroups: + uses: - nanopb_headers - name: nanopb_headers headers: diff --git a/grpc.gemspec b/grpc.gemspec index f04a14167b..9957f050d0 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -369,6 +369,10 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h ) + s.files += %w( third_party/nanopb/pb.h ) + s.files += %w( third_party/nanopb/pb_common.h ) + s.files += %w( third_party/nanopb/pb_decode.h ) + s.files += %w( third_party/nanopb/pb_encode.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h ) diff --git a/package.xml b/package.xml index e21fa6212c..c970a918d2 100644 --- a/package.xml +++ b/package.xml @@ -383,6 +383,10 @@ + + + + diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ea0ebf1704..771e7fc66b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1424,9 +1424,13 @@ src/core/tsi/transport_security_adapter.h \ src/core/tsi/transport_security_grpc.c \ src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ +third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ +third_party/nanopb/pb_common.h \ third_party/nanopb/pb_decode.c \ -third_party/nanopb/pb_encode.c +third_party/nanopb/pb_decode.h \ +third_party/nanopb/pb_encode.c \ +third_party/nanopb/pb_encode.h # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 8a659280a4..33e3901e38 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -494,6 +494,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6879047f55..d5ef822172 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -1454,6 +1454,18 @@ src\core\ext\filters\client_channel\lb_policy\grpclb\proto\grpc\lb\v1 + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + src\core\ext\filters\client_channel\resolver\fake diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 945ca2f3d7..38e8a3a52a 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -462,6 +462,10 @@ + + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index deb3433168..cceb79936c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -1292,6 +1292,18 @@ src\core\ext\filters\client_channel\lb_policy\grpclb\proto\grpc\lb\v1 + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + + + third_party\nanopb + src\core\ext\census -- cgit v1.2.3 From c353643c378cecf96a580eee901a8be3308e18b9 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 23 Aug 2017 14:03:30 -0700 Subject: Log error when stream compression fails --- src/core/ext/transport/chttp2/transport/writing.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 0ae10ee883..b6bf0450ee 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -323,10 +323,12 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); if (is_last_data_frame && s->stream_compression_ctx != NULL) { - GPR_ASSERT(grpc_stream_compress( - s->stream_compression_ctx, &s->flow_controlled_buffer, - s->compressed_data_buffer, NULL, MAX_SIZE_T, - GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + if (!grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)) { + gpr_log(GPR_ERROR, "Stream compression failed."); + } grpc_stream_compression_context_destroy( s->stream_compression_ctx); s->stream_compression_ctx = NULL; -- cgit v1.2.3 From f5f5ed0305d1d4d59b85f9a00786a876be098f1a Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 24 Aug 2017 11:46:03 -0700 Subject: Fix syntax error on classifiers --- setup.py | 2 +- src/python/grpcio_health_checking/setup.py | 2 +- src/python/grpcio_reflection/setup.py | 2 +- tools/distrib/python/grpcio_tools/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 403c13562c..c90ca8652a 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] # Environment variable to determine whether or not the Cython extension should # *use* Cython or use the generated C files. Note that this requires the C files diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 0299b4cca9..1f5e9c5130 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -34,7 +34,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index bed2311b59..9360550afb 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -35,7 +35,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PACKAGE_DIRECTORIES = { '': '.', diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 5c0329bff0..e0e9226211 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -47,7 +47,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', -], +] PY3 = sys.version_info.major == 3 -- cgit v1.2.3 From 5920abc5358e854a0f4a2217bf8be230977d1853 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 24 Aug 2017 12:26:05 -0700 Subject: Revert "Don't clear alarm in jobset when running performance tests" This reverts commit c15d32bbe89a2bf950992ded06d1b3da7f1f39a6. --- tools/run_tests/python_utils/jobset.py | 13 ++++--------- tools/run_tests/run_performance_tests.py | 10 +++++----- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f..044c6f3aa4 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -367,10 +367,9 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time, clear_alarms): + stop_on_failure, add_env, quiet_success, max_time): self._running = set() self._check_cancelled = check_cancelled - self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -474,10 +473,7 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - # Clear the alarms when finished to avoid a race condition causing job - # failures. Don't do this when running multi-VM tests because clearing - # the alarms causes the test to stall - if platform_string() != 'windows' and self._clear_alarms: + if platform_string() != 'windows': signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -507,8 +503,7 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1, - clear_alarms=True): + max_time=-1): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -520,7 +515,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time, clear_alarms) + quiet_success, max_time) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 0db5e4ef83..b9277c919b 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) + [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures @@ -556,7 +556,7 @@ for scenario in scenarios: jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) - scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) -- cgit v1.2.3 From abbeefa8a462530b29f2341bd798c6f3f47f1350 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Thu, 24 Aug 2017 12:28:14 -0700 Subject: PHP: backport master features into v1.6.x branch --- package.xml | 12 +++------ src/php/ext/grpc/call_credentials.c | 2 +- src/php/ext/grpc/channel.c | 40 +++++++++++++++--------------- src/php/ext/grpc/channel_credentials.c | 8 ++++-- src/php/ext/grpc/php_grpc.c | 42 ++++++++++++++++++++++++++------ src/php/ext/grpc/php_grpc.h | 6 +++-- src/php/lib/Grpc/ServerStreamingCall.php | 28 ++++++++++++++++----- src/php/lib/Grpc/UnaryCall.php | 28 ++++++++++++++++----- src/php/tests/unit_tests/ChannelTest.php | 15 ++++++------ templates/package.xml.template | 12 +++------ 10 files changed, 124 insertions(+), 69 deletions(-) diff --git a/package.xml b/package.xml index c970a918d2..ce67849273 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ grpc-packages@google.com yes - 2017-05-22 + 2017-08-24 1.6.0RC1 @@ -22,13 +22,9 @@ Apache 2.0 -- Fixed some memory leaks #9559, #10996 -- Disabled cares dependency from gRPC C Core #10940 -- De-coupled protobuf dependency #11112 -- Fixed extension reported version #10842 -- Added config.w32 for Windows support #8161 -- Fixed PHP distrib test after cc files were added #11193 -- Fixed protoc plugin comment escape bug #11025 +- License changed to Apache 2.0 +- Added support for php_namespace option in codegen plugin #11886 +- Updated gRPC C Core library version 1.6 diff --git a/src/php/ext/grpc/call_credentials.c b/src/php/ext/grpc/call_credentials.c index f46091d709..1eee8645df 100644 --- a/src/php/ext/grpc/call_credentials.c +++ b/src/php/ext/grpc/call_credentials.c @@ -179,7 +179,7 @@ void plugin_get_metadata(void *ptr, grpc_auth_metadata_context context, grpc_metadata_array metadata; bool cleanup = true; - if (Z_TYPE_P(retval) != IS_ARRAY) { + if (retval == NULL || Z_TYPE_P(retval) != IS_ARRAY) { cleanup = false; code = GRPC_STATUS_INVALID_ARGUMENT; } else if (!create_metadata_array(retval, &metadata)) { diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index f1187e8722..dc3acc89bb 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -66,6 +66,8 @@ PHP_GRPC_FREE_WRAPPED_FUNC_START(wrapped_grpc_channel) if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), p->wrapper->key, key_len, rsrc))) { grpc_channel_destroy(p->wrapper->wrapped); + free(p->wrapper->target); + free(p->wrapper->args_hashstr); } gpr_mu_unlock(&global_persistent_list_mu); } @@ -158,7 +160,7 @@ void create_and_add_channel_to_persistent_list( grpc_channel_args args, wrapped_grpc_channel_credentials *creds, char *key, - php_grpc_int key_len) { + php_grpc_int key_len TSRMLS_DC) { php_grpc_zend_resource new_rsrc; channel_persistent_le_t *le; // this links each persistent list entry to a destructor @@ -187,10 +189,8 @@ void create_and_add_channel_to_persistent_list( * credentials. * * If the $args array contains a "force_new" key mapping to a boolean value - * of "true", a new underlying grpc_channel will be created regardless. If - * there are any opened channels on the same hostname, user must manually - * call close() on those dangling channels before the end of the PHP - * script. + * of "true", a new and separate underlying grpc_channel will be created + * and returned. This will not affect existing channels. * * @param string $target The hostname to associate with this channel * @param array $args_array The arguments to pass to the Channel @@ -273,19 +273,15 @@ PHP_METHOD(Channel, __construct) { } channel->wrapper = malloc(sizeof(grpc_channel_wrapper)); channel->wrapper->key = key; - channel->wrapper->target = target; - channel->wrapper->args_hashstr = sha1str; + channel->wrapper->target = strdup(target); + channel->wrapper->args_hashstr = strdup(sha1str); if (creds != NULL && creds->hashstr != NULL) { channel->wrapper->creds_hashstr = creds->hashstr; } gpr_mu_init(&channel->wrapper->mu); smart_str_free(&buf); - if (force_new) { - php_grpc_delete_persistent_list_entry(key, key_len TSRMLS_CC); - } - - if (creds != NULL && creds->has_call_creds) { + if (force_new || (creds != NULL && creds->has_call_creds)) { // If the ChannelCredentials object was composed with a CallCredentials // object, there is no way we can tell them apart. Do NOT persist // them. They should be individually destroyed. @@ -293,7 +289,7 @@ PHP_METHOD(Channel, __construct) { } else if (!(PHP_GRPC_PERSISTENT_LIST_FIND(&EG(persistent_list), key, key_len, rsrc))) { create_and_add_channel_to_persistent_list( - channel, target, args, creds, key, key_len); + channel, target, args, creds, key, key_len TSRMLS_CC); } else { // Found a previously stored channel in the persistent list channel_persistent_le_t *le = (channel_persistent_le_t *)rsrc->ptr; @@ -303,7 +299,7 @@ PHP_METHOD(Channel, __construct) { strcmp(creds->hashstr, le->channel->creds_hashstr) != 0)) { // somehow hash collision create_and_add_channel_to_persistent_list( - channel, target, args, creds, key, key_len); + channel, target, args, creds, key, key_len TSRMLS_CC); } else { channel->wrapper = le->channel; } @@ -416,12 +412,14 @@ PHP_METHOD(Channel, close) { gpr_mu_lock(&channel->wrapper->mu); if (channel->wrapper->wrapped != NULL) { grpc_channel_destroy(channel->wrapper->wrapped); + free(channel->wrapper->target); + free(channel->wrapper->args_hashstr); channel->wrapper->wrapped = NULL; - } - php_grpc_delete_persistent_list_entry(channel->wrapper->key, - strlen(channel->wrapper->key) - TSRMLS_CC); + php_grpc_delete_persistent_list_entry(channel->wrapper->key, + strlen(channel->wrapper->key) + TSRMLS_CC); + } gpr_mu_unlock(&channel->wrapper->mu); } @@ -449,12 +447,11 @@ static void php_grpc_channel_plink_dtor(php_grpc_zend_resource *rsrc gpr_mu_lock(&le->channel->mu); if (le->channel->wrapped != NULL) { grpc_channel_destroy(le->channel->wrapped); - free(le->channel->key); - free(le->channel); + free(le->channel->target); + free(le->channel->args_hashstr); } gpr_mu_unlock(&le->channel->mu); } - free(le); } ZEND_BEGIN_ARG_INFO_EX(arginfo_construct, 0, 0, 2) @@ -496,6 +493,7 @@ GRPC_STARTUP_FUNCTION(channel) { INIT_CLASS_ENTRY(ce, "Grpc\\Channel", channel_methods); ce.create_object = create_wrapped_grpc_channel; grpc_ce_channel = zend_register_internal_class(&ce TSRMLS_CC); + gpr_mu_init(&global_persistent_list_mu); le_plink = zend_register_list_destructors_ex( NULL, php_grpc_channel_plink_dtor, "Persistent Channel", module_number); PHP_GRPC_INIT_HANDLER(wrapped_grpc_channel, channel_ce_handlers); diff --git a/src/php/ext/grpc/channel_credentials.c b/src/php/ext/grpc/channel_credentials.c index 19e1cefb6f..86e4f46c67 100644 --- a/src/php/ext/grpc/channel_credentials.c +++ b/src/php/ext/grpc/channel_credentials.c @@ -135,6 +135,8 @@ PHP_METHOD(ChannelCredentials, createSsl) { pem_key_cert_pair.private_key = pem_key_cert_pair.cert_chain = NULL; + grpc_set_ssl_roots_override_callback(get_ssl_roots_override); + /* "|s!s!s!" == 3 optional nullable strings */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|s!s!s!", &pem_root_certs, &root_certs_length, @@ -148,7 +150,7 @@ PHP_METHOD(ChannelCredentials, createSsl) { } php_grpc_int hashkey_len = root_certs_length + cert_chain_length; - char hashkey[hashkey_len]; + char *hashkey = emalloc(hashkey_len); if (root_certs_length > 0) { strcpy(hashkey, pem_root_certs); } @@ -164,6 +166,7 @@ PHP_METHOD(ChannelCredentials, createSsl) { pem_key_cert_pair.private_key == NULL ? NULL : &pem_key_cert_pair, NULL); zval *creds_object = grpc_php_wrap_channel_credentials(creds, hashstr, false TSRMLS_CC); + efree(hashkey); RETURN_DESTROY_ZVAL(creds_object); } @@ -177,6 +180,8 @@ PHP_METHOD(ChannelCredentials, createComposite) { zval *cred1_obj; zval *cred2_obj; + grpc_set_ssl_roots_override_callback(get_ssl_roots_override); + /* "OO" == 2 Objects */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "OO", &cred1_obj, grpc_ce_channel_credentials, &cred2_obj, @@ -245,7 +250,6 @@ void grpc_init_channel_credentials(TSRMLS_D) { zend_class_entry ce; INIT_CLASS_ENTRY(ce, "Grpc\\ChannelCredentials", channel_credentials_methods); - grpc_set_ssl_roots_override_callback(get_ssl_roots_override); ce.create_object = create_wrapped_grpc_channel_credentials; grpc_ce_channel_credentials = zend_register_internal_class(&ce TSRMLS_CC); PHP_GRPC_INIT_HANDLER(wrapped_grpc_channel_credentials, diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c index a96daf7d9b..4ed56de993 100644 --- a/src/php/ext/grpc/php_grpc.c +++ b/src/php/ext/grpc/php_grpc.c @@ -34,7 +34,8 @@ #include #include "php_grpc.h" -// ZEND_DECLARE_MODULE_GLOBALS(grpc) +ZEND_DECLARE_MODULE_GLOBALS(grpc) +static PHP_GINIT_FUNCTION(grpc); /* {{{ grpc_functions[] * @@ -55,13 +56,17 @@ zend_module_entry grpc_module_entry = { grpc_functions, PHP_MINIT(grpc), PHP_MSHUTDOWN(grpc), - NULL, + PHP_RINIT(grpc), NULL, PHP_MINFO(grpc), #if ZEND_MODULE_API_NO >= 20010901 PHP_GRPC_VERSION, #endif - STANDARD_MODULE_PROPERTIES}; + PHP_MODULE_GLOBALS(grpc), + PHP_GINIT(grpc), + NULL, + NULL, + STANDARD_MODULE_PROPERTIES_EX}; /* }}} */ #ifdef COMPILE_DL_GRPC @@ -99,7 +104,6 @@ PHP_MINIT_FUNCTION(grpc) { REGISTER_INI_ENTRIES(); */ /* Register call error constants */ - grpc_init(); REGISTER_LONG_CONSTANT("Grpc\\CALL_OK", GRPC_CALL_OK, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("Grpc\\CALL_ERROR", GRPC_CALL_ERROR, @@ -227,7 +231,6 @@ PHP_MINIT_FUNCTION(grpc) { grpc_init_channel_credentials(TSRMLS_C); grpc_init_call_credentials(TSRMLS_C); grpc_init_server_credentials(TSRMLS_C); - grpc_php_init_completion_queue(TSRMLS_C); return SUCCESS; } /* }}} */ @@ -240,9 +243,12 @@ PHP_MSHUTDOWN_FUNCTION(grpc) { */ // WARNING: This function IS being called by PHP when the extension // is unloaded but the logs were somehow suppressed. - grpc_shutdown_timeval(TSRMLS_C); - grpc_php_shutdown_completion_queue(TSRMLS_C); - grpc_shutdown(); + if (GRPC_G(initialized)) { + grpc_shutdown_timeval(TSRMLS_C); + grpc_php_shutdown_completion_queue(TSRMLS_C); + grpc_shutdown(); + GRPC_G(initialized) = 0; + } return SUCCESS; } /* }}} */ @@ -259,6 +265,26 @@ PHP_MINFO_FUNCTION(grpc) { */ } /* }}} */ + +/* {{{ PHP_RINIT_FUNCTION + */ +PHP_RINIT_FUNCTION(grpc) { + if (!GRPC_G(initialized)) { + grpc_init(); + grpc_php_init_completion_queue(TSRMLS_C); + GRPC_G(initialized) = 1; + } + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_GINIT_FUNCTION + */ +static PHP_GINIT_FUNCTION(grpc) { + grpc_globals->initialized = 0; +} +/* }}} */ + /* The previous line is meant for vim and emacs, so it can correctly fold and unfold functions in source code. See the corresponding marks just before function definition, where the functions purpose is also documented. Please diff --git a/src/php/ext/grpc/php_grpc.h b/src/php/ext/grpc/php_grpc.h index 32329178dc..e30f011c39 100644 --- a/src/php/ext/grpc/php_grpc.h +++ b/src/php/ext/grpc/php_grpc.h @@ -49,14 +49,16 @@ PHP_MINIT_FUNCTION(grpc); PHP_MSHUTDOWN_FUNCTION(grpc); /* Displays information about the module */ PHP_MINFO_FUNCTION(grpc); +/* Code that runs at request start */ +PHP_RINIT_FUNCTION(grpc); /* Declare any global variables you may need between the BEGIN and END macros here: - +*/ ZEND_BEGIN_MODULE_GLOBALS(grpc) + zend_bool initialized; ZEND_END_MODULE_GLOBALS(grpc) -*/ /* In every utility function you add that needs to use variables in php_grpc_globals, call TSRMLS_FETCH(); after declaring other diff --git a/src/php/lib/Grpc/ServerStreamingCall.php b/src/php/lib/Grpc/ServerStreamingCall.php index 269a99ab59..f8fddfea01 100644 --- a/src/php/lib/Grpc/ServerStreamingCall.php +++ b/src/php/lib/Grpc/ServerStreamingCall.php @@ -40,13 +40,11 @@ class ServerStreamingCall extends AbstractCall if (array_key_exists('flags', $options)) { $message_array['flags'] = $options['flags']; } - $event = $this->call->startBatch([ + $this->call->startBatch([ OP_SEND_INITIAL_METADATA => $metadata, - OP_RECV_INITIAL_METADATA => true, OP_SEND_MESSAGE => $message_array, OP_SEND_CLOSE_FROM_CLIENT => true, ]); - $this->metadata = $event->metadata; } /** @@ -54,9 +52,15 @@ class ServerStreamingCall extends AbstractCall */ public function responses() { - $response = $this->call->startBatch([ - OP_RECV_MESSAGE => true, - ])->message; + $batch = [OP_RECV_MESSAGE => true]; + if ($this->metadata === null) { + $batch[OP_RECV_INITIAL_METADATA] = true; + } + $read_event = $this->call->startBatch($batch); + if ($this->metadata === null) { + $this->metadata = $read_event->metadata; + } + $response = $read_event->message; while ($response !== null) { yield $this->_deserializeResponse($response); $response = $this->call->startBatch([ @@ -81,4 +85,16 @@ class ServerStreamingCall extends AbstractCall return $status_event->status; } + + /** + * @return mixed The metadata sent by the server + */ + public function getMetadata() + { + if ($this->metadata === null) { + $event = $this->call->startBatch([OP_RECV_INITIAL_METADATA => true]); + $this->metadata = $event->metadata; + } + return $this->metadata; + } } diff --git a/src/php/lib/Grpc/UnaryCall.php b/src/php/lib/Grpc/UnaryCall.php index dd55d02165..e8496492b8 100644 --- a/src/php/lib/Grpc/UnaryCall.php +++ b/src/php/lib/Grpc/UnaryCall.php @@ -40,13 +40,11 @@ class UnaryCall extends AbstractCall if (isset($options['flags'])) { $message_array['flags'] = $options['flags']; } - $event = $this->call->startBatch([ + $this->call->startBatch([ OP_SEND_INITIAL_METADATA => $metadata, - OP_RECV_INITIAL_METADATA => true, OP_SEND_MESSAGE => $message_array, OP_SEND_CLOSE_FROM_CLIENT => true, ]); - $this->metadata = $event->metadata; } /** @@ -56,14 +54,32 @@ class UnaryCall extends AbstractCall */ public function wait() { - $event = $this->call->startBatch([ + $batch = [ OP_RECV_MESSAGE => true, OP_RECV_STATUS_ON_CLIENT => true, - ]); - + ]; + if ($this->metadata === null) { + $batch[OP_RECV_INITIAL_METADATA] = true; + } + $event = $this->call->startBatch($batch); + if ($this->metadata === null) { + $this->metadata = $event->metadata; + } $status = $event->status; $this->trailing_metadata = $status->metadata; return [$this->_deserializeResponse($event->message), $status]; } + + /** + * @return mixed The metadata sent by the server + */ + public function getMetadata() + { + if ($this->metadata === null) { + $event = $this->call->startBatch([OP_RECV_INITIAL_METADATA => true]); + $this->metadata = $event->metadata; + } + return $this->metadata; + } } diff --git a/src/php/tests/unit_tests/ChannelTest.php b/src/php/tests/unit_tests/ChannelTest.php index 400df0fb66..c375a16269 100644 --- a/src/php/tests/unit_tests/ChannelTest.php +++ b/src/php/tests/unit_tests/ChannelTest.php @@ -517,8 +517,6 @@ class ChannelTest extends PHPUnit_Framework_TestCase $state = $this->channel2->getConnectivityState(); $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - // any dangling old connection to the same host must be - // manually closed $this->channel1->close(); $this->channel2->close(); } @@ -529,6 +527,7 @@ class ChannelTest extends PHPUnit_Framework_TestCase $this->channel1 = new Grpc\Channel('localhost:1', []); $this->channel2 = new Grpc\Channel('localhost:1', ["force_new" => true]); + // channel3 shares with channel1 $this->channel3 = new Grpc\Channel('localhost:1', []); // try to connect on channel2 @@ -540,29 +539,31 @@ class ChannelTest extends PHPUnit_Framework_TestCase $state = $this->channel2->getConnectivityState(); $this->assertConnecting($state); $state = $this->channel3->getConnectivityState(); - $this->assertConnecting($state); + $this->assertEquals(GRPC\CHANNEL_IDLE, $state); $this->channel1->close(); $this->channel2->close(); } + /** + * @expectedException RuntimeException + */ public function testPersistentChannelForceNewOldChannelClose() { $this->channel1 = new Grpc\Channel('localhost:1', []); $this->channel2 = new Grpc\Channel('localhost:1', ["force_new" => true]); + // channel3 shares with channel1 $this->channel3 = new Grpc\Channel('localhost:1', []); $this->channel1->close(); $state = $this->channel2->getConnectivityState(); $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - $state = $this->channel3->getConnectivityState(); - $this->assertEquals(GRPC\CHANNEL_IDLE, $state); - $this->channel2->close(); - $this->channel3->close(); + // channel3 already closed + $state = $this->channel3->getConnectivityState(); } public function testPersistentChannelForceNewNewChannelClose() diff --git a/templates/package.xml.template b/templates/package.xml.template index 27cc4d01a9..a378b58e0c 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ grpc-packages@google.com yes - 2017-05-22 + 2017-08-24 ${settings.php_version.php()} @@ -24,13 +24,9 @@ Apache 2.0 - - Fixed some memory leaks #9559, #10996 - - Disabled cares dependency from gRPC C Core #10940 - - De-coupled protobuf dependency #11112 - - Fixed extension reported version #10842 - - Added config.w32 for Windows support #8161 - - Fixed PHP distrib test after cc files were added #11193 - - Fixed protoc plugin comment escape bug #11025 + - License changed to Apache 2.0 + - Added support for php_namespace option in codegen plugin #11886 + - Updated gRPC C Core library version 1.6 -- cgit v1.2.3 From 3da8c5defbc9a83d3db0bf84fb0dce012802d9b3 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 24 Aug 2017 12:35:04 -0700 Subject: Let alarms at end of jobset.py trigger isntead of clearing --- tools/run_tests/python_utils/jobset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 044c6f3aa4..50fe7d7df8 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -473,8 +473,10 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - if platform_string() != 'windows': - signal.alarm(0) + global have_alarm + if platform_string() != 'windows' and have_alarm: + signal.alarm(1) + signal.pause() return not self.cancelled() and self._failures == 0 -- cgit v1.2.3 From dfde2881487f05ec4e528c40510029833ed5c8c9 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 29 Aug 2017 11:31:50 -0700 Subject: 1.6.0-pre1 -> 1.6.0 --- BUILD | 2 +- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 30 files changed, 34 insertions(+), 34 deletions(-) diff --git a/BUILD b/BUILD index 26a8af9fc2..4d1121a79d 100644 --- a/BUILD +++ b/BUILD @@ -38,7 +38,7 @@ g_stands_for = "garcia" core_version = "4.0.0" -version = "1.6.0-pre1" +version = "1.6.0" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fea507987..ebbaa6a9ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.6.0-pre1") +set(PACKAGE_VERSION "1.6.0") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index b47fd222d5..801f8a7d6e 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,8 @@ Q = @ endif CORE_VERSION = 4.0.0 -CPP_VERSION = 1.6.0-pre1 -CSHARP_VERSION = 1.6.0-pre1 +CPP_VERSION = 1.6.0 +CSHARP_VERSION = 1.6.0 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 905d0e4cd9..723c8fe446 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 4.0.0 g_stands_for: garcia - version: 1.6.0-pre1 + version: 1.6.0 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 083305762a..222d3e4f0f 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index e15d4d9bd7..2d2f31d323 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 50c3f36d62..38a159404f 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index b8a4d92f05..12591cda33 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.6.0-pre1' + version = '1.6.0' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.json b/package.json index 4dd51237ce..94e6391a8d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", diff --git a/package.xml b/package.xml index ce67849273..30018620dd 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-08-24 - 1.6.0RC1 - 1.6.0RC1 + 1.6.0 + 1.6.0 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index de30303665..c65185c77e 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.6.0-pre1"; } +grpc::string Version() { return "1.6.0"; } } diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 178b036953..23f6e40909 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.6.0-pre1 + 1.6.0 3.3.0 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 619955423b..d402c323c1 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -38,6 +38,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.6.0-pre1"; + public const string CurrentVersion = "1.6.0"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index f47f8daa4b..b7aa22b292 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.6.0-pre1 +set VERSION=1.6.0 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 125b1dfdd2..455b930525 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.6.0-pre1" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.6.0-pre1" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 30e6df6b38..66ab2a292f 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.6.0-pre1", + "grpc": "^1.6.0", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 2a5c172353..b0b1e8679a 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.6.0-pre1", + "version": "1.6.0", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 7fc065c60c..0e180a4711 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.6.0-pre1' + v = '1.6.0' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index bd02990391..3521537578 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.6.0-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.6.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 7a20adf8c3..1a46570117 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.6.0RC1" +#define PHP_GRPC_VERSION "1.6.0" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 4be88e1a4f..28f5a1a8a2 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.6.0rc1""" +__version__ = """1.6.0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 76bf737cfe..ce35bda91c 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 75b0e47da1..fb5c5f5f65 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 4194086f40..73e7d2e497 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index ca09db0bf0..99510bbe09 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index fedb824c54..66834d20e1 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.6.0.pre1' + VERSION = '1.6.0' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 4de06c9db6..73fa15f85f 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.6.0.pre1' + VERSION = '1.6.0' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f1ad853700..db3b39ae68 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.6.0rc1' +VERSION='1.6.0' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 478a8da685..f9225688ad 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0-pre1 +PROJECT_NUMBER = 1.6.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 3756dc92ac..1b41266c71 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0-pre1 +PROJECT_NUMBER = 1.6.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a -- cgit v1.2.3 From 738b1bb424292b4138e82a3c2085ab4a3a95507e Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 29 Aug 2017 12:45:53 -0700 Subject: Get rid of have_alarm var in jobset.py --- tools/run_tests/python_utils/jobset.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 50fe7d7df8..6151a7276a 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,10 +71,8 @@ def platform_string(): if platform_string() == 'windows': pass else: - have_alarm = False def alarm_handler(unused_signum, unused_frame): - global have_alarm - have_alarm = False + pass signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -454,10 +452,7 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - global have_alarm - if not have_alarm: - have_alarm = True - signal.alarm(10) + signal.alarm(10) signal.pause() def cancelled(self): @@ -473,10 +468,8 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - global have_alarm - if platform_string() != 'windows' and have_alarm: - signal.alarm(1) - signal.pause() + if platform_string() != 'windows': + signal.alarm(0) return not self.cancelled() and self._failures == 0 -- cgit v1.2.3 From f282c8f525788e6f15331bb6c94471012a7e5346 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 30 Aug 2017 14:40:15 -0700 Subject: Surface call deadline to Objective C API --- src/objective-c/GRPCClient/GRPCCall.h | 5 +++++ src/objective-c/GRPCClient/GRPCCall.m | 3 ++- src/objective-c/GRPCClient/private/GRPCChannel.h | 1 + src/objective-c/GRPCClient/private/GRPCChannel.m | 8 +++++++- src/objective-c/GRPCClient/private/GRPCHost.h | 1 + src/objective-c/GRPCClient/private/GRPCHost.m | 6 +++++- .../GRPCClient/private/GRPCWrappedCall.h | 3 ++- .../GRPCClient/private/GRPCWrappedCall.m | 10 +++++++--- src/objective-c/tests/GRPCClientTests.m | 21 +++++++++++++++++++++ 9 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 4d90cfd384..bcf4ae9cc7 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -169,6 +169,11 @@ extern id const kGRPCTrailersKey; */ @property (atomic, copy, readwrite) NSString *serverName; +/** + * The deadline for the RPC call in milliseconds. If set to 0, the deadline will be infinity. + */ +@property UInt64 deadline; + /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of * NSMutableDictionary's interface. It will become a NSMutableDictionary later on. diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 436c19e354..0f8f93cdae 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -423,7 +423,8 @@ static NSString * const kBearerPrefix = @"Bearer "; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host serverName:_serverName - path:_path]; + path:_path + deadline:_deadline]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); [self sendHeaders:_requestHeaders]; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index e2aa5bd036..67d78c3a44 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,5 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName + deadline:(UInt64)deadline completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 52dbc70b99..72dc961d09 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,18 +182,24 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); } grpc_slice path_slice = grpc_slice_from_copied_string(path.UTF8String); + gpr_timespec deadline_ms = deadline == 0 ? + gpr_inf_future(GPR_CLOCK_REALTIME) : + gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(deadline, GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, path_slice, serverName ? &host_slice : NULL, - gpr_inf_future(GPR_CLOCK_REALTIME), NULL); + deadline_ms, NULL); if (serverName) { grpc_slice_unref(host_slice); } diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 0c1d715240..7513cd0b8d 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,6 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 23794c1fed..b847637ac5 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,6 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName + deadline:(UInt64)deadline completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. @@ -130,7 +131,10 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } channel = _channel; } - return [channel unmanagedCallWithPath:path serverName:serverName completionQueue:queue]; + return [channel unmanagedCallWithPath:path + serverName:serverName + deadline:deadline + completionQueue:queue]; } - (BOOL)setTLSPEMRootCerts:(nullable NSString *)pemRootCerts diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 64075591a3..1995f1d0b5 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -76,7 +76,8 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path NS_DESIGNATED_INITIALIZER; + path:(NSString *)path + deadline:(UInt64)deadline NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index 87dc33af88..b855a96bee 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -238,12 +238,13 @@ } - (instancetype)init { - return [self initWithHost:nil serverName:nil path:nil]; + return [self initWithHost:nil serverName:nil path:nil deadline:0]; } - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName - path:(NSString *)path { + path:(NSString *)path + deadline:(UInt64)deadline { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; @@ -255,7 +256,10 @@ // queue. Currently we use a singleton queue. _queue = [GRPCCompletionQueue completionQueue]; - _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path serverName:serverName completionQueue:_queue]; + _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path + serverName:serverName + deadline:deadline + completionQueue:_queue]; if (_call == NULL) { return nil; } diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 9afe507121..d9c6932ea8 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -422,4 +422,25 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testDeadline { + __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress + path:kEmptyCallMethod.HTTPPath + requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + + id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTAssert(0, @"Failure: response received; Expect: no response received."); + } completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Failure: no error received; Expect: receive deadline exceeded."); + XCTAssertEqual(errorOrNil.code, GRPCErrorCodeDeadlineExceeded); + [completion fulfill]; + }]; + + call.deadline = 1; + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end -- cgit v1.2.3 From 1ebe1e89155159a933ffadc58a07d8aa5e49b844 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 30 Aug 2017 14:52:57 -0700 Subject: PHP: fix c++ error after c core 1.6 uptake --- config.m4 | 2 +- templates/config.m4.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.m4 b/config.m4 index f6f8531b2f..393250fff1 100644 --- a/config.m4 +++ b/config.m4 @@ -12,7 +12,7 @@ if test "$PHP_GRPC" != "no"; then LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) diff --git a/templates/config.m4.template b/templates/config.m4.template index f91893c2bd..cd93fbd0fb 100644 --- a/templates/config.m4.template +++ b/templates/config.m4.template @@ -14,7 +14,7 @@ LIBS="-lpthread $LIBS" CFLAGS="-Wall -Werror -Wno-parentheses-equality -Wno-unused-value -std=c11" - CXXFLAGS="-std=c++11" + CXXFLAGS="-std=c++11 -fno-exceptions -fno-rtti" GRPC_SHARED_LIBADD="-lpthread $GRPC_SHARED_LIBADD" PHP_REQUIRE_CXX() PHP_ADD_LIBRARY(pthread) -- cgit v1.2.3 From c7c8e3cb1712438c64b1212911b6c22165982426 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 30 Aug 2017 15:36:47 -0700 Subject: Rename deadline to timeout --- src/objective-c/GRPCClient/GRPCCall.h | 4 ++-- src/objective-c/GRPCClient/GRPCCall.m | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.m | 12 ++++++------ src/objective-c/GRPCClient/private/GRPCHost.h | 2 +- src/objective-c/GRPCClient/private/GRPCHost.m | 4 ++-- src/objective-c/GRPCClient/private/GRPCWrappedCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.m | 6 +++--- src/objective-c/tests/GRPCClientTests.m | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index bcf4ae9cc7..26c920b973 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,9 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The deadline for the RPC call in milliseconds. If set to 0, the deadline will be infinity. + * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. */ -@property UInt64 deadline; +@property UInt64 timeout; /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 0f8f93cdae..d6c3a3c165 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -424,7 +424,7 @@ static NSString * const kBearerPrefix = @"Bearer "; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host serverName:_serverName path:_path - deadline:_deadline]; + timeout:_timeout]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); [self sendHeaders:_requestHeaders]; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 67d78c3a44..9dff86c271 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,6 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 72dc961d09..39b4e6c1a7 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,18 +182,18 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); } grpc_slice path_slice = grpc_slice_from_copied_string(path.UTF8String); - gpr_timespec deadline_ms = deadline == 0 ? - gpr_inf_future(GPR_CLOCK_REALTIME) : - gpr_time_add( - gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_millis(deadline, GPR_TIMESPAN)); + gpr_timespec deadline_ms = timeout == 0 ? + gpr_inf_future(GPR_CLOCK_REALTIME) : + gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_millis(timeout, GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index 7513cd0b8d..a9b3032c9c 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,7 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index b847637ac5..a0791827db 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,7 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - deadline:(UInt64)deadline + timeout:(UInt64)timeout completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. @@ -133,7 +133,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; } return [channel unmanagedCallWithPath:path serverName:serverName - deadline:deadline + timeout:timeout completionQueue:queue]; } diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index 1995f1d0b5..c0a6c71c72 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -77,7 +77,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - deadline:(UInt64)deadline NS_DESIGNATED_INITIALIZER; + timeout:(UInt64)timeout NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index b855a96bee..7366f31931 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -238,13 +238,13 @@ } - (instancetype)init { - return [self initWithHost:nil serverName:nil path:nil deadline:0]; + return [self initWithHost:nil serverName:nil path:nil timeout:0]; } - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - deadline:(UInt64)deadline { + timeout:(UInt64)timeout { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; @@ -258,7 +258,7 @@ _call = [[GRPCHost hostWithAddress:host] unmanagedCallWithPath:path serverName:serverName - deadline:deadline + timeout:timeout completionQueue:_queue]; if (_call == NULL) { return nil; diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d9c6932ea8..d050319533 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -422,7 +422,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -- (void)testDeadline { +- (void)testTimeout { __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress @@ -437,7 +437,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [completion fulfill]; }]; - call.deadline = 1; + call.timeout = 1; [call startWithWriteable:responsesWriteable]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; -- cgit v1.2.3 From 9c437c2067e9e0142ad5799696388c2acb25f3d5 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 31 Aug 2017 11:23:03 +0200 Subject: Use docfx to generate C# API reference --- src/csharp/doc/.gitignore | 2 + src/csharp/doc/README.md | 9 +++- src/csharp/doc/docfx.json | 37 +++++++++++++ src/csharp/doc/grpc_csharp_public.shfbproj | 83 ------------------------------ src/csharp/doc/toc.yml | 3 ++ 5 files changed, 50 insertions(+), 84 deletions(-) create mode 100644 src/csharp/doc/.gitignore create mode 100644 src/csharp/doc/docfx.json delete mode 100644 src/csharp/doc/grpc_csharp_public.shfbproj create mode 100644 src/csharp/doc/toc.yml diff --git a/src/csharp/doc/.gitignore b/src/csharp/doc/.gitignore new file mode 100644 index 0000000000..09ee235efc --- /dev/null +++ b/src/csharp/doc/.gitignore @@ -0,0 +1,2 @@ +html +obj diff --git a/src/csharp/doc/README.md b/src/csharp/doc/README.md index 585500b5ca..46cce013a1 100644 --- a/src/csharp/doc/README.md +++ b/src/csharp/doc/README.md @@ -1,2 +1,9 @@ +DocFX-generated C# API Reference +-------------------------------- -SandCastle project files to generate HTML reference documentation. \ No newline at end of file +Install docfx based on instructions here: https://github.com/dotnet/docfx + +``` +# generate docfx documentation into ./html directory +$ docfx +``` diff --git a/src/csharp/doc/docfx.json b/src/csharp/doc/docfx.json new file mode 100644 index 0000000000..7219d0e7a6 --- /dev/null +++ b/src/csharp/doc/docfx.json @@ -0,0 +1,37 @@ +{ + "metadata": [ + { + "src": [ + { + "files": ["Grpc.Core/Grpc.Core.csproj", + "Grpc.Auth/Grpc.Auth.csproj", + "Grpc.Core.Testing/Grpc.Core.Testing.csproj", + "Grpc.HealthCheck/Grpc.HealthCheck.csproj", + "Grpc.Reflection/Grpc.HealthCheck.csproj"], + "exclude": [ "**/bin/**", "**/obj/**" ], + "cwd": ".." + } + ], + "properties": { "TargetFramework": "net45" }, + "dest": "obj/api" + } + ], + "build": { + "content": [ + { + "files": [ "**/*.yml" ], + "cwd": "obj/api", + "dest": "api" + }, + { + "files": [ "toc.yml"], + } + ], + "globalMetadata": { + "_appTitle": "gRPC C#", + "_enableSearch": true, + "_disableContribution": true + }, + "dest": "html" + } +} diff --git a/src/csharp/doc/grpc_csharp_public.shfbproj b/src/csharp/doc/grpc_csharp_public.shfbproj deleted file mode 100644 index fab953da35..0000000000 --- a/src/csharp/doc/grpc_csharp_public.shfbproj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - Debug - AnyCPU - 2.0 - {77e3da09-fc92-486f-a90a-99ca788e8b59} - 2015.6.5.0 - - Documentation - Documentation - Documentation - - .NET Framework 4.5 - ..\..\..\doc\ref\csharp\html - en-US - - - - - - - OnlyWarningsAndErrors - Website - False - True - False - True - 1.0.0.0 - 2 - False - Standard - Blank - True - VS2013 - False - MemberName - gRPC C# - AboveNamespaces - Documentation - - Provides OAuth2 based authentication for gRPC. <c>Grpc.Auth</c> currently consists of a set of very lightweight wrappers and uses C# <a href="https://www.nuget.org/packages/Google.Apis.Auth/">Google.Apis.Auth</a> library. - Main namespace for gRPC C# functionality. Contains concepts representing both client side and server side gRPC logic. - -<seealso cref="Grpc.Core.Channel"/> -<seealso cref="Grpc.Core.Server"/> - Provides functionality to redirect gRPC logs to application-specified destination. - Various utilities for gRPC C#. - - Summary, Parameter, AutoDocumentCtors, Namespace, TypeParameter, AutoDocumentDispose - - - - - - - - - - - - - - - - - - - - - - - - - - - OnBuildSuccess - - \ No newline at end of file diff --git a/src/csharp/doc/toc.yml b/src/csharp/doc/toc.yml new file mode 100644 index 0000000000..c3a1e415ab --- /dev/null +++ b/src/csharp/doc/toc.yml @@ -0,0 +1,3 @@ +- name: API Documentation + href: obj/api/ + homepage: obj/api/Grpc.Core.yml -- cgit v1.2.3 From 76e0c1ddd536342bdc5059209da38d5406c7e717 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 31 Aug 2017 14:59:17 -0700 Subject: Guarantee that Z_FINISH is only applied at the end of stream --- src/core/ext/transport/chttp2/transport/writing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b6bf0450ee..410c154206 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -322,7 +322,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( (send_bytes == s->compressed_data_buffer->length && s->flow_controlled_buffer.length == 0 && s->fetching_send_message == NULL); - if (is_last_data_frame && s->stream_compression_ctx != NULL) { + if (is_last_data_frame && s->send_trailing_metadata != NULL && + s->stream_compression_ctx != NULL) { if (!grpc_stream_compress( s->stream_compression_ctx, &s->flow_controlled_buffer, s->compressed_data_buffer, NULL, MAX_SIZE_T, -- cgit v1.2.3 From 0ff222a23b94a34d03d1cfa4b5c148406e441f4c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 1 Sep 2017 09:41:43 -0700 Subject: Add pollset_kick stats --- src/core/lib/debug/stats_data.c | 243 ++++++++++++++++++++- src/core/lib/debug/stats_data.h | 128 ++++++++--- src/core/lib/debug/stats_data.yaml | 12 + src/core/lib/iomgr/ev_epoll1_linux.c | 25 ++- .../lib/iomgr/ev_epoll_limited_pollers_linux.c | 5 +- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 5 +- src/core/lib/iomgr/ev_epollex_linux.c | 9 +- src/core/lib/iomgr/ev_epollsig_linux.c | 5 +- src/core/lib/iomgr/ev_poll_posix.c | 46 ++-- src/core/lib/iomgr/ev_posix.c | 4 +- src/core/lib/iomgr/ev_posix.h | 2 +- src/core/lib/iomgr/pollset.h | 2 +- .../google_default/google_default_credentials.c | 3 +- src/core/lib/surface/call.c | 5 +- src/core/lib/surface/completion_queue.c | 11 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/endpoint_tests.c | 9 +- test/core/iomgr/fd_posix_test.c | 16 +- test/core/iomgr/resolve_address_test.c | 6 +- test/core/iomgr/tcp_client_posix_test.c | 6 +- test/core/iomgr/tcp_posix_test.c | 11 +- test/core/iomgr/tcp_server_posix_test.c | 4 +- test/core/iomgr/udp_server_test.c | 8 +- test/core/security/oauth2_utils.c | 3 +- .../security/print_google_default_creds_token.c | 3 +- test/core/security/verify_jwt.c | 3 +- test/core/surface/concurrent_connectivity_test.c | 3 +- test/core/util/port_server_client.c | 9 +- test/cpp/qps/report.cc | 4 +- 30 files changed, 477 insertions(+), 117 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 9277ee57b2..445dcb67a3 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -24,6 +24,12 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "server_calls_created", "syscall_poll", "syscall_wait", + "pollset_kick", + "pollset_kicked_without_poller", + "pollset_kicked_again", + "pollset_kick_wakeup_fd", + "pollset_kick_wakeup_cv", + "pollset_kick_own_thread", "histogram_slow_lookups", "syscall_write", "syscall_read", @@ -47,10 +53,226 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_queue_drained", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { - "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", - "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "call_initial_size", "poll_events_returned", "tcp_write_size", + "tcp_write_iov_size", "tcp_read_size", "tcp_read_offer", + "tcp_read_iov_size", "http2_send_message_size", }; const double grpc_stats_table_0[64] = {0, + 1, + 2, + 3, + 4, + 5, + 6.03034139457, + 7.27300346702, + 8.77173877401, + 10.5793158863, + 12.759377303, + 15.3886802239, + 18.5597990724, + 22.3843849243, + 26.9970966002, + 32.5603418323, + 39.2699954346, + 47.3622958068, + 57.1221625891, + 68.8932283217, + 83.0899373109, + 100.212137688, + 120.862680427, + 145.768644968, + 175.806938756, + 212.035172047, + 255.72889502, + 308.426508286, + 371.98342802, + 448.637412817, + 541.087352333, + 652.588291771, + 787.066037897, + 949.255381718, + 1144.86680448, + 1380.78753647, + 1665.32404765, + 2008.49450799, + 2422.38151446, + 2921.55750402, + 3523.59783062, + 4249.69957117, + 5125.4278477, + 6181.61594298, + 7455.45090126, + 8991.78283702, + 10844.7040506, + 13079.4535497, + 15774.7140318, + 19025.3822027, + 22945.9099689, + 27674.3341444, + 33377.1365516, + 40255.1056359, + 48550.4059718, + 58555.104571, + 70621.4541917, + 85174.2957114, + 102726.016236, + 123894.589602, + 149425.334448, + 180217.155944, + 217354.195101, + 262144.0}; +const uint8_t grpc_stats_table_1[124] = { + 0, 2, 2, 4, 4, 6, 6, 8, 8, 11, 11, 11, 13, 13, + 15, 15, 17, 17, 20, 20, 20, 21, 23, 23, 26, 26, 26, 28, + 28, 30, 30, 32, 32, 35, 35, 35, 37, 37, 38, 41, 41, 41, + 43, 43, 45, 45, 47, 47, 50, 50, 50, 52, 52, 54, 54, 56, + 56, 58, 58, 60, 60, 62, 62, 65, 65, 65, 67, 67, 69, 69, + 71, 71, 73, 73, 76, 76, 76, 78, 78, 80, 80, 82, 82, 84, + 84, 86, 86, 88, 88, 91, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 101, 103, 103, 106, 106, 106, 108, 108, 110, 110, 112, + 112, 115, 115, 115, 117, 117, 118, 121, 121, 121, 124, 124}; +const double grpc_stats_table_2[128] = {0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29.0367057593, + 30.1117957626, + 31.2266911945, + 32.382865859, + 33.5818481283, + 34.8252229627, + 36.1146340061, + 37.4517857586, + 38.8384458298, + 40.2764472753, + 41.7676910202, + 43.3141483714, + 44.9178636242, + 46.580956764, + 48.3056262695, + 50.0941520181, + 51.9488983005, + 53.8723169454, + 55.8669505614, + 57.9354358972, + 60.0805073281, + 62.3050004699, + 64.6118559278, + 67.0041231836, + 69.4849646266, + 72.0576597342, + 74.7256094075, + 77.4923404661, + 80.3615103113, + 83.3369117602, + 86.4224780597, + 89.6222880862, + 92.940571737, + 96.3817155226, + 99.9502683646, + 103.65094761, + 107.488645265, + 111.468434465, + 115.595576179, + 119.875526164, + 124.313942178, + 128.916691457, + 133.689858475, + 138.639752984, + 143.772918355, + 149.09614023, + 154.616455489, + 160.341161557, + 166.277826044, + 172.434296755, + 178.818712061, + 185.439511656, + 192.305447719, + 199.425596476, + 206.809370205, + 214.466529676, + 222.407197051, + 230.641869269, + 239.181431919, + 248.037173633, + 257.220801006, + 266.744454071, + 276.62072235, + 286.862661493, + 297.48381054, + 308.498209814, + 319.920419488, + 331.765538824, + 344.04922614, + 356.787719506, + 369.997858208, + 383.697105013, + 397.903569249, + 412.636030746, + 427.913964659, + 443.757567219, + 460.187782422, + 477.226329722, + 494.895732741, + 513.219349041, + 532.221401003, + 551.927007848, + 572.36221884, + 593.554047722, + 615.530508428, + 638.320652111, + 661.954605552, + 686.46361098, + 711.880067376, + 738.237573297, + 765.570971297, + 793.91639398, + 823.311311768, + 853.794582433, + 885.406502465, + 918.188860339, + 952.184991756, + 987.439836931, + 1024.0}; +const uint8_t grpc_stats_table_3[166] = { + 0, 2, 2, 4, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, + 14, 15, 17, 17, 18, 20, 20, 22, 22, 24, 24, 25, 27, 27, + 29, 29, 31, 31, 34, 34, 34, 36, 36, 38, 38, 39, 40, 42, + 42, 43, 44, 46, 46, 47, 49, 49, 50, 52, 52, 54, 54, 55, + 57, 57, 59, 59, 61, 61, 63, 63, 65, 65, 68, 68, 68, 70, + 70, 71, 72, 73, 75, 75, 76, 78, 78, 79, 81, 81, 82, 84, + 84, 86, 86, 87, 89, 89, 91, 91, 93, 93, 95, 95, 97, 97, + 100, 100, 100, 102, 102, 103, 104, 105, 107, 107, 108, 109, 111, 111, + 113, 113, 114, 116, 116, 117, 119, 119, 121, 121, 123, 123, 125, 125, + 127, 127, 129, 129, 131, 131, 134, 134, 134, 135, 136, 137, 139, 139, + 140, 141, 143, 143, 144, 146, 146, 148, 148, 149, 151, 151, 153, 153, + 155, 155, 157, 157, 159, 159, 161, 161, 163, 163, 166, 166}; +const double grpc_stats_table_4[64] = {0, 1, 2, 3, @@ -114,13 +336,13 @@ const double grpc_stats_table_0[64] = {0, 10005134.9318, 12956014.428, 16777216.0}; -const uint8_t grpc_stats_table_1[87] = { +const uint8_t grpc_stats_table_5[87] = { 0, 1, 3, 3, 4, 6, 6, 7, 9, 9, 10, 12, 12, 13, 15, 15, 16, 18, 18, 19, 21, 21, 22, 24, 24, 25, 27, 27, 28, 30, 30, 31, 32, 34, 34, 36, 36, 37, 39, 39, 40, 42, 42, 43, 44, 46, 46, 47, 49, 49, 51, 51, 52, 53, 55, 55, 56, 58, 58, 59, 61, 61, 63, 63, 64, 65, 67, 67, 68, 70, 70, 71, 73, 73, 75, 75, 76, 77, 79, 79, 80, 82, 82, 83, 85, 85, 87}; -const double grpc_stats_table_2[64] = {0, +const double grpc_stats_table_6[64] = {0, 1, 2, 3, @@ -184,12 +406,13 @@ const double grpc_stats_table_2[64] = {0, 860.147148411, 938.504491184, 1024.0}; -const uint8_t grpc_stats_table_3[52] = { +const uint8_t grpc_stats_table_7[52] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 52}; -const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320}; -const double *const grpc_stats_histo_bucket_boundaries[6] = { - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0}; +const int grpc_stats_histo_buckets[8] = {64, 128, 64, 64, 64, 64, 64, 64}; +const int grpc_stats_histo_start[8] = {0, 64, 192, 256, 320, 384, 448, 512}; +const double *const grpc_stats_histo_bucket_boundaries[8] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4, + grpc_stats_table_6, grpc_stats_table_4}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 4d1078dfdb..17e5f22e6c 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -28,6 +28,12 @@ typedef enum { GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, + GRPC_STATS_COUNTER_POLLSET_KICK, + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER, + GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD, + GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV, + GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, GRPC_STATS_COUNTER_SYSCALL_WRITE, GRPC_STATS_COUNTER_SYSCALL_READ, @@ -53,6 +59,8 @@ typedef enum { } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; typedef enum { + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -63,19 +71,23 @@ typedef enum { } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; typedef enum { - GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_FIRST_SLOT = 0, + GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED_BUCKETS = 128, + GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_FIRST_SLOT = 192, GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 64, + GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_FIRST_SLOT = 256, GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 128, + GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_TCP_READ_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 192, + GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_FIRST_SLOT = 384, GRPC_STATS_HISTOGRAM_TCP_READ_OFFER_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 256, + GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_FIRST_SLOT = 448, GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, + GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 512, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 384 + GRPC_STATS_HISTOGRAM_BUCKETS = 576 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -85,6 +97,19 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT) +#define GRPC_STATS_INC_POLLSET_KICK(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK) +#define GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_POLLSET_KICKED_WITHOUT_POLLER) +#define GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICKED_AGAIN) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD) +#define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV) +#define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD) #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS) #define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \ @@ -138,6 +163,55 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED) #define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) +#define GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 6.000000) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, (int)_val.dbl); \ + } else { \ + if (_val.uint < 4688247212092686336ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_table_1[((_val.uint - 4618441417868443648ull) >> 49)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_0, 64)); \ + } \ + } \ + } while (false) +#define GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, value) \ + do { \ + union { \ + double dbl; \ + uint64_t uint; \ + } _val; \ + _val.dbl = (double)(value); \ + if (_val.dbl < 0) _val.dbl = 0; \ + if (_val.dbl < 29.000000) { \ + GRPC_STATS_INC_HISTOGRAM((exec_ctx), \ + GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + (int)_val.dbl); \ + } else { \ + if (_val.uint < 4652218415073722368ull) { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_table_3[((_val.uint - 4628855992006737920ull) >> 47)]); \ + } else { \ + GRPC_STATS_INC_HISTOGRAM( \ + (exec_ctx), GRPC_STATS_HISTOGRAM_POLL_EVENTS_RETURNED, \ + grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ + grpc_stats_table_2, 128)); \ + } \ + } \ + } while (false) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ do { \ union { \ @@ -153,12 +227,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -177,12 +251,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -201,12 +275,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -225,12 +299,12 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) @@ -249,12 +323,12 @@ typedef enum { if (_val.uint < 4652218415073722368ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ - grpc_stats_table_3[((_val.uint - 4622945017495814144ull) >> 49)]); \ + grpc_stats_table_7[((_val.uint - 4622945017495814144ull) >> 49)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_2, 64)); \ + grpc_stats_table_6, 64)); \ } \ } \ } while (false) @@ -274,21 +348,25 @@ typedef enum { if (_val.uint < 4715268809856909312ull) { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ - grpc_stats_table_1[((_val.uint - 4617315517961601024ull) >> 50)]); \ + grpc_stats_table_5[((_val.uint - 4617315517961601024ull) >> 50)]); \ } else { \ GRPC_STATS_INC_HISTOGRAM( \ (exec_ctx), GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, \ grpc_stats_histo_find_bucket_slow((exec_ctx), _val.dbl, \ - grpc_stats_table_0, 64)); \ + grpc_stats_table_4, 64)); \ } \ } \ } while (false) extern const double grpc_stats_table_0[64]; -extern const uint8_t grpc_stats_table_1[87]; -extern const double grpc_stats_table_2[64]; -extern const uint8_t grpc_stats_table_3[52]; -extern const int grpc_stats_histo_buckets[6]; -extern const int grpc_stats_histo_start[6]; -extern const double *const grpc_stats_histo_bucket_boundaries[6]; +extern const uint8_t grpc_stats_table_1[124]; +extern const double grpc_stats_table_2[128]; +extern const uint8_t grpc_stats_table_3[166]; +extern const double grpc_stats_table_4[64]; +extern const uint8_t grpc_stats_table_5[87]; +extern const double grpc_stats_table_6[64]; +extern const uint8_t grpc_stats_table_7[52]; +extern const int grpc_stats_histo_buckets[8]; +extern const int grpc_stats_histo_start[8]; +extern const double *const grpc_stats_histo_bucket_boundaries[8]; #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a0d042a688..4e0bc5de58 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -18,9 +18,21 @@ # overall - counter: client_calls_created - counter: server_calls_created +- histogram: call_initial_size + max: 262144 + buckets: 64 # polling - counter: syscall_poll - counter: syscall_wait +- histogram: poll_events_returned + max: 1024 + buckets: 128 +- counter: pollset_kick +- counter: pollset_kicked_without_poller +- counter: pollset_kicked_again +- counter: pollset_kick_wakeup_fd +- counter: pollset_kick_wakeup_cv +- counter: pollset_kick_own_thread # stats system - counter: histogram_slow_lookups # tcp diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 7f053fa728..6a3c2654a8 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -502,12 +502,14 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static grpc_error *pollset_kick_all(grpc_pollset *pollset) { +static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { GPR_TIMER_BEGIN("pollset_kick_all", 0); grpc_error *error = GRPC_ERROR_NONE; if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: break; @@ -550,7 +552,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutdown_closure = closure; pollset->shutting_down = true; - GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(exec_ctx, pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); GPR_TIMER_END("pollset_shutdown", 0); } @@ -646,6 +648,8 @@ static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + GRPC_STATS_INC_POLL_EVENTS_RETURNED(exec_ctx, r); + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "ps: %p poll got %d events", ps, r); } @@ -971,9 +975,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return error; } -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *ret_err = GRPC_ERROR_NONE; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_strvec log; @@ -1005,6 +1010,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { grpc_pollset_worker *root_worker = pollset->root_worker; if (root_worker == NULL) { + GRPC_STATS_INC_POLLSET_KICKED_WITHOUT_POLLER(exec_ctx); pollset->kicked_without_poller = true; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked_without_poller"); @@ -1013,12 +1019,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } grpc_pollset_worker *next_worker = root_worker->next; if (root_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; } else if (next_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1029,6 +1037,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, // there is no next worker root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( &g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", root_worker); } @@ -1036,6 +1045,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (next_worker->kick_state == UNKICKED) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1053,10 +1063,12 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } SET_KICK_STATE(root_worker, KICKED); if (root_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&root_worker->cv); } goto done; } else { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker, root_worker); @@ -1066,23 +1078,27 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); GPR_ASSERT(next_worker->kick_state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } goto done; } } else if (specific_worker->kick_state == KICKED) { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } goto done; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker); } @@ -1090,6 +1106,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } else if (specific_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick active poller"); } @@ -1097,6 +1114,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; } else if (specific_worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick waiting worker"); } @@ -1104,6 +1122,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); goto done; } else { + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kick non-waiting worker"); } diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c index e2e3cd9003..ee3250297f 100644 --- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -1112,12 +1112,13 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker != NULL) { if (worker == GRPC_POLLSET_KICK_BROADCAST) { if (pollset_has_workers(p)) { @@ -1265,7 +1266,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index ddb8c74d2d..ace491550b 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -632,9 +632,10 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); grpc_error *error = GRPC_ERROR_NONE; const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; @@ -731,7 +732,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying epoll set. In such a case, we diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0e42f76af3..165a562bd7 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -561,6 +561,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (worker->pollable != &pollset->pollable) { gpr_mu_lock(&worker->pollable->po.mu); } @@ -599,7 +600,8 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -662,13 +664,14 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { pollable *p = pollset->current_pollable; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } - grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + grpc_error *error = pollset_kick_inner(exec_ctx, pollset, p, specific_worker); if (p != &pollset->pollable) { gpr_mu_unlock(&p->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 59c7cdc285..6cc9645526 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1020,10 +1020,11 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { } /* p->mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { GPR_TIMER_BEGIN("pollset_kick", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); const char *err_desc = "Kick Failure"; grpc_pollset_worker *worker = specific_worker; if (worker != NULL) { @@ -1157,7 +1158,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = true; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() because it would release the underlying polling island. In such a case, we diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index fbd265f3ce..78817144af 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -209,7 +209,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, #define GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP 2 /* As per pollset_kick, with an extended set of flags (defined above) -- mostly for fd_posix's use. */ -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) GRPC_MUST_USE_RESULT; @@ -365,36 +365,39 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, return notifier; } -static grpc_error *pollset_kick_locked(grpc_fd_watcher *watcher) { +static grpc_error *pollset_kick_locked(grpc_exec_ctx *exec_ctx, + grpc_fd_watcher *watcher) { gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); - grpc_error *err = pollset_kick_ext(watcher->pollset, watcher->worker, - GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); + grpc_error *err = + pollset_kick_ext(exec_ctx, watcher->pollset, watcher->worker, + GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP); gpr_mu_unlock(&watcher->pollset->mu); return err; } -static void maybe_wake_one_watcher_locked(grpc_fd *fd) { +static void maybe_wake_one_watcher_locked(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { if (fd->inactive_watcher_root.next != &fd->inactive_watcher_root) { - pollset_kick_locked(fd->inactive_watcher_root.next); + pollset_kick_locked(exec_ctx, fd->inactive_watcher_root.next); } else if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } else if (fd->write_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } -static void wake_all_watchers_locked(grpc_fd *fd) { +static void wake_all_watchers_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_fd_watcher *watcher; for (watcher = fd->inactive_watcher_root.next; watcher != &fd->inactive_watcher_root; watcher = watcher->next) { - pollset_kick_locked(watcher); + pollset_kick_locked(exec_ctx, watcher); } if (fd->read_watcher) { - pollset_kick_locked(fd->read_watcher); + pollset_kick_locked(exec_ctx, fd->read_watcher); } if (fd->write_watcher && fd->write_watcher != fd->read_watcher) { - pollset_kick_locked(fd->write_watcher); + pollset_kick_locked(exec_ctx, fd->write_watcher); } } @@ -435,7 +438,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, if (!has_watchers(fd)) { close_fd_locked(exec_ctx, fd); } else { - wake_all_watchers_locked(fd); + wake_all_watchers_locked(exec_ctx, fd); } gpr_mu_unlock(&fd->mu); UNREF_BY(fd, 2, reason); /* drop the reference */ @@ -479,7 +482,7 @@ static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* already ready ==> queue the closure to run immediately */ *st = CLOSURE_NOT_READY; GRPC_CLOSURE_SCHED(exec_ctx, closure, fd_shutdown_error(fd)); - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } else { /* upcallptr was set to a different closure. This is an error! */ gpr_log(GPR_ERROR, @@ -648,7 +651,7 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher, } } if (kick) { - maybe_wake_one_watcher_locked(fd); + maybe_wake_one_watcher_locked(exec_ctx, fd); } if (fd_is_orphaned(fd) && !has_watchers(fd) && !fd->closed) { close_fd_locked(exec_ctx, fd); @@ -712,11 +715,12 @@ static void kick_append_error(grpc_error **composite, grpc_error *error) { *composite = grpc_error_add_child(*composite, error); } -static grpc_error *pollset_kick_ext(grpc_pollset *p, +static grpc_error *pollset_kick_ext(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker, uint32_t flags) { GPR_TIMER_BEGIN("pollset_kick_ext", 0); grpc_error *error = GRPC_ERROR_NONE; + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); /* pollset->mu already held */ if (specific_worker != NULL) { @@ -782,9 +786,9 @@ static grpc_error *pollset_kick_ext(grpc_pollset *p, return error; } -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { - return pollset_kick_ext(p, specific_worker, 0); + return pollset_kick_ext(exec_ctx, p, specific_worker, 0); } /* global state management */ @@ -847,7 +851,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } pollset->fds[pollset->fd_count++] = fd; GRPC_FD_REF(fd, "multipoller"); - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); exit: gpr_mu_unlock(&pollset->mu); } @@ -1070,7 +1074,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* check shutdown conditions */ if (pollset->shutting_down) { if (pollset_has_workers(pollset)) { - pollset_kick(pollset, NULL); + pollset_kick(exec_ctx, pollset, NULL); } else if (!pollset->called_shutdown && !pollset_has_observers(pollset)) { pollset->called_shutdown = 1; gpr_mu_unlock(&pollset->mu); @@ -1099,7 +1103,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(!pollset->shutting_down); pollset->shutting_down = 1; pollset->shutdown_done = closure; - pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset_has_workers(pollset)) { GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pollset->idle_jobs); } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 424b40e425..32363765e3 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -214,9 +214,9 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return g_event_engine->pollset_work(exec_ctx, pollset, worker, now, deadline); } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - return g_event_engine->pollset_kick(pollset, specific_worker); + return g_event_engine->pollset_kick(exec_ctx, pollset, specific_worker); } void grpc_pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1108e46ef8..1ff2ff1413 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -54,7 +54,7 @@ typedef struct grpc_event_engine_vtable { grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); - grpc_error *(*pollset_kick)(grpc_pollset *pollset, + grpc_error *(*pollset_kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); void (*pollset_add_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, struct grpc_fd *fd); diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index a609a3877a..a0f6b3a9d3 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -76,7 +76,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* Break one polling thread out of polling work for this pollset. If specific_worker is non-NULL, then kick that worker. */ -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index a2a8e289ee..691d66df69 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -79,7 +79,8 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, detector->is_done = 1; GRPC_LOG_IF_ERROR( "Pollset kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&detector->pollent), NULL)); + grpc_pollset_kick(exec_ctx, + grpc_polling_entity_pollset(&detector->pollent), NULL)); gpr_mu_unlock(g_polling_mu); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index e68c201134..2599765510 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -322,8 +322,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, grpc_channel_get_channel_stack(args->channel); grpc_call *call; GPR_TIMER_BEGIN("grpc_call_create", 0); - gpr_arena *arena = - gpr_arena_create(grpc_channel_get_call_size_estimate(args->channel)); + size_t initial_size = grpc_channel_get_call_size_estimate(args->channel); + GRPC_STATS_INC_CALL_INITIAL_SIZE(exec_ctx, initial_size); + gpr_arena *arena = gpr_arena_create(initial_size); call = gpr_arena_alloc(arena, sizeof(grpc_call) + channel_stack->call_stack_size); gpr_ref_init(&call->ext_ref, 1); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 10e4e5ab0c..a15b09e55d 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -54,7 +54,7 @@ typedef struct { bool can_listen; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); - grpc_error *(*kick)(grpc_pollset *pollset, + grpc_error *(*kick)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker); grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, @@ -130,7 +130,8 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, } static grpc_error *non_polling_poller_kick( - grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { non_polling_poller *p = (non_polling_poller *)pollset; if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; if (specific_worker != NULL) { @@ -645,7 +646,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, if (is_first) { gpr_mu_lock(cq->mu); grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); if (kick_error != GRPC_ERROR_NONE) { @@ -731,7 +732,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, } grpc_error *kick_error = - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker); + cq->poller_vtable->kick(exec_ctx, POLLSET_FROM_CQ(cq), pluck_worker); gpr_mu_unlock(cq->mu); @@ -930,7 +931,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, if (cq_event_queue_num_items(&cqd->queue) > 0 && gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); - cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); + cq->poller_vtable->kick(&exec_ctx, POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index b8b96d673c..8a53903763 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index a9d7abdcff..c7455bd8df 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -52,7 +52,7 @@ static void on_finish(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { g_done = 1; GPR_ASSERT(GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&g_pops), NULL))); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&g_pops), NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c index 11b45e8e08..353d79b11d 100644 --- a/test/core/iomgr/endpoint_tests.c +++ b/test/core/iomgr/endpoint_tests.c @@ -125,7 +125,8 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Read handler done"); gpr_mu_lock(g_mu); state->read_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } else if (error == GRPC_ERROR_NONE) { grpc_endpoint_read(exec_ctx, state->read_ep, &state->incoming, @@ -160,7 +161,8 @@ static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx, gpr_log(GPR_INFO, "Write handler done"); gpr_mu_lock(g_mu); state->write_done = 1 + (error == GRPC_ERROR_NONE); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL)); gpr_mu_unlock(g_mu); } @@ -252,7 +254,8 @@ static void inc_on_failure(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_mu_lock(g_mu); *(int *)arg += (error != GRPC_ERROR_NONE); - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 5791d17af6..881277a8d6 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -178,8 +178,8 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */, gpr_mu_lock(g_mu); sv->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -297,8 +297,8 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx, grpc_fd_orphan(exec_ctx, cl->em_fd, NULL, NULL, false /* already_closed */, "c"); cl->done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Write as much as possible, then register notify_on_write. */ @@ -417,8 +417,8 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = first_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -429,8 +429,8 @@ static void second_read_callback(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); fdc->cb_that_ran = second_read_callback; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 7f5c4073d8..1110c04b6e 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -106,7 +106,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *argsp, GPR_ASSERT(args->addrs->naddrs > 0); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } @@ -115,7 +116,8 @@ static void must_fail(grpc_exec_ctx *exec_ctx, void *argsp, grpc_error *err) { GPR_ASSERT(err != GRPC_ERROR_NONE); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 00ea495bbe..1032da942b 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -53,8 +53,10 @@ static gpr_timespec test_deadline(void) { static void finish_connection() { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(&exec_ctx, g_pollset, NULL))); + grpc_exec_ctx_finish(&exec_ctx); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index c45068e7ec..8cfc1bcab1 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -147,7 +147,8 @@ static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, gpr_log(GPR_INFO, "Read %" PRIuPTR " bytes of %" PRIuPTR, read_bytes, state->target_read_bytes); if (state->read_bytes >= state->target_read_bytes) { - GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } else { grpc_endpoint_read(exec_ctx, state->ep, &state->incoming, &state->read_cb); @@ -295,8 +296,8 @@ static void write_done(grpc_exec_ctx *exec_ctx, gpr_mu_lock(g_mu); gpr_log(GPR_INFO, "Signalling write done"); state->write_done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -406,8 +407,8 @@ static void write_test(size_t num_bytes, size_t slice_size) { void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *errors) { int *done = arg; *done = 1; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); } /* Do a read_test, then release fd and try to read/write again. Verify that diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 2371721a60..4d84608376 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -159,8 +159,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index aa34857dbd..1d051bea62 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -61,8 +61,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { g_number_of_reads++; g_number_of_bytes_read += (int)byte_count; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -70,8 +70,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { gpr_mu_lock(g_mu); g_number_of_writes++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index fdbc6ea741..d240403a29 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -60,7 +60,8 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *arg, request->token = token; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&request->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&request->pops), + NULL)); gpr_mu_unlock(request->mu); } diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c index e1385a80ca..3144717a85 100644 --- a/test/core/security/print_google_default_creds_token.c +++ b/test/core/security/print_google_default_creds_token.c @@ -57,7 +57,8 @@ static void on_metadata_response(grpc_exec_ctx *exec_ctx, void *arg, sync->is_done = true; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&sync->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&sync->pops), + NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c index 259038f4d6..5faa6352a8 100644 --- a/test/core/security/verify_jwt.c +++ b/test/core/security/verify_jwt.c @@ -66,7 +66,8 @@ static void on_jwt_verification_done(grpc_exec_ctx *exec_ctx, void *user_data, gpr_mu_lock(sync->mu); sync->is_done = 1; - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(sync->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, sync->pollset, NULL)); gpr_mu_unlock(sync->mu); } diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index 08079b6091..ec2cd8610b 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -108,7 +108,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *vargs, grpc_endpoint *tcp, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Connected")); grpc_endpoint_destroy(exec_ctx, tcp); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index d5739effb3..1b015c95e9 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -54,7 +54,8 @@ static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->done = 1; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } @@ -153,7 +154,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = 0; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); return; } @@ -189,7 +191,8 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, pr->port = port; GRPC_LOG_IF_ERROR( "pollset_kick", - grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL)); + grpc_pollset_kick(exec_ctx, grpc_polling_entity_pollset(&pr->pops), + NULL)); gpr_mu_unlock(pr->mu); } diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 3c99bda144..42ebeff41d 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -107,8 +107,8 @@ void GprLogReporter::ReportCoreStats(const char* name, int idx, grpc_stats_counter_name[i], data.counters[i]); } for (int i = 0; i < GRPC_STATS_HISTOGRAM_COUNT; i++) { - gpr_log(GPR_DEBUG, "%s[%d].%s = %lf/%lf/%lf (50/95/99%%-ile)", name, idx, - grpc_stats_histogram_name[i], + gpr_log(GPR_DEBUG, "%s[%d].%s = %.1lf/%.1lf/%.1lf (50/95/99%%-ile)", name, + idx, grpc_stats_histogram_name[i], grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 50), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 95), grpc_stats_histo_percentile(&data, (grpc_stats_histograms)i, 99)); -- cgit v1.2.3 From f7acbc40e7e806a8ca3e4793e6ed6831435ab10d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 10:06:28 +0200 Subject: fix C# distribtest dockerfiles --- tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile | 13 +++++-------- tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile | 13 +++++-------- .../dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile | 11 ++++++++--- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index d13eecaa55..02ec4c278a 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -14,18 +14,15 @@ FROM debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index 71845b590b..758f314572 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -14,18 +14,15 @@ FROM 32bit/debian:jessie -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean -# make sure we have nuget 2.12+ (in case there's an older cached docker image) -RUN apt-get update && apt-get install -y nuget - -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip && apt-get clean diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 6604caa42c..0f40f18e38 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -17,11 +17,16 @@ FROM ubuntu:16.04 RUN apt-get update && apt-get install -y \ mono-devel \ ca-certificates-mono \ - nuget + nuget \ + && apt-get clean # make sure we have nuget 2.12+ (in case there's an older cached docker image) RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list -RUN apt-get update && apt-get install -y nuget +RUN apt-get update && apt-get install -y nuget && apt-get clean -RUN apt-get update && apt-get install -y unzip +# Prevent "Error: SendFailure (Error writing headers)" when fetching nuget packages +# See https://github.com/tianon/docker-brew-ubuntu-core/issues/86 +RUN apt-get update && apt-get install -y tzdata && apt-get clean + +RUN apt-get update && apt-get install -y unzip && apt-get clean -- cgit v1.2.3 From 980744c436793ce63cd539c862959fe653df9c35 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 16:36:12 +0200 Subject: fix C# windows x64 distrib test --- tools/run_tests/artifacts/distribtest_targets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fa461efa85..76407810c0 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -105,7 +105,9 @@ class CSharpDistribTest(object): use_workspace=True) elif self.platform == 'windows': if self.arch == 'x64': - environ={'MSBUILD_EXTRA_ARGS': '/p:Platform=x64', + # Use double leading / as the first occurence gets removed by msys bash + # when invoking the .bat file (side-effect of posix path conversion) + environ={'MSBUILD_EXTRA_ARGS': '//p:Platform=x64', 'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\x64\\Debug'} else: environ={'DISTRIBTEST_OUTPATH': 'DistribTest\\bin\\Debug'} -- cgit v1.2.3 From 60c03d9b2a1ecb5d2c494b08e09a606830e2088e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 30 Aug 2017 16:42:16 +0200 Subject: fix nonexistent version warning for System.Threading.ThreadPool --- src/csharp/Grpc.Core/Grpc.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index dde800aadd..08f0e096f1 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -65,7 +65,7 @@ - + -- cgit v1.2.3 From 9eb36afb1124c9bf1753e78f0efa47bf41d72d71 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:40:06 +0200 Subject: dont set NetStandardImplicitPackageVersion --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 1 - src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj | 1 - src/csharp/Grpc.Core/Grpc.Core.csproj | 1 - src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj | 1 - src/csharp/Grpc.Reflection/Grpc.Reflection.csproj | 1 - 5 files changed, 5 deletions(-) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index abf326459c..bbcbd95be5 100755 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -15,7 +15,6 @@ gRPC RPC Protocol HTTP/2 Auth OAuth2 https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj index 9ad6fd0c61..4d6767fa98 100755 --- a/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj +++ b/src/csharp/Grpc.Core.Testing/Grpc.Core.Testing.csproj @@ -15,7 +15,6 @@ gRPC test testing https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 08f0e096f1..d9950b2f20 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -14,7 +14,6 @@ gRPC RPC Protocol HTTP/2 https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 3eb90434f3..681719d124 100755 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -14,7 +14,6 @@ gRPC health check https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true diff --git a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj index b77fd69aee..704eea5c17 100755 --- a/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj +++ b/src/csharp/Grpc.Reflection/Grpc.Reflection.csproj @@ -14,7 +14,6 @@ gRPC reflection https://github.com/grpc/grpc https://github.com/grpc/grpc/blob/master/LICENSE - 1.6.0 true true true -- cgit v1.2.3 From ec6f7280849aa95f30de32fb85d1e27f7c5688fd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:46:48 +0200 Subject: NUnit.ConsoleRunner dependency is useless --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 6df68fda58..be3e312b2a 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -21,7 +21,6 @@ - -- cgit v1.2.3 From a37b72e9fdee4130af45073714938a39bc83b8e1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:55:46 +0200 Subject: dont set RuntimeFrameworkVersion unneccessarily --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj | 1 - src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj | 1 - src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj | 1 - src/csharp/Grpc.Examples/Grpc.Examples.csproj | 1 - src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj | 1 - .../Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj | 1 - .../Grpc.IntegrationTesting.QpsWorker.csproj | 1 - .../Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj | 1 - .../Grpc.IntegrationTesting.StressClient.csproj | 1 - src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 1 - src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj | 1 - src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj | 1 - 13 files changed, 13 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index be3e312b2a..82fb080e6e 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Core.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj index 74deed6584..db4e3ef4e3 100755 --- a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj +++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj @@ -8,7 +8,6 @@ Grpc.Examples.MathClient Exe Grpc.Examples.MathClient - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj index 1abf261498..b12b418d01 100755 --- a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj +++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj @@ -8,7 +8,6 @@ Grpc.Examples.MathServer Exe Grpc.Examples.MathServer - 1.0.4 true diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d2a13ed6e1..32362aad87 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Examples.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj index 491d313f17..baa3b4ce6c 100755 --- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj +++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj @@ -7,7 +7,6 @@ net45;netcoreapp1.0 Grpc.Examples Grpc.Examples - 1.0.4 true diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 2ccf46b9b9..a9920ff68e 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.HealthCheck.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index c67beea7cd..0f3971e8d6 100755 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.Client $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index e452257b1b..92ed20b039 100755 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -10,7 +10,6 @@ Grpc.IntegrationTesting.QpsWorker true $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index a1fb316fdb..4480ca1bba 100755 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.Server $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj index f64bea3d2b..fef23633cc 100755 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.StressClient $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index f5077fe0f7..719f1eebba 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj index 17797e1e1e..0da24c6179 100644 --- a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj +++ b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj @@ -9,7 +9,6 @@ Exe Grpc.Microbenchmarks $(PackageTargetFallback);portable-net45 - 1.0.4 true diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index cf756c68ad..33a9106cae 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -9,7 +9,6 @@ Exe Grpc.Reflection.Tests $(PackageTargetFallback);portable-net45 - 1.0.4 true -- cgit v1.2.3 From f32d3a79b561fc887b91e2bdd5051cf06df502e2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 13:59:23 +0200 Subject: System.Linq.Expressions dependency in redundant --- src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index 719f1eebba..b0de1dc560 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -30,10 +30,6 @@ - - - - -- cgit v1.2.3 From 398435122a74f3f6a9f282531bfa91739ac39808 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 14:22:48 +0200 Subject: PackageTargetFallback not necessary --- src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 - src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj | 1 - src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj | 1 - .../Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj | 1 - .../Grpc.IntegrationTesting.QpsWorker.csproj | 1 - .../Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj | 1 - .../Grpc.IntegrationTesting.StressClient.csproj | 1 - src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj | 1 - src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj | 1 - src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj | 1 - 10 files changed, 10 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 82fb080e6e..18993a93e0 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Core.Tests Exe Grpc.Core.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index 32362aad87..3ccc9adfaf 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Examples.Tests Exe Grpc.Examples.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index a9920ff68e..9da0539dcb 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -8,7 +8,6 @@ Grpc.HealthCheck.Tests Exe Grpc.HealthCheck.Tests - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index 0f3971e8d6..35713156ea 100755 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.Client Exe Grpc.IntegrationTesting.Client - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj index 92ed20b039..3ecefe3bc4 100755 --- a/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj +++ b/src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj @@ -9,7 +9,6 @@ Exe Grpc.IntegrationTesting.QpsWorker true - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index 4480ca1bba..1092b2c21e 100755 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.Server Exe Grpc.IntegrationTesting.Server - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj index fef23633cc..22272547f6 100755 --- a/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj +++ b/src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting.StressClient Exe Grpc.IntegrationTesting.StressClient - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index b0de1dc560..c02c9844e3 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -8,7 +8,6 @@ Grpc.IntegrationTesting Exe Grpc.IntegrationTesting - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj index 0da24c6179..108357e4eb 100644 --- a/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj +++ b/src/csharp/Grpc.Microbenchmarks/Grpc.Microbenchmarks.csproj @@ -8,7 +8,6 @@ Grpc.Microbenchmarks Exe Grpc.Microbenchmarks - $(PackageTargetFallback);portable-net45 true diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index 33a9106cae..d368697124 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -8,7 +8,6 @@ Grpc.Reflection.Tests Exe Grpc.Reflection.Tests - $(PackageTargetFallback);portable-net45 true -- cgit v1.2.3 From c3e2510bd492f6bd30a7f5fa538c0a549058b59d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 5 Sep 2017 15:02:38 +0200 Subject: csharp_coreclr_x64 dockerfile not useful --- .../dockerfile/test/csharp_coreclr_x64/Dockerfile | 114 --------------------- tools/run_tests/artifacts/package_targets.py | 2 +- 2 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 tools/dockerfile/test/csharp_coreclr_x64/Dockerfile diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile deleted file mode 100644 index 4ccfbc43c3..0000000000 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:jessie - -# Install Git and basic packages. -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - golang \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - perl \ - strace \ - python-dev \ - python-setuptools \ - python-yaml \ - telnet \ - unzip \ - wget \ - zip && apt-get clean - -#================ -# Build profiling -RUN apt-get update && apt-get install -y time && apt-get clean - -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install pip --upgrade -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 - -#================ -# C# dependencies - -# Update to a newer version of mono -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -RUN echo "deb http://download.mono-project.com/repo/debian jessie main" | tee /etc/apt/sources.list.d/mono-official.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list -RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list - -# Install dependencies -RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ - mono-devel \ - ca-certificates-mono \ - nuget \ - && apt-get clean - -RUN nuget update -self - -# Install dotnet SDK based on https://www.microsoft.com/net/core#debian -RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003131 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 -RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet -# dotnet-dev-1.0.1 -RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 -RUN mkdir -p /opt/dotnet && tar zxf dotnet101.tar.gz -C /opt/dotnet -RUN ln -s /opt/dotnet/dotnet /usr/local/bin - -# Trigger the population of the local package cache -ENV NUGET_XMLDOC_MODE skip -RUN mkdir warmup \ - && cd warmup \ - && dotnet new \ - && cd .. \ - && rm -rf warmup - -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - - -RUN mkdir /var/local/jenkins - -# Define the default command. -CMD ["bash"] diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index 0da13864f0..671d0f7b45 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -78,7 +78,7 @@ class CSharpPackage: if self.linux: return create_docker_jobspec( self.name, - 'tools/dockerfile/test/csharp_coreclr_x64', + 'tools/dockerfile/test/csharp_jessie_x64', 'src/csharp/build_packages_dotnetcli.sh') else: return create_jobspec(self.name, -- cgit v1.2.3 From 561dc32247534d9204d9f68e92259759875c6d93 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 5 Sep 2017 13:00:46 -0700 Subject: Add stats for server request serving path --- src/core/lib/debug/stats_data.c | 48 +++++++++++++++++++++++++++++++------- src/core/lib/debug/stats_data.h | 23 ++++++++++++++---- src/core/lib/debug/stats_data.yaml | 6 +++++ src/core/lib/surface/server.c | 5 ++++ 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 57cbafc817..73703208da 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -48,10 +48,13 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_scheduled_to_self", "executor_wakeup_initiated", "executor_queue_drained", + "server_requested_calls", + "server_slowpath_requests_queued", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { - "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", - "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "tcp_write_size", "tcp_write_iov_size", "tcp_read_size", + "tcp_read_offer", "tcp_read_iov_size", "http2_send_message_size", + "server_cqs_checked", }; const int grpc_stats_table_0[64] = { 0, 1, 2, 3, 4, 6, 8, 11, @@ -81,6 +84,8 @@ const uint8_t grpc_stats_table_3[104] = { 24, 24, 25, 25, 26, 27, 28, 28, 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, 34, 34, 35, 36, 36, 37, 38, 38, 39, 39, 40, 40, 41, 41, 42, 42, 42, 43, 44, 45, 45, 46, 47, 47, 48, 48, 49, 49, 50, 50, 51, 51}; +const int grpc_stats_table_4[8] = {0, 1, 2, 4, 8, 16, 32, 64}; +const uint8_t grpc_stats_table_5[4] = {0, 1, 2, 3}; void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { @@ -233,12 +238,39 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_0, 64)); } -const int grpc_stats_histo_buckets[6] = {64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[6] = {0, 64, 128, 192, 256, 320}; -const int *const grpc_stats_histo_bucket_boundaries[6] = { +void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { + value = GPR_CLAMP(value, 0, 64); + if (value < 3) { + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4627448617123184640ull) { + int bucket = + grpc_stats_table_5[((_val.uint - 4613937818241073152ull) >> 52)] + 3; + _bkt.dbl = grpc_stats_table_4[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_4, 8)); +} +const int grpc_stats_histo_buckets[7] = {64, 64, 64, 64, 64, 64, 8}; +const int grpc_stats_histo_start[7] = {0, 64, 128, 192, 256, 320, 384}; +const int *const grpc_stats_histo_bucket_boundaries[7] = { + grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, - grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0}; -void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_table_4}; +void (*const grpc_stats_inc_histogram[7])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_tcp_write_size, grpc_stats_inc_tcp_write_iov_size, grpc_stats_inc_tcp_read_size, grpc_stats_inc_tcp_read_offer, - grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size}; + grpc_stats_inc_tcp_read_iov_size, grpc_stats_inc_http2_send_message_size, + grpc_stats_inc_server_cqs_checked}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 9b2d43a03c..f5dd486873 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -50,6 +50,8 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_SCHEDULED_TO_SELF, GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED, GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED, + GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS, + GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED, GRPC_STATS_COUNTER_COUNT } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; @@ -60,6 +62,7 @@ typedef enum { GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, GRPC_STATS_HISTOGRAM_COUNT } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; @@ -76,7 +79,9 @@ typedef enum { GRPC_STATS_HISTOGRAM_TCP_READ_IOV_SIZE_BUCKETS = 64, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_FIRST_SLOT = 320, GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_SIZE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 384 + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 384, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8, + GRPC_STATS_HISTOGRAM_BUCKETS = 392 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -139,6 +144,11 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED) #define GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) +#define GRPC_STATS_INC_SERVER_REQUESTED_CALLS(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS) +#define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value)) void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x); @@ -157,10 +167,13 @@ void grpc_stats_inc_tcp_read_iov_size(grpc_exec_ctx *exec_ctx, int x); #define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(exec_ctx, value) \ grpc_stats_inc_http2_send_message_size((exec_ctx), (int)(value)) void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[6]; -extern const int grpc_stats_histo_start[6]; -extern const int *const grpc_stats_histo_bucket_boundaries[6]; -extern void (*const grpc_stats_inc_histogram[6])(grpc_exec_ctx *exec_ctx, +#define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \ + grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value)) +void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x); +extern const int grpc_stats_histo_buckets[7]; +extern const int grpc_stats_histo_start[7]; +extern const int *const grpc_stats_histo_bucket_boundaries[7]; +extern void (*const grpc_stats_inc_histogram[7])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a0d042a688..71dc32a8ea 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -65,3 +65,9 @@ - counter: executor_scheduled_to_self - counter: executor_wakeup_initiated - counter: executor_queue_drained +# server +- counter: server_requested_calls +- histogram: server_cqs_checked + buckets: 8 + max: 64 +- counter: server_slowpath_requests_queued diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 66dcc299aa..786d754b49 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -29,6 +29,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/slice/slice_internal.h" @@ -538,6 +539,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, if (request_id == -1) { continue; } else { + GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, i); gpr_mu_lock(&calld->mu_state); calld->state = ACTIVATED; gpr_mu_unlock(&calld->mu_state); @@ -548,6 +550,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, } /* no cq to take the request found: queue it on the slow list */ + GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx); gpr_mu_lock(&server->mu_call); gpr_mu_lock(&calld->mu_state); calld->state = PENDING; @@ -1429,6 +1432,7 @@ grpc_call_error grpc_server_request_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = gpr_malloc(sizeof(*rc)); + GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_call(" "server=%p, call=%p, details=%p, initial_metadata=%p, " @@ -1475,6 +1479,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = gpr_malloc(sizeof(*rc)); registered_method *rm = rmp; + GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" "server=%p, rmp=%p, call=%p, deadline=%p, initial_metadata=%p, " -- cgit v1.2.3 From 130e07061f6eed9bbb04f58eeab091fa515b0572 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 Sep 2017 18:02:51 -0700 Subject: Add interface abstraction and identity module --- build.yaml | 2 + .../transport/chttp2/transport/chttp2_transport.c | 6 +- src/core/ext/transport/chttp2/transport/writing.c | 2 +- src/core/lib/compression/stream_compression.c | 173 +++------------- src/core/lib/compression/stream_compression.h | 34 +++- src/core/lib/compression/stream_compression_gzip.c | 218 +++++++++++++++++++++ src/core/lib/compression/stream_compression_gzip.h | 26 +++ .../lib/compression/stream_compression_identity.c | 77 ++++++++ .../lib/compression/stream_compression_identity.h | 26 +++ 9 files changed, 410 insertions(+), 154 deletions(-) create mode 100644 src/core/lib/compression/stream_compression_gzip.c create mode 100644 src/core/lib/compression/stream_compression_gzip.h create mode 100644 src/core/lib/compression/stream_compression_identity.c create mode 100644 src/core/lib/compression/stream_compression_identity.h diff --git a/build.yaml b/build.yaml index 7d8440ce44..0939796bcb 100644 --- a/build.yaml +++ b/build.yaml @@ -197,6 +197,7 @@ filegroups: - src/core/lib/compression/stream_compression.c - src/core/lib/debug/stats.c - src/core/lib/debug/stats_data.c + - src/core/lib/compression/stream_compression_gzip.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -349,6 +350,7 @@ filegroups: - src/core/lib/compression/stream_compression.h - src/core/lib/debug/stats.h - src/core/lib/debug/stats_data.h + - src/core/lib/compression/stream_compression_gzip.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 7541bd5c92..0e00995d11 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1731,7 +1731,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, @@ -1804,7 +1804,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->frame_storage, @@ -2694,7 +2694,7 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 711938b278..b877e9f126 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -329,7 +329,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (s->stream_compression_ctx == NULL) { s->stream_compression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_COMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); } s->uncompressed_data_size = s->flow_controlled_buffer.length; GPR_ASSERT(grpc_stream_compress( diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index df13d53e06..274ee100ba 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -16,176 +16,57 @@ * */ -#include #include #include "src/core/lib/compression/stream_compression.h" -#include "src/core/lib/iomgr/exec_ctx.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/compression/stream_compression_gzip.h" -#define OUTPUT_BLOCK_SIZE (1024) - -static bool gzip_flate(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, int flush, - bool *end_of_context) { - GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH); - /* Full flush is not allowed when inflating. */ - GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH))); - - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - int r; - bool eoc = false; - size_t original_max_output_size = max_output_size; - while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) { - size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size - : OUTPUT_BLOCK_SIZE; - grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size); - ctx->zs.avail_out = (uInt)slice_size; - ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out); - while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) { - grpc_slice slice = grpc_slice_buffer_take_first(in); - ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice); - ctx->zs.next_in = GRPC_SLICE_START_PTR(slice); - r = ctx->flate(&ctx->zs, Z_NO_FLUSH); - if (r < 0 && r != Z_BUF_ERROR) { - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } else if (r == Z_STREAM_END && ctx->flate == inflate) { - eoc = true; - } - if (ctx->zs.avail_in > 0) { - grpc_slice_buffer_undo_take_first( - in, - grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in, - GRPC_SLICE_LENGTH(slice))); - } - grpc_slice_unref_internal(&exec_ctx, slice); - } - if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) { - GPR_ASSERT(in->length == 0); - r = ctx->flate(&ctx->zs, flush); - if (flush == Z_SYNC_FLUSH) { - switch (r) { - case Z_OK: - /* Maybe flush is not complete; just made some partial progress. */ - if (ctx->zs.avail_out > 0) { - flush = 0; - } - break; - case Z_BUF_ERROR: - case Z_STREAM_END: - flush = 0; - break; - default: - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } - } else if (flush == Z_FINISH) { - switch (r) { - case Z_OK: - case Z_BUF_ERROR: - /* Wait for the next loop to assign additional output space. */ - GPR_ASSERT(ctx->zs.avail_out == 0); - break; - case Z_STREAM_END: - flush = 0; - break; - default: - gpr_log(GPR_ERROR, "zlib error (%d)", r); - grpc_slice_unref_internal(&exec_ctx, slice_out); - grpc_exec_ctx_finish(&exec_ctx); - return false; - } - } - } - - if (ctx->zs.avail_out == 0) { - grpc_slice_buffer_add(out, slice_out); - } else if (ctx->zs.avail_out < slice_size) { - slice_out.data.refcounted.length -= ctx->zs.avail_out; - grpc_slice_buffer_add(out, slice_out); - } else { - grpc_slice_unref_internal(&exec_ctx, slice_out); - } - max_output_size -= (slice_size - ctx->zs.avail_out); - } - grpc_exec_ctx_finish(&exec_ctx); - if (end_of_context) { - *end_of_context = eoc; - } - if (output_size) { - *output_size = original_max_output_size - max_output_size; - } - return true; -} +extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; bool grpc_stream_compress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, grpc_stream_compression_flush flush) { - GPR_ASSERT(ctx->flate == deflate); - int gzip_flush; - switch (flush) { - case GRPC_STREAM_COMPRESSION_FLUSH_NONE: - gzip_flush = 0; - break; - case GRPC_STREAM_COMPRESSION_FLUSH_SYNC: - gzip_flush = Z_SYNC_FLUSH; - break; - case GRPC_STREAM_COMPRESSION_FLUSH_FINISH: - gzip_flush = Z_FINISH; - break; - default: - gzip_flush = 0; - } - return gzip_flate(ctx, in, out, output_size, max_output_size, gzip_flush, - NULL); + return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, flush); } bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, bool *end_of_context) { - GPR_ASSERT(ctx->flate == inflate); - return gzip_flate(ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, - end_of_context); + return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, end_of_context); } grpc_stream_compression_context *grpc_stream_compression_context_create( grpc_stream_compression_method method) { - grpc_stream_compression_context *ctx = - gpr_zalloc(sizeof(grpc_stream_compression_context)); - int r; - if (ctx == NULL) { - return NULL; - } - if (method == GRPC_STREAM_COMPRESSION_DECOMPRESS) { - r = inflateInit2(&ctx->zs, 0x1F); - ctx->flate = inflate; - } else { - r = deflateInit2(&ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, - Z_DEFAULT_STRATEGY); - ctx->flate = deflate; - } - if (r != Z_OK) { - gpr_free(ctx); - return NULL; + switch (method) { + case GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS: + case GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS: + return grpc_stream_compression_identity_vtable.context_create(method); + case GRPC_STREAM_COMPRESSION_GZIP_COMPRESS: + case GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS: + return grpc_stream_compression_gzip_vtable.context_create(method); + default: + gpr_log(GPR_ERROR, "Unknown stream compression method: %d", method); + return NULL; } - - return ctx; } void grpc_stream_compression_context_destroy( grpc_stream_compression_context *ctx) { - if (ctx->flate == inflate) { - inflateEnd(&ctx->zs); + ctx->vtable->context_destroy(ctx); +} + +int grpc_stream_compression_method_parse( + grpc_slice value, bool is_compress, grpc_stream_compression_method *method) { + if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { + *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + return 1; + } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { + *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; + return 1; } else { - deflateEnd(&ctx->zs); + return 0; } - gpr_free(ctx); } + diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index 844dff81a3..fa07247c31 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -24,15 +24,20 @@ #include #include +#include "src/core/lib/transport/static_metadata.h" + +typedef struct grpc_stream_compression_vtable grpc_stream_compression_vtable; + /* Stream compression/decompression context */ typedef struct grpc_stream_compression_context { - z_stream zs; - int (*flate)(z_stream *zs, int flush); + const grpc_stream_compression_vtable *vtable; } grpc_stream_compression_context; typedef enum grpc_stream_compression_method { - GRPC_STREAM_COMPRESSION_COMPRESS = 0, - GRPC_STREAM_COMPRESSION_DECOMPRESS, + GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS = 0, + GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS, + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS, + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS, GRPC_STREAM_COMPRESSION_METHOD_COUNT } grpc_stream_compression_method; @@ -43,6 +48,21 @@ typedef enum grpc_stream_compression_flush { GRPC_STREAM_COMPRESSION_FLUSH_COUNT } grpc_stream_compression_flush; +struct grpc_stream_compression_vtable { +bool (*compress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush); +bool (*decompress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context); +grpc_stream_compression_context *(*context_create)( + grpc_stream_compression_method method); +void (*context_destroy)( + grpc_stream_compression_context *ctx); +}; + /** * Compress bytes provided in \a in with a given context, with an optional flush * at the end of compression. Emits at most \a max_output_size compressed bytes @@ -87,4 +107,10 @@ grpc_stream_compression_context *grpc_stream_compression_context_create( void grpc_stream_compression_context_destroy( grpc_stream_compression_context *ctx); +/** + * Parse stream compression method based on algorithm name + */ +int grpc_stream_compression_method_parse( + grpc_slice value, bool is_compress, grpc_stream_compression_method *method); + #endif diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c new file mode 100644 index 0000000000..2d03836a0d --- /dev/null +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -0,0 +1,218 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" + +#define OUTPUT_BLOCK_SIZE (1024) + +typedef struct grpc_stream_compression_context_gzip { + grpc_stream_compression_context base; + + z_stream zs; + int (*flate)(z_stream *zs, int flush); +} grpc_stream_compression_context_gzip; + +static bool gzip_flate(grpc_stream_compression_context_gzip *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, int flush, + bool *end_of_context) { + GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH); + /* Full flush is not allowed when inflating. */ + GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH))); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + int r; + bool eoc = false; + size_t original_max_output_size = max_output_size; + while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) { + size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size + : OUTPUT_BLOCK_SIZE; + grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size); + ctx->zs.avail_out = (uInt)slice_size; + ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out); + while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) { + grpc_slice slice = grpc_slice_buffer_take_first(in); + ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice); + ctx->zs.next_in = GRPC_SLICE_START_PTR(slice); + r = ctx->flate(&ctx->zs, Z_NO_FLUSH); + if (r < 0 && r != Z_BUF_ERROR) { + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } else if (r == Z_STREAM_END && ctx->flate == inflate) { + eoc = true; + } + if (ctx->zs.avail_in > 0) { + grpc_slice_buffer_undo_take_first( + in, + grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in, + GRPC_SLICE_LENGTH(slice))); + } + grpc_slice_unref_internal(&exec_ctx, slice); + } + if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) { + GPR_ASSERT(in->length == 0); + r = ctx->flate(&ctx->zs, flush); + if (flush == Z_SYNC_FLUSH) { + switch (r) { + case Z_OK: + /* Maybe flush is not complete; just made some partial progress. */ + if (ctx->zs.avail_out > 0) { + flush = 0; + } + break; + case Z_BUF_ERROR: + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } else if (flush == Z_FINISH) { + switch (r) { + case Z_OK: + case Z_BUF_ERROR: + /* Wait for the next loop to assign additional output space. */ + GPR_ASSERT(ctx->zs.avail_out == 0); + break; + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } + } + + if (ctx->zs.avail_out == 0) { + grpc_slice_buffer_add(out, slice_out); + } else if (ctx->zs.avail_out < slice_size) { + slice_out.data.refcounted.length -= ctx->zs.avail_out; + grpc_slice_buffer_add(out, slice_out); + } else { + grpc_slice_unref_internal(&exec_ctx, slice_out); + } + max_output_size -= (slice_size - ctx->zs.avail_out); + } + grpc_exec_ctx_finish(&exec_ctx); + if (end_of_context) { + *end_of_context = eoc; + } + if (output_size) { + *output_size = original_max_output_size - max_output_size; + } + return true; +} + +static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + GPR_ASSERT(gzip_ctx->flate == deflate); + int gzip_flush; + switch (flush) { + case GRPC_STREAM_COMPRESSION_FLUSH_NONE: + gzip_flush = 0; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_SYNC: + gzip_flush = Z_SYNC_FLUSH; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_FINISH: + gzip_flush = Z_FINISH; + break; + default: + gzip_flush = 0; + } + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, gzip_flush, + NULL); +} + +static bool grpc_stream_decompress_gzip(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + GPR_ASSERT(gzip_ctx->flate == inflate); + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, + end_of_context); +} + +static grpc_stream_compression_context *grpc_stream_compression_context_create_gzip( + grpc_stream_compression_method method) { + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + grpc_stream_compression_context_gzip *gzip_ctx = + gpr_zalloc(sizeof(grpc_stream_compression_context_gzip)); + int r; + if (gzip_ctx == NULL) { + return NULL; + } + if (method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS) { + r = inflateInit2(&gzip_ctx->zs, 0x1F); + gzip_ctx->flate = inflate; + } else { + r = deflateInit2(&gzip_ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, + Z_DEFAULT_STRATEGY); + gzip_ctx->flate = deflate; + } + if (r != Z_OK) { + gpr_free(gzip_ctx); + return NULL; + } + + return (grpc_stream_compression_context *)gzip_ctx; +} + +static void grpc_stream_compression_context_destroy_gzip( + grpc_stream_compression_context *ctx) { + if (ctx == NULL) { + return; + } + grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip*)ctx; + if (gzip_ctx->flate == inflate) { + inflateEnd(&gzip_ctx->zs); + } else { + deflateEnd(&gzip_ctx->zs); + } + gpr_free(ctx); +} + +const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable = { + .compress = grpc_stream_compress_gzip, + .decompress = grpc_stream_decompress_gzip, + .context_create = grpc_stream_compression_context_create_gzip, + .context_destroy = grpc_stream_compression_context_destroy_gzip +}; diff --git a/src/core/lib/compression/stream_compression_gzip.h b/src/core/lib/compression/stream_compression_gzip.h new file mode 100644 index 0000000000..7cf49a0de9 --- /dev/null +++ b/src/core/lib/compression/stream_compression_gzip.h @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_GZIP_H +#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_GZIP_H + +#include "src/core/lib/compression/stream_compression.h" + +extern const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable; + +#endif diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c new file mode 100644 index 0000000000..1395b762d4 --- /dev/null +++ b/src/core/lib/compression/stream_compression_identity.c @@ -0,0 +1,77 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" + +#define OUTPUT_BLOCK_SIZE (1024) + +static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { + if (max_output_size >= in->length) { + *output_size = in->length; + grpc_slice_buffer_move_into(in, out); + } else { + *output_size = max_output_size; + grpc_slice_buffer_move_first(in, max_output_size, out); + } +} + +static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + return true; +} + +static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context) { + if (ctx == NULL) { + return false; + } + grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + return true; +} + +static grpc_stream_compression_context *grpc_stream_compression_context_create_identity( + grpc_stream_compression_method method) { + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); + /* No context needed in this case. Use fake context instead. */ + return (grpc_stream_compression_context *)1; +} + +static void grpc_stream_compression_context_destroy_identity( + grpc_stream_compression_context *ctx) { + return; +} + +const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable = { + .compress = grpc_stream_compress_identity, + .decompress = grpc_stream_decompress_identity, + .context_create = grpc_stream_compression_context_create_identity, + .context_destroy = grpc_stream_compression_context_destroy_identity +}; diff --git a/src/core/lib/compression/stream_compression_identity.h b/src/core/lib/compression/stream_compression_identity.h new file mode 100644 index 0000000000..0d770dd32b --- /dev/null +++ b/src/core/lib/compression/stream_compression_identity.h @@ -0,0 +1,26 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_IDENTITY_H +#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_IDENTITY_H + +#include "src/core/lib/compression/stream_compression.h" + +extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; + +#endif -- cgit v1.2.3 From bf5484e785f93e1183a15415239a729d3d120af1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 1 Sep 2017 18:03:03 -0700 Subject: build_project --- CMakeLists.txt | 12 ++++++++++++ Makefile | 12 ++++++++++++ binding.gyp | 2 ++ build.yaml | 6 ++++-- config.m4 | 2 ++ config.w32 | 2 ++ gRPC-Core.podspec | 6 ++++++ grpc.gemspec | 4 ++++ grpc.gyp | 8 ++++++++ package.xml | 4 ++++ src/python/grpcio/grpc_core_dependencies.py | 2 ++ tools/doxygen/Doxyfile.c++.internal | 2 ++ tools/doxygen/Doxyfile.core.internal | 4 ++++ tools/run_tests/generated/sources_and_headers.json | 6 ++++++ 14 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b22aa1899e..ee4d2bcbab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -968,6 +968,8 @@ add_library(grpc src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1319,6 +1321,8 @@ add_library(grpc_cronet src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1638,6 +1642,8 @@ add_library(grpc_test_util src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -1901,6 +1907,8 @@ add_library(grpc_test_util_unsecure src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -2150,6 +2158,8 @@ add_library(grpc_unsecure src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c @@ -2851,6 +2861,8 @@ add_library(grpc++_cronet src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c src/core/lib/compression/stream_compression.c + src/core/lib/compression/stream_compression_gzip.c + src/core/lib/compression/stream_compression_identity.c src/core/lib/debug/stats.c src/core/lib/debug/stats_data.c src/core/lib/http/format_request.c diff --git a/Makefile b/Makefile index 743ca41dc6..a3db8106da 100644 --- a/Makefile +++ b/Makefile @@ -2915,6 +2915,8 @@ LIBGRPC_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3264,6 +3266,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3580,6 +3584,8 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -3832,6 +3838,8 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -4057,6 +4065,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ @@ -4741,6 +4751,8 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ diff --git a/binding.gyp b/binding.gyp index 946edd8139..0547a82512 100644 --- a/binding.gyp +++ b/binding.gyp @@ -667,6 +667,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/build.yaml b/build.yaml index 0939796bcb..0de5b9859d 100644 --- a/build.yaml +++ b/build.yaml @@ -195,9 +195,10 @@ filegroups: - src/core/lib/compression/compression.c - src/core/lib/compression/message_compress.c - src/core/lib/compression/stream_compression.c + - src/core/lib/compression/stream_compression_gzip.c + - src/core/lib/compression/stream_compression_identity.c - src/core/lib/debug/stats.c - src/core/lib/debug/stats_data.c - - src/core/lib/compression/stream_compression_gzip.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -348,9 +349,10 @@ filegroups: - src/core/lib/compression/algorithm_metadata.h - src/core/lib/compression/message_compress.h - src/core/lib/compression/stream_compression.h + - src/core/lib/compression/stream_compression_gzip.h + - src/core/lib/compression/stream_compression_identity.h - src/core/lib/debug/stats.h - src/core/lib/debug/stats_data.h - - src/core/lib/compression/stream_compression_gzip.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h diff --git a/config.m4 b/config.m4 index d9d25cb006..fbae82de7f 100644 --- a/config.m4 +++ b/config.m4 @@ -96,6 +96,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/stream_compression.c \ + src/core/lib/compression/stream_compression_gzip.c \ + src/core/lib/compression/stream_compression_identity.c \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats_data.c \ src/core/lib/http/format_request.c \ diff --git a/config.w32 b/config.w32 index 640115e861..3dacc31e6a 100644 --- a/config.w32 +++ b/config.w32 @@ -73,6 +73,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\compression\\compression.c " + "src\\core\\lib\\compression\\message_compress.c " + "src\\core\\lib\\compression\\stream_compression.c " + + "src\\core\\lib\\compression\\stream_compression_gzip.c " + + "src\\core\\lib\\compression\\stream_compression_identity.c " + "src\\core\\lib\\debug\\stats.c " + "src\\core\\lib\\debug\\stats_data.c " + "src\\core\\lib\\http\\format_request.c " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 66368b1b4b..8fea17c5db 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -329,6 +329,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', 'src/core/lib/compression/stream_compression.h', + 'src/core/lib/compression/stream_compression_gzip.h', + 'src/core/lib/compression/stream_compression_identity.h', 'src/core/lib/debug/stats.h', 'src/core/lib/debug/stats_data.h', 'src/core/lib/http/format_request.h', @@ -479,6 +481,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -826,6 +830,8 @@ Pod::Spec.new do |s| 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', 'src/core/lib/compression/stream_compression.h', + 'src/core/lib/compression/stream_compression_gzip.h', + 'src/core/lib/compression/stream_compression_identity.h', 'src/core/lib/debug/stats.h', 'src/core/lib/debug/stats_data.h', 'src/core/lib/http/format_request.h', diff --git a/grpc.gemspec b/grpc.gemspec index d3779a9991..166c914c52 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -261,6 +261,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/compression/algorithm_metadata.h ) s.files += %w( src/core/lib/compression/message_compress.h ) s.files += %w( src/core/lib/compression/stream_compression.h ) + s.files += %w( src/core/lib/compression/stream_compression_gzip.h ) + s.files += %w( src/core/lib/compression/stream_compression_identity.h ) s.files += %w( src/core/lib/debug/stats.h ) s.files += %w( src/core/lib/debug/stats_data.h ) s.files += %w( src/core/lib/http/format_request.h ) @@ -415,6 +417,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/compression/compression.c ) s.files += %w( src/core/lib/compression/message_compress.c ) s.files += %w( src/core/lib/compression/stream_compression.c ) + s.files += %w( src/core/lib/compression/stream_compression_gzip.c ) + s.files += %w( src/core/lib/compression/stream_compression_identity.c ) s.files += %w( src/core/lib/debug/stats.c ) s.files += %w( src/core/lib/debug/stats_data.c ) s.files += %w( src/core/lib/http/format_request.c ) diff --git a/grpc.gyp b/grpc.gyp index 40938a4564..2e2e0b7dfd 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -233,6 +233,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -533,6 +535,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -738,6 +742,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', @@ -928,6 +934,8 @@ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/package.xml b/package.xml index 1cca4fbddd..33b8378a35 100644 --- a/package.xml +++ b/package.xml @@ -271,6 +271,8 @@ + + @@ -425,6 +427,8 @@ + + diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 1cbf345ab6..9025a07ba9 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -72,6 +72,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', 'src/core/lib/compression/stream_compression.c', + 'src/core/lib/compression/stream_compression_gzip.c', + 'src/core/lib/compression/stream_compression_identity.c', 'src/core/lib/debug/stats.c', 'src/core/lib/debug/stats_data.c', 'src/core/lib/http/format_request.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 91c149eec9..71f500b53f 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -942,6 +942,8 @@ src/core/lib/channel/handshaker_registry.h \ src/core/lib/compression/algorithm_metadata.h \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.h \ src/core/lib/debug/trace.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 26d982acd7..63993c808b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1077,6 +1077,10 @@ src/core/lib/compression/message_compress.c \ src/core/lib/compression/message_compress.h \ src/core/lib/compression/stream_compression.c \ src/core/lib/compression/stream_compression.h \ +src/core/lib/compression/stream_compression_gzip.c \ +src/core/lib/compression/stream_compression_gzip.h \ +src/core/lib/compression/stream_compression_identity.c \ +src/core/lib/compression/stream_compression_identity.h \ src/core/lib/debug/stats.c \ src/core/lib/debug/stats.h \ src/core/lib/debug/stats_data.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fbd47389f7..cca4cff0fb 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7856,6 +7856,8 @@ "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression_gzip.c", + "src/core/lib/compression/stream_compression_identity.c", "src/core/lib/debug/stats.c", "src/core/lib/debug/stats_data.c", "src/core/lib/http/format_request.c", @@ -8009,6 +8011,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", @@ -8141,6 +8145,8 @@ "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/debug/stats.h", "src/core/lib/debug/stats_data.h", "src/core/lib/http/format_request.h", -- cgit v1.2.3 From 8e14acc76bacdf67fa852a96e44381df238e3907 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 13:39:07 -0700 Subject: Remove duplicate sentences on send path --- .../transport/chttp2/transport/chttp2_transport.c | 14 ++-- src/core/ext/transport/chttp2/transport/internal.h | 9 ++- src/core/ext/transport/chttp2/transport/writing.c | 89 +++++++++------------- 3 files changed, 45 insertions(+), 67 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 0e00995d11..e1fb9dd915 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -727,10 +727,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_destroy_internal(exec_ctx, &s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage); - if (s->compressed_data_buffer) { - grpc_slice_buffer_destroy_internal(exec_ctx, s->compressed_data_buffer); - gpr_free(s->compressed_data_buffer); - } + grpc_slice_buffer_destroy_internal(exec_ctx, &s->compressed_data_buffer); if (s->decompressed_data_buffer) { grpc_slice_buffer_destroy_internal(exec_ctx, s->decompressed_data_buffer); gpr_free(s->decompressed_data_buffer); @@ -1300,12 +1297,11 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; /* Identify stream compression */ - if ((s->stream_compression_send_enabled = - (op_payload->send_initial_metadata.send_initial_metadata->idx.named - .content_encoding != NULL)) == true) { - s->compressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer)); - grpc_slice_buffer_init(s->compressed_data_buffer); + if (op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding == NULL || + grpc_stream_compression_method_parse(GRPC_MDVALUE(op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding->md), true, &s->stream_compression_method) == 0) { + s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } + grpc_slice_buffer_init(&s->compressed_data_buffer); s->send_initial_metadata_finished = add_closure_barrier(on_complete); s->send_initial_metadata = diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 3c41a8958f..e5e4dd3344 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -557,10 +557,11 @@ struct grpc_chttp2_stream { grpc_chttp2_write_cb *finish_after_write; size_t sending_bytes; - /** Whether stream compression send is enabled */ + /* Stream compression method to be used. */ + grpc_stream_compression_method stream_compression_method; + /* Stream decompression method to be used. */ + grpc_stream_compression_method stream_decompression_method; bool stream_compression_recv_enabled; - /** Whether stream compression recv is enabled */ - bool stream_compression_send_enabled; /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed */ bool unprocessed_incoming_frames_decompressed; @@ -570,7 +571,7 @@ struct grpc_chttp2_stream { grpc_stream_compression_context *stream_compression_ctx; /** Buffer storing data that is compressed but not sent */ - grpc_slice_buffer *compressed_data_buffer; + grpc_slice_buffer compressed_data_buffer; /** Amount of uncompressed bytes sent out when compressed_data_buffer is * emptied */ size_t uncompressed_data_size; diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b877e9f126..b6bc864c29 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -288,8 +288,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (sent_initial_metadata) { /* send any body bytes, if allowed by flow control */ if (s->flow_controlled_buffer.length > 0 || - (s->stream_compression_send_enabled && - s->compressed_data_buffer->length > 0)) { + s->compressed_data_buffer.length > 0) { uint32_t stream_remote_window = (uint32_t)GPR_MAX( 0, s->flow_control.remote_window_delta + @@ -302,56 +301,40 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (max_outgoing > 0) { bool is_last_data_frame = false; bool is_last_frame = false; - if (s->stream_compression_send_enabled) { - while ((s->flow_controlled_buffer.length > 0 || - s->compressed_data_buffer->length > 0) && - max_outgoing > 0) { - if (s->compressed_data_buffer->length > 0) { - uint32_t send_bytes = (uint32_t)GPR_MIN( - max_outgoing, s->compressed_data_buffer->length); - is_last_data_frame = - (send_bytes == s->compressed_data_buffer->length && - s->flow_controlled_buffer.length == 0 && - s->fetching_send_message == NULL); - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - grpc_chttp2_encode_data(s->id, s->compressed_data_buffer, - send_bytes, is_last_frame, - &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, - &s->flow_control, send_bytes); - max_outgoing -= send_bytes; - if (s->compressed_data_buffer->length == 0) { - s->sending_bytes += s->uncompressed_data_size; - } - } else { - if (s->stream_compression_ctx == NULL) { - s->stream_compression_ctx = - grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); - } - s->uncompressed_data_size = s->flow_controlled_buffer.length; - GPR_ASSERT(grpc_stream_compress( - s->stream_compression_ctx, &s->flow_controlled_buffer, - s->compressed_data_buffer, NULL, MAX_SIZE_T, - GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); + while ((s->flow_controlled_buffer.length > 0 || + s->compressed_data_buffer.length > 0) && + max_outgoing > 0) { + if (s->compressed_data_buffer.length > 0) { + uint32_t send_bytes = (uint32_t)GPR_MIN( + max_outgoing, s->compressed_data_buffer.length); + is_last_data_frame = + (send_bytes == s->compressed_data_buffer.length && + s->flow_controlled_buffer.length == 0 && + s->fetching_send_message == NULL); + is_last_frame = + is_last_data_frame && s->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty(s->send_trailing_metadata); + grpc_chttp2_encode_data(s->id, &s->compressed_data_buffer, + send_bytes, is_last_frame, + &s->stats.outgoing, &t->outbuf); + grpc_chttp2_flowctl_sent_data(&t->flow_control, + &s->flow_control, send_bytes); + max_outgoing -= send_bytes; + if (s->compressed_data_buffer.length == 0) { + s->sending_bytes += s->uncompressed_data_size; } + } else { + if (s->stream_compression_ctx == NULL) { + s->stream_compression_ctx = + grpc_stream_compression_context_create( + s->stream_compression_method); + } + s->uncompressed_data_size = s->flow_controlled_buffer.length; + GPR_ASSERT(grpc_stream_compress( + s->stream_compression_ctx, &s->flow_controlled_buffer, + &s->compressed_data_buffer, NULL, MAX_SIZE_T, + GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); } - } else { - uint32_t send_bytes = (uint32_t)GPR_MIN( - max_outgoing, s->flow_controlled_buffer.length); - is_last_data_frame = s->fetching_send_message == NULL && - send_bytes == s->flow_controlled_buffer.length; - is_last_frame = - is_last_data_frame && s->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty(s->send_trailing_metadata); - grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, - send_bytes, is_last_frame, - &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, &s->flow_control, - send_bytes); - s->sending_bytes += send_bytes; } t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; @@ -371,8 +354,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } now_writing = true; if (s->flow_controlled_buffer.length > 0 || - (s->stream_compression_send_enabled && - s->compressed_data_buffer->length > 0)) { + s->compressed_data_buffer.length > 0) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:fork"); grpc_chttp2_list_add_writable_stream(t, s); } @@ -387,8 +369,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (s->send_trailing_metadata != NULL && s->fetching_send_message == NULL && s->flow_controlled_buffer.length == 0 && - (!s->stream_compression_send_enabled || - s->compressed_data_buffer->length == 0)) { + s->compressed_data_buffer.length == 0) { GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "sending trailing_metadata")); if (grpc_metadata_batch_is_empty(s->send_trailing_metadata)) { grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, 0, true, -- cgit v1.2.3 From 03600fc37df35399063ea9d89e9b745835a41396 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 13:42:44 -0700 Subject: clang-format --- .../transport/chttp2/transport/chttp2_transport.c | 9 ++++- src/core/ext/transport/chttp2/transport/writing.c | 4 +- src/core/lib/compression/stream_compression.c | 19 ++++++---- src/core/lib/compression/stream_compression.h | 22 +++++------ src/core/lib/compression/stream_compression_gzip.c | 44 +++++++++++++--------- .../lib/compression/stream_compression_identity.c | 36 ++++++++++-------- .../lib/compression/stream_compression_identity.h | 3 +- 7 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index e1fb9dd915..64a5e342fe 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1297,8 +1297,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE; /* Identify stream compression */ - if (op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding == NULL || - grpc_stream_compression_method_parse(GRPC_MDVALUE(op_payload->send_initial_metadata.send_initial_metadata->idx.named.content_encoding->md), true, &s->stream_compression_method) == 0) { + if (op_payload->send_initial_metadata.send_initial_metadata->idx.named + .content_encoding == NULL || + grpc_stream_compression_method_parse( + GRPC_MDVALUE( + op_payload->send_initial_metadata.send_initial_metadata->idx + .named.content_encoding->md), + true, &s->stream_compression_method) == 0) { s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } grpc_slice_buffer_init(&s->compressed_data_buffer); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index b6bc864c29..fff98a6201 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -317,8 +317,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_chttp2_encode_data(s->id, &s->compressed_data_buffer, send_bytes, is_last_frame, &s->stats.outgoing, &t->outbuf); - grpc_chttp2_flowctl_sent_data(&t->flow_control, - &s->flow_control, send_bytes); + grpc_chttp2_flowctl_sent_data(&t->flow_control, &s->flow_control, + send_bytes); max_outgoing -= send_bytes; if (s->compressed_data_buffer.length == 0) { s->sending_bytes += s->uncompressed_data_size; diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index 274ee100ba..411489f029 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -21,20 +21,23 @@ #include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/compression/stream_compression_gzip.h" -extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; +extern const grpc_stream_compression_vtable + grpc_stream_compression_identity_vtable; bool grpc_stream_compress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, grpc_stream_compression_flush flush) { - return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, flush); + return ctx->vtable->compress(ctx, in, out, output_size, max_output_size, + flush); } bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, bool *end_of_context) { - return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, end_of_context); + return ctx->vtable->decompress(ctx, in, out, output_size, max_output_size, + end_of_context); } grpc_stream_compression_context *grpc_stream_compression_context_create( @@ -58,15 +61,17 @@ void grpc_stream_compression_context_destroy( } int grpc_stream_compression_method_parse( - grpc_slice value, bool is_compress, grpc_stream_compression_method *method) { + grpc_slice value, bool is_compress, + grpc_stream_compression_method *method) { if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) { - *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS + : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; return 1; } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) { - *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; + *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS + : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS; return 1; } else { return 0; } } - diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index fa07247c31..6d073280fa 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -49,18 +49,16 @@ typedef enum grpc_stream_compression_flush { } grpc_stream_compression_flush; struct grpc_stream_compression_vtable { -bool (*compress)(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush); -bool (*decompress)(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context); -grpc_stream_compression_context *(*context_create)( - grpc_stream_compression_method method); -void (*context_destroy)( - grpc_stream_compression_context *ctx); + bool (*compress)(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, + grpc_slice_buffer *out, size_t *output_size, + size_t max_output_size, grpc_stream_compression_flush flush); + bool (*decompress)(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context); + grpc_stream_compression_context *(*context_create)( + grpc_stream_compression_method method); + void (*context_destroy)(grpc_stream_compression_context *ctx); }; /** diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 2d03836a0d..5d21bf27ae 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -132,13 +132,16 @@ static bool gzip_flate(grpc_stream_compression_context_gzip *ctx, } static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + grpc_stream_compression_flush flush) { if (ctx == NULL) { return false; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; GPR_ASSERT(gzip_ctx->flate == deflate); int gzip_flush; switch (flush) { @@ -159,21 +162,26 @@ static bool grpc_stream_compress_gzip(grpc_stream_compression_context *ctx, } static bool grpc_stream_decompress_gzip(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + bool *end_of_context) { if (ctx == NULL) { return false; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip *)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; GPR_ASSERT(gzip_ctx->flate == inflate); - return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, - end_of_context); + return gzip_flate(gzip_ctx, in, out, output_size, max_output_size, + Z_SYNC_FLUSH, end_of_context); } -static grpc_stream_compression_context *grpc_stream_compression_context_create_gzip( +static grpc_stream_compression_context * +grpc_stream_compression_context_create_gzip( grpc_stream_compression_method method) { - GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || + method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_stream_compression_context_gzip *gzip_ctx = gpr_zalloc(sizeof(grpc_stream_compression_context_gzip)); int r; @@ -201,7 +209,8 @@ static void grpc_stream_compression_context_destroy_gzip( if (ctx == NULL) { return; } - grpc_stream_compression_context_gzip *gzip_ctx = (grpc_stream_compression_context_gzip*)ctx; + grpc_stream_compression_context_gzip *gzip_ctx = + (grpc_stream_compression_context_gzip *)ctx; if (gzip_ctx->flate == inflate) { inflateEnd(&gzip_ctx->zs); } else { @@ -211,8 +220,7 @@ static void grpc_stream_compression_context_destroy_gzip( } const grpc_stream_compression_vtable grpc_stream_compression_gzip_vtable = { - .compress = grpc_stream_compress_gzip, - .decompress = grpc_stream_decompress_gzip, - .context_create = grpc_stream_compression_context_create_gzip, - .context_destroy = grpc_stream_compression_context_destroy_gzip -}; + .compress = grpc_stream_compress_gzip, + .decompress = grpc_stream_decompress_gzip, + .context_create = grpc_stream_compression_context_create_gzip, + .context_destroy = grpc_stream_compression_context_destroy_gzip}; diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 1395b762d4..133c3250d2 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -25,7 +25,10 @@ #define OUTPUT_BLOCK_SIZE (1024) -static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { +static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size) { if (max_output_size >= in->length) { *output_size = in->length; grpc_slice_buffer_move_into(in, out); @@ -36,9 +39,11 @@ static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_sli } static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - grpc_stream_compression_flush flush) { + grpc_slice_buffer *in, + grpc_slice_buffer *out, + size_t *output_size, + size_t max_output_size, + grpc_stream_compression_flush flush) { if (ctx == NULL) { return false; } @@ -46,10 +51,10 @@ static bool grpc_stream_compress_identity(grpc_stream_compression_context *ctx, return true; } -static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx, - grpc_slice_buffer *in, grpc_slice_buffer *out, - size_t *output_size, size_t max_output_size, - bool *end_of_context) { +static bool grpc_stream_decompress_identity( + grpc_stream_compression_context *ctx, grpc_slice_buffer *in, + grpc_slice_buffer *out, size_t *output_size, size_t max_output_size, + bool *end_of_context) { if (ctx == NULL) { return false; } @@ -57,9 +62,11 @@ static bool grpc_stream_decompress_identity(grpc_stream_compression_context *ctx return true; } -static grpc_stream_compression_context *grpc_stream_compression_context_create_identity( +static grpc_stream_compression_context * +grpc_stream_compression_context_create_identity( grpc_stream_compression_method method) { - GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); + GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || + method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); /* No context needed in this case. Use fake context instead. */ return (grpc_stream_compression_context *)1; } @@ -70,8 +77,7 @@ static void grpc_stream_compression_context_destroy_identity( } const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable = { - .compress = grpc_stream_compress_identity, - .decompress = grpc_stream_decompress_identity, - .context_create = grpc_stream_compression_context_create_identity, - .context_destroy = grpc_stream_compression_context_destroy_identity -}; + .compress = grpc_stream_compress_identity, + .decompress = grpc_stream_decompress_identity, + .context_create = grpc_stream_compression_context_create_identity, + .context_destroy = grpc_stream_compression_context_destroy_identity}; diff --git a/src/core/lib/compression/stream_compression_identity.h b/src/core/lib/compression/stream_compression_identity.h index 0d770dd32b..41926e949e 100644 --- a/src/core/lib/compression/stream_compression_identity.h +++ b/src/core/lib/compression/stream_compression_identity.h @@ -21,6 +21,7 @@ #include "src/core/lib/compression/stream_compression.h" -extern const grpc_stream_compression_vtable grpc_stream_compression_identity_vtable; +extern const grpc_stream_compression_vtable + grpc_stream_compression_identity_vtable; #endif -- cgit v1.2.3 From 68a1a1530be8573e109c4ec6754941a682f8b5dd Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 14:47:33 -0700 Subject: Build fix --- test/core/compression/stream_compression_test.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index e576507aaf..26c957ded4 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -59,10 +59,10 @@ static void test_stream_compression_simple_compress_decompress() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -91,10 +91,10 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -139,10 +139,10 @@ test_stream_compression_simple_compress_decompress_with_large_data() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -172,7 +172,7 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -181,7 +181,7 @@ static void test_stream_compression_drop_context() { grpc_stream_compression_context_destroy(compress_ctx); compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); slice = grpc_slice_from_static_string(test_str2); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -205,7 +205,7 @@ static void test_stream_compression_drop_context() { grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); bool end_of_context; size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, @@ -219,7 +219,7 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&sink); decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, ~(size_t)0, &end_of_context)); GPR_ASSERT(end_of_context == true); @@ -240,7 +240,7 @@ static void test_stream_compression_sync_flush() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -249,7 +249,7 @@ static void test_stream_compression_sync_flush() { grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_DECOMPRESS); + GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); bool end_of_context; size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, -- cgit v1.2.3 From c3229b777ce58f49b092a92a07b18155d1ae7799 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 17:57:19 -0700 Subject: Remove duplicate sentences on recv path --- .../transport/chttp2/transport/chttp2_transport.c | 37 +++++++++------------- .../ext/transport/chttp2/transport/hpack_parser.c | 13 ++------ src/core/ext/transport/chttp2/transport/internal.h | 9 +++--- .../lib/compression/stream_compression_identity.c | 1 + 4 files changed, 23 insertions(+), 37 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 64a5e342fe..ad0521cc42 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -693,6 +693,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_schedule_on_exec_ctx); grpc_slice_buffer_init(&s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_init(&s->frame_storage); + grpc_slice_buffer_init(&s->compressed_data_buffer); + grpc_slice_buffer_init(&s->decompressed_data_buffer); s->pending_byte_stream = false; GRPC_CLOSURE_INIT(&s->reset_byte_stream, reset_byte_stream, s, grpc_combiner_scheduler(t->combiner)); @@ -728,10 +730,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, &s->unprocessed_incoming_frames_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage); grpc_slice_buffer_destroy_internal(exec_ctx, &s->compressed_data_buffer); - if (s->decompressed_data_buffer) { - grpc_slice_buffer_destroy_internal(exec_ctx, s->decompressed_data_buffer); - gpr_free(s->decompressed_data_buffer); - } + grpc_slice_buffer_destroy_internal(exec_ctx, &s->decompressed_data_buffer); grpc_chttp2_list_remove_stalled_by_transport(t, s); grpc_chttp2_list_remove_stalled_by_stream(t, s); @@ -1306,7 +1305,6 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, true, &s->stream_compression_method) == 0) { s->stream_compression_method = GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS; } - grpc_slice_buffer_init(&s->compressed_data_buffer); s->send_initial_metadata_finished = add_closure_barrier(on_complete); s->send_initial_metadata = @@ -1725,18 +1723,17 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, &s->frame_storage); s->unprocessed_incoming_frames_decompressed = false; } - if (s->stream_compression_recv_enabled && - !s->unprocessed_incoming_frames_decompressed) { - GPR_ASSERT(s->decompressed_data_buffer->length == 0); + if (!s->unprocessed_incoming_frames_decompressed) { + GPR_ASSERT(s->decompressed_data_buffer.length == 0); bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer, NULL, + &s->decompressed_data_buffer, NULL, GRPC_HEADER_SIZE_IN_BYTES, &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, @@ -1747,7 +1744,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, "Stream decompression error."); } else { error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, s->decompressed_data_buffer, NULL, + exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, s->recv_message); if (end_of_context) { grpc_stream_compression_context_destroy( @@ -1755,10 +1752,6 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->stream_decompression_ctx = NULL; } } - } else { - error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, - &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; @@ -1797,7 +1790,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } bool pending_data = s->pending_byte_stream || s->unprocessed_incoming_frames_buffer.length > 0; - if (s->stream_compression_recv_enabled && s->read_closed && + if (s->read_closed && s->frame_storage.length > 0 && !pending_data && !s->seen_error && s->recv_trailing_metadata_finished != NULL) { /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and @@ -1805,7 +1798,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->frame_storage, @@ -1818,6 +1811,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } else { if (s->unprocessed_incoming_frames_buffer.length > 0) { s->unprocessed_incoming_frames_decompressed = true; + pending_data = true; } if (end_of_context) { grpc_stream_compression_context_destroy(s->stream_decompression_ctx); @@ -2690,16 +2684,15 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, grpc_error *error; if (s->unprocessed_incoming_frames_buffer.length > 0) { - if (s->stream_compression_recv_enabled && - !s->unprocessed_incoming_frames_decompressed) { + if (!s->unprocessed_incoming_frames_decompressed) { bool end_of_context; if (!s->stream_decompression_ctx) { s->stream_decompression_ctx = grpc_stream_compression_context_create( - GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); + s->stream_decompression_method); } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer, NULL, MAX_SIZE_T, + &s->decompressed_data_buffer, NULL, MAX_SIZE_T, &end_of_context)) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream decompression error."); @@ -2707,7 +2700,7 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, } GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0); grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, - s->decompressed_data_buffer); + &s->decompressed_data_buffer); s->unprocessed_incoming_frames_decompressed = true; if (end_of_context) { grpc_stream_compression_context_destroy(s->stream_decompression_ctx); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index c21d76ba71..3b338da288 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1659,16 +1659,9 @@ static void parse_stream_compression_md(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_metadata_batch *initial_metadata) { - if (initial_metadata->idx.named.content_encoding != NULL) { - grpc_slice content_encoding = - GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md); - if (!grpc_slice_eq(content_encoding, GRPC_MDSTR_IDENTITY)) { - if (grpc_slice_eq(content_encoding, GRPC_MDSTR_GZIP)) { - s->stream_compression_recv_enabled = true; - s->decompressed_data_buffer = gpr_malloc(sizeof(grpc_slice_buffer)); - grpc_slice_buffer_init(s->decompressed_data_buffer); - } - } + if (initial_metadata->idx.named.content_encoding == NULL || + grpc_stream_compression_method_parse(GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, &s->stream_decompression_method) == 0) { + s->stream_decompression_method = GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; } } diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index e5e4dd3344..33cc962f64 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -561,10 +561,6 @@ struct grpc_chttp2_stream { grpc_stream_compression_method stream_compression_method; /* Stream decompression method to be used. */ grpc_stream_compression_method stream_decompression_method; - bool stream_compression_recv_enabled; - /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed - */ - bool unprocessed_incoming_frames_decompressed; /** Stream compression decompress context */ grpc_stream_compression_context *stream_decompression_ctx; /** Stream compression compress context */ @@ -576,7 +572,10 @@ struct grpc_chttp2_stream { * emptied */ size_t uncompressed_data_size; /** Temporary buffer storing decompressed data */ - grpc_slice_buffer *decompressed_data_buffer; + grpc_slice_buffer decompressed_data_buffer; + /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed + */ + bool unprocessed_incoming_frames_decompressed; }; /** Transport writing call flow: diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 133c3250d2..91ee06d783 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -59,6 +59,7 @@ static bool grpc_stream_decompress_identity( return false; } grpc_stream_compression_pass_through(in, out, output_size, max_output_size); + *end_of_context = false; return true; } -- cgit v1.2.3 From 93197df3d5c5ad49c7fcbaa05b005e6ad7837560 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 5 Sep 2017 19:19:44 -0700 Subject: Add patch to identity --- src/core/lib/compression/stream_compression_gzip.c | 3 ++- .../lib/compression/stream_compression_identity.c | 21 ++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 5d21bf27ae..502bf44a91 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -19,7 +19,7 @@ #include #include -#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/compression/stream_compression_gzip.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" @@ -201,6 +201,7 @@ grpc_stream_compression_context_create_gzip( return NULL; } + gzip_ctx->base.vtable = &grpc_stream_compression_gzip_vtable; return (grpc_stream_compression_context *)gzip_ctx; } diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 91ee06d783..19eb6a9a00 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -19,21 +19,30 @@ #include #include -#include "src/core/lib/compression/stream_compression.h" +#include "src/core/lib/compression/stream_compression_identity.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" #define OUTPUT_BLOCK_SIZE (1024) +/* Singleton context used for all identity streams. */ +static grpc_stream_compression_context identity_ctx = { + .vtable = &grpc_stream_compression_identity_vtable +}; + static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, size_t *output_size, size_t max_output_size) { if (max_output_size >= in->length) { - *output_size = in->length; + if (output_size) { + *output_size = in->length; + } grpc_slice_buffer_move_into(in, out); } else { - *output_size = max_output_size; + if (output_size) { + *output_size = max_output_size; + } grpc_slice_buffer_move_first(in, max_output_size, out); } } @@ -59,7 +68,9 @@ static bool grpc_stream_decompress_identity( return false; } grpc_stream_compression_pass_through(in, out, output_size, max_output_size); - *end_of_context = false; + if (end_of_context) { + *end_of_context = false; + } return true; } @@ -69,7 +80,7 @@ grpc_stream_compression_context_create_identity( GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS || method == GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS); /* No context needed in this case. Use fake context instead. */ - return (grpc_stream_compression_context *)1; + return (grpc_stream_compression_context *)&identity_ctx; } static void grpc_stream_compression_context_destroy_identity( -- cgit v1.2.3 From f4c95fa4ebfa5cd8dfea81f949fce2412125b085 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 6 Sep 2017 11:58:29 -0700 Subject: Patch for fuzzer test failure --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 7 ++++++- src/core/ext/transport/chttp2/transport/frame_data.c | 4 ++-- src/core/ext/transport/chttp2/transport/internal.h | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index ad0521cc42..2a26bbb0c8 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -696,6 +696,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_slice_buffer_init(&s->compressed_data_buffer); grpc_slice_buffer_init(&s->decompressed_data_buffer); s->pending_byte_stream = false; + s->decompressed_header_bytes = 0; GRPC_CLOSURE_INIT(&s->reset_byte_stream, reset_byte_stream, s, grpc_combiner_scheduler(t->combiner)); @@ -1734,7 +1735,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, &s->decompressed_data_buffer, NULL, - GRPC_HEADER_SIZE_IN_BYTES, + GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); @@ -1743,6 +1744,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Stream decompression error."); } else { + s->decompressed_header_bytes += s->decompressed_data_buffer.length; + if (s->decompressed_header_bytes == GRPC_HEADER_SIZE_IN_BYTES) { + s->decompressed_header_bytes = 0; + } error = grpc_deframe_unprocessed_incoming_frames( exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, s->recv_message); diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 222d2177b2..73aaab1802 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -210,7 +210,7 @@ grpc_error *grpc_deframe_unprocessed_incoming_frames( if (cur != end) { grpc_slice_buffer_undo_take_first( - &s->unprocessed_incoming_frames_buffer, + slices, grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); } grpc_slice_unref_internal(exec_ctx, slice); @@ -277,7 +277,7 @@ grpc_error *grpc_deframe_unprocessed_incoming_frames( p->state = GRPC_CHTTP2_DATA_FH_0; cur += p->frame_size; grpc_slice_buffer_undo_take_first( - &s->unprocessed_incoming_frames_buffer, + slices, grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); grpc_slice_unref_internal(exec_ctx, slice); return GRPC_ERROR_NONE; diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 33cc962f64..b64e9a0cd6 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -576,6 +576,8 @@ struct grpc_chttp2_stream { /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed */ bool unprocessed_incoming_frames_decompressed; + /** gRPC header bytes that are already decompressed */ + size_t decompressed_header_bytes; }; /** Transport writing call flow: -- cgit v1.2.3 From c2a61a2780e56c474a12f5570133fe166ec9840f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 6 Sep 2017 22:07:42 -0700 Subject: write_buffering patch --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 2a26bbb0c8..9b80e1d336 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1757,6 +1757,9 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->stream_decompression_ctx = NULL; } } + } else { + error = grpc_deframe_unprocessed_incoming_frames( + exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; -- cgit v1.2.3 From 1e99b2388bf5e13c79143463f6d503275fd63cb7 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 7 Sep 2017 09:47:58 -0700 Subject: clang-format --- .../transport/chttp2/transport/chttp2_transport.c | 27 +++++++++++----------- .../ext/transport/chttp2/transport/hpack_parser.c | 7 ++++-- .../lib/compression/stream_compression_identity.c | 3 +-- test/core/compression/stream_compression_test.c | 19 +++++++++------ 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9b80e1d336..721f694506 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1732,11 +1732,12 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, grpc_stream_compression_context_create( s->stream_decompression_method); } - if (!grpc_stream_decompress(s->stream_decompression_ctx, - &s->unprocessed_incoming_frames_buffer, - &s->decompressed_data_buffer, NULL, - GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, - &end_of_context)) { + if (!grpc_stream_decompress( + s->stream_decompression_ctx, + &s->unprocessed_incoming_frames_buffer, + &s->decompressed_data_buffer, NULL, + GRPC_HEADER_SIZE_IN_BYTES - s->decompressed_header_bytes, + &end_of_context)) { grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); grpc_slice_buffer_reset_and_unref_internal( @@ -1749,8 +1750,8 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, s->decompressed_header_bytes = 0; } error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, NULL, - s->recv_message); + exec_ctx, &s->data_parser, s, &s->decompressed_data_buffer, + NULL, s->recv_message); if (end_of_context) { grpc_stream_compression_context_destroy( s->stream_decompression_ctx); @@ -1759,7 +1760,8 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, } } else { error = grpc_deframe_unprocessed_incoming_frames( - exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); + exec_ctx, &s->data_parser, s, + &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); } if (error != GRPC_ERROR_NONE) { s->seen_error = true; @@ -1798,9 +1800,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } bool pending_data = s->pending_byte_stream || s->unprocessed_incoming_frames_buffer.length > 0; - if (s->read_closed && - s->frame_storage.length > 0 && !pending_data && !s->seen_error && - s->recv_trailing_metadata_finished != NULL) { + if (s->read_closed && s->frame_storage.length > 0 && !pending_data && + !s->seen_error && s->recv_trailing_metadata_finished != NULL) { /* Maybe some SYNC_FLUSH data is left in frame_storage. Consume them and * maybe decompress the next 5 bytes in the stream. */ bool end_of_context; @@ -2700,8 +2701,8 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, } if (!grpc_stream_decompress(s->stream_decompression_ctx, &s->unprocessed_incoming_frames_buffer, - &s->decompressed_data_buffer, NULL, MAX_SIZE_T, - &end_of_context)) { + &s->decompressed_data_buffer, NULL, + MAX_SIZE_T, &end_of_context)) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream decompression error."); return error; diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 3b338da288..2028877cac 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1660,8 +1660,11 @@ static void parse_stream_compression_md(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s, grpc_metadata_batch *initial_metadata) { if (initial_metadata->idx.named.content_encoding == NULL || - grpc_stream_compression_method_parse(GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, &s->stream_decompression_method) == 0) { - s->stream_decompression_method = GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; + grpc_stream_compression_method_parse( + GRPC_MDVALUE(initial_metadata->idx.named.content_encoding->md), false, + &s->stream_decompression_method) == 0) { + s->stream_decompression_method = + GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS; } } diff --git a/src/core/lib/compression/stream_compression_identity.c b/src/core/lib/compression/stream_compression_identity.c index 19eb6a9a00..3dfcf53b85 100644 --- a/src/core/lib/compression/stream_compression_identity.c +++ b/src/core/lib/compression/stream_compression_identity.c @@ -27,8 +27,7 @@ /* Singleton context used for all identity streams. */ static grpc_stream_compression_context identity_ctx = { - .vtable = &grpc_stream_compression_identity_vtable -}; + .vtable = &grpc_stream_compression_identity_vtable}; static void grpc_stream_compression_pass_through(grpc_slice_buffer *in, grpc_slice_buffer *out, diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index 26c957ded4..afed6cd6b5 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -59,7 +59,8 @@ static void test_stream_compression_simple_compress_decompress() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -91,7 +92,8 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -139,7 +141,8 @@ test_stream_compression_simple_compress_decompress_with_large_data() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_stream_compression_context *decompress_ctx = grpc_stream_compression_context_create( GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); @@ -172,7 +175,8 @@ static void test_stream_compression_drop_context() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -180,8 +184,8 @@ static void test_stream_compression_drop_context() { GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); grpc_stream_compression_context_destroy(compress_ctx); - compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + compress_ctx = grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); slice = grpc_slice_from_static_string(test_str2); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, @@ -240,7 +244,8 @@ static void test_stream_compression_sync_flush() { grpc_slice_buffer_init(&relay); grpc_slice_buffer_init(&sink); grpc_stream_compression_context *compress_ctx = - grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_GZIP_COMPRESS); grpc_slice slice = grpc_slice_from_static_string(test_str); grpc_slice_buffer_add(&source, slice); GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, -- cgit v1.2.3 From f5a7f7f559ec6cbc30227cf051603ade22b556f8 Mon Sep 17 00:00:00 2001 From: Michael Bausor Date: Thu, 7 Sep 2017 09:47:50 -0700 Subject: Determine type name using php namespace option --- src/compiler/php_generator.cc | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/compiler/php_generator.cc b/src/compiler/php_generator.cc index 67967d0bd7..9fa14d1684 100644 --- a/src/compiler/php_generator.cc +++ b/src/compiler/php_generator.cc @@ -33,7 +33,7 @@ using std::map; namespace grpc_php_generator { namespace { -grpc::string MessageIdentifierName(const grpc::string &name) { +grpc::string ConvertToPhpNamespace(const grpc::string &name) { std::vector tokens = grpc_generator::tokenize(name, "."); std::ostringstream oss; for (unsigned int i = 0; i < tokens.size(); i++) { @@ -43,14 +43,32 @@ grpc::string MessageIdentifierName(const grpc::string &name) { return oss.str(); } +grpc::string PackageName(const FileDescriptor *file) { + if (file->options().has_php_namespace()) { + return file->options().php_namespace(); + } else { + return ConvertToPhpNamespace(file->package()); + } +} + +grpc::string MessageIdentifierName(const grpc::string &name, const FileDescriptor *file) { + std::vector tokens = grpc_generator::tokenize(name, "."); + std::ostringstream oss; + oss << PackageName(file) + << "\\" + << grpc_generator::CapitalizeFirstLetter(tokens[tokens.size() - 1]); + return oss.str(); +} + + void PrintMethod(const MethodDescriptor *method, Printer *out) { const Descriptor *input_type = method->input_type(); const Descriptor *output_type = method->output_type(); map vars; vars["service_name"] = method->service()->full_name(); vars["name"] = method->name(); - vars["input_type_id"] = MessageIdentifierName(input_type->full_name()); - vars["output_type_id"] = MessageIdentifierName(output_type->full_name()); + vars["input_type_id"] = MessageIdentifierName(input_type->full_name(), input_type->file()); + vars["output_type_id"] = MessageIdentifierName(output_type->full_name(), output_type->file()); out->Print("/**\n"); out->Print(GetPHPComments(method, " *").c_str()); @@ -149,12 +167,7 @@ grpc::string GenerateFile(const FileDescriptor *file, } map vars; - grpc::string php_namespace; - if (file->options().has_php_namespace()) { - php_namespace = file->options().php_namespace(); - } else { - php_namespace = MessageIdentifierName(file->package()); - } + grpc::string php_namespace = PackageName(file); vars["package"] = php_namespace; out.Print(vars, "namespace $package$;\n\n"); -- cgit v1.2.3 From 9cd90f43dcfeabff95cb7b61fb029ff7e4a899b3 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 7 Sep 2017 09:51:13 -0700 Subject: fix bazel build --- BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/BUILD b/BUILD index 67e727bd52..e147dec4d4 100644 --- a/BUILD +++ b/BUILD @@ -574,6 +574,8 @@ grpc_cc_library( "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression_gzip.c", + "src/core/lib/compression/stream_compression_identity.c", "src/core/lib/http/format_request.c", "src/core/lib/http/httpcli.c", "src/core/lib/http/parser.c", @@ -706,6 +708,8 @@ grpc_cc_library( "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", "src/core/lib/compression/stream_compression.h", + "src/core/lib/compression/stream_compression_gzip.h", + "src/core/lib/compression/stream_compression_identity.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", -- cgit v1.2.3 From 677395558d4f28ca3fad53d6e082552f0ce2eb5e Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 20 Jul 2017 10:02:47 -0700 Subject: Allow SerializationTraits to use grpc::ByteBuffer, not just grpc_byte_buffer --- BUILD | 2 + CMakeLists.txt | 15 +++ Makefile | 15 +++ build.yaml | 2 + gRPC-Core.podspec | 1 + grpc.def | 20 +-- grpc.gemspec | 1 + include/grpc++/impl/codegen/byte_buffer.h | 141 +++++++++++++++++++++ include/grpc++/impl/codegen/call.h | 113 +++++++++++++---- include/grpc++/impl/codegen/method_handler_impl.h | 9 +- include/grpc++/impl/codegen/rpc_service_method.h | 14 +- include/grpc++/impl/codegen/serialization_traits.h | 25 ++-- include/grpc++/impl/codegen/slice.h | 78 ++++++++++++ include/grpc++/support/byte_buffer.h | 68 +--------- include/grpc++/support/slice.h | 80 +----------- include/grpc/byte_buffer.h | 64 +--------- include/grpc/impl/codegen/byte_buffer.h | 86 +++++++++++++ package.xml | 1 + src/core/lib/iomgr/resource_quota.c | 1 + .../server/health/default_health_check_service.cc | 1 + src/cpp/util/byte_buffer_cc.cc | 37 ++++-- src/cpp/util/slice_cc.cc | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 +++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++----- test/core/surface/public_headers_must_be_c89.c | 1 + test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + tools/run_tests/generated/sources_and_headers.json | 4 + 31 files changed, 564 insertions(+), 330 deletions(-) create mode 100644 include/grpc++/impl/codegen/byte_buffer.h create mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index f74db0a1c8..70d9635bc5 100644 --- a/BUILD +++ b/BUILD @@ -989,6 +989,7 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1487,6 +1488,7 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index f77cd5b131..01d6904c56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1255,6 +1255,7 @@ target_link_libraries(grpc ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1563,6 +1564,7 @@ target_link_libraries(grpc_cronet ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1841,6 +1843,7 @@ target_link_libraries(grpc_test_util ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2103,6 +2106,7 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2400,6 +2404,7 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2715,6 +2720,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2725,6 +2731,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3149,6 +3156,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3159,6 +3167,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3529,6 +3538,7 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3556,6 +3566,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3669,6 +3680,7 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3696,6 +3708,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3900,6 +3913,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3910,6 +3924,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index d933c2a5ee..02fa235146 100644 --- a/Makefile +++ b/Makefile @@ -3165,6 +3165,7 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3471,6 +3472,7 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3748,6 +3750,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3999,6 +4002,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4271,6 +4275,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4559,6 +4564,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4569,6 +4575,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5000,6 +5007,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5010,6 +5018,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5363,6 +5372,7 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5390,6 +5400,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5478,6 +5489,7 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5505,6 +5517,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5712,6 +5725,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5722,6 +5736,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index 089b8423a8..3ebaa24e25 100644 --- a/build.yaml +++ b/build.yaml @@ -502,6 +502,7 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: + - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -969,6 +970,7 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h + - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index fdab4b3f81..8acd4380f5 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,6 +141,7 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', + 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index 37cc7e20f0..30d89857fa 100644 --- a/grpc.def +++ b/grpc.def @@ -1,14 +1,4 @@ EXPORTS - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -144,6 +134,16 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index c441107643..3c093dc99e 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -145,6 +145,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) + s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h new file mode 100644 index 0000000000..e6231ca243 --- /dev/null +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -0,0 +1,141 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#include +#include +#include +#include +#include + +#include + +namespace grpc { + +template +class CallOpRecvMessage; +class MethodHandler; +namespace internal { +template +class MessageDeserializer; +} + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + } + } + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } + } + + /// Make a duplicate copy of the internals of this byte + /// buffer so that we have our own owned version of it. + /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable + void Duplicate() { + buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); + } + + /// Forget underlying byte buffer without destroying + /// Use this only for un-owned byte buffers + void Release() { buffer_ = nullptr; } + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + /// Is this ByteBuffer valid? + bool Valid() const { return (buffer_ != nullptr); } + + private: + friend class SerializationTraits; + friend class CallOpSendMessage; + template + friend class CallOpRecvMessage; + friend class CallOpGenericRecvMessage; + friend class MethodHandler; + template + friend class internal::MessageDeserializer; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + grpc_byte_buffer* c_buffer() { return buffer_; } + grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } + + // DEPRECATED: Implicit conversion to transparently + // support deprecated SerializationTraits API + // No need to inline since deprecated + operator grpc_byte_buffer*(); + operator const grpc_byte_buffer*() const; + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer.buffer_); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, + bool* own_buffer) { + *buffer = source; + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 8e70225f86..bbd277f288 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -struct grpc_byte_buffer; - namespace grpc { class ByteBuffer; @@ -272,7 +271,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_(nullptr) {} + CallOpSendMessage() : send_buf_() {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -285,33 +284,64 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (send_buf_ == nullptr) return; + if (!send_buf_.Valid()) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_; + op->data.send_message.send_message = send_buf_.c_buffer(); // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { - g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); - send_buf_ = nullptr; - } + void FinishOp(bool* status) { send_buf_.Clear(); } private: - grpc_byte_buffer* send_buf_; + template + class MessageSerializer; + + ByteBuffer send_buf_; WriteOptions write_options_; }; +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + static_cast(*(static_cast(nullptr))), + static_cast(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), + own_buf); + } +}; + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + static_cast(*(static_cast(nullptr))), + static_cast<::grpc::ByteBuffer*>(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf, own_buf); + } +}; + template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - SerializationTraits::Serialize(message, &send_buf_, &own_buf); + MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); + send_buf_.Duplicate(); } return result; } @@ -321,6 +351,38 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } +namespace internal { +template +class MessageDeserializer; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + static_cast( + *(static_cast(nullptr))), + static_cast(nullptr)))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize(bbuf, message); + } +}; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + static_cast(nullptr), + static_cast(nullptr)))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize( + const_cast(bbuf).c_buffer(), message); + } +}; +} // namespace internal + template class CallOpRecvMessage { public: @@ -343,18 +405,20 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = *status = - SerializationTraits::Deserialize(recv_buf_, message_).ok(); + internal::MessageDeserializer::Deserialize(recv_buf_, message_) + .ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -367,14 +431,14 @@ class CallOpRecvMessage { private: R* message_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(grpc_byte_buffer* buf) = 0; + virtual Status Deserialize(const ByteBuffer& buf) = 0; virtual ~DeserializeFunc() {} }; @@ -382,8 +446,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(grpc_byte_buffer* buf) override { - return SerializationTraits::Deserialize(buf, message_); + Status Deserialize(const ByteBuffer& buf) override { + return grpc::internal::MessageDeserializer::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -419,18 +483,19 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -443,7 +508,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 15e24bdcdc..8125e0a651 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H +#include #include #include #include @@ -37,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -123,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index 7165774172..d356012ad6 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,14 +25,11 @@ #include #include +#include #include #include #include -extern "C" { -struct grpc_byte_buffer; -} - namespace grpc { class ServerContext; class StreamContextInterface; @@ -43,11 +40,14 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context), request(req) {} + : call(c), server_context(context) { + request.set_buffer(req); + } + ~HandlerParameter() { request.Release(); } Call* call; ServerContext* server_context; - // Handler required to grpc_byte_buffer_destroy this - grpc_byte_buffer* request; + // Handler required to destroy these contents + ByteBuffer request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index b72d474126..7fc4e24172 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,15 +26,24 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg, -/// int max_receive_message_size); +/// 1. static Status Serialize(const Message& msg, +/// ByteBuffer* buffer, +/// bool* own_buffer); +/// AND/OR +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// The former is preferred; the latter is deprecated /// -/// Serialize is required to convert message to a grpc_byte_buffer, and -/// to store a pointer to that byte buffer at *buffer. *own_buffer should +/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// Message* msg); +/// AND/OR +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg); +/// The former is preferred; the latter is deprecated +/// +/// Serialize is required to convert message to a ByteBuffer, and +/// return that byte buffer through *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index e682bdef6a..c185bf4fd0 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,11 +19,89 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H +#include #include #include +#include + namespace grpc { +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const; + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index df16c6a75e..81fa3b0a18 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H +#include #include #include #include @@ -27,71 +28,4 @@ #include #include -#include - -namespace grpc { - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer(); - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear(); - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - private: - friend class SerializationTraits; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - // For \a SerializationTraits's usage. - grpc_byte_buffer* buffer() const { return buffer_; } - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, - bool* own_buffer) { - *buffer = grpc_byte_buffer_copy(source.buffer()); - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index bbf97f280e..10db10d79c 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,86 +19,8 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H +#include #include #include -namespace grpc { - -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const { return grpc_slice_ref(slice_); } - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 55e191da31..7669582af2 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,69 +19,7 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h new file mode 100644 index 0000000000..fc33305713 --- /dev/null +++ b/include/grpc/impl/codegen/byte_buffer.h @@ -0,0 +1,86 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 4b1f42dd02..c3daa8f847 100644 --- a/package.xml +++ b/package.xml @@ -155,6 +155,7 @@ + diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index a31d9eef93..b096679a4c 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 815b607032..d2cba6d662 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index b1ff25252a..1ebe22b6a4 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,11 +16,15 @@ * */ +#include #include +#include #include namespace grpc { +static internal::GrpcLibraryInitializer g_gli_initializer; + ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -29,6 +33,7 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -37,19 +42,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } -ByteBuffer::~ByteBuffer() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - } -} - -void ByteBuffer::Clear() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } -} - Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -80,7 +72,9 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - Clear(); // first remove existing data + if (this != &buf) { + Clear(); // first remove existing data + } if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -93,4 +87,19 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } +ByteBuffer::operator grpc_byte_buffer*() { + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + return buffer_; +} + +ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } + } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 486d0cdf0e..3ae17e8052 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,4 +50,6 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} +grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } + } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 0402ce34fb..0d223700af 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,16 +22,6 @@ #include "rb_grpc_imports.generated.h" -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -167,6 +157,16 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -329,16 +329,6 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -474,6 +464,16 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index e3704e592b..a13d8aa804 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include -#include #include #include #include #include #include +#include #include #include #include @@ -47,36 +47,6 @@ #include #include -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -482,6 +452,36 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0d7f68c0ad..d36d116afb 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index cac01a7307..8fb51bc663 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - grpc_byte_buffer* send_buffer = nullptr; + ByteBuffer send_buffer; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,8 +101,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer != nullptr); - grpc_byte_buffer_destroy(send_buffer); + EXPECT_TRUE(send_buffer.Valid()); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 62f113907d..d81b7b4d11 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -881,6 +882,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 49919415bd..263c6c1afb 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,6 +883,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index e4cc1c7461..135b8923a0 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index c2daf900fb..ce5800b6f8 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index eefd80e46f..ddc9d392fa 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8324,6 +8324,7 @@ "gpr_codegen" ], "headers": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8337,6 +8338,7 @@ "language": "c", "name": "grpc_codegen", "src": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9215,6 +9217,7 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9249,6 +9252,7 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", -- cgit v1.2.3 From 4657a4818b56e11d6b11654933dcd80a434f9f93 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 8 Sep 2017 14:49:38 -0700 Subject: Warm pip cache with libraries used by virtualenv --- tools/internal_ci/helper_scripts/prepare_build_linux_rc | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 91627d60cb..2ade8dac51 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -26,6 +26,7 @@ sudo service docker restart # Populate xdg-cache-home to workaround https://github.com/grpc/grpc/issues/11968 sudo mkdir -p /tmp/xdg-cache-home +PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install setuptools wheel PYTHONWARNINGS=ignore XDG_CACHE_HOME=/tmp/xdg-cache-home sudo -E pip install coverage==4.4 pylint==1.6.5 # Download Docker images from DockerHub -- cgit v1.2.3 From 9dba7eb6b8edfb8035f1166be6e501f90a5e33d0 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 8 Sep 2017 15:32:51 -0700 Subject: Fix error unref problem --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 721f694506..39c2e7baaa 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2596,7 +2596,7 @@ static void reset_byte_stream(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_UNREF(s->byte_stream_error); s->byte_stream_error = GRPC_ERROR_NONE; grpc_chttp2_cancel_stream(exec_ctx, s->t, s, GRPC_ERROR_REF(error)); - s->byte_stream_error = error; + s->byte_stream_error = GRPC_ERROR_REF(error); } } -- cgit v1.2.3 From 2426163b314d9dc7f841b433aed55fe3c6f17350 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 8 Sep 2017 16:12:05 -0700 Subject: Switch to three threads per cq for unconstrained benchmarks --- tools/run_tests/generated/tests.json | 64 +++++++++++++------------- tools/run_tests/performance/scenario_config.py | 2 + 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ad39f2d6d0..1f97309a97 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -49905,7 +49905,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -49955,7 +49955,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50005,7 +50005,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50105,7 +50105,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50205,7 +50205,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50255,7 +50255,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50305,7 +50305,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50355,7 +50355,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50759,7 +50759,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50809,7 +50809,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50859,7 +50859,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -50959,7 +50959,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51059,7 +51059,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51109,7 +51109,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51159,7 +51159,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51209,7 +51209,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51884,7 +51884,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -51960,7 +51960,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52036,7 +52036,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52188,7 +52188,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52340,7 +52340,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52416,7 +52416,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52492,7 +52492,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -52568,7 +52568,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53180,7 +53180,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53256,7 +53256,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53332,7 +53332,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53484,7 +53484,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53636,7 +53636,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53712,7 +53712,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_client_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_CLIENT\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53788,7 +53788,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"SYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"SYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -53864,7 +53864,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_from_server_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"security_params\": null, \"threads_per_cq\": 3, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"blend\", \"name\": \"grpc.optimization_target\"}, {\"int_value\": 1, \"name\": \"grpc.minimal_stack\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING_FROM_SERVER\", \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"threads_per_cq\": 3, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 86236b19bf..b97650de67 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -452,6 +452,8 @@ class CXXLanguage: unconstrained_client=synchronicity, secure=secure, minimal_stack=not secure, + server_threads_per_cq=3, + client_threads_per_cq=3, categories=smoketest_categories+[SCALABLE]) # TODO(vjpai): Re-enable this test. It has a lot of timeouts -- cgit v1.2.3 From eda5a4db1e88e484d0c45401c345cbc199d633fd Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 9 Sep 2017 15:52:11 +0000 Subject: Fix metadata memory leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gRPC Core has two styles for passing metadata: as an integer count along with a grpc_metadata* pointer, which is used for passing metadata into the core, and as a grpc_metadata_array, which is used for passing metadata out of the core. The Cython layer of gRPC Python was using a single data structure wrapping grpc_metadata_array for both purposes, but this was complex because the core manages the slices contained in grpc_metadata_array objects (at least those of which it is aware), so the Cython layer had to keep track of whether or not the core was aware of the slices it was using (and it was also defective, leaking slices). This is solved by realigning with the Cython layer’s intended design of mirroring as closely as possible in Python the gRPC Core API: we use one structure for passing metadata into the core (what is now called cygrpc.Metadata) and second, different structure for receiving metadata out of the core (what was called cygrpc.Metadata but is now cygrpc.MetadataArray, reflecting that it wraps the core’s grpc_metadata_array). All bug fixes should contain added tests preventing regression but this doesn't because I don't know at this time how to write a does-not-leak test for Python that fits well into our existing body of tests. Phooey. Thanks to Dominik Janků (djanku@email.cz) for investigation and an earlier draft of a solution. --- .../grpc/_cython/_cygrpc/completion_queue.pyx.pxi | 11 +- .../grpc/_cython/_cygrpc/credentials.pyx.pxi | 9 +- src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi | 1 + .../grpcio/grpc/_cython/_cygrpc/records.pxd.pxi | 13 ++- .../grpcio/grpc/_cython/_cygrpc/records.pyx.pxi | 118 ++++++++++++--------- .../grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 2 +- 6 files changed, 83 insertions(+), 71 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi index 28c30e5d35..237f430799 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi @@ -41,9 +41,8 @@ cdef class CompletionQueue: cdef object user_tag = None cdef Call operation_call = None cdef CallDetails request_call_details = None - cdef Metadata request_metadata = None + cdef object request_metadata = None cdef Operations batch_operations = None - cdef Operation batch_operation = None if event.type == GRPC_QUEUE_TIMEOUT: return Event( event.type, False, None, None, None, None, False, None) @@ -63,14 +62,8 @@ cdef class CompletionQueue: operation_call = tag.operation_call request_call_details = tag.request_call_details if tag.request_metadata is not None: - request_metadata = tag.request_metadata - request_metadata._claim_slice_ownership() + request_metadata = tuple(tag.request_metadata) batch_operations = tag.batch_operations - if tag.batch_operations is not None: - for op in batch_operations.operations: - batch_operation = op - if batch_operation._received_metadata is not None: - batch_operation._received_metadata._claim_slice_ownership() if tag.is_new_request: # Stuff in the tag not explicitly handled by us needs to live through # the life of the call diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 98d7a9820d..57816f1cab 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -76,7 +76,7 @@ cdef class CredentialsMetadataPlugin: """ Args: plugin_callback (callable): Callback accepting a service URL (str/bytes) - and callback object (accepting a Metadata, + and callback object (accepting a MetadataArray, grpc_status_code, and a str/bytes error message). This argument when called should be non-blocking and eventually call the callback object with the appropriate status code/details and metadata (if @@ -129,8 +129,7 @@ cdef void plugin_get_metadata( def python_callback( Metadata metadata, grpc_status_code status, bytes error_details): - cb(user_data, metadata.c_metadata_array.metadata, - metadata.c_metadata_array.count, status, error_details) + cb(user_data, metadata.c_metadata, metadata.c_count, status, error_details) called_flag[0] = True cdef CredentialsMetadataPlugin self = state cdef AuthMetadataContext cy_context = AuthMetadataContext() @@ -139,8 +138,8 @@ cdef void plugin_get_metadata( self.plugin_callback(cy_context, python_callback) except Exception as error: if not called_flag[0]: - cb(user_data, Metadata([]).c_metadata_array.metadata, - 0, StatusCode.unknown, traceback.format_exc().encode()) + cb(user_data, NULL, 0, StatusCode.unknown, + traceback.format_exc().encode()) cdef void plugin_destroy_c_plugin_state(void *state) with gil: cpython.Py_DECREF(state) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 5950bfa0e6..28cf114451 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -59,6 +59,7 @@ cdef extern from "grpc/grpc.h": grpc_slice grpc_slice_malloc(size_t length) nogil grpc_slice grpc_slice_from_copied_string(const char *source) nogil grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t len) nogil + grpc_slice grpc_slice_copy(grpc_slice s) nogil # Declare functions for function-like macros (because Cython)... void *grpc_slice_start_ptr "GRPC_SLICE_START_PTR" (grpc_slice s) nogil diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi index 8ace6aeb52..9c40ebf0c2 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pxd.pxi @@ -37,7 +37,7 @@ cdef class OperationTag: cdef Server shutting_down_server cdef Call operation_call cdef CallDetails request_call_details - cdef Metadata request_metadata + cdef MetadataArray request_metadata cdef Operations batch_operations cdef bint is_new_request @@ -51,7 +51,7 @@ cdef class Event: # For Server.request_call cdef readonly bint is_new_request cdef readonly CallDetails request_call_details - cdef readonly Metadata request_metadata + cdef readonly object request_metadata # For server calls cdef readonly Call operation_call @@ -92,15 +92,20 @@ cdef class Metadatum: cdef class Metadata: + cdef grpc_metadata *c_metadata + cdef readonly size_t c_count + + +cdef class MetadataArray: + cdef grpc_metadata_array c_metadata_array - cdef void _claim_slice_ownership(self) cdef class Operation: cdef grpc_op c_op cdef ByteBuffer _received_message - cdef Metadata _received_metadata + cdef MetadataArray _received_metadata cdef grpc_status_code _received_status_code cdef grpc_slice _status_details cdef int _received_cancelled diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 1b2ddd2469..0a2a6eee05 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -238,7 +238,7 @@ cdef class Event: def __cinit__(self, grpc_completion_type type, bint success, object tag, Call operation_call, CallDetails request_call_details, - Metadata request_metadata, + object request_metadata, bint is_new_request, Operations batch_operations): self.type = type @@ -437,48 +437,79 @@ cdef class Metadatum: cdef class _MetadataIterator: cdef size_t i - cdef Metadata metadata + cdef size_t _length + cdef object _metadatum_indexable - def __cinit__(self, Metadata metadata not None): + def __cinit__(self, length, metadatum_indexable): + self._length = length + self._metadatum_indexable = metadatum_indexable self.i = 0 - self.metadata = metadata def __iter__(self): return self def __next__(self): - if self.i < len(self.metadata): - result = self.metadata[self.i] + if self.i < self._length: + result = self._metadatum_indexable[self.i] self.i = self.i + 1 return result else: raise StopIteration +# TODO(https://github.com/grpc/grpc/issues/7950): Eliminate this; just use an +# ordinary sequence of pairs of bytestrings all the way down to the +# grpc_call_start_batch call. cdef class Metadata: + """Metadata being passed from application to core.""" def __cinit__(self, metadata_iterable): + metadata_sequence = tuple(metadata_iterable) + cdef size_t count = len(metadata_sequence) with nogil: grpc_init() - grpc_metadata_array_init(&self.c_metadata_array) - metadata = list(metadata_iterable) - for metadatum in metadata: - if not isinstance(metadatum, Metadatum): - raise TypeError("expected list of Metadatum") - self.c_metadata_array.count = len(metadata) - self.c_metadata_array.capacity = len(metadata) + self.c_metadata = gpr_malloc( + count * sizeof(grpc_metadata)) + self.c_count = count + for index, metadatum in enumerate(metadata_sequence): + self.c_metadata[index].key = grpc_slice_copy( + (metadatum).c_metadata.key) + self.c_metadata[index].value = grpc_slice_copy( + (metadatum).c_metadata.value) + + def __dealloc__(self): + with nogil: + for index in range(self.c_count): + grpc_slice_unref(self.c_metadata[index].key) + grpc_slice_unref(self.c_metadata[index].value) + gpr_free(self.c_metadata) + grpc_shutdown() + + def __len__(self): + return self.c_count + + def __getitem__(self, size_t index): + if index < self.c_count: + key = _slice_bytes(self.c_metadata[index].key) + value = _slice_bytes(self.c_metadata[index].value) + return Metadatum(key, value) + else: + raise IndexError() + + def __iter__(self): + return _MetadataIterator(self.c_count, self) + + +cdef class MetadataArray: + """Metadata being passed from core to application.""" + + def __cinit__(self): with nogil: - self.c_metadata_array.metadata = gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata) - ) - for i in range(self.c_metadata_array.count): - (metadata[i])._copy_metadatum(&self.c_metadata_array.metadata[i]) + grpc_init() + grpc_metadata_array_init(&self.c_metadata_array) def __dealloc__(self): with nogil: - # this frees the allocated memory for the grpc_metadata_array (although - # it'd be nice if that were documented somewhere...) - # TODO(atash): document this in the C core grpc_metadata_array_destroy(&self.c_metadata_array) grpc_shutdown() @@ -493,21 +524,7 @@ cdef class Metadata: return Metadatum(key=key, value=value) def __iter__(self): - return _MetadataIterator(self) - - cdef void _claim_slice_ownership(self): - cdef grpc_metadata_array new_c_metadata_array - grpc_metadata_array_init(&new_c_metadata_array) - new_c_metadata_array.metadata = gpr_malloc( - self.c_metadata_array.count*sizeof(grpc_metadata)) - new_c_metadata_array.count = self.c_metadata_array.count - for i in range(self.c_metadata_array.count): - new_c_metadata_array.metadata[i].key = _copy_slice( - self.c_metadata_array.metadata[i].key) - new_c_metadata_array.metadata[i].value = _copy_slice( - self.c_metadata_array.metadata[i].value) - grpc_metadata_array_destroy(&self.c_metadata_array) - self.c_metadata_array = new_c_metadata_array + return _MetadataIterator(self.c_metadata_array.count, self) cdef class Operation: @@ -547,14 +564,13 @@ cdef class Operation: if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): raise TypeError("self must be an operation receiving metadata") - return self._received_metadata - - @property - def received_metadata_or_none(self): - if (self.c_op.type != GRPC_OP_RECV_INITIAL_METADATA and - self.c_op.type != GRPC_OP_RECV_STATUS_ON_CLIENT): - return None - return self._received_metadata + # TODO(https://github.com/grpc/grpc/issues/7950): Drop the "all Cython + # objects must be legitimate for use from Python at any time" policy in + # place today, shift the policy toward "Operation objects are only usable + # while their calls are active", and move this making-a-copy-because-this- + # data-needs-to-live-much-longer-than-the-call-from-which-it-arose to the + # lowest Python layer. + return tuple(self._received_metadata) @property def received_status_code(self): @@ -601,9 +617,8 @@ def operation_send_initial_metadata(Metadata metadata, int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_SEND_INITIAL_METADATA op.c_op.flags = flags - op.c_op.data.send_initial_metadata.count = metadata.c_metadata_array.count - op.c_op.data.send_initial_metadata.metadata = ( - metadata.c_metadata_array.metadata) + op.c_op.data.send_initial_metadata.count = metadata.c_count + op.c_op.data.send_initial_metadata.metadata = metadata.c_metadata op.references.append(metadata) op.is_valid = True return op @@ -631,9 +646,8 @@ def operation_send_status_from_server( op.c_op.type = GRPC_OP_SEND_STATUS_FROM_SERVER op.c_op.flags = flags op.c_op.data.send_status_from_server.trailing_metadata_count = ( - metadata.c_metadata_array.count) - op.c_op.data.send_status_from_server.trailing_metadata = ( - metadata.c_metadata_array.metadata) + metadata.c_count) + op.c_op.data.send_status_from_server.trailing_metadata = metadata.c_metadata op.c_op.data.send_status_from_server.status = code grpc_slice_unref(op._status_details) op._status_details = _slice_from_bytes(details) @@ -646,7 +660,7 @@ def operation_receive_initial_metadata(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_INITIAL_METADATA op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_initial_metadata.receive_initial_metadata = ( &op._received_metadata.c_metadata_array) op.is_valid = True @@ -669,7 +683,7 @@ def operation_receive_status_on_client(int flags): cdef Operation op = Operation() op.c_op.type = GRPC_OP_RECV_STATUS_ON_CLIENT op.c_op.flags = flags - op._received_metadata = Metadata([]) + op._received_metadata = MetadataArray() op.c_op.data.receive_status_on_client.trailing_metadata = ( &op._received_metadata.c_metadata_array) op.c_op.data.receive_status_on_client.status = ( diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index dd276fd57b..b8db27469f 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -44,7 +44,7 @@ cdef class Server: cdef OperationTag operation_tag = OperationTag(tag) operation_tag.operation_call = Call() operation_tag.request_call_details = CallDetails() - operation_tag.request_metadata = Metadata([]) + operation_tag.request_metadata = MetadataArray() operation_tag.references.extend([self, call_queue, server_queue]) operation_tag.is_new_request = True operation_tag.batch_operations = Operations([]) -- cgit v1.2.3 From 5a2a792509af457322b8fc63eff3ec1ee360b0a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 24 Apr 2017 13:31:20 +0200 Subject: add cmake distribtest --- test/distrib/cpp/run_distrib_test.sh | 37 ------------ test/distrib/cpp/run_distrib_test_cmake.sh | 66 ++++++++++++++++++++++ test/distrib/cpp/run_distrib_test_routeguide.sh | 37 ++++++++++++ .../distribtest/cpp_jessie_x64/Dockerfile | 2 + tools/run_tests/artifacts/distribtest_targets.py | 12 ++-- 5 files changed, 112 insertions(+), 42 deletions(-) delete mode 100755 test/distrib/cpp/run_distrib_test.sh create mode 100755 test/distrib/cpp/run_distrib_test_cmake.sh create mode 100755 test/distrib/cpp/run_distrib_test_routeguide.sh diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh deleted file mode 100755 index b043075d93..0000000000 --- a/test/distrib/cpp/run_distrib_test.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -# change to grpc repo root -cd $(dirname $0)/../../.. - -cd third_party/protobuf && ./autogen.sh && \ -./configure && make -j4 && make check && make install && ldconfig - -cd ../.. && make -j4 && make install - -cd examples/cpp/helloworld - -make - -make clean - -cd ../../../examples/cpp/route_guide - -make - -make clean - diff --git a/test/distrib/cpp/run_distrib_test_cmake.sh b/test/distrib/cpp/run_distrib_test_cmake.sh new file mode 100755 index 0000000000..a81de57810 --- /dev/null +++ b/test/distrib/cpp/run_distrib_test_cmake.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd $(dirname $0)/../../.. + +echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +apt-get update +#apt-get install -t jessie-backports -y libc-ares-dev # we need specifically version 1.12 +apt-get install -t jessie-backports -y libssl-dev + +# Install c-ares +cd third_party/cares/cares +git fetch origin +git checkout cares-1_13_0 +mkdir -p cmake/build +cd cmake/build +cmake -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../../../../.. + +# Install zlib +cd third_party/zlib +mkdir -p cmake/build +cd cmake/build +cmake -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../../../.. + +# Install protobuf +cd third_party/protobuf +mkdir -p cmake/build +cd cmake/build +cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release .. +make -j4 install +cd ../../../.. + +# TODO: Install boringssl + +# Install gRPC +mkdir -p cmake/build +cd cmake/build +cmake -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release ../.. +make -j4 install +cd ../.. + +# Build helloworld example using cmake +cd examples/cpp/helloworld +mkdir -p cmake/build +cd cmake/build +cmake ../.. +make + diff --git a/test/distrib/cpp/run_distrib_test_routeguide.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh new file mode 100755 index 0000000000..b043075d93 --- /dev/null +++ b/test/distrib/cpp/run_distrib_test_routeguide.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +cd third_party/protobuf && ./autogen.sh && \ +./configure && make -j4 && make check && make install && ldconfig + +cd ../.. && make -j4 && make install + +cd examples/cpp/helloworld + +make + +make clean + +cd ../../../examples/cpp/route_guide + +make + +make clean + diff --git a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile index ff66bca0ae..0e8186d40c 100644 --- a/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/cpp_jessie_x64/Dockerfile @@ -27,4 +27,6 @@ RUN apt-get update && apt-get install -y \ pkg-config \ unzip && apt-get clean +RUN apt-get update && apt-get install -y cmake golang && apt-get clean + CMD ["bash"] diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index fa461efa85..fb1be383cd 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -255,12 +255,13 @@ class PHPDistribTest(object): class CppDistribTest(object): """Tests Cpp make intall by building examples.""" - def __init__(self, platform, arch, docker_suffix=None): - self.name = 'cpp_%s_%s_%s' % (platform, arch, docker_suffix) + def __init__(self, platform, arch, docker_suffix=None, testcase=None): + self.name = 'cpp_%s_%s_%s_%s' % (platform, arch, docker_suffix, testcase) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix] + self.testcase = testcase + self.labels = ['distribtest', 'cpp', platform, arch, docker_suffix, testcase] def pre_build_jobspecs(self): return [] @@ -271,7 +272,7 @@ class CppDistribTest(object): 'tools/dockerfile/distribtest/cpp_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/cpp/run_distrib_test.sh') + 'test/distrib/cpp/run_distrib_test_%s.sh' % self.testcase) else: raise Exception("Not supported yet.") @@ -281,7 +282,8 @@ class CppDistribTest(object): def targets(): """Gets list of supported targets""" - return [CppDistribTest('linux', 'x64', 'jessie'), + return [CppDistribTest('linux', 'x64', 'jessie', 'routeguide'), + CppDistribTest('linux', 'x64', 'jessie', 'cmake'), CSharpDistribTest('linux', 'x64', 'wheezy'), CSharpDistribTest('linux', 'x64', 'jessie'), CSharpDistribTest('linux', 'x86', 'jessie'), -- cgit v1.2.3 From 4caadb9205bcb9f745cda2e3c310b27837cfd636 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 25 Aug 2017 12:59:11 +0200 Subject: fix helloworld cmake build --- examples/cpp/helloworld/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/cpp/helloworld/CMakeLists.txt b/examples/cpp/helloworld/CMakeLists.txt index 8f098c91a6..71a8db4f24 100644 --- a/examples/cpp/helloworld/CMakeLists.txt +++ b/examples/cpp/helloworld/CMakeLists.txt @@ -2,7 +2,11 @@ cmake_minimum_required(VERSION 2.8) # Project -project(HelloWorld CXX) +project(HelloWorld C CXX) + +if(NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +endif() # Protobuf set(protobuf_MODULE_COMPATIBLE TRUE) -- cgit v1.2.3 From f567ab0d2b3b761a3647126c4c5a2ec2549f4c73 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 25 Aug 2017 16:13:30 +0200 Subject: fix public header path for installed grpc --- CMakeLists.txt | 70 +++++++++++++++++++-------------------- templates/CMakeLists.txt.template | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5131bf39b4..a82e9ef6d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -832,7 +832,7 @@ endif() target_include_directories(gpr - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -924,7 +924,7 @@ endif() target_include_directories(gpr_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1219,7 +1219,7 @@ endif() target_include_directories(grpc - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1525,7 +1525,7 @@ endif() target_include_directories(grpc_cronet - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -1803,7 +1803,7 @@ endif() target_include_directories(grpc_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2063,7 +2063,7 @@ endif() target_include_directories(grpc_test_util_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2357,7 +2357,7 @@ endif() target_include_directories(grpc_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2446,7 +2446,7 @@ endif() target_include_directories(reconnect_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2488,7 +2488,7 @@ endif() target_include_directories(test_tcp_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2569,7 +2569,7 @@ endif() target_include_directories(grpc++ - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -2769,7 +2769,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_core_stats - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3060,7 +3060,7 @@ endif() target_include_directories(grpc++_cronet - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3259,7 +3259,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_error_details - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3324,7 +3324,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_proto_reflection_desc_db - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3385,7 +3385,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_reflection - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3443,7 +3443,7 @@ endif() target_include_directories(grpc++_test_config - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3521,7 +3521,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_test_util - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3659,7 +3659,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc++_test_util_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3799,7 +3799,7 @@ endif() target_include_directories(grpc++_unsecure - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -3989,7 +3989,7 @@ endif() target_include_directories(grpc_benchmark - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4048,7 +4048,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(grpc_cli_libs - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4108,7 +4108,7 @@ endif() target_include_directories(grpc_plugin_support - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4186,7 +4186,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(http2_client_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4241,7 +4241,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_client_helper - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4311,7 +4311,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_client_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4362,7 +4362,7 @@ endif() target_include_directories(interop_server_helper - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4431,7 +4431,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(interop_server_lib - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4482,7 +4482,7 @@ endif() target_include_directories(interop_server_main - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4570,7 +4570,7 @@ protobuf_generate_grpc_cpp( ) target_include_directories(qps - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4617,7 +4617,7 @@ endif() target_include_directories(grpc_csharp_ext - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4712,7 +4712,7 @@ endif() target_include_directories(ares - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4750,7 +4750,7 @@ endif() target_include_directories(bad_client_test - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4791,7 +4791,7 @@ endif() target_include_directories(bad_ssl_test_server - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4892,7 +4892,7 @@ endif() target_include_directories(end2end_tests - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src @@ -4993,7 +4993,7 @@ endif() target_include_directories(end2end_nosec_tests - PUBLIC $ $ + PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${BORINGSSL_ROOT_DIR}/include PRIVATE ${PROTOBUF_ROOT_DIR}/src diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 6adff889fe..ad79873d56 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -515,7 +515,7 @@ % endfor target_include_directories(${lib.name} - PUBLIC <%text>$ $ + PUBLIC <%text>$ $ PRIVATE <%text>${CMAKE_CURRENT_SOURCE_DIR} PRIVATE <%text>${BORINGSSL_ROOT_DIR}/include PRIVATE <%text>${PROTOBUF_ROOT_DIR}/src -- cgit v1.2.3 From b2cf73e1f81a9c9eeff6dab8f8980c1711784796 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 28 Aug 2017 12:18:45 +0200 Subject: prevent submodule headers from influencing the build --- test/distrib/cpp/run_distrib_test_cmake.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/distrib/cpp/run_distrib_test_cmake.sh b/test/distrib/cpp/run_distrib_test_cmake.sh index a81de57810..ead8cc10bc 100755 --- a/test/distrib/cpp/run_distrib_test_cmake.sh +++ b/test/distrib/cpp/run_distrib_test_cmake.sh @@ -31,6 +31,7 @@ cd cmake/build cmake -DCMAKE_BUILD_TYPE=Release ../.. make -j4 install cd ../../../../.. +rm -rf third_party/cares/cares # wipe out to prevent influencing the grpc build # Install zlib cd third_party/zlib @@ -39,6 +40,7 @@ cd cmake/build cmake -DCMAKE_BUILD_TYPE=Release ../.. make -j4 install cd ../../../.. +rm -rf third_party/zlib # wipe out to prevent influencing the grpc build # Install protobuf cd third_party/protobuf @@ -47,8 +49,7 @@ cd cmake/build cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release .. make -j4 install cd ../../../.. - -# TODO: Install boringssl +rm -rf third_party/protobuf # wipe out to prevent influencing the grpc build # Install gRPC mkdir -p cmake/build -- cgit v1.2.3 From ae6fd66cef94bf9d4d89bcdcf64a7463ef0e77f2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 28 Aug 2017 12:21:46 +0200 Subject: fix zlib and openssl package mode --- CMakeLists.txt | 18 +++++++----------- templates/CMakeLists.txt.template | 18 +++++++----------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a82e9ef6d0..dd68016be8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,10 +123,8 @@ if("${gRPC_ZLIB_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_ZLIB_PROVIDER}" STREQUAL "package") - find_package(ZLIB) - if(TARGET ZLIB::ZLIB) - set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB) - endif() + find_package(ZLIB REQUIRED) + set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES}) set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n find_package(ZLIB)\nendif()") endif() @@ -145,7 +143,7 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_CARES_PROVIDER}" STREQUAL "package") - find_package(c-ares CONFIG) + find_package(c-ares REQUIRED CONFIG) if(TARGET c-ares::cares) set(_gRPC_CARES_LIBRARIES c-ares::cares) endif() @@ -189,7 +187,7 @@ if("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package") - find_package(Protobuf ${gRPC_PROTOBUF_PACKAGE_TYPE}) + find_package(Protobuf REQUIRED ${gRPC_PROTOBUF_PACKAGE_TYPE}) if(Protobuf_FOUND OR PROTOBUF_FOUND) if(TARGET protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME}) set(_gRPC_PROTOBUF_LIBRARIES protobuf::${_gRPC_PROTOBUF_LIBRARY_NAME}) @@ -234,11 +232,9 @@ if("${gRPC_SSL_PROVIDER}" STREQUAL "module") set(gRPC_INSTALL FALSE) endif() elseif("${gRPC_SSL_PROVIDER}" STREQUAL "package") - find_package(OpenSSL) - if(TARGET OpenSSL::SSL) - set(_gRPC_SSL_LIBRARIES OpenSSL::SSL) - endif() - set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n find_package(OpenSSL)\nendif()") + find_package(OpenSSL REQUIRED) + set(_gRPC_SSL_LIBRARIES ${OPENSSL_LIBRARIES}) + set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n find_package(OpenSSL)\nendif()") endif() if("${gRPC_GFLAGS_PROVIDER}" STREQUAL "module") diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index ad79873d56..f2ce9b5a90 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -168,10 +168,8 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_ZLIB_PROVIDER}" STREQUAL "package") - find_package(ZLIB) - if(TARGET ZLIB::ZLIB) - set(_gRPC_ZLIB_LIBRARIES ZLIB::ZLIB) - endif() + find_package(ZLIB REQUIRED) + set(_gRPC_ZLIB_LIBRARIES <%text>${ZLIB_LIBRARIES}) set(_gRPC_FIND_ZLIB "if(NOT ZLIB_FOUND)\n find_package(ZLIB)\nendif()") endif() @@ -190,7 +188,7 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_CARES_PROVIDER}" STREQUAL "package") - find_package(c-ares CONFIG) + find_package(c-ares REQUIRED CONFIG) if(TARGET c-ares::cares) set(_gRPC_CARES_LIBRARIES c-ares::cares) endif() @@ -234,7 +232,7 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_PROTOBUF_PROVIDER}" STREQUAL "package") - find_package(Protobuf <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}) + find_package(Protobuf REQUIRED <%text>${gRPC_PROTOBUF_PACKAGE_TYPE}) if(Protobuf_FOUND OR PROTOBUF_FOUND) if(TARGET protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}) set(_gRPC_PROTOBUF_LIBRARIES protobuf::<%text>${_gRPC_PROTOBUF_LIBRARY_NAME}) @@ -279,11 +277,9 @@ set(gRPC_INSTALL FALSE) endif() elseif("<%text>${gRPC_SSL_PROVIDER}" STREQUAL "package") - find_package(OpenSSL) - if(TARGET OpenSSL::SSL) - set(_gRPC_SSL_LIBRARIES OpenSSL::SSL) - endif() - set(_gRPC_FIND_SSL "if(NOT OpenSSL_FOUND)\n find_package(OpenSSL)\nendif()") + find_package(OpenSSL REQUIRED) + set(_gRPC_SSL_LIBRARIES <%text>${OPENSSL_LIBRARIES}) + set(_gRPC_FIND_SSL "if(NOT OPENSSL_FOUND)\n find_package(OpenSSL)\nendif()") endif() if("<%text>${gRPC_GFLAGS_PROVIDER}" STREQUAL "module") -- cgit v1.2.3 From aba0a0a54412fe59bc2090334ae26c99bb148097 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Sun, 10 Sep 2017 18:28:03 -0700 Subject: Add fallback (use backends from resolver if can't reach balancer) to grpclb. --- include/grpc++/support/channel_arguments.h | 6 + include/grpc/impl/codegen/grpc_types.h | 6 +- .../client_channel/lb_policy/grpclb/grpclb.c | 197 ++++++++++++++++----- .../ext/filters/client_channel/lb_policy_factory.c | 2 +- .../ext/filters/client_channel/lb_policy_factory.h | 2 +- src/cpp/common/channel_arguments.cc | 4 + test/cpp/end2end/grpclb_end2end_test.cc | 74 +++++++- 7 files changed, 244 insertions(+), 47 deletions(-) diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 7b6befeaf1..9dc505f008 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,6 +64,12 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); + /// Set the grpclb fallback timeout (in ms) for the channel. If this amount + /// of time has passed but we have not gotten any non-empty \a serverlist from + /// the balancer, we will fall back to use the backend address(es) returned by + /// the resolver. + void SetGrpclbFallbackTimeout(int fallback_timeout); + /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 59b90af03a..4745fddd71 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -287,7 +287,11 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" +/* Timeout in milliseconds to wait for the serverlist from the grpclb load + balancer before using fallback backend addresses from the resolver. + If 0, fallback will never be used. */ +#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 5aafed1374..18979829bd 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -122,6 +122,7 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 +#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -298,6 +299,10 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; + /** timeout in milliseconds for before using fallback backend addresses. + * 0 means not using fallback. */ + int lb_fallback_timeout_ms; + /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -324,6 +329,9 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; + /** stores the backend addresses from the resolver */ + grpc_lb_addresses *fallback_backend_addresses; + /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -344,6 +352,9 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; + /** is \a lb_fallback_timer active? */ + bool fallback_timer_active; + /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -366,6 +377,9 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; + /* LB fallback timer callback. */ + grpc_closure lb_on_fallback; + grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -389,6 +403,9 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; + /** LB fallback timer */ + grpc_timer lb_fallback_timer; + bool initial_request_sent; bool seen_initial_response; @@ -535,6 +552,32 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } +/* Returns the backend addresses extracted from the given addresses */ +static grpc_lb_addresses *extract_backend_addresses_locked( + grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { + /* first pass: count the number of backend addresses */ + size_t num_backends = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (!addresses->addresses[i].is_balancer) { + ++num_backends; + } + } + /* second pass: actually populate the addresses and (empty) LB tokens */ + grpc_lb_addresses *backend_addresses = + grpc_lb_addresses_create(num_backends, &lb_token_vtable); + size_t num_copied = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) continue; + const grpc_resolved_address *addr = &addresses->addresses[i].address; + grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, + addr->len, false /* is_balancer */, + NULL /* balancer_name */, + (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); + ++num_copied; + } + return backend_addresses; +} + static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -602,35 +645,38 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. - } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); + // Check for drops if we are not using fallback backend addresses. + if (glb_policy->serverlist != NULL) { + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, - wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); + } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked( + server->load_balance_token, wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + gpr_free(wc_arg->free_when_done); + return false; + } gpr_free(wc_arg->free_when_done); - return false; + return true; } - gpr_free(wc_arg->free_when_done); - return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -668,8 +714,18 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses = - process_serverlist_locked(exec_ctx, glb_policy->serverlist); + grpc_lb_addresses *addresses; + if (glb_policy->serverlist != NULL) { + GPR_ASSERT(glb_policy->serverlist->num_servers > 0); + addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); + } else { + // If rr_handover_locked() is invoked when we haven't received any + // serverlist from the balancer, we use the fallback backends returned by + // the resolver. Note that the fallback backend list may be empty, in which + // case the new round_robin policy will keep the requested picks pending. + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); + } GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -775,8 +831,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - GPR_ASSERT(glb_policy->serverlist != NULL && - glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -917,13 +971,7 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. - * TODO(roth): For now, we ignore non-balancer addresses, but in the - * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. In the - * fallback case, we should use the LB policy indicated by - * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is - * unset, we should default to pick_first). */ + /* Count the number of gRPC-LB addresses. There must be at least one. */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -959,6 +1007,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); + glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( + arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, + INT_MAX}); + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -967,6 +1020,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + /* Extract the backend addresses (may be empty) from the resolver for + * fallback. */ + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); @@ -1010,6 +1068,9 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } + if (glb_policy->fallback_backend_addresses != NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); @@ -1150,10 +1211,28 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + /* start a timer to fall back */ + if (glb_policy->lb_fallback_timeout_ms > 0 && + glb_policy->serverlist == NULL) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_time_add( + now, + gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); + GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); + GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, + glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + glb_policy->fallback_timer_active = true; + grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, + &glb_policy->lb_on_fallback, now); + } + glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1607,6 +1686,15 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); + } else { + /* or dispose of the fallback */ + grpc_lb_addresses_destroy(exec_ctx, + glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = NULL; + if (glb_policy->fallback_timer_active) { + grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); + glb_policy->fallback_timer_active = false; + } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1617,9 +1705,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Received empty server list. Picks will stay pending until " - "a response with > 0 servers is received"); + gpr_log(GPR_INFO, "Received empty server list, ignoring."); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1666,6 +1752,26 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); } +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + glb_lb_policy *glb_policy = arg; + /* If we receive a serverlist after the timer fires but before this callback + * actually runs, don't do anything. */ + if (glb_policy->serverlist != NULL) return; + glb_policy->fallback_timer_active = false; + if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, + "Falling back to use backends from resolver (grpclb %p)", + (void *)glb_policy); + } + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + rr_handover_locked(exec_ctx, glb_policy); + } + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "grpclb_fallback_timer"); +} + static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1786,6 +1892,17 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } + + // Propagate update to fallback_backend_addresses if a non-empty serverlist + // hasn't been received from the balancer. + if (glb_policy->serverlist == NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + if (glb_policy->rr_policy != NULL) { + rr_handover_locked(exec_ctx, glb_policy); + } + } } // Invoked as part of the update process. It continues watching the LB channel diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index cdcaf17544..918bab745c 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - void* address, size_t address_len, + const void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index 9d9fb143df..cf0f8cb615 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - void *address, size_t address_len, + const void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f130aecd4b..f89f5f1f03 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,6 +86,10 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } +void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { + SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); +} + void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 570a3d1067..17a094f7a2 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,8 +368,9 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub() { + void ResetStub(int fallback_timeout = 0) { ChannelArguments args; + args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -441,10 +442,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector GetBackendPorts() const { + const std::vector GetBackendPorts(const size_t start_index = 0) const { std::vector backend_ports; - for (const auto& bs : backend_servers_) { - backend_ports.push_back(bs.port_); + for (size_t i = start_index; i < backend_servers_.size(); ++i) { + backend_ports.push_back(backend_servers_[i].port_); } return backend_ports; } @@ -615,6 +616,71 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } +TEST_F(SingleBalancerTest, Fallback) { + const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); + const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); + const size_t kNumBackendInResolution = backends_.size() / 2; + + ResetStub(kFallbackTimeoutMs); + std::vector addresses; + addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); + } + SetNextResolution(addresses); + + // Send non-empty serverlist only after kServerlistDelayMs + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends( + GetBackendPorts(kNumBackendInResolution /* start_index */), {}), + kServerlistDelayMs); + + // The first request. The client will block while it's still trying to + // contact the balancer. + gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); + CheckRpcSendOk(kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); + + // Fallback is used: each backend returned by the resolver should have + // gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + + // Wait until update has been processed, as signaled by the backend returned + // by the balancer receiving a request. + do { + CheckRpcSendOk(1); + } while ( + backend_servers_[kNumBackendInResolution].service_->request_count() == 0); + for (size_t i = 0; i < backends_.size(); ++i) { + backend_servers_[i].service_->ResetCounters(); + } + + // Send out the second request. + gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); + CheckRpcSendOk(backends_.size() - kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); + + // Serverlist is used: each backend returned by the balancer should + // have gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + + balancers_[0]->NotifyDoneWithServerlists(); + // The balancer got a single request. + EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); + // and sent a single response. + EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); +} + TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( -- cgit v1.2.3 From b29296042a308060574d5cd119d6b05c00660624 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 11 Sep 2017 09:31:11 -0700 Subject: More cleanup in client_channel code. --- .../ext/filters/client_channel/client_channel.c | 314 ++++++++++----------- 1 file changed, 153 insertions(+), 161 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index dad5c4fce5..e5f4a8a813 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -1016,13 +1016,11 @@ static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - grpc_error *error) { +// Invoked when a pick is completed, on both success or failure. +static void pick_done_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, + grpc_error *error) { call_data *calld = (call_data *)elem->call_data; channel_data *chand = (channel_data *)elem->channel_data; - grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent, - chand->interested_parties); if (calld->connected_subchannel == NULL) { // Failed to create subchannel. GRPC_ERROR_UNREF(calld->error); @@ -1044,12 +1042,116 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -/** Return true if subchannel is available immediately (in which case - subchannel_ready_locked() should not be called), or false otherwise (in - which case subchannel_ready_locked() should be called when the subchannel - is available). */ -static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem); +// A wrapper around pick_done_locked() that is used in cases where +// either (a) the pick was deferred pending a resolver result or (b) the +// pick was done asynchronously. Removes the call's polling entity from +// chand->interested_parties before invoking pick_done_locked(). +static void async_pick_done_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem, grpc_error *error) { + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent, + chand->interested_parties); + pick_done_locked(exec_ctx, elem, error); +} + +// Note: This runs under the client_channel combiner, but will NOT be +// holding the call combiner. +static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_call_element *elem = (grpc_call_element *)arg; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (calld->lb_policy != NULL) { + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p", + chand, calld, calld->lb_policy); + } + grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy, + &calld->connected_subchannel, + GRPC_ERROR_REF(error)); + } + GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel"); +} + +// Callback invoked by grpc_lb_policy_pick_locked() for async picks. +// Unrefs the LB policy and invokes async_pick_done_locked(). +static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_call_element *elem = (grpc_call_element *)arg; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously", + chand, calld); + } + GPR_ASSERT(calld->lb_policy != NULL); + GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); + calld->lb_policy = NULL; + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); +} + +// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked(). +// If the pick was completed synchronously, unrefs the LB policy and +// returns true. +static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx, + grpc_call_element *elem) { + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p", + chand, calld, chand->lb_policy); + } + apply_service_config_to_call_locked(exec_ctx, elem); + // If the application explicitly set wait_for_ready, use that. + // Otherwise, if the service config specified a value for this + // method, use that. + uint32_t initial_metadata_flags = + calld->initial_metadata_batch->payload->send_initial_metadata + .send_initial_metadata_flags; + const bool wait_for_ready_set_from_api = + initial_metadata_flags & + GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; + const bool wait_for_ready_set_from_service_config = + calld->method_params != NULL && + calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET; + if (!wait_for_ready_set_from_api && wait_for_ready_set_from_service_config) { + if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) { + initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY; + } else { + initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY; + } + } + const grpc_lb_policy_pick_args inputs = { + calld->initial_metadata_batch->payload->send_initial_metadata + .send_initial_metadata, + initial_metadata_flags, &calld->lb_token_mdelem}; + // Keep a ref to the LB policy in calld while the pick is pending. + GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel"); + calld->lb_policy = chand->lb_policy; + GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem, + grpc_combiner_scheduler(chand->combiner)); + const bool pick_done = grpc_lb_policy_pick_locked( + exec_ctx, chand->lb_policy, &inputs, &calld->connected_subchannel, + calld->subchannel_call_context, NULL, &calld->lb_pick_closure); + if (pick_done) { + /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ + if (GRPC_TRACER_ON(grpc_client_channel_trace)) { + gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously", + chand, calld); + } + GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); + calld->lb_policy = NULL; + } else { + GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel"); + grpc_call_combiner_set_notify_on_cancel( + exec_ctx, calld->call_combiner, + GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure, + pick_callback_cancel_locked, elem, + grpc_combiner_scheduler(chand->combiner))); + } + return pick_done; +} typedef struct { grpc_call_element *elem; @@ -1069,17 +1171,17 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx, gpr_free(args); return; } - args->finished = true; - grpc_call_element *elem = args->elem; - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; // If we don't yet have a resolver result, then a closure for // pick_after_resolver_result_done_locked() will have been added to // chand->waiting_for_resolver_result_closures, and it may not be invoked // until after this call has been destroyed. We mark the operation as // finished, so that when pick_after_resolver_result_done_locked() // is called, it will be a no-op. We also immediately invoke - // subchannel_ready_locked() to propagate the error back to the caller. + // async_pick_done_locked() to propagate the error back to the caller. + args->finished = true; + grpc_call_element *elem = args->elem; + channel_data *chand = (channel_data *)elem->channel_data; + call_data *calld = (call_data *)elem->call_data; if (GRPC_TRACER_ON(grpc_client_channel_trace)) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick waiting for resolver result", @@ -1087,12 +1189,12 @@ static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx, } // Note: Although we are not in the call combiner here, we are // basically stealing the call combiner from the pending pick, so - // it's safe to call subchannel_ready_locked() here -- we are + // it's safe to call async_pick_done_locked() here -- we are // essentially calling it here instead of calling it in // pick_after_resolver_result_done_locked(). - subchannel_ready_locked(exec_ctx, elem, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Pick cancelled", &error, 1)); + async_pick_done_locked(exec_ctx, elem, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "Pick cancelled", &error, 1)); } static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx, @@ -1117,14 +1219,19 @@ static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx, gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data", chand, calld); } - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); } else { if (GRPC_TRACER_ON(grpc_client_channel_trace)) { gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick", chand, calld); } - if (pick_subchannel_locked(exec_ctx, elem)) { - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE); + if (pick_callback_start_locked(exec_ctx, elem)) { + // Even if the LB policy returns a result synchronously, we have + // already added our polling entity to chand->interested_parties + // in order to wait for the resolver result, so we need to + // remove it here. Therefore, we call async_pick_done_locked() + // instead of pick_done_locked(). + async_pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE); } } } @@ -1152,154 +1259,38 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx, grpc_combiner_scheduler(chand->combiner))); } -// Note: This runs under the client_channel combiner, but will NOT be -// holding the call combiner. -static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - grpc_call_element *elem = (grpc_call_element *)arg; - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - if (error != GRPC_ERROR_NONE && calld->lb_policy != NULL) { - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p", - chand, calld, calld->lb_policy); - } - grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy, - &calld->connected_subchannel, - GRPC_ERROR_REF(error)); - } - GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel"); -} - -// Callback invoked by grpc_lb_policy_pick_locked() for async picks. -// Unrefs the LB policy and invokes subchannel_ready_locked(). -static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *ignored) { grpc_call_element *elem = (grpc_call_element *)arg; - channel_data *chand = (channel_data *)elem->channel_data; call_data *calld = (call_data *)elem->call_data; - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously", - chand, calld); - } - GPR_ASSERT(calld->lb_policy != NULL); - GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); - calld->lb_policy = NULL; - subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error)); -} - -// Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked(). -// If the pick was completed synchronously, unrefs the LB policy and -// returns true. -static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem, - const grpc_lb_policy_pick_args *inputs) { channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p", - chand, calld, chand->lb_policy); - } - // Keep a ref to the LB policy in calld while the pick is pending. - GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel"); - calld->lb_policy = chand->lb_policy; - GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem, - grpc_combiner_scheduler(chand->combiner)); - const bool pick_done = grpc_lb_policy_pick_locked( - exec_ctx, chand->lb_policy, inputs, &calld->connected_subchannel, - calld->subchannel_call_context, NULL, &calld->lb_pick_closure); - if (pick_done) { - /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ - if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously", - chand, calld); + GPR_ASSERT(calld->connected_subchannel == NULL); + if (chand->lb_policy != NULL) { + // We already have an LB policy, so ask it for a pick. + if (pick_callback_start_locked(exec_ctx, elem)) { + // Pick completed synchronously. + pick_done_locked(exec_ctx, elem, GRPC_ERROR_NONE); + return; } - GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel"); - calld->lb_policy = NULL; } else { - GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel"); - grpc_call_combiner_set_notify_on_cancel( - exec_ctx, calld->call_combiner, - GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure, - pick_callback_cancel_locked, elem, - grpc_combiner_scheduler(chand->combiner))); - } - return pick_done; -} - -static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx, - grpc_call_element *elem) { - GPR_TIMER_BEGIN("pick_subchannel", 0); - channel_data *chand = (channel_data *)elem->channel_data; - call_data *calld = (call_data *)elem->call_data; - bool pick_done = false; - if (chand->lb_policy != NULL) { - apply_service_config_to_call_locked(exec_ctx, elem); - // If the application explicitly set wait_for_ready, use that. - // Otherwise, if the service config specified a value for this - // method, use that. - uint32_t initial_metadata_flags = - calld->initial_metadata_batch->payload->send_initial_metadata - .send_initial_metadata_flags; - const bool wait_for_ready_set_from_api = - initial_metadata_flags & - GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; - const bool wait_for_ready_set_from_service_config = - calld->method_params != NULL && - calld->method_params->wait_for_ready != WAIT_FOR_READY_UNSET; - if (!wait_for_ready_set_from_api && - wait_for_ready_set_from_service_config) { - if (calld->method_params->wait_for_ready == WAIT_FOR_READY_TRUE) { - initial_metadata_flags |= GRPC_INITIAL_METADATA_WAIT_FOR_READY; - } else { - initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY; - } + // We do not yet have an LB policy, so wait for a resolver result. + if (chand->resolver == NULL) { + pick_done_locked(exec_ctx, elem, + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected")); + return; } - const grpc_lb_policy_pick_args inputs = { - calld->initial_metadata_batch->payload->send_initial_metadata - .send_initial_metadata, - initial_metadata_flags, &calld->lb_token_mdelem}; - pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs); - } else if (chand->resolver != NULL) { if (!chand->started_resolving) { start_resolving_locked(exec_ctx, chand); } pick_after_resolver_result_start_locked(exec_ctx, elem); - } else { - subchannel_ready_locked( - exec_ctx, elem, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected")); - } - GPR_TIMER_END("pick_subchannel", 0); - return pick_done; -} - -static void start_pick_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error_ignored) { - GPR_TIMER_BEGIN("start_pick_locked", 0); - grpc_call_element *elem = (grpc_call_element *)arg; - call_data *calld = (call_data *)elem->call_data; - channel_data *chand = (channel_data *)elem->channel_data; - GPR_ASSERT(calld->connected_subchannel == NULL); - if (pick_subchannel_locked(exec_ctx, elem)) { - // Pick was returned synchronously. - if (calld->connected_subchannel == NULL) { - GRPC_ERROR_UNREF(calld->error); - calld->error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Call dropped by load balancing policy"); - waiting_for_pick_batches_fail(exec_ctx, elem, - GRPC_ERROR_REF(calld->error)); - } else { - // Create subchannel call. - create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE); - } - } else { - // Pick will be done asynchronously. Add the call's polling entity to - // the channel's interested_parties, so that I/O for the resolver - // and LB policy can be done under it. - grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent, - chand->interested_parties); } - GPR_TIMER_END("start_pick_locked", 0); + // We need to wait for either a resolver result or for an async result + // from the LB policy. Add the polling entity from call_data to the + // channel_data's interested_parties, so that the I/O of the LB policy + // and resolver can be done under it. The polling entity will be + // removed in async_pick_done_locked(). + grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent, + chand->interested_parties); } static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { @@ -1394,7 +1385,8 @@ static void cc_start_transport_stream_op_batch( // combiner to start a pick. if (batch->send_initial_metadata) { if (GRPC_TRACER_ON(grpc_client_channel_trace)) { - gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld); + gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering client_channel combiner", + chand, calld); } GRPC_CLOSURE_SCHED( exec_ctx, -- cgit v1.2.3 From 1c32292c7be078303f4bb59e635319773c5012c2 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 09:05:59 -0700 Subject: Address reviewer comments --- include/grpc++/impl/codegen/call.h | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index bbd277f288..0bf33cb88f 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -303,14 +303,18 @@ class CallOpSendMessage { WriteOptions write_options_; }; +namespace internal { +template +T Example(); +} // namespace internal + template class CallOpSendMessage::MessageSerializer< M, typename std::enable_if::Serialize( - static_cast(*(static_cast(nullptr))), - static_cast(nullptr), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Serialize( + internal::Example(), + internal::Example(), + internal::Example()))>::value>::type> { public: static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, bool* own_buf) { @@ -322,11 +326,10 @@ class CallOpSendMessage::MessageSerializer< template class CallOpSendMessage::MessageSerializer< M, typename std::enable_if::Serialize( - static_cast(*(static_cast(nullptr))), - static_cast<::grpc::ByteBuffer*>(nullptr), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Serialize( + internal::Example(), + internal::Example<::grpc::ByteBuffer*>(), + internal::Example()))>::value>::type> { public: static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, bool* own_buf) { -- cgit v1.2.3 From c75ae78b0806f8ecea3ddfedbf4c2497331bbca5 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 11 Sep 2017 19:21:34 +0000 Subject: Add more Cython-layer tests I wrote these in the course of a bug hunt. I haven't yet caught and fixed the bug, but that's no reason not to check in perfectly good tests. :-) --- src/python/grpcio_tests/tests/tests.json | 2 + .../grpcio_tests/tests/unit/_cython/_common.py | 118 +++++++++++++++++++ ...ssages_server_completion_queue_per_call_test.py | 131 +++++++++++++++++++++ ...messages_single_server_completion_queue_test.py | 126 ++++++++++++++++++++ 4 files changed, 377 insertions(+) create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_common.py create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py create mode 100644 src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index d61297b918..4c078e6c22 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -26,6 +26,8 @@ "unit._credentials_test.CredentialsTest", "unit._cython._cancel_many_calls_test.CancelManyCallsTest", "unit._cython._channel_test.ChannelTest", + "unit._cython._no_messages_server_completion_queue_per_call_test.Test", + "unit._cython._no_messages_single_server_completion_queue_test.Test", "unit._cython._read_some_but_not_all_responses_test.ReadSomeButNotAllResponsesTest", "unit._cython.cygrpc_test.InsecureServerInsecureClient", "unit._cython.cygrpc_test.SecureServerSecureClient", diff --git a/src/python/grpcio_tests/tests/unit/_cython/_common.py b/src/python/grpcio_tests/tests/unit/_cython/_common.py new file mode 100644 index 0000000000..ac66d1db3d --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_common.py @@ -0,0 +1,118 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common utilities for tests of the Cython layer of gRPC Python.""" + +import collections +import threading + +from grpc._cython import cygrpc + +RPC_COUNT = 4000 + +INFINITE_FUTURE = cygrpc.Timespec(float('+inf')) +EMPTY_FLAGS = 0 + +INVOCATION_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'client-md-key', b'client-md-key'), + cygrpc.Metadatum(b'client-md-key-bin', b'\x00\x01' * 3000),)) + +INITIAL_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'server-initial-md-key', b'server-initial-md-value'), + cygrpc.Metadatum(b'server-initial-md-key-bin', b'\x00\x02' * 3000),)) + +TRAILING_METADATA = cygrpc.Metadata( + (cygrpc.Metadatum(b'server-trailing-md-key', b'server-trailing-md-value'), + cygrpc.Metadatum(b'server-trailing-md-key-bin', b'\x00\x03' * 3000),)) + + +class QueueDriver(object): + + def __init__(self, condition, completion_queue): + self._condition = condition + self._completion_queue = completion_queue + self._due = collections.defaultdict(int) + self._events = collections.defaultdict(list) + + def add_due(self, tags): + if not self._due: + + def in_thread(): + while True: + event = self._completion_queue.poll() + with self._condition: + self._events[event.tag].append(event) + self._due[event.tag] -= 1 + self._condition.notify_all() + if self._due[event.tag] <= 0: + self._due.pop(event.tag) + if not self._due: + return + + thread = threading.Thread(target=in_thread) + thread.start() + for tag in tags: + self._due[tag] += 1 + + def event_with_tag(self, tag): + with self._condition: + while True: + if self._events[tag]: + return self._events[tag].pop(0) + else: + self._condition.wait() + + +def execute_many_times(behavior): + return tuple(behavior() for _ in range(RPC_COUNT)) + + +class OperationResult( + collections.namedtuple('OperationResult', ( + 'start_batch_result', 'completion_type', 'success',))): + pass + + +SUCCESSFUL_OPERATION_RESULT = OperationResult( + cygrpc.CallError.ok, cygrpc.CompletionType.operation_complete, True) + + +class RpcTest(object): + + def setUp(self): + self.server_completion_queue = cygrpc.CompletionQueue() + self.server = cygrpc.Server(cygrpc.ChannelArgs([])) + self.server.register_completion_queue(self.server_completion_queue) + port = self.server.add_http2_port(b'[::]:0') + self.server.start() + self.channel = cygrpc.Channel('localhost:{}'.format(port).encode(), + cygrpc.ChannelArgs([])) + + self._server_shutdown_tag = 'server_shutdown_tag' + self.server_condition = threading.Condition() + self.server_driver = QueueDriver(self.server_condition, + self.server_completion_queue) + with self.server_condition: + self.server_driver.add_due({ + self._server_shutdown_tag, + }) + + self.client_condition = threading.Condition() + self.client_completion_queue = cygrpc.CompletionQueue() + self.client_driver = QueueDriver(self.client_condition, + self.client_completion_queue) + + def tearDown(self): + self.server.shutdown(self.server_completion_queue, + self._server_shutdown_tag) + self.server.cancel_all_calls() diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py new file mode 100644 index 0000000000..14cc66675c --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_server_completion_queue_per_call_test.py @@ -0,0 +1,131 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test a corner-case at the level of the Cython API.""" + +import threading +import unittest + +from grpc._cython import cygrpc + +from tests.unit._cython import _common + + +class Test(_common.RpcTest, unittest.TestCase): + + def _do_rpcs(self): + server_call_condition = threading.Condition() + server_call_completion_queue = cygrpc.CompletionQueue() + server_call_driver = _common.QueueDriver(server_call_condition, + server_call_completion_queue) + + server_request_call_tag = 'server_request_call_tag' + server_send_initial_metadata_tag = 'server_send_initial_metadata_tag' + server_complete_rpc_tag = 'server_complete_rpc_tag' + + with self.server_condition: + server_request_call_start_batch_result = self.server.request_call( + server_call_completion_queue, self.server_completion_queue, + server_request_call_tag) + self.server_driver.add_due({ + server_request_call_tag, + }) + + client_call = self.channel.create_call( + None, _common.EMPTY_FLAGS, self.client_completion_queue, + b'/twinkies', None, _common.INFINITE_FUTURE) + client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' + client_complete_rpc_tag = 'client_complete_rpc_tag' + with self.client_condition: + client_receive_initial_metadata_start_batch_result = ( + client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_receive_initial_metadata( + _common.EMPTY_FLAGS), + ]), client_receive_initial_metadata_tag)) + client_complete_rpc_start_batch_result = client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_send_initial_metadata( + _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), + cygrpc.operation_send_close_from_client( + _common.EMPTY_FLAGS), + cygrpc.operation_receive_status_on_client( + _common.EMPTY_FLAGS), + ]), client_complete_rpc_tag) + self.client_driver.add_due({ + client_receive_initial_metadata_tag, + client_complete_rpc_tag, + }) + + server_request_call_event = self.server_driver.event_with_tag( + server_request_call_tag) + + with server_call_condition: + server_send_initial_metadata_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_send_initial_metadata( + _common.INITIAL_METADATA, _common.EMPTY_FLAGS), + ], server_send_initial_metadata_tag)) + server_call_driver.add_due({ + server_send_initial_metadata_tag, + }) + server_send_initial_metadata_event = server_call_driver.event_with_tag( + server_send_initial_metadata_tag) + + with server_call_condition: + server_complete_rpc_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_receive_close_on_server( + _common.EMPTY_FLAGS), + cygrpc.operation_send_status_from_server( + _common.TRAILING_METADATA, cygrpc.StatusCode.ok, + b'test details', _common.EMPTY_FLAGS), + ], server_complete_rpc_tag)) + server_call_driver.add_due({ + server_complete_rpc_tag, + }) + server_complete_rpc_event = server_call_driver.event_with_tag( + server_complete_rpc_tag) + + client_receive_initial_metadata_event = self.client_driver.event_with_tag( + client_receive_initial_metadata_tag) + client_complete_rpc_event = self.client_driver.event_with_tag( + client_complete_rpc_tag) + + return (_common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.type, + server_complete_rpc_event.success),) + + def test_rpcs(self): + expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * + 5] * _common.RPC_COUNT + actuallys = _common.execute_many_times(self._do_rpcs) + self.assertSequenceEqual(expecteds, actuallys) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py new file mode 100644 index 0000000000..1e44bcc4dc --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_cython/_no_messages_single_server_completion_queue_test.py @@ -0,0 +1,126 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test a corner-case at the level of the Cython API.""" + +import threading +import unittest + +from grpc._cython import cygrpc + +from tests.unit._cython import _common + + +class Test(_common.RpcTest, unittest.TestCase): + + def _do_rpcs(self): + server_request_call_tag = 'server_request_call_tag' + server_send_initial_metadata_tag = 'server_send_initial_metadata_tag' + server_complete_rpc_tag = 'server_complete_rpc_tag' + + with self.server_condition: + server_request_call_start_batch_result = self.server.request_call( + self.server_completion_queue, self.server_completion_queue, + server_request_call_tag) + self.server_driver.add_due({ + server_request_call_tag, + }) + + client_call = self.channel.create_call( + None, _common.EMPTY_FLAGS, self.client_completion_queue, + b'/twinkies', None, _common.INFINITE_FUTURE) + client_receive_initial_metadata_tag = 'client_receive_initial_metadata_tag' + client_complete_rpc_tag = 'client_complete_rpc_tag' + with self.client_condition: + client_receive_initial_metadata_start_batch_result = ( + client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_receive_initial_metadata( + _common.EMPTY_FLAGS), + ]), client_receive_initial_metadata_tag)) + client_complete_rpc_start_batch_result = client_call.start_client_batch( + cygrpc.Operations([ + cygrpc.operation_send_initial_metadata( + _common.INVOCATION_METADATA, _common.EMPTY_FLAGS), + cygrpc.operation_send_close_from_client( + _common.EMPTY_FLAGS), + cygrpc.operation_receive_status_on_client( + _common.EMPTY_FLAGS), + ]), client_complete_rpc_tag) + self.client_driver.add_due({ + client_receive_initial_metadata_tag, + client_complete_rpc_tag, + }) + + server_request_call_event = self.server_driver.event_with_tag( + server_request_call_tag) + + with self.server_condition: + server_send_initial_metadata_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_send_initial_metadata( + _common.INITIAL_METADATA, _common.EMPTY_FLAGS), + ], server_send_initial_metadata_tag)) + self.server_driver.add_due({ + server_send_initial_metadata_tag, + }) + server_send_initial_metadata_event = self.server_driver.event_with_tag( + server_send_initial_metadata_tag) + + with self.server_condition: + server_complete_rpc_start_batch_result = ( + server_request_call_event.operation_call.start_server_batch([ + cygrpc.operation_receive_close_on_server( + _common.EMPTY_FLAGS), + cygrpc.operation_send_status_from_server( + _common.TRAILING_METADATA, cygrpc.StatusCode.ok, + b'test details', _common.EMPTY_FLAGS), + ], server_complete_rpc_tag)) + self.server_driver.add_due({ + server_complete_rpc_tag, + }) + server_complete_rpc_event = self.server_driver.event_with_tag( + server_complete_rpc_tag) + + client_receive_initial_metadata_event = self.client_driver.event_with_tag( + client_receive_initial_metadata_tag) + client_complete_rpc_event = self.client_driver.event_with_tag( + client_complete_rpc_tag) + + return (_common.OperationResult(server_request_call_start_batch_result, + server_request_call_event.type, + server_request_call_event.success), + _common.OperationResult( + client_receive_initial_metadata_start_batch_result, + client_receive_initial_metadata_event.type, + client_receive_initial_metadata_event.success), + _common.OperationResult(client_complete_rpc_start_batch_result, + client_complete_rpc_event.type, + client_complete_rpc_event.success), + _common.OperationResult( + server_send_initial_metadata_start_batch_result, + server_send_initial_metadata_event.type, + server_send_initial_metadata_event.success), + _common.OperationResult(server_complete_rpc_start_batch_result, + server_complete_rpc_event.type, + server_complete_rpc_event.success),) + + def test_rpcs(self): + expecteds = [(_common.SUCCESSFUL_OPERATION_RESULT,) * + 5] * _common.RPC_COUNT + actuallys = _common.execute_many_times(self._do_rpcs) + self.assertSequenceEqual(expecteds, actuallys) + + +if __name__ == '__main__': + unittest.main(verbosity=2) -- cgit v1.2.3 From afa47005015295e65b932403142938f9bb9b72fc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 11 Sep 2017 12:22:49 -0700 Subject: Use Apple internal type for timeout --- src/objective-c/GRPCClient/GRPCCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.h | 2 +- src/objective-c/GRPCClient/private/GRPCChannel.m | 4 ++-- src/objective-c/GRPCClient/private/GRPCHost.h | 2 +- src/objective-c/GRPCClient/private/GRPCHost.m | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.h | 2 +- src/objective-c/GRPCClient/private/GRPCWrappedCall.m | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 26c920b973..03552779d8 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -172,7 +172,7 @@ extern id const kGRPCTrailersKey; /** * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. */ -@property UInt64 timeout; +@property NSTimeInterval timeout; /** * The container of the request headers of an RPC conforms to this protocol, which is a subset of diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h index 9dff86c271..d37182f754 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.h +++ b/src/objective-c/GRPCClient/private/GRPCChannel.h @@ -63,6 +63,6 @@ struct grpc_channel_credentials; - (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path serverName:(nonnull NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(nonnull GRPCCompletionQueue *)queue; @end diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 39b4e6c1a7..2eb0211550 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -182,7 +182,7 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { - (grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { grpc_slice host_slice; if (serverName) { @@ -193,7 +193,7 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { gpr_inf_future(GPR_CLOCK_REALTIME) : gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_millis(timeout, GPR_TIMESPAN)); + gpr_time_from_millis((int64_t)(timeout * 1000), GPR_TIMESPAN)); grpc_call *call = grpc_channel_create_call(_unmanagedChannel, NULL, GRPC_PROPAGATE_DEFAULTS, queue.unmanagedQueue, diff --git a/src/objective-c/GRPCClient/private/GRPCHost.h b/src/objective-c/GRPCClient/private/GRPCHost.h index a9b3032c9c..58171211b0 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.h +++ b/src/objective-c/GRPCClient/private/GRPCHost.h @@ -55,7 +55,7 @@ struct grpc_channel_credentials; /** Create a grpc_call object to the provided path on this host. */ - (nullable struct grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue; // TODO: There's a race when a new RPC is coming through just as an existing one is getting diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index a0791827db..f73e9cbc50 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -121,7 +121,7 @@ static GRPCConnectivityMonitor *connectivityMonitor = nil; - (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path serverName:(NSString *)serverName - timeout:(UInt64)timeout + timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { GRPCChannel *channel; // This is racing -[GRPCHost disconnect]. diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index c0a6c71c72..1cd9da8f3e 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -77,7 +77,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - timeout:(UInt64)timeout NS_DESIGNATED_INITIALIZER; + timeout:(NSTimeInterval)timeout NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index 7366f31931..b0b1223b64 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -244,7 +244,7 @@ - (instancetype)initWithHost:(NSString *)host serverName:(NSString *)serverName path:(NSString *)path - timeout:(UInt64)timeout { + timeout:(NSTimeInterval)timeout { if (!path || !host) { [NSException raise:NSInvalidArgumentException format:@"path and host cannot be nil."]; -- cgit v1.2.3 From 54f3c2eefe65a2f5586b8c1ee8b7c9981502fe85 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 13:12:04 -0700 Subject: Automatically fetch schema from BigQuery --- tools/profiling/microbenchmarks/bm2bq.py | 43 ++++++-------------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 0797f3e21d..6c5040de56 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -22,42 +22,15 @@ import sys import json import csv import bm_json +import json +import subprocess + +columns = [] -columns = [ - ('jenkins_build', 'integer'), - ('jenkins_job', 'string'), - ('date', 'timestamp'), - ('cpu_scaling_enabled', 'boolean'), - ('num_cpus', 'integer'), - ('mhz_per_cpu', 'integer'), - ('library_build_type', 'string'), - ('name', 'string'), - ('fixture', 'string'), - ('client_mutator', 'string'), - ('server_mutator', 'string'), - ('request_size', 'integer'), - ('response_size', 'integer'), - ('request_count', 'integer'), - ('iterations', 'integer'), - ('time_unit', 'string'), - ('real_time', 'integer'), - ('cpu_time', 'integer'), - ('bytes_per_second', 'float'), - ('allocs_per_iteration', 'float'), - ('locks_per_iteration', 'float'), - ('writes_per_iteration', 'float'), - ('bandwidth_kilobits', 'integer'), - ('cli_transport_stalls_per_iteration', 'float'), - ('cli_stream_stalls_per_iteration', 'float'), - ('svr_transport_stalls_per_iteration', 'float'), - ('svr_stream_stalls_per_iteration', 'float'), - ('atm_cas_per_iteration', 'float'), - ('atm_add_per_iteration', 'float'), - ('end_of_stream', 'boolean'), - ('header_bytes_per_iteration', 'float'), - ('framing_bytes_per_iteration', 'float'), - ('nows_per_iteration', 'float'), -] +for row in json.loads( + subprocess.check_output([ + 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: + columns.append((row['name'], row['type'])) SANITIZE = { 'integer': int, -- cgit v1.2.3 From 4b7fe94a9728686065a0cd4275ca64e24dc32c3c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 13:30:13 -0700 Subject: Automatically generate BigQuery schema details for all counters --- src/core/lib/debug/stats_data_bq_schema.sql | 33 +++++++++++++++++++++++++++++ tools/codegen/core/gen_stats_data.py | 6 ++++++ 2 files changed, 39 insertions(+) create mode 100644 src/core/lib/debug/stats_data_bq_schema.sql diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql new file mode 100644 index 0000000000..b669555fa0 --- /dev/null +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -0,0 +1,33 @@ +client_calls_created_per_iteration:INTEGER, +server_calls_created_per_iteration:INTEGER, +syscall_poll_per_iteration:INTEGER, +syscall_wait_per_iteration:INTEGER, +histogram_slow_lookups_per_iteration:INTEGER, +syscall_write_per_iteration:INTEGER, +syscall_read_per_iteration:INTEGER, +tcp_backup_pollers_created_per_iteration:INTEGER, +tcp_backup_poller_polls_per_iteration:INTEGER, +http2_op_batches_per_iteration:INTEGER, +http2_op_cancel_per_iteration:INTEGER, +http2_op_send_initial_metadata_per_iteration:INTEGER, +http2_op_send_message_per_iteration:INTEGER, +http2_op_send_trailing_metadata_per_iteration:INTEGER, +http2_op_recv_initial_metadata_per_iteration:INTEGER, +http2_op_recv_message_per_iteration:INTEGER, +http2_op_recv_trailing_metadata_per_iteration:INTEGER, +http2_settings_writes_per_iteration:INTEGER, +http2_pings_sent_per_iteration:INTEGER, +http2_writes_begun_per_iteration:INTEGER, +http2_writes_offloaded_per_iteration:INTEGER, +http2_writes_continued_per_iteration:INTEGER, +http2_partial_writes_per_iteration:INTEGER, +combiner_locks_initiated_per_iteration:INTEGER, +combiner_locks_scheduled_items_per_iteration:INTEGER, +combiner_locks_scheduled_final_items_per_iteration:INTEGER, +combiner_locks_offloaded_per_iteration:INTEGER, +executor_scheduled_short_items_per_iteration:INTEGER, +executor_scheduled_long_items_per_iteration:INTEGER, +executor_scheduled_to_self_per_iteration:INTEGER, +executor_wakeup_initiated_per_iteration:INTEGER, +executor_queue_drained_per_iteration:INTEGER, +executor_push_retries_per_iteration:INTEGER diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af..cb01321ed3 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,9 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: + columns = [] + for counter in inst_map['Counter']: + columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + print >>S, ',\n'.join('%s:%s' % x for x in columns) -- cgit v1.2.3 From 8e87c74fe95e445dac4dc66d4f464d572fcbaa5e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 14:06:09 -0700 Subject: Fix compilation for C++ --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 9d71d3990d..5c9405f583 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -59,7 +59,8 @@ static void pollset_destroy(grpc_exec_ctx* exec_ctx, grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } -static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { +static grpc_error* pollset_kick(grpc_exec_ctx* exec_ctx, grpc_pollset* p, + grpc_pollset_worker* worker) { return GRPC_ERROR_NONE; } -- cgit v1.2.3 From 4433973242e21db18291263a5b41ee292002eb6f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 14:26:34 -0700 Subject: Address review comments --- include/grpc++/impl/codegen/byte_buffer.h | 2 +- include/grpc++/impl/codegen/call.h | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h index e6231ca243..87d390c688 100644 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015 gRPC authors. + * Copyright 2017 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 0bf33cb88f..0652d69d12 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -361,11 +361,9 @@ class MessageDeserializer; template class MessageDeserializer< M, typename std::enable_if::Deserialize( - static_cast( - *(static_cast(nullptr))), - static_cast(nullptr)))>::value>::type> { + ::grpc::Status, decltype(SerializationTraits::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { public: static Status Deserialize(const ByteBuffer& bbuf, M* message) { return SerializationTraits::Deserialize(bbuf, message); @@ -376,8 +374,8 @@ template class MessageDeserializer< M, typename std::enable_if::Deserialize( - static_cast(nullptr), - static_cast(nullptr)))>::value>::type> { + internal::Example(), + internal::Example()))>::value>::type> { public: static Status Deserialize(const ByteBuffer& bbuf, M* message) { return SerializationTraits::Deserialize( -- cgit v1.2.3 From 20104ce596b8ccbe4cd167e245654f56e8d86edc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 11 Sep 2017 15:26:57 -0700 Subject: Add a clamp to remove ubsan failure --- src/core/ext/transport/chttp2/transport/flow_control.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index cec99f6fb6..39aa521029 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -483,7 +483,8 @@ grpc_chttp2_flowctl_action grpc_chttp2_flowctl_get_bdp_action( if (grpc_bdp_estimator_get_bw(&tfc->bdp_estimator, &bw_dbl)) { // we target the max of BDP or bandwidth in microseconds. int32_t frame_size = (int32_t)GPR_CLAMP( - GPR_MAX((int32_t)bw_dbl / 1000, bdp), 16384, 16777215); + GPR_MAX((int32_t)GPR_CLAMP(bw_dbl, 0, INT_MAX) / 1000, bdp), 16384, + 16777215); grpc_chttp2_flowctl_urgency frame_size_urgency = delta_is_significant( tfc, frame_size, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE); if (frame_size_urgency != GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED) { -- cgit v1.2.3 From f6e61bf7686bfce40d4f3912fd9eeb2ba6987e62 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 11 Sep 2017 15:48:51 -0700 Subject: Add comment for timeout; assert on negative timeout value; fix timeout test --- src/objective-c/GRPCClient/GRPCCall.h | 4 +++- src/objective-c/GRPCClient/private/GRPCChannel.m | 4 ++++ src/objective-c/tests/GRPCClientTests.m | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 03552779d8..3b0ef56c3a 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,7 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. + * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. If set to + * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded. A negative value is + * not allowed. */ @property NSTimeInterval timeout; diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m index 2eb0211550..b78b14f2af 100644 --- a/src/objective-c/GRPCClient/private/GRPCChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCChannel.m @@ -184,6 +184,10 @@ static grpc_channel_args *BuildChannelArgs(NSDictionary *dictionary) { serverName:(NSString *)serverName timeout:(NSTimeInterval)timeout completionQueue:(GRPCCompletionQueue *)queue { + GPR_ASSERT(timeout >= 0); + if (timeout < 0) { + timeout = 0; + } grpc_slice host_slice; if (serverName) { host_slice = grpc_slice_from_copied_string(serverName.UTF8String); diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index d050319533..4be3d1c5e8 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -437,7 +437,7 @@ static GRPCProtoMethod *kUnaryCallMethod; [completion fulfill]; }]; - call.timeout = 1; + call.timeout = 0.001; [call startWithWriteable:responsesWriteable]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; -- cgit v1.2.3 From 41388890860f29cd6e147bde163f57322a408a2f Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 11 Sep 2017 22:47:38 +0000 Subject: Ignore all Python 3 virtual environments How soon could Python 4 happen anyway? :-P --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5e38f5fa01..5ccad2e4f2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ htmlcov/ dist/ *.egg py27/ -py34/ +py3[0-9]*/ # Node installation output node_modules -- cgit v1.2.3 From a6721a0979029a2492631e075946556ce7c0eaf2 Mon Sep 17 00:00:00 2001 From: "K.K. Yap" Date: Mon, 11 Sep 2017 16:20:04 -0700 Subject: Comment that IPv6 Any accepts both IPv4 and IPv6 connections. --- include/grpc++/server_builder.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 21ae70d13a..bbf45b3e74 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -136,8 +136,10 @@ class ServerBuilder { /// It can be invoked multiple times. /// /// \param addr_uri The address to try to bind to the server in URI form. If - /// the scheme name is omitted, "dns:///" is assumed. Valid values include - /// dns:///localhost:1234, / 192.168.1.1:31416, dns:///[::1]:27182, etc.). + /// the scheme name is omitted, "dns:///" is assumed. To bind to any address, + /// please use IPv6 any, i.e., [::]:, which also accepts IPv4 + /// connections. Valid values include dns:///localhost:1234, / + /// 192.168.1.1:31416, dns:///[::1]:27182, etc.). /// \params creds The credentials associated with the server. /// \param selected_port[out] If not `nullptr`, gets populated with the port /// number bound to the \a grpc::Server for the corresponding endpoint after -- cgit v1.2.3 From 6d11e9092667565a181d051932e04c50ab1db099 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 23 Aug 2017 11:43:40 -0700 Subject: Add Kokoro test filtering for MacOS/Windows --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 7 +++++++ tools/internal_ci/helper_scripts/prepare_build_windows.bat | 8 ++++++++ tools/run_tests/python_utils/filter_pull_request_tests.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bd250957f4..2a1225683d 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -32,6 +32,13 @@ ulimit -a sudo easy_install --upgrade google-api-python-client export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json +# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ] && [ -n "$RUN_TESTS_FLAGS" ]; then + brew install jq + ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + export RUN_TESTS_FLAGS="$RUN_TESTS_FLAGS --filter_pr_tests --base_branch origin/$ghprbTargetBranch" +fi + # required to build protobuf brew install gflags diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 69e087e1db..f625755db9 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -18,6 +18,14 @@ set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% bash tools/internal_ci/helper_scripts/gen_report_index.sh +@rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests +if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( + chocolatey install -y jq + for /f "usebackq delims=" %%x in (`curl -s https://api.github.com/repos/grpc/grpc/pulls/%KOKORO_GITHUB_PULL_REQUEST_NUMBER% ^| jq -r .base.ref`) do ( + set RUN_TESTS_FLAGS=%RUN_TESTS_FLAGS% --filter_pr_tests --base_branch origin/%%x + ) +) + @rem Update DNS settings to: @rem 1. allow resolving metadata.google.internal hostname @rem 2. make fetching default GCE credential by oauth2client work diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 032c0701e2..4ad981237b 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -128,7 +128,7 @@ def _get_changed_files(base_branch): # Get file changes between branch and merge-base of specified branch # Not combined to be Windows friendly base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() - return check_output(["git", "diff", base_commit, "--name-only"]).splitlines() + return check_output(["git", "diff", base_commit, "--name-only", "HEAD"]).splitlines() def _can_skip_tests(file_names, triggers): -- cgit v1.2.3 From bfcbad3b86c7912968dc8e64f2121c920dad4dfb Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Tue, 12 Sep 2017 11:52:16 +0800 Subject: cmake: fix cares configuration --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5131bf39b4..187c700233 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,7 +134,8 @@ if("${gRPC_CARES_PROVIDER}" STREQUAL "module") if(NOT CARES_ROOT_DIR) set(CARES_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares) endif() - set(CARES_STATIC ON) + set(CARES_SHARED OFF CACHE BOOL "disable shared library") + set(CARES_STATIC ON CACHE BOOL "link cares statically") set(CARES_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares") add_subdirectory(third_party/cares/cares) if(TARGET c-ares) -- cgit v1.2.3 From 4b7ef4d52546e05a1b33aac1eeb218409c4b0a3a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 11 Sep 2017 23:09:22 -0700 Subject: Build a wall and make 'u' pay for it --- src/core/lib/iomgr/ev_epoll1_linux.c | 143 +++++++++++++++++------------------ 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de..4efd705fa8 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -160,18 +160,18 @@ struct grpc_pollset_worker { (worker)->kick_state_mutator = __LINE__; \ } while (false) -#define MAX_NEIGHBOURHOODS 1024 +#define MAX_NEIGHBORHOODS 1024 -typedef struct pollset_neighbourhood { +typedef struct pollset_neighborhood { gpr_mu mu; grpc_pollset *active_root; char pad[GPR_CACHELINE_SIZE]; -} pollset_neighbourhood; +} pollset_neighborhood; struct grpc_pollset { gpr_mu mu; - pollset_neighbourhood *neighbourhood; - bool reassigning_neighbourhood; + pollset_neighborhood *neighborhood; + bool reassigning_neighborhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; @@ -384,8 +384,8 @@ GPR_TLS_DECL(g_current_thread_worker); /* The designated poller */ static gpr_atm g_active_poller; -static pollset_neighbourhood *g_neighbourhoods; -static size_t g_num_neighbourhoods; +static pollset_neighborhood *g_neighborhoods; +static size_t g_num_neighborhoods; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -424,8 +424,8 @@ static worker_remove_result worker_remove(grpc_pollset *pollset, } } -static size_t choose_neighbourhood(void) { - return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +static size_t choose_neighborhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighborhoods; } static grpc_error *pollset_global_init(void) { @@ -441,11 +441,11 @@ static grpc_error *pollset_global_init(void) { &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } - g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); - g_neighbourhoods = (pollset_neighbourhood *)gpr_zalloc( - sizeof(*g_neighbourhoods) * g_num_neighbourhoods); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_init(&g_neighbourhoods[i].mu); + g_num_neighborhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBORHOODS); + g_neighborhoods = (pollset_neighborhood *)gpr_zalloc( + sizeof(*g_neighborhoods) * g_num_neighborhoods); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_init(&g_neighborhoods[i].mu); } return GRPC_ERROR_NONE; } @@ -454,17 +454,17 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); - for (size_t i = 0; i < g_num_neighbourhoods; i++) { - gpr_mu_destroy(&g_neighbourhoods[i].mu); + for (size_t i = 0; i < g_num_neighborhoods; i++) { + gpr_mu_destroy(&g_neighborhoods[i].mu); } - gpr_free(g_neighbourhoods); + gpr_free(g_neighborhoods); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; - pollset->reassigning_neighbourhood = false; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; + pollset->reassigning_neighborhood = false; pollset->root_worker = NULL; pollset->kicked_without_poller = false; pollset->seen_inactive = true; @@ -477,26 +477,26 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - if (pollset->neighbourhood != neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (pollset->neighborhood != neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } pollset->prev->next = pollset->next; pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = + if (pollset == pollset->neighborhood->active_root) { + pollset->neighborhood->active_root = pollset->next == pollset ? NULL : pollset->next; } } - gpr_mu_unlock(&pollset->neighbourhood->mu); + gpr_mu_unlock(&pollset->neighborhood->mu); } gpr_mu_unlock(&pollset->mu); gpr_mu_destroy(&pollset->mu); @@ -675,16 +675,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, // pollset has been observed to be inactive, we need to move back to the // active list bool is_reassigning = false; - if (!pollset->reassigning_neighbourhood) { + if (!pollset->reassigning_neighborhood) { is_reassigning = true; - pollset->reassigning_neighbourhood = true; - pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + pollset->reassigning_neighborhood = true; + pollset->neighborhood = &g_neighborhoods[choose_neighborhood()]; } - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighborhood *neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) - retry_lock_neighbourhood: - gpr_mu_lock(&neighbourhood->mu); + retry_lock_neighborhood: + gpr_mu_lock(&neighborhood->mu); gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", @@ -692,17 +692,17 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, is_reassigning); } if (pollset->seen_inactive) { - if (neighbourhood != pollset->neighbourhood) { - gpr_mu_unlock(&neighbourhood->mu); - neighbourhood = pollset->neighbourhood; + if (neighborhood != pollset->neighborhood) { + gpr_mu_unlock(&neighborhood->mu); + neighborhood = pollset->neighborhood; gpr_mu_unlock(&pollset->mu); - goto retry_lock_neighbourhood; + goto retry_lock_neighborhood; } /* In the brief time we released the pollset locks above, the worker MAY have been kicked. In this case, the worker should get out of this pollset ASAP and hence this should neither add the pollset to - neighbourhood nor mark the pollset as active. + neighborhood nor mark the pollset as active. On a side note, the only way a worker's kick state could have changed at this point is if it were "kicked specifically". Since the worker has @@ -710,25 +710,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, not visible in the "kick any" path yet */ if (worker->kick_state == UNKICKED) { pollset->seen_inactive = false; - if (neighbourhood->active_root == NULL) { - neighbourhood->active_root = pollset->next = pollset->prev = pollset; + if (neighborhood->active_root == NULL) { + neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ if (worker->kick_state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } } else { - pollset->next = neighbourhood->active_root; + pollset->next = neighborhood->active_root; pollset->prev = pollset->next->prev; pollset->next->prev = pollset->prev->next = pollset; } } } if (is_reassigning) { - GPR_ASSERT(pollset->reassigning_neighbourhood); - pollset->reassigning_neighbourhood = false; + GPR_ASSERT(pollset->reassigning_neighborhood); + pollset->reassigning_neighborhood = false; } - gpr_mu_unlock(&neighbourhood->mu); + gpr_mu_unlock(&neighborhood->mu); } worker_insert(pollset, worker); @@ -763,7 +763,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } /* We release pollset lock in this function at a couple of places: - * 1. Briefly when assigning pollset to a neighbourhood + * 1. Briefly when assigning pollset to a neighborhood * 2. When doing gpr_cv_wait() * It is possible that 'kicked_without_poller' was set to true during (1) and * 'shutting_down' is set to true during (1) or (2). If either of them is @@ -781,12 +781,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; } -static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { - GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); +static bool check_neighborhood_for_available_poller( + pollset_neighborhood *neighborhood) { + GPR_TIMER_BEGIN("check_neighborhood_for_available_poller", 0); bool found_worker = false; do { - grpc_pollset *inspect = neighbourhood->active_root; + grpc_pollset *inspect = neighborhood->active_root; if (inspect == NULL) { break; } @@ -831,8 +831,8 @@ static bool check_neighbourhood_for_available_poller( gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect); } inspect->seen_inactive = true; - if (inspect == neighbourhood->active_root) { - neighbourhood->active_root = + if (inspect == neighborhood->active_root) { + neighborhood->active_root = inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; @@ -841,7 +841,7 @@ static bool check_neighbourhood_for_available_poller( } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - GPR_TIMER_END("check_neighbourhood_for_available_poller", 0); + GPR_TIMER_END("check_neighborhood_for_available_poller", 0); return found_worker; } @@ -873,32 +873,31 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); - size_t poller_neighbourhood_idx = - (size_t)(pollset->neighbourhood - g_neighbourhoods); + size_t poller_neighborhood_idx = + (size_t)(pollset->neighborhood - g_neighborhoods); gpr_mu_unlock(&pollset->mu); bool found_worker = false; - bool scan_state[MAX_NEIGHBOURHOODS]; - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - if (gpr_mu_trylock(&neighbourhood->mu)) { - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + bool scan_state[MAX_NEIGHBORHOODS]; + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + if (gpr_mu_trylock(&neighborhood->mu)) { + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); scan_state[i] = true; } else { scan_state[i] = false; } } - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + for (size_t i = 0; !found_worker && i < g_num_neighborhoods; i++) { if (scan_state[i]) continue; - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); + pollset_neighborhood *neighborhood = + &g_neighborhoods[(poller_neighborhood_idx + i) % + g_num_neighborhoods]; + gpr_mu_lock(&neighborhood->mu); + found_worker = check_neighborhood_for_available_poller(neighborhood); + gpr_mu_unlock(&neighborhood->mu); } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); -- cgit v1.2.3 From 4fa33649753f64283c82f73f5ca59daaa8895671 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 12:56:45 +0200 Subject: bump version to 1.6.1 --- BUILD | 2 +- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Core/Version.csproj.include | 2 +- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/composer.json | 2 +- src/php/ext/grpc/version.h | 2 +- src/python/grpcio/grpc/_grpcio_metadata.py | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 36 insertions(+), 36 deletions(-) diff --git a/BUILD b/BUILD index 4d1121a79d..cbd7e46b97 100644 --- a/BUILD +++ b/BUILD @@ -38,7 +38,7 @@ g_stands_for = "garcia" core_version = "4.0.0" -version = "1.6.0" +version = "1.6.1" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index ebbaa6a9ec..5225c6e34e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.6.0") +set(PACKAGE_VERSION "1.6.1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 801f8a7d6e..5feff15071 100644 --- a/Makefile +++ b/Makefile @@ -411,8 +411,8 @@ Q = @ endif CORE_VERSION = 4.0.0 -CPP_VERSION = 1.6.0 -CSHARP_VERSION = 1.6.0 +CPP_VERSION = 1.6.1 +CSHARP_VERSION = 1.6.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 723c8fe446..954d6a3ecd 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 4.0.0 g_stands_for: garcia - version: 1.6.0 + version: 1.6.1 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 222d3e4f0f..1b90beffc4 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 2d2f31d323..f060274b5f 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 38a159404f..76bf400424 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 12591cda33..3c310c8cd9 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.6.0' + version = '1.6.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' diff --git a/package.json b/package.json index 94e6391a8d..dd383548f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", diff --git a/package.xml b/package.xml index 30018620dd..0735c02944 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-08-24 - 1.6.0 - 1.6.0 + 1.6.1 + 1.6.1 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index c65185c77e..35c1ec6f5e 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include namespace grpc { -grpc::string Version() { return "1.6.0"; } +grpc::string Version() { return "1.6.1"; } } diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 23f6e40909..616f390f88 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ - 1.6.0 + 1.6.1 3.3.0 diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index d402c323c1..52f8e2642a 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.6.0.0"; + public const string CurrentAssemblyFileVersion = "1.6.1.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.6.0"; + public const string CurrentVersion = "1.6.1"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index b7aa22b292..a6581ac733 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.6.0 +set VERSION=1.6.1 @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 455b930525..76d6544c07 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -39,7 +39,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.6.0" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.6.1" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.6.1" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 66ab2a292f..92e748e72f 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.6.0", + "grpc": "^1.6.1", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index b0b1e8679a..023cb1e464 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.6.0", + "version": "1.6.1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 0e180a4711..32f6c39e6c 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.6.0' + v = '1.6.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 3521537578..64e08f9151 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.6.0" +#define GRPC_OBJC_VERSION_STRING @"1.6.1" diff --git a/src/php/composer.json b/src/php/composer.json index 5d492ccb86..df3dff3922 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.6.0", + "version": "1.6.1", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index 1a46570117..67f0c4b5ee 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.6.0" +#define PHP_GRPC_VERSION "1.6.1" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 28f5a1a8a2..8707a5a9a6 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.6.0""" +__version__ = """1.6.1""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index ce35bda91c..5a76a66e6c 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index fb5c5f5f65..41f7e344ba 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 73e7d2e497..63865b1bb3 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 99510bbe09..ea0278c7e5 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 66834d20e1..680e5d1454 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.6.0' + VERSION = '1.6.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 73fa15f85f..2d2db88575 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.6.0' + VERSION = '1.6.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index db3b39ae68..0630445fd4 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.6.0' +VERSION='1.6.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index f9225688ad..cefce74f98 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0 +PROJECT_NUMBER = 1.6.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 1b41266c71..c9bdd1454d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.6.0 +PROJECT_NUMBER = 1.6.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a -- cgit v1.2.3 From 29828c58d33fbbc6503784f0d03064ced6a22841 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:42:55 +0200 Subject: differentiate run_tests suite runtime --- tools/run_tests/run_tests_matrix.py | 50 +++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 00680b02d3..957e7b569e 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -29,9 +29,11 @@ from python_utils.filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) +_DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 + # Set the timeout high to allow enough time for sanitizers and pre-building # clang docker. -_RUNTESTS_TIMEOUT = 4*60*60 +_CPP_RUNTESTS_TIMEOUT = 4*60*60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -51,8 +53,11 @@ def _report_filename_internal_ci(name): def _docker_jobspec(name, runtests_args=[], runtests_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a docker container""" + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT test_job = jobset.JobSpec( cmdline=['python', 'tools/run_tests/run_tests.py', '--use_docker', @@ -62,15 +67,18 @@ def _docker_jobspec(name, runtests_args=[], runtests_envs={}, '--report_suite_name', '%s' % name] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _workspace_jobspec(name, runtests_args=[], workspace_name=None, - runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS): + runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): """Run a single instance of run_tests.py in a separate workspace""" if not workspace_name: workspace_name = 'workspace_%s' % name + if not timeout_seconds: + timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT env = {'WORKSPACE_NAME': workspace_name} env.update(runtests_envs) test_job = jobset.JobSpec( @@ -82,14 +90,15 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, '--report_suite_name', '%s' % name] + runtests_args, environ=env, shortname='run_tests_%s' % name, - timeout_seconds=_RUNTESTS_TIMEOUT) + timeout_seconds=timeout_seconds) return test_job def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', arch=None, compiler=None, labels=[], extra_args=[], extra_envs={}, - inner_jobs=_DEFAULT_INNER_JOBS): + inner_jobs=_DEFAULT_INNER_JOBS, + timeout_seconds=None): result = [] for language in languages: for platform in platforms: @@ -110,10 +119,12 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', runtests_args += extra_args if platform == 'linux': job = _docker_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) else: job = _workspace_jobspec(name=name, runtests_args=runtests_args, - runtests_envs=extra_envs, inner_jobs=inner_jobs) + runtests_envs=extra_envs, inner_jobs=inner_jobs, + timeout_seconds=timeout_seconds) job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) @@ -136,7 +147,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos', 'windows'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['csharp', 'node', 'python'], configs=['dbg', 'opt'], @@ -151,7 +163,8 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux', 'macos'], labels=['basictests', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['ruby', 'php'], configs=['dbg', 'opt'], @@ -174,13 +187,15 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], configs=['asan', 'tsan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) return test_jobs @@ -207,7 +222,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) compiler=compiler, labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # portability C on Windows 64-bit (x86 is the default) test_jobs += _generate_jobs(languages=['c'], @@ -246,10 +262,11 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) configs=['dbg'], platforms=['linux'], labels=['portability', 'corelang'], extra_args=extra_args, - extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) + extra_envs={'GRPC_DNS_RESOLVER': 'ares'}, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) # TODO(zyc): Turn on this test after adding c-ares support on windows. - # C with the c-ares DNS resolver on Windonws + # C with the c-ares DNS resolver on Windows # test_jobs += _generate_jobs(languages=['c'], # configs=['dbg'], platforms=['windows'], # labels=['portability', 'corelang'], @@ -292,7 +309,8 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) iomgr_platform='uv', labels=['portability', 'corelang'], extra_args=extra_args, - inner_jobs=inner_jobs) + inner_jobs=inner_jobs, + timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['node'], configs=['dbg'], -- cgit v1.2.3 From 44a2563cab51ea20cf729a98400e2f8287c953fb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:55:23 +0200 Subject: add missing template for grpcio_testing versining --- .../python/grpcio_testing/grpc_version.py.template | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 templates/src/python/grpcio_testing/grpc_version.py.template diff --git a/templates/src/python/grpcio_testing/grpc_version.py.template b/templates/src/python/grpcio_testing/grpc_version.py.template new file mode 100644 index 0000000000..74db811d60 --- /dev/null +++ b/templates/src/python/grpcio_testing/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2017 gRPC authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! + + VERSION='${settings.python_version.pep440()}' -- cgit v1.2.3 From ea7807876cb7a4c53e610f516d6dbc09d1ef2d2e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 12 Sep 2017 16:56:05 +0200 Subject: regenerate --- src/python/grpcio_testing/grpc_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 41a75d46f6..1638b38c2e 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -12,6 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.5.0.dev0' +VERSION='1.6.1' -- cgit v1.2.3 From d6c4e937ce3f13151ec070ace6d73712f6d11f26 Mon Sep 17 00:00:00 2001 From: Jay Lee Date: Wed, 13 Sep 2017 00:56:08 +0800 Subject: update template --- templates/CMakeLists.txt.template | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 6adff889fe..cb6a82c380 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -179,7 +179,8 @@ if(NOT CARES_ROOT_DIR) set(CARES_ROOT_DIR <%text>${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares) endif() - set(CARES_STATIC ON) + set(CARES_SHARED OFF CACHE BOOL "disable shared library") + set(CARES_STATIC ON CACHE BOOL "link cares statically") set(CARES_INCLUDE_DIR "<%text>${CMAKE_CURRENT_SOURCE_DIR}/third_party/cares/cares") add_subdirectory(third_party/cares/cares) if(TARGET c-ares) -- cgit v1.2.3 From f11a40d23b982129a56fc5e167ebc213afe52e6d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ tools/run_tests/performance/bq_upload_result.py | 2 + tools/run_tests/performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++++---- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af..f665b5808e 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac23196..31819d6159 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 0000000000..1ed5be731d --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 0000000000..c5d884344f --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7..a28d27fece 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file -- cgit v1.2.3 From f9da3a7bc40171c249fd1ffe56ed09d330520b38 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 8 Mar 2017 00:12:50 -0800 Subject: Start setting up files for autorun of php_to_cpp qps tests --- tools/gce/linux_performance_worker_init.sh | 6 +++++ tools/run_tests/performance/run_worker_php.sh | 37 ++++++++++++++++++++++++++ tools/run_tests/performance/scenario_config.py | 28 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c3..04844506c9 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,12 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies + +sudo apt-get install -y php5 php5-dev phpunit php-pear zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 0000000000..b4d1a1cccd --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +cd $(dirname $0)/../../.. + +# The proxy worker for PHP is implemented in Ruby +ruby src/ruby/qps/proxy-worker.rb $@ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index cbcc4d0ecc..15bce80fa4 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -798,6 +798,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'ruby' + + class JavaLanguage: def __init__(self): @@ -995,6 +1022,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), -- cgit v1.2.3 From 649c95eadc313e211b54f1abb7f9826f15a2ac36 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 03:40:02 -0700 Subject: Make sure to get all PHP dependences set up properly --- tools/gce/linux_performance_worker_init.sh | 2 +- tools/run_tests/performance/run_worker_php.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 04844506c9..b952fd1b35 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -130,7 +130,7 @@ gem install bundler # PHP dependencies -sudo apt-get install -y php5 php5-dev phpunit php-pear zlib1g-dev +sudo apt-get install -y php5 php5-dev phpunit php-pear unzip zlib1g-dev curl -sS https://getcomposer.org/installer | php sudo mv composer.phar /usr/local/bin/composer diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index b4d1a1cccd..ebedae8795 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -31,7 +31,14 @@ source ~/.rvm/scripts/rvm set -ex -cd $(dirname $0)/../../.. +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +curl -sS https://getcomposer.org/installer | php +php composer.phar install # The proxy worker for PHP is implemented in Ruby +cd $repo ruby src/ruby/qps/proxy-worker.rb $@ -- cgit v1.2.3 From 644676fb12f756a7efa34c27dd42809da8cbf311 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Sun, 10 Sep 2017 19:06:56 -0700 Subject: vjpai's PR --- tools/gce/linux_performance_worker_init.sh | 5 +++ tools/run_tests/performance/run_worker_php.sh | 45 ++++++++++++++++++++++++++ tools/run_tests/performance/scenario_config.py | 28 ++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c3..88d8de7402 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,11 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 0000000000..ba26f38a77 --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +curl -sS https://getcomposer.org/installer | php +php composer.phar install + +# The proxy worker for PHP is implemented in Ruby +cd $repo +ruby src/ruby/qps/proxy-worker.rb $@ + diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 86236b19bf..329b9f8c47 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -797,6 +797,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -994,6 +1021,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), -- cgit v1.2.3 From 3c77ee8ad1e1c864c86d51126d9a74a62899133e Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 12 Sep 2017 11:45:03 -0700 Subject: dir bug --- tools/run_tests/performance/run_worker_php.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index ba26f38a77..1da74cd0db 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -36,10 +36,8 @@ repo=$(dirname $0)/../../.. # First set up all dependences needed for PHP QPS test cd $repo cd src/php/tests/qps -curl -sS https://getcomposer.org/installer | php -php composer.phar install - +composer install # The proxy worker for PHP is implemented in Ruby -cd $repo +cd ../../../.. ruby src/ruby/qps/proxy-worker.rb $@ -- cgit v1.2.3 From 9b2a91306beafe5829ecfcf658988657e0eff1af Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 04:05:33 -0700 Subject: php7 is new default for ubuntu 16.04 --- tools/gce/linux_performance_worker_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index b952fd1b35..210bbbd3a2 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -130,7 +130,7 @@ gem install bundler # PHP dependencies -sudo apt-get install -y php5 php5-dev phpunit php-pear unzip zlib1g-dev +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev curl -sS https://getcomposer.org/installer | php sudo mv composer.phar /usr/local/bin/composer -- cgit v1.2.3 From 11b48c766e00290bd7481132fd20806f5317d2fe Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 13 Mar 2017 04:26:19 -0700 Subject: typo --- tools/run_tests/performance/scenario_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 15bce80fa4..262001d23e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -822,7 +822,7 @@ class PhpLanguage: server_language='c++', async_server_threads=1) def __str__(self): - return 'ruby' + return 'php' class JavaLanguage: -- cgit v1.2.3 From d5e390b509221ef1574b6b02d91155efd459c30b Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 12 Sep 2017 11:56:59 -0700 Subject: No newline at the end of file --- tools/run_tests/performance/run_worker_php.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 15cee151f8..1da74cd0db 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -39,4 +39,5 @@ cd src/php/tests/qps composer install # The proxy worker for PHP is implemented in Ruby cd ../../../.. -ruby src/ruby/qps/proxy-worker.rb $@ \ No newline at end of file +ruby src/ruby/qps/proxy-worker.rb $@ + -- cgit v1.2.3 From 3a24364b48dcce2f7d84a1c8e8b755cf8925e997 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 12 Sep 2017 12:06:27 -0700 Subject: Fix a memory leak in cares wrapper --- .../filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 0d71f3560e..2e2b411ab8 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -174,7 +174,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET6_ADDRSTRLEN]; ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN); @@ -195,7 +195,7 @@ static void on_hostbyname_done_cb(void *arg, int status, int timeouts, grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, hr->is_balancer /* is_balancer */, - hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + hr->is_balancer ? hr->host : NULL /* balancer_name */, NULL /* user_data */); char output[INET_ADDRSTRLEN]; ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN); -- cgit v1.2.3 From 0852acd8b3700b052676b21b096fc641fc4194aa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:03 -0700 Subject: Revert "Let alarms trigger at end of jobset.py instead of clearing them" --- tools/run_tests/python_utils/jobset.py | 22 ++++++++++++++++------ tools/run_tests/run_performance_tests.py | 8 ++++---- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 6151a7276a..08d652ae3f 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,8 +71,10 @@ def platform_string(): if platform_string() == 'windows': pass else: + have_alarm = False def alarm_handler(unused_signum, unused_frame): - pass + global have_alarm + have_alarm = False signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -365,9 +367,10 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time): + stop_on_failure, add_env, quiet_success, max_time, clear_alarms): self._running = set() self._check_cancelled = check_cancelled + self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -452,7 +455,10 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - signal.alarm(10) + global have_alarm + if not have_alarm: + have_alarm = True + signal.alarm(10) signal.pause() def cancelled(self): @@ -468,7 +474,10 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - if platform_string() != 'windows': + # Clear the alarms when finished to avoid a race condition causing job + # failures. Don't do this when running multi-VM tests because clearing + # the alarms causes the test to stall + if platform_string() != 'windows' and self._clear_alarms: signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -498,7 +507,8 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1): + max_time=-1, + clear_alarms=True): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -510,7 +520,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time) + quiet_success, max_time, clear_alarms) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 3bfd736c51..9b20fae78f 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1) + [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10) + prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10) + build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_gra perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures -- cgit v1.2.3 From 5d6156033e2fff01abd6b01b49626fed95a6d81e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- tools/run_tests/performance/massage_qps_stats.py | 118 ++++++++++++--------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 ++++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e..985c1f51c9 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731d..30a94ca7cd 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344f..400a0c82bf 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece..84b3cb7c0a 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", -- cgit v1.2.3 From 3c8f6dbffd715d0dd9ff03555b15a20c12c3f3ae Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 13:19:49 -0700 Subject: Remove leftover instance of clear_alarms --- tools/run_tests/run_performance_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 9b20fae78f..6e407b4a41 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -556,7 +556,7 @@ def main(): jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) - scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) -- cgit v1.2.3 From 69ce380ccc018a5618d35f5942f102c61b1e91f9 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 13:30:51 -0700 Subject: Revert "Merge pull request #12513 from grpc/revert-12289-sig_hand" This reverts commit 81ad19d4be0e12d519b026a37982f14ff87a0847, reversing changes made to ed63aac1952b8f19c7f1b4186cae139613c77603. --- tools/run_tests/python_utils/jobset.py | 22 ++++++---------------- tools/run_tests/run_performance_tests.py | 8 ++++---- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f..6151a7276a 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -71,10 +71,8 @@ def platform_string(): if platform_string() == 'windows': pass else: - have_alarm = False def alarm_handler(unused_signum, unused_frame): - global have_alarm - have_alarm = False + pass signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None) signal.signal(signal.SIGALRM, alarm_handler) @@ -367,10 +365,9 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, quiet_success, max_time, clear_alarms): + stop_on_failure, add_env, quiet_success, max_time): self._running = set() self._check_cancelled = check_cancelled - self._clear_alarms = clear_alarms self._cancelled = False self._failures = 0 self._completed = 0 @@ -455,10 +452,7 @@ class Jobset(object): if platform_string() == 'windows': time.sleep(0.1) else: - global have_alarm - if not have_alarm: - have_alarm = True - signal.alarm(10) + signal.alarm(10) signal.pause() def cancelled(self): @@ -474,10 +468,7 @@ class Jobset(object): while self._running: if self.cancelled(): pass # poll cancellation self.reap() - # Clear the alarms when finished to avoid a race condition causing job - # failures. Don't do this when running multi-VM tests because clearing - # the alarms causes the test to stall - if platform_string() != 'windows' and self._clear_alarms: + if platform_string() != 'windows': signal.alarm(0) return not self.cancelled() and self._failures == 0 @@ -507,8 +498,7 @@ def run(cmdlines, add_env={}, skip_jobs=False, quiet_success=False, - max_time=-1, - clear_alarms=True): + max_time=-1): if skip_jobs: resultset = {} skipped_job_result = JobResult() @@ -520,7 +510,7 @@ def run(cmdlines, js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - quiet_success, max_time, clear_alarms) + quiet_success, max_time) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 6e407b4a41..1bbab9e894 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -183,7 +183,7 @@ def archive_repo(languages): jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( - [archive_job], newline_on_success=True, maxjobs=1, clear_alarms=False) + [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', @@ -215,7 +215,7 @@ def prepare_remote_hosts(hosts, prepare_local=False): timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( - prepare_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', @@ -248,7 +248,7 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), bui timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( - build_jobs, newline_on_success=True, maxjobs=10, clear_alarms=False) + build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', @@ -414,7 +414,7 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_gra perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) - failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) + failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures -- cgit v1.2.3 From 40c346ced94ecdb5a5d818d160f86e1be1c77046 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 12 Sep 2017 22:39:30 +0200 Subject: Adding --base to generate_projects.py --- tools/buildgen/generate_projects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index f885b26697..559ea1624b 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -34,6 +34,7 @@ argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) argp.add_argument('--output_merged', default=None, type=str) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) +argp.add_argument('--base', default='.', type=str) args = argp.parse_args() json = args.build_files @@ -69,7 +70,7 @@ jobs = [] for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': - out_dir = '.' + root[len('templates'):] + out_dir = args.base + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) -- cgit v1.2.3 From dd2c3cf23afe0c1e11770157a8fdedb3c4146d2b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Tue, 12 Sep 2017 13:54:37 -0700 Subject: fix handling of service config choosing when percentage is zero --- .../ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b7082..371c59b8cf 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } -- cgit v1.2.3 From 73a61708965d35e3b96d7a4f8feeba85601d5aa2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 10:54:20 -0700 Subject: Upload core stats to qps_test --- tools/codegen/core/gen_stats_data.py | 87 ++ tools/run_tests/performance/bq_upload_result.py | 2 + tools/run_tests/performance/massage_qps_stats.py | 105 ++ .../performance/massage_qps_stats_helpers.py | 57 + .../performance/scenario_result_schema.json | 1296 ++++++++++++++++---- 5 files changed, 1314 insertions(+), 233 deletions(-) create mode 100644 tools/run_tests/performance/massage_qps_stats.py create mode 100644 tools/run_tests/performance/massage_qps_stats_helpers.py diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8e4ef594af..f665b5808e 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % ( len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + +# patch qps_test bigquery schema +RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] + +with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: + qps_schema = json.loads(f.read()) + +def FindNamed(js, name): + for el in js: + if el['name'] == name: + return el + +def RemoveCoreFields(js): + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + +RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) +RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + +def AddCoreFields(js): + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + +AddCoreFields(FindNamed(qps_schema, 'clientStats')) +AddCoreFields(FindNamed(qps_schema, 'serverStats')) + +with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + +# and generate a helper script to massage scenario results into the format we'd +# like to query +with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >>P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >>P, line.rstrip() + + print >>P + print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >>P + + print >>P, 'import massage_qps_stats_helpers' + + print >>P, 'def massage_qps_stats(scenario_result):' + print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >>P, ' if "coreStats" not in stats: return' + print >>P, ' core_stats = stats["coreStats"]' + print >>P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( + histogram.name, pctl, histogram.name, pctl, histogram.name) diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 630ac23196..31819d6159 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -24,6 +24,7 @@ import os import sys import time import uuid +import massage_qps_stats gcp_utils_dir = os.path.abspath(os.path.join( @@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result): scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None) scenario_result['summary'].pop('successfulRequestsPerSecond', None) scenario_result['summary'].pop('failedRequestsPerSecond', None) + massage_qps_stats.massage_qps_stats(scenario_result) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py new file mode 100644 index 0000000000..1ed5be731d --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -0,0 +1,105 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Autogenerated by tools/codegen/core/gen_stats_data.py + +import massage_qps_stats_helpers +def massage_qps_stats(scenario_result): + for stats in scenario_result["serverStats"] + scenario_result["clientStats"]: + if "coreStats" not in stats: return + core_stats = stats["coreStats"] + del stats["coreStats"] + stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") + stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") + stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") + stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") + stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write") + stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read") + stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created") + stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls") + stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches") + stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel") + stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata") + stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message") + stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata") + stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata") + stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message") + stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata") + stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes") + stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent") + stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun") + stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") + stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") + stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") + stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") + stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") + stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded") + stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items") + stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items") + stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self") + stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") + stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") + stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) + stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py new file mode 100644 index 0000000000..c5d884344f --- /dev/null +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -0,0 +1,57 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections + +def threshold_for_count_below(buckets, boundaries, count_below): + count_so_far = 0 + for lower_idx in range(0, len(buckets)): + count_so_far += buckets[lower_idx] + if count_so_far >= count_below: + break + if count_so_far == count_below: + # this bucket hits the threshold exactly... we should be midway through + # any run of zero values following the bucket + for upper_idx in range(lower_idx + 1, num_buckets): + if buckets[upper_idx] != 0: + break + return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 + else: + # treat values as uniform throughout the bucket, and find where this value + # should lie + lower_bound = boundaries[lower_idx] + upper_bound = boundaries[lower_idx + 1] + return (upper_bound - + (upper_bound - lower_bound) * (count_so_far - count_below) / + float(buckets[lower_idx])) + +def percentile(buckets, pctl, boundaries): + return threshold_for_count_below( + buckets, boundaries, sum(buckets) * pctl / 100.0) + +def counter(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + return int(stat.get('count', 0)) + +Histogram = collections.namedtuple('Histogram', 'buckets boundaries') +def histogram(core_stats, name): + for stat in core_stats['metrics']: + if stat['name'] == name: + buckets = [] + boundaries = [] + for b in stat['histogram']['buckets']: + buckets.append(int(b.get('count', 0))) + boundaries.append(int(b.get('start', 0))) + return Histogram(buckets=buckets, boundaries=boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index d7e2e292a7..a28d27fece 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -1,269 +1,1099 @@ [ { - "name": "metadata", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "buildNumber", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "buildUrl", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "jobName", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "gitActualCommit", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "created", - "type": "TIMESTAMP", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "buildNumber", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "buildUrl", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "jobName", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "gitActualCommit", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "created", + "type": "TIMESTAMP" } - ] - }, + ], + "mode": "NULLABLE", + "name": "metadata", + "type": "RECORD" + }, { - "name": "scenario", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "clientConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numClients", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "serverConfig", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "numServers", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "warmupSeconds", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "benchmarkSeconds", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "name", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "clientConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numClients", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "serverConfig", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "numServers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "warmupSeconds", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "benchmarkSeconds", + "type": "INTEGER" } - ] - }, + ], + "mode": "NULLABLE", + "name": "scenario", + "type": "RECORD" + }, { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, { - "name": "clientStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "latencies", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "latencies", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "clientStats", + "type": "RECORD" + }, { - "name": "serverStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "timeElapsed", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeElapsed", + "type": "FLOAT" + }, { - "name": "timeUser", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeUser", + "type": "FLOAT" + }, { - "name": "timeSystem", - "type": "FLOAT", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "timeSystem", + "type": "FLOAT" + }, { - "name": "cqPollCount", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "cqPollCount", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_calls_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_poll", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_wait", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_histogram_slow_lookups", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_syscall_read", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_pollers_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_backup_poller_polls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_batches", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_cancel", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_op_recv_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_settings_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_pings_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_begun", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_writes_continued", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_partial_writes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_scheduled_final_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_combiner_locks_offloaded", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_short_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_long_items", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_scheduled_to_self", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_wakeup_initiated", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_queue_drained", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_executor_push_retries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_write_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_tcp_read_offer_iov_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_size_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_initial_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_message_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_trailing_metadata_per_write_99p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_http2_send_flowctl_per_write_99p", + "type": "FLOAT" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverStats", + "type": "RECORD" + }, { - "name": "serverCores", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverCores", + "type": "STRING" + }, { - "name": "summary", - "type": "RECORD", - "mode": "NULLABLE", "fields": [ { - "name": "qps", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "qpsPerServerCore", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientSystemTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientUserTime", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency50", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency90", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency95", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency99", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "latency999", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverPollsPerRequest", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "serverQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "clientQueriesPerCpuSec", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "qps", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "qpsPerServerCore", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientSystemTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientUserTime", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency50", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency90", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency95", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency99", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "latency999", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverPollsPerRequest", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "serverQueriesPerCpuSec", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "clientQueriesPerCpuSec", + "type": "FLOAT" } - ] - }, + ], + "mode": "NULLABLE", + "name": "summary", + "type": "RECORD" + }, { - "name": "clientSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "clientSuccess", + "type": "STRING" + }, { - "name": "serverSuccess", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "serverSuccess", + "type": "STRING" + }, { - "name": "requestResults", - "type": "STRING", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "requestResults", + "type": "STRING" + }, { - "name": "serverCpuStats", - "type": "RECORD", - "mode": "REPEATED", "fields": [ { - "name": "totalCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" - }, + "mode": "NULLABLE", + "name": "totalCpuTime", + "type": "INTEGER" + }, { - "name": "idleCpuTime", - "type": "INTEGER", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "idleCpuTime", + "type": "INTEGER" } - ] - }, + ], + "mode": "REPEATED", + "name": "serverCpuStats", + "type": "RECORD" + }, { - "name": "serverCpuUsage", - "type": "FLOAT", - "mode": "NULLABLE" + "mode": "NULLABLE", + "name": "serverCpuUsage", + "type": "FLOAT" } -] +] \ No newline at end of file -- cgit v1.2.3 From 27147b85f8eeede4f918705e42a91093d450757c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 13:15:58 -0700 Subject: Review feedback --- tools/codegen/core/gen_stats_data.py | 9 +- tools/run_tests/performance/massage_qps_stats.py | 118 ++++++++++++--------- .../performance/massage_qps_stats_helpers.py | 4 +- .../performance/scenario_result_schema.json | 70 ++++++++++++ 4 files changed, 145 insertions(+), 56 deletions(-) diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index f665b5808e..985c1f51c9 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -395,8 +395,9 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: for counter in inst_map['Counter']: print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name) - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name) + print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % ( - histogram.name, pctl, histogram.name, pctl, histogram.name) + print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 1ed5be731d..30a94ca7cd 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -53,53 +53,71 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets) - stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries) - stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets) - stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries) - stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets) - stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries) - stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets) - stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries) - stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets) - stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries) - stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets) - stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries) - stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets) - stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries) - stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets) - stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets) - stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries) - stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets) - stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) - stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries) + stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") + stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size") + stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size") + stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size") + stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer") + stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size") + stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size") + stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write") + stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write") + stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write") + stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write") + stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets) + stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) + h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") + stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) + stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) + stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) + stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) + stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index c5d884344f..400a0c82bf 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -14,7 +14,7 @@ import collections -def threshold_for_count_below(buckets, boundaries, count_below): +def _threshold_for_count_below(buckets, boundaries, count_below): count_so_far = 0 for lower_idx in range(0, len(buckets)): count_so_far += buckets[lower_idx] @@ -37,7 +37,7 @@ def threshold_for_count_below(buckets, boundaries, count_below): float(buckets[lower_idx])) def percentile(buckets, pctl, boundaries): - return threshold_for_count_below( + return _threshold_for_count_below( buckets, boundaries, sum(buckets) * pctl / 100.0) def counter(core_stats, name): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index a28d27fece..84b3cb7c0a 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -275,6 +275,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -524,6 +534,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", @@ -717,6 +752,16 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_server_requested_calls", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_slowpath_requests_queued", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_tcp_write_size", @@ -966,6 +1011,31 @@ "mode": "NULLABLE", "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_bkts", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_50p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_95p", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "core_server_cqs_checked_99p", + "type": "FLOAT" } ], "mode": "REPEATED", -- cgit v1.2.3 From 52778c4729ae4e59eadae3bb49728967f397eb6f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:00:11 -0700 Subject: Adding pointer conversions. Renaming a few variables and type names to avoid C++ compilation issues --- src/core/lib/channel/channel_args.c | 4 +- src/core/lib/channel/channel_stack_builder.c | 16 ++++---- src/core/lib/compression/stream_compression.c | 3 +- src/core/lib/iomgr/combiner.c | 3 +- src/core/lib/iomgr/error.c | 34 ++++++++-------- src/core/lib/iomgr/ev_epoll1_linux.c | 6 +-- src/core/lib/iomgr/ev_epollex_linux.c | 45 +++++++++++---------- src/core/lib/iomgr/polling_entity.h | 4 +- src/core/lib/iomgr/resource_quota.c | 6 +-- src/core/lib/iomgr/socket_factory_posix.c | 4 +- src/core/lib/iomgr/socket_mutator.c | 4 +- src/core/lib/iomgr/timer_manager.c | 2 +- src/core/lib/iomgr/udp_server.c | 2 +- src/core/lib/slice/slice.c | 4 +- src/core/lib/surface/call.c | 58 ++++++++++++++------------- 15 files changed, 101 insertions(+), 94 deletions(-) diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 16d0737b40..9215707f1c 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -212,7 +212,7 @@ void grpc_channel_args_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_args *a) { grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, a->args[i].key)) { @@ -226,7 +226,7 @@ grpc_compression_algorithm grpc_channel_args_get_compression_algorithm( grpc_stream_compression_algorithm grpc_channel_args_get_stream_compression_algorithm(const grpc_channel_args *a) { size_t i; - if (a == NULL) return 0; + if (a == NULL) return GRPC_STREAM_COMPRESS_NONE; for (i = 0; i < a->num_args; ++i) { if (a->args[i].type == GRPC_ARG_INTEGER && !strcmp(GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index df45f13c81..b663ebfb52 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -214,13 +214,13 @@ bool grpc_channel_stack_builder_prepend_filter( static void add_after(filter_node *before, const grpc_channel_filter *filter, grpc_post_filter_create_init_func post_init_func, void *user_data) { - filter_node *new = (filter_node *)gpr_malloc(sizeof(*new)); - new->next = before->next; - new->prev = before; - new->next->prev = new->prev->next = new; - new->filter = filter; - new->init = post_init_func; - new->init_arg = user_data; + filter_node *new_node = (filter_node *)gpr_malloc(sizeof(*new_node)); + new_node->next = before->next; + new_node->prev = before; + new_node->next->prev = new_node->prev->next = new_node; + new_node->filter = filter; + new_node->init = post_init_func; + new_node->init_arg = user_data; } bool grpc_channel_stack_builder_add_filter_before( @@ -268,7 +268,7 @@ grpc_error *grpc_channel_stack_builder_finish( // create an array of filters const grpc_channel_filter **filters = - gpr_malloc(sizeof(*filters) * num_filters); + (const grpc_channel_filter **)gpr_malloc(sizeof(*filters) * num_filters); size_t i = 0; for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) { filters[i++] = p->filter; diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index df13d53e06..ba9302794f 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -159,7 +159,8 @@ bool grpc_stream_decompress(grpc_stream_compression_context *ctx, grpc_stream_compression_context *grpc_stream_compression_context_create( grpc_stream_compression_method method) { grpc_stream_compression_context *ctx = - gpr_zalloc(sizeof(grpc_stream_compression_context)); + (grpc_stream_compression_context *)gpr_zalloc( + sizeof(grpc_stream_compression_context)); int r; if (ctx == NULL) { return NULL; diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 360967f3ba..f899b25f10 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -356,7 +356,8 @@ static void combiner_finally_exec(grpc_exec_ctx *exec_ctx, static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure, grpc_error *error) { - combiner_finally_exec(exec_ctx, closure, GRPC_ERROR_REF(error)); + combiner_finally_exec(exec_ctx, (grpc_closure *)closure, + GRPC_ERROR_REF(error)); } grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner) { diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index dcd175a2e1..d5e00a8f64 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -278,13 +278,13 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, memcpy((*err)->arena + slot, &value, sizeof(value)); } -static void internal_add_error(grpc_error **err, grpc_error *new) { - grpc_linked_error new_last = {new, UINT8_MAX}; +static void internal_add_error(grpc_error **err, grpc_error *new_err) { + grpc_linked_error new_last = {new_err, UINT8_MAX}; uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); if (slot == UINT8_MAX) { - gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new, - grpc_error_string(new)); - GRPC_ERROR_UNREF(new); + gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, + new_err, grpc_error_string(new_err)); + GRPC_ERROR_UNREF(new_err); return; } if ((*err)->first_err == UINT8_MAX) { @@ -321,8 +321,8 @@ grpc_error *grpc_error_create(const char *file, int line, grpc_slice desc, uint8_t initial_arena_capacity = (uint8_t)( DEFAULT_ERROR_CAPACITY + (uint8_t)(num_referencing * SLOTS_PER_LINKED_ERROR) + SURPLUS_CAPACITY); - grpc_error *err = - gpr_malloc(sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); + grpc_error *err = (grpc_error *)gpr_malloc( + sizeof(*err) + initial_arena_capacity * sizeof(intptr_t)); if (err == NULL) { // TODO(ctiller): make gpr_malloc return NULL return GRPC_ERROR_OOM; } @@ -432,10 +432,10 @@ static grpc_error *copy_error_and_unref(grpc_error *in) { grpc_error *grpc_error_set_int(grpc_error *src, grpc_error_ints which, intptr_t value) { GPR_TIMER_BEGIN("grpc_error_set_int", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_int(&new, which, value); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_int(&new_err, which, value); GPR_TIMER_END("grpc_error_set_int", 0); - return new; + return new_err; } typedef struct { @@ -477,10 +477,10 @@ bool grpc_error_get_int(grpc_error *err, grpc_error_ints which, intptr_t *p) { grpc_error *grpc_error_set_str(grpc_error *src, grpc_error_strs which, grpc_slice str) { GPR_TIMER_BEGIN("grpc_error_set_str", 0); - grpc_error *new = copy_error_and_unref(src); - internal_set_str(&new, which, str); + grpc_error *new_err = copy_error_and_unref(src); + internal_set_str(&new_err, which, str); GPR_TIMER_END("grpc_error_set_str", 0); - return new; + return new_err; } bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, @@ -507,10 +507,10 @@ bool grpc_error_get_str(grpc_error *err, grpc_error_strs which, grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) { GPR_TIMER_BEGIN("grpc_error_add_child", 0); - grpc_error *new = copy_error_and_unref(src); - internal_add_error(&new, child); + grpc_error *new_err = copy_error_and_unref(src); + internal_add_error(&new_err, child); GPR_TIMER_END("grpc_error_add_child", 0); - return new; + return new_err; } static const char *no_error_string = "\"No Error\""; @@ -733,7 +733,7 @@ const char *grpc_error_string(grpc_error *err) { void *p = (void *)gpr_atm_acq_load(&err->atomics.error_string); if (p != NULL) { GPR_TIMER_END("grpc_error_string", 0); - return p; + return (const char *)p; } kv_pairs kvs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5bc7e878de..01d28220c7 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; -static const char *kick_state_string(kick_state st) { +static const char *kick_state_string(kick_state_t st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state st) { } struct grpc_pollset_worker { - kick_state kick_state; + kick_state_t kick_state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 277347ac70..df69025f1a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable { +typedef struct pollable_t { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable; +} pollable_t; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return ""; } -static char *pollable_desc(pollable *p) { +static char *pollable_desc(pollable_t *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable *p) { return out; } -static pollable g_empty_pollable; +static pollable_t g_empty_pollable; -static void pollable_init(pollable *p, polling_obj_type type); -static void pollable_destroy(pollable *p); +static void pollable_init(pollable_t *p, polling_obj_type type); +static void pollable_destroy(pollable_t *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p); +static grpc_error *pollable_materialize(pollable_t *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable pollable; + pollable_t pollable; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable *pollable; + pollable_t *pollable; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable pollable; - pollable *current_pollable; + pollable_t pollable; + pollable_t *current_pollable; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable *p, polling_obj_type type) { +static void pollable_init(pollable_t *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable *p) { +static void pollable_destroy(pollable_t *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable *p) { +static grpc_error *pollable_materialize(pollable_t *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -599,7 +599,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,7 +664,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; + pollable_t *p = pollset->current_pollable; if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } @@ -744,7 +744,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { return p != &g_empty_pollable && p != &pollset->pollable; } @@ -762,8 +762,9 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p got pollset_wakeup %p", pollset, data_ptr); } - append_error(&error, grpc_wakeup_fd_consume_wakeup( - (void *)((~(intptr_t)1) & (intptr_t)data_ptr)), + append_error(&error, + grpc_wakeup_fd_consume_wakeup( + (grpc_wakeup_fd *)((~(intptr_t)1) & (intptr_t)data_ptr)), err_desc); } else { grpc_fd *fd = (grpc_fd *)data_ptr; @@ -800,7 +801,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable *p, gpr_timespec now, + pollable_t *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -1028,7 +1029,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable */ + /* empty pollable --> single fd pollable_t */ pollset_kick_all(exec_ctx, pollset); pollset->current_pollable = &fd->pollable; if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 971fd88b42..940f10f152 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,6 +22,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" +typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; + /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an * abstract interface. No ownership is taken. */ @@ -31,7 +33,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } tag; + pops_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 6c58986b53..4895e0d1c9 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -656,7 +656,7 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { if (channel_args->args[i].type == GRPC_ARG_POINTER) { return grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); + (grpc_resource_quota *)channel_args->args[i].value.pointer.p); } else { gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer"); } @@ -666,12 +666,12 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( } static void *rq_copy(void *rq) { - grpc_resource_quota_ref(rq); + grpc_resource_quota_ref((grpc_resource_quota *)rq); return rq; } static void rq_destroy(grpc_exec_ctx *exec_ctx, void *rq) { - grpc_resource_quota_unref_internal(exec_ctx, rq); + grpc_resource_quota_unref_internal(exec_ctx, (grpc_resource_quota *)rq); } static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); } diff --git a/src/core/lib/iomgr/socket_factory_posix.c b/src/core/lib/iomgr/socket_factory_posix.c index 0f82dea570..c81566575e 100644 --- a/src/core/lib/iomgr/socket_factory_posix.c +++ b/src/core/lib/iomgr/socket_factory_posix.c @@ -69,11 +69,11 @@ void grpc_socket_factory_unref(grpc_socket_factory *factory) { } static void *socket_factory_arg_copy(void *p) { - return grpc_socket_factory_ref(p); + return grpc_socket_factory_ref((grpc_socket_factory *)p); } static void socket_factory_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_factory_unref(p); + grpc_socket_factory_unref((grpc_socket_factory *)p); } static int socket_factory_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c index 5d6c2c400e..300ac75b38 100644 --- a/src/core/lib/iomgr/socket_mutator.c +++ b/src/core/lib/iomgr/socket_mutator.c @@ -60,11 +60,11 @@ void grpc_socket_mutator_unref(grpc_socket_mutator *mutator) { } static void *socket_mutator_arg_copy(void *p) { - return grpc_socket_mutator_ref(p); + return grpc_socket_mutator_ref((grpc_socket_mutator *)p); } static void socket_mutator_arg_destroy(grpc_exec_ctx *exec_ctx, void *p) { - grpc_socket_mutator_unref(p); + grpc_socket_mutator_unref((grpc_socket_mutator *)p); } static int socket_mutator_cmp(void *a, void *b) { diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index ae2c0bf0ae..04ca44563d 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -276,7 +276,7 @@ static void timer_thread(void *completed_thread_ptr) { GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); timer_main_loop(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); - timer_thread_cleanup(completed_thread_ptr); + timer_thread_cleanup((completed_thread *)completed_thread_ptr); } static void start_threads(void) { diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 9a02c1d1bb..00b2e68bb5 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -118,7 +118,7 @@ static grpc_socket_factory *get_socket_factory(const grpc_channel_args *args) { const grpc_arg *arg = grpc_channel_args_find(args, GRPC_ARG_SOCKET_FACTORY); if (arg) { GPR_ASSERT(arg->type == GRPC_ARG_POINTER); - return arg->value.pointer.p; + return (grpc_socket_factory *)arg->value.pointer.p; } } return NULL; diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 321a21a10b..0764eda052 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -174,8 +174,8 @@ static const grpc_slice_refcount_vtable new_with_len_vtable = { grpc_slice grpc_slice_new_with_len(void *p, size_t len, void (*destroy)(void *, size_t)) { grpc_slice slice; - new_with_len_slice_refcount *rc = - gpr_malloc(sizeof(new_with_len_slice_refcount)); + new_with_len_slice_refcount *rc = (new_with_len_slice_refcount *)gpr_malloc( + sizeof(new_with_len_slice_refcount)); gpr_ref_init(&rc->refs, 1); rc->rc.vtable = &new_with_len_vtable; rc->rc.sub_refcount = &rc->rc; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 93c512df69..f4e11025b1 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call; +} parent_call_t; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call; +} child_call_t; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call* */ gpr_atm parent_call_atm; - child_call *child_call; + /* parent_call_t* */ gpr_atm parent_call_atm; + child_call_t *child_call; /* client or server call */ bool is_client; @@ -293,32 +293,32 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl); static void add_batch_error(grpc_exec_ctx *exec_ctx, batch_control *bctl, grpc_error *error, bool has_cancelled); -static void add_init_error(grpc_error **composite, grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; +static void add_init_error(grpc_error **composite, grpc_error *new_err) { + if (new_err == GRPC_ERROR_NONE) return; if (*composite == GRPC_ERROR_NONE) *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed"); - *composite = grpc_error_add_child(*composite, new); + *composite = grpc_error_add_child(*composite, new_err); } void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call *get_or_create_parent_call(grpc_call *call) { - parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_or_create_parent_call(grpc_call *call) { + parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call *get_parent_call(grpc_call *call) { - return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call_t *get_parent_call(grpc_call *call) { + return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -378,15 +378,15 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; if (args->parent_call != NULL) { - child_call *cc = call->child_call = - gpr_arena_alloc(arena, sizeof(child_call)); + child_call_t *cc = call->child_call = + (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); call->child_call->parent = args->parent_call; GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - parent_call *pc = get_or_create_parent_call(args->parent_call); + parent_call_t *pc = get_or_create_parent_call(args->parent_call); gpr_mu_lock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call *pc = get_parent_call(c); + parent_call_t *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status(call, set_status_value_directly, &c->final_info.final_status, - NULL); + get_final_status((grpc_call *)call, set_status_value_directly, + &c->final_info.final_status, NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call *cc = c->child_call; + child_call_t *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call *pc = get_parent_call(cc->parent); + parent_call_t *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -1309,7 +1309,7 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call *pc = get_parent_call(call); + parent_call_t *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); @@ -1345,7 +1345,8 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, if (bctl->completion_data.notify_tag.is_closure) { /* unrefs bctl->error */ bctl->call = NULL; - GRPC_CLOSURE_RUN(exec_ctx, bctl->completion_data.notify_tag.tag, error); + GRPC_CLOSURE_RUN( + exec_ctx, (grpc_closure *)bctl->completion_data.notify_tag.tag, error); GRPC_CALL_INTERNAL_UNREF(exec_ctx, call, "completion"); } else { /* unrefs bctl->error */ @@ -1474,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, bctlp); + process_data_after_md(exec_ctx, (batch_control *)bctlp); } } @@ -1679,11 +1680,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, if (nops == 0) { if (!is_notify_tag_closure) { GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag)); - grpc_cq_end_op(exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, - free_no_op_completion, NULL, - gpr_malloc(sizeof(grpc_cq_completion))); + grpc_cq_end_op( + exec_ctx, call->cq, notify_tag, GRPC_ERROR_NONE, + free_no_op_completion, NULL, + (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion))); } else { - GRPC_CLOSURE_SCHED(exec_ctx, notify_tag, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(exec_ctx, (grpc_closure *)notify_tag, GRPC_ERROR_NONE); } error = GRPC_CALL_OK; goto done; -- cgit v1.2.3 From 090aca55bda549a1c80bae8f4770de330b88b4bd Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 12:14:13 -0700 Subject: Using already type casted variables --- src/core/lib/surface/call.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index f4e11025b1..de5aa7b6be 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -549,8 +549,8 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } - get_final_status((grpc_call *)call, set_status_value_directly, - &c->final_info.final_status, NULL); + get_final_status(c, set_status_value_directly, &c->final_info.final_status, + NULL); c->final_info.stats.latency = gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time); @@ -1475,7 +1475,7 @@ static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp, * acq_load is in receiving_initial_metadata_ready() */ if (error != GRPC_ERROR_NONE || call->receiving_stream == NULL || !gpr_atm_rel_cas(&call->recv_state, RECV_NONE, (gpr_atm)bctlp)) { - process_data_after_md(exec_ctx, (batch_control *)bctlp); + process_data_after_md(exec_ctx, bctl); } } -- cgit v1.2.3 From 97dab51a3f356a1cb324c9d7e01ed12ba84e5ed4 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 14:27:46 -0700 Subject: Renaming pops_tag enum --- src/core/lib/iomgr/polling_entity.c | 18 +++++++++--------- src/core/lib/iomgr/polling_entity.h | 8 ++++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/lib/iomgr/polling_entity.c b/src/core/lib/iomgr/polling_entity.c index 74d8794af5..8591a5518e 100644 --- a/src/core/lib/iomgr/polling_entity.c +++ b/src/core/lib/iomgr/polling_entity.c @@ -25,7 +25,7 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset_set( grpc_pollset_set *pollset_set) { grpc_polling_entity pollent; pollent.pollent.pollset_set = pollset_set; - pollent.tag = POPS_POLLSET_SET; + pollent.tag = GRPC_POLLS_POLLSET_SET; return pollent; } @@ -33,12 +33,12 @@ grpc_polling_entity grpc_polling_entity_create_from_pollset( grpc_pollset *pollset) { grpc_polling_entity pollent; pollent.pollent.pollset = pollset; - pollent.tag = POPS_POLLSET; + pollent.tag = GRPC_POLLS_POLLSET; return pollent; } grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { return pollent->pollent.pollset; } return NULL; @@ -46,23 +46,23 @@ grpc_pollset *grpc_polling_entity_pollset(grpc_polling_entity *pollent) { grpc_pollset_set *grpc_polling_entity_pollset_set( grpc_polling_entity *pollent) { - if (pollent->tag == POPS_POLLSET_SET) { + if (pollent->tag == GRPC_POLLS_POLLSET_SET) { return pollent->pollent.pollset_set; } return NULL; } bool grpc_polling_entity_is_empty(const grpc_polling_entity *pollent) { - return pollent->tag == POPS_NONE; + return pollent->tag == GRPC_POLLS_NONE; } void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_add_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_add_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); @@ -75,10 +75,10 @@ void grpc_polling_entity_add_to_pollset_set(grpc_exec_ctx *exec_ctx, void grpc_polling_entity_del_from_pollset_set(grpc_exec_ctx *exec_ctx, grpc_polling_entity *pollent, grpc_pollset_set *pss_dst) { - if (pollent->tag == POPS_POLLSET) { + if (pollent->tag == GRPC_POLLS_POLLSET) { GPR_ASSERT(pollent->pollent.pollset != NULL); grpc_pollset_set_del_pollset(exec_ctx, pss_dst, pollent->pollent.pollset); - } else if (pollent->tag == POPS_POLLSET_SET) { + } else if (pollent->tag == GRPC_POLLS_POLLSET_SET) { GPR_ASSERT(pollent->pollent.pollset_set != NULL); grpc_pollset_set_del_pollset_set(exec_ctx, pss_dst, pollent->pollent.pollset_set); diff --git a/src/core/lib/iomgr/polling_entity.h b/src/core/lib/iomgr/polling_entity.h index 940f10f152..a161e1fea6 100644 --- a/src/core/lib/iomgr/polling_entity.h +++ b/src/core/lib/iomgr/polling_entity.h @@ -22,7 +22,11 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" -typedef enum pops_tag { POPS_NONE, POPS_POLLSET, POPS_POLLSET_SET } pops_tag; +typedef enum grpc_pollset_tag { + GRPC_POLLS_NONE, + GRPC_POLLS_POLLSET, + GRPC_POLLS_POLLSET_SET +} grpc_pollset_tag; /* A grpc_polling_entity is a pollset-or-pollset_set container. It allows * functions that accept a pollset XOR a pollset_set to do so through an @@ -33,7 +37,7 @@ typedef struct grpc_polling_entity { grpc_pollset *pollset; grpc_pollset_set *pollset_set; } pollent; - pops_tag tag; + grpc_pollset_tag tag; } grpc_polling_entity; grpc_polling_entity grpc_polling_entity_create_from_pollset_set( -- cgit v1.2.3 From 72b36736ceaa7a267722cbcfe99a94bb879e3204 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 13 Sep 2017 00:05:29 +0200 Subject: The input files also need to follow the base argument... --- tools/buildgen/generate_projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 559ea1624b..d29cd02152 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -85,7 +85,7 @@ for template in reversed(sorted(templates)): test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) - cmd.append(root + '/' + f) + cmd.append(args.base + '/' + root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) jobset.run(pre_jobs, maxjobs=args.jobs) -- cgit v1.2.3 From 49119a4caf48c20ed3655e0e5637129c9ed94481 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 8 Sep 2017 15:29:19 -0700 Subject: Fix for max_concurrent_streams issue - Call mark_stream_closed before sending trailing metadata --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 8 +++++++- src/core/ext/transport/chttp2/transport/frame_rst_stream.c | 3 +++ src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 + src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3fd701fe2f..5fb943d3ca 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,6 +1974,10 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, + GRPC_ERROR_REF(due_to_error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2103,7 +2107,6 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2288,6 +2291,9 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 0133b6efa2..d77097811a 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,6 +103,9 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } + if (!s->write_closed) { + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 82ff2c8e2c..e4f2ae8475 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,6 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index fa224a49a4..7512fe122d 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,6 +386,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, @@ -444,6 +446,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( @@ -519,10 +523,6 @@ void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_REF(error)); s->sending_bytes = 0; } - if (s->sent_trailing_metadata) { - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_REF(error)); - } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end"); } grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &t->outbuf); -- cgit v1.2.3 From 6988dbb45696f385cb3945e3c10b5ba67872ec35 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:34:09 -0700 Subject: There is no need to remove fail_pending_writes from mark_stream_closed anymore --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 8 +------- src/core/ext/transport/chttp2/transport/frame_rst_stream.c | 3 --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 2 +- src/core/ext/transport/chttp2/transport/writing.c | 8 ++++---- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 5fb943d3ca..3fd701fe2f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1974,10 +1974,6 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { s->seen_error = true; } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, - GRPC_ERROR_REF(due_to_error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } @@ -2107,6 +2103,7 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, if (close_writes && !s->write_closed) { s->write_closed_error = GRPC_ERROR_REF(error); s->write_closed = true; + grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); } if (s->read_closed && s->write_closed) { became_closed = true; @@ -2291,9 +2288,6 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); } diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index d77097811a..0133b6efa2 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -103,9 +103,6 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); gpr_free(message); } - if (!s->write_closed) { - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); - } grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index e4f2ae8475..35d52963e2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,7 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); + // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 7512fe122d..3fa38db4db 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -386,13 +386,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); } result.early_results_scheduled |= update_list(exec_ctx, t, s, @@ -446,13 +446,13 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( } s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; - grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, - GRPC_ERROR_NONE); if (!t->is_client && !s->read_closed) { grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_rst_stream_create( s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_NONE); now_writing = true; result.early_results_scheduled = true; grpc_chttp2_complete_closure_step( -- cgit v1.2.3 From a6ede3bea1cde14ffd5fc7f9b80a27dcdee345d9 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 11 Sep 2017 15:37:27 -0700 Subject: Remove unnecessary comment --- src/core/ext/transport/chttp2/transport/hpack_parser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 35d52963e2..82ff2c8e2c 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1650,7 +1650,6 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); - // grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_NONE); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); -- cgit v1.2.3 From 2be06b43e7532ef2e9d1e5709ceb162957e6a926 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 12 Sep 2017 15:24:49 -0700 Subject: Make unmeasured CPU costs be 1, not 0 Allows capturing 0 as a CPU cost, which we'll be able to leverage to increase concurrency (once data has caught up in a few days) --- tools/run_tests/python_utils/jobset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 08d652ae3f..80e79444ea 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -224,7 +224,7 @@ class JobResult(object): self.retries = 0 self.message = '' self.cpu_estimated = 1 - self.cpu_measured = 0 + self.cpu_measured = 1 def read_from_start(f): -- cgit v1.2.3 From b2f490c565e87822245c486ce5b04b17adacd52d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 16:25:39 -0700 Subject: Include limits.h since we're using INT_MAX (affects some compilers) --- src/core/ext/transport/chttp2/transport/flow_control.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index 39aa521029..0f078e79e9 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -18,6 +18,7 @@ #include "src/core/ext/transport/chttp2/transport/internal.h" +#include #include #include -- cgit v1.2.3 From 707145c78a7e1d65ddf14836adbbb7032172a3a0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 10:02:58 +0200 Subject: mingw already installed on win kokoro workers --- tools/internal_ci/windows/grpc_build_artifacts.bat | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 17d9571d43..c6e523697c 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -19,8 +19,6 @@ rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits -pacman -S --noconfirm mingw64/mingw-w64-x86_64-gcc mingw32/mingw-w64-i686-gcc - @rem enter repo root cd /d %~dp0\..\..\.. -- cgit v1.2.3 From e97bc51d3966aaa6d8bbcd6ec9238063bba8dad2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 13 Sep 2017 12:26:32 +0200 Subject: already installed on mac workers --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 13 +------------ tools/internal_ci/macos/grpc_build_artifacts.sh | 12 ------------ 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index f7fbec93ff..dd2741e595 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -31,23 +31,16 @@ ulimit -a pip install google-api-python-client --user python export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json -# required to build protobuf -brew install gflags - set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -rvm install ruby-2.4 +rvm use ruby-2.4 rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex -gem install bundler - # cocoapods export LANG=en_US.UTF-8 -gem install cocoapods -gem install xcpretty pod repo update # needed by python # python @@ -56,10 +49,6 @@ pip install virtualenv --user python pip install -U six tox setuptools --user python export PYTHONPATH=/Library/Python/3.4/site-packages -# python 3.4 -wget -q https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg -sudo installer -pkg python-3.4.4-macosx10.6.pkg -target / - # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_8.2.1.app/Contents/Developer diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 603c15f210..09784e3bb4 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -20,25 +20,13 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_macos_rc -# python 3.5 -wget -q https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg -sudo installer -pkg python-3.5.2-macosx10.6.pkg -target / - # install cython for all python versions python2.7 -m pip install cython setuptools wheel python3.4 -m pip install cython setuptools wheel python3.5 -m pip install cython setuptools wheel python3.6 -m pip install cython setuptools wheel -# node-gyp (needed for node artifacts) -npm install -g node-gyp - -# php dependencies: pecl -curl -O http://pear.php.net/go-pear.phar -sudo php -d detect_unicode=0 go-pear.phar - # needed to build ruby artifacts -gem install rake-compiler wget https://raw.githubusercontent.com/grpc/grpc/master/tools/distrib/build_ruby_environment_macos.sh bash build_ruby_environment_macos.sh -- cgit v1.2.3 From e6d2e8f16bd2ebbfd112b09e589205191211a2c8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 06:10:18 -0700 Subject: Fix case on internal type names Should allow microbenchmarks to get running again --- tools/profiling/microbenchmarks/bm2bq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 6c5040de56..9f9b672f75 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -30,7 +30,7 @@ columns = [] for row in json.loads( subprocess.check_output([ 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: - columns.append((row['name'], row['type'])) + columns.append((row['name'], row['type'].lower())) SANITIZE = { 'integer': int, -- cgit v1.2.3 From 70bbb6b1176626346efc7c7549fb027896f8d8c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:04:30 -0700 Subject: Fix schema generation --- src/core/lib/debug/stats_data_bq_schema.sql | 70 ++++++++++++++--------------- tools/codegen/core/gen_stats_data.py | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 7291bbf070..5354769291 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,35 +1,35 @@ -client_calls_created_per_iteration:INTEGER, -server_calls_created_per_iteration:INTEGER, -syscall_poll_per_iteration:INTEGER, -syscall_wait_per_iteration:INTEGER, -histogram_slow_lookups_per_iteration:INTEGER, -syscall_write_per_iteration:INTEGER, -syscall_read_per_iteration:INTEGER, -tcp_backup_pollers_created_per_iteration:INTEGER, -tcp_backup_poller_polls_per_iteration:INTEGER, -http2_op_batches_per_iteration:INTEGER, -http2_op_cancel_per_iteration:INTEGER, -http2_op_send_initial_metadata_per_iteration:INTEGER, -http2_op_send_message_per_iteration:INTEGER, -http2_op_send_trailing_metadata_per_iteration:INTEGER, -http2_op_recv_initial_metadata_per_iteration:INTEGER, -http2_op_recv_message_per_iteration:INTEGER, -http2_op_recv_trailing_metadata_per_iteration:INTEGER, -http2_settings_writes_per_iteration:INTEGER, -http2_pings_sent_per_iteration:INTEGER, -http2_writes_begun_per_iteration:INTEGER, -http2_writes_offloaded_per_iteration:INTEGER, -http2_writes_continued_per_iteration:INTEGER, -http2_partial_writes_per_iteration:INTEGER, -combiner_locks_initiated_per_iteration:INTEGER, -combiner_locks_scheduled_items_per_iteration:INTEGER, -combiner_locks_scheduled_final_items_per_iteration:INTEGER, -combiner_locks_offloaded_per_iteration:INTEGER, -executor_scheduled_short_items_per_iteration:INTEGER, -executor_scheduled_long_items_per_iteration:INTEGER, -executor_scheduled_to_self_per_iteration:INTEGER, -executor_wakeup_initiated_per_iteration:INTEGER, -executor_queue_drained_per_iteration:INTEGER, -executor_push_retries_per_iteration:INTEGER, -server_requested_calls_per_iteration:INTEGER, -server_slowpath_requests_queued_per_iteration:INTEGER +client_calls_created_per_iteration:FLOAT, +server_calls_created_per_iteration:FLOAT, +syscall_poll_per_iteration:FLOAT, +syscall_wait_per_iteration:FLOAT, +histogram_slow_lookups_per_iteration:FLOAT, +syscall_write_per_iteration:FLOAT, +syscall_read_per_iteration:FLOAT, +tcp_backup_pollers_created_per_iteration:FLOAT, +tcp_backup_poller_polls_per_iteration:FLOAT, +http2_op_batches_per_iteration:FLOAT, +http2_op_cancel_per_iteration:FLOAT, +http2_op_send_initial_metadata_per_iteration:FLOAT, +http2_op_send_message_per_iteration:FLOAT, +http2_op_send_trailing_metadata_per_iteration:FLOAT, +http2_op_recv_initial_metadata_per_iteration:FLOAT, +http2_op_recv_message_per_iteration:FLOAT, +http2_op_recv_trailing_metadata_per_iteration:FLOAT, +http2_settings_writes_per_iteration:FLOAT, +http2_pings_sent_per_iteration:FLOAT, +http2_writes_begun_per_iteration:FLOAT, +http2_writes_offloaded_per_iteration:FLOAT, +http2_writes_continued_per_iteration:FLOAT, +http2_partial_writes_per_iteration:FLOAT, +combiner_locks_initiated_per_iteration:FLOAT, +combiner_locks_scheduled_items_per_iteration:FLOAT, +combiner_locks_scheduled_final_items_per_iteration:FLOAT, +combiner_locks_offloaded_per_iteration:FLOAT, +executor_scheduled_short_items_per_iteration:FLOAT, +executor_scheduled_long_items_per_iteration:FLOAT, +executor_scheduled_to_self_per_iteration:FLOAT, +executor_wakeup_initiated_per_iteration:FLOAT, +executor_queue_drained_per_iteration:FLOAT, +executor_push_retries_per_iteration:FLOAT, +server_requested_calls_per_iteration:FLOAT, +server_slowpath_requests_queued_per_iteration:FLOAT diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 877a5d9d43..8359734c84 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -405,6 +405,6 @@ with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: columns = [] for counter in inst_map['Counter']: - columns.append(('%s_per_iteration' % counter.name, 'INTEGER')) + columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) -- cgit v1.2.3 From f2cbd6c81eab267fb8c04121ff76ffbbe689dfa1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:31:16 -0700 Subject: Fix qps_test histogram python code --- tools/run_tests/performance/massage_qps_stats_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py index 400a0c82bf..a2fe4ae6c3 100644 --- a/tools/run_tests/performance/massage_qps_stats_helpers.py +++ b/tools/run_tests/performance/massage_qps_stats_helpers.py @@ -23,7 +23,7 @@ def _threshold_for_count_below(buckets, boundaries, count_below): if count_so_far == count_below: # this bucket hits the threshold exactly... we should be midway through # any run of zero values following the bucket - for upper_idx in range(lower_idx + 1, num_buckets): + for upper_idx in range(lower_idx + 1, len(buckets)): if buckets[upper_idx] != 0: break return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0 -- cgit v1.2.3 From 480f5d861264f00ee2226b2aa000bbeb268f0a29 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:36:07 -0700 Subject: Add missing increments --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 322fa160d0..8a984e24eb 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -512,14 +512,17 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICK(exec_ctx); switch (worker->kick_state) { case KICKED: + GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { gpr_cv_signal(&worker->cv); } break; case DESIGNATED_POLLER: + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD(exec_ctx); SET_KICK_STATE(worker, KICKED); append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_kick_all"); -- cgit v1.2.3 From f7617bb8f6bd4b6993d6cf5bf70a14a5819f5fd6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 09:47:28 -0700 Subject: Really use measured CPU costs (with a small delta) for run_tests --- tools/run_tests/run_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b66c5f7f71..591a0bebbd 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -80,7 +80,7 @@ def get_bqtest_data(limit=None): SELECT filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, - MAX(cpu_measured) as cpu + MAX(cpu_measured) + 0.01 as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, @@ -92,9 +92,7 @@ SELECT AND platform = '"""+platform_string()+"""' AND NOT REGEXP_MATCH(job_name, '.*portability.*') ) GROUP BY - filtered_test_name -HAVING - flaky OR cpu > 0""" + filtered_test_name""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) -- cgit v1.2.3 From 5160155005c7d0f1ee9b855dfbe6aa45d8602bf1 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 12 Sep 2017 09:31:07 -0700 Subject: Upload Kokoro MacOS master results --- tools/internal_ci/macos/grpc_basictests_dbg.cfg | 2 +- tools/internal_ci/macos/grpc_basictests_opt.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index f058f0c7e4..53bda1ff0a 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index 5048baaf48..d359eb601a 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -27,5 +27,5 @@ action { env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4" + value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } -- cgit v1.2.3 From b0ed7374b3a1a653612f0f5b5548964c56a93f35 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 16:24:42 -0700 Subject: Enable filter if either AGE or IDLE is specified; don't require both. --- src/core/ext/filters/max_age/max_age_filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c index 450f67746f..0ac803ed41 100644 --- a/src/core/ext/filters/max_age/max_age_filter.c +++ b/src/core/ext/filters/max_age/max_age_filter.c @@ -402,7 +402,7 @@ static bool maybe_add_max_age_filter(grpc_exec_ctx* exec_ctx, bool enable = grpc_channel_arg_get_integer( grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_AGE_MS), - MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX && + MAX_CONNECTION_AGE_INTEGER_OPTIONS) != INT_MAX || grpc_channel_arg_get_integer( grpc_channel_args_find(channel_args, GRPC_ARG_MAX_CONNECTION_IDLE_MS), MAX_CONNECTION_IDLE_INTEGER_OPTIONS) != INT_MAX; -- cgit v1.2.3 From 4a7fca5b655f5e9114c81f48b3615847c5cb783d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 12 Sep 2017 14:31:11 -0700 Subject: Give guidance on prefixing struct/enum/#define names --- doc/c-style-guide.md | 14 +++++++++++--- src/core/lib/iomgr/ev_poll_posix.c | 6 +++--- src/core/lib/iomgr/wakeup_fd_cv.c | 14 +++++++------- src/core/lib/iomgr/wakeup_fd_cv.h | 4 ++-- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/doc/c-style-guide.md b/doc/c-style-guide.md index 369bd56a46..2cfa41dd9e 100644 --- a/doc/c-style-guide.md +++ b/doc/c-style-guide.md @@ -32,14 +32,14 @@ Header Files # endif ``` - Header files should be self-contained and end in .h. -- All header files should have a #define guard to prevent multiple inclusion. +- All header files should have a `#define` guard to prevent multiple inclusion. To guarantee uniqueness they should be based on the file's path. For public headers: `include/grpc/grpc.h` → `GRPC_GRPC_H` For private headers: - `src/core/channel/channel_stack.h` → - `GRPC_INTERNAL_CORE_CHANNEL_CHANNEL_STACK_H` + `src/core/lib/channel/channel_stack.h` → + `GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H` Variable Initialization ----------------------- @@ -72,8 +72,16 @@ Symbol Names - Non-static functions must be prefixed by `grpc_` - Static functions must *not* be prefixed by `grpc_` +- Typenames of `struct`s , `union`s, and `enum`s must be prefixed by `grpc_` if + they are declared in a header file. They must not be prefixed by `grpc_` if + they are declared in a source file. - Enumeration values and `#define` names must be uppercase. All other values must be lowercase. +- Enumeration values or `#define` names defined in a header file must be + prefixed with `GRPC_` (except for `#define` macros that are being used to + substitute functions; those should follow the general rules for + functions). Enumeration values or `#define`s defined in source files must not + be prefixed with `GRPC_`. - Multiple word identifiers use underscore as a delimiter, *never* camel case. E.g. `variable_name`. diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index bcf1d9001b..7f44eda138 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1539,7 +1539,7 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { for (i = 0; i < nfds; i++) { fds[i].revents = 0; if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { - idx = FD_TO_IDX(fds[i].fd); + idx = GRPC_FD_TO_IDX(fds[i].fd); fd_cvs[i].cv = &pollcv_cv; fd_cvs[i].prev = NULL; fd_cvs[i].next = g_cvfds.cvfds[idx].cvs; @@ -1602,8 +1602,8 @@ static int cvfd_poll(struct pollfd *fds, nfds_t nfds, int timeout) { idx = 0; for (i = 0; i < nfds; i++) { if (fds[i].fd < 0 && (fds[i].events & POLLIN)) { - remove_cvn(&g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i])); - if (g_cvfds.cvfds[FD_TO_IDX(fds[i].fd)].is_set) { + remove_cvn(&g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].cvs, &(fd_cvs[i])); + if (g_cvfds.cvfds[GRPC_FD_TO_IDX(fds[i].fd)].is_set) { fds[i].revents = POLLIN; if (res >= 0) res++; } diff --git a/src/core/lib/iomgr/wakeup_fd_cv.c b/src/core/lib/iomgr/wakeup_fd_cv.c index 5e0b1d1704..268e0175dd 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.c +++ b/src/core/lib/iomgr/wakeup_fd_cv.c @@ -57,7 +57,7 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { g_cvfds.free_fds = g_cvfds.free_fds->next_free; g_cvfds.cvfds[idx].cvs = NULL; g_cvfds.cvfds[idx].is_set = 0; - fd_info->read_fd = IDX_TO_FD(idx); + fd_info->read_fd = GRPC_IDX_TO_FD(idx); fd_info->write_fd = -1; gpr_mu_unlock(&g_cvfds.mu); return GRPC_ERROR_NONE; @@ -66,8 +66,8 @@ static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { cv_node* cvn; gpr_mu_lock(&g_cvfds.mu); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 1; - cvn = g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs; + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 1; + cvn = g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs; while (cvn) { gpr_cv_signal(cvn->cv); cvn = cvn->next; @@ -78,7 +78,7 @@ static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { static grpc_error* cv_fd_consume(grpc_wakeup_fd* fd_info) { gpr_mu_lock(&g_cvfds.mu); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 0; + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].is_set = 0; gpr_mu_unlock(&g_cvfds.mu); return GRPC_ERROR_NONE; } @@ -89,9 +89,9 @@ static void cv_fd_destroy(grpc_wakeup_fd* fd_info) { } gpr_mu_lock(&g_cvfds.mu); // Assert that there are no active pollers - GPR_ASSERT(!g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs); - g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds; - g_cvfds.free_fds = &g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)]; + GPR_ASSERT(!g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].cvs); + g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds; + g_cvfds.free_fds = &g_cvfds.cvfds[GRPC_FD_TO_IDX(fd_info->read_fd)]; gpr_mu_unlock(&g_cvfds.mu); } diff --git a/src/core/lib/iomgr/wakeup_fd_cv.h b/src/core/lib/iomgr/wakeup_fd_cv.h index 46e84f5843..dc170ad5b4 100644 --- a/src/core/lib/iomgr/wakeup_fd_cv.h +++ b/src/core/lib/iomgr/wakeup_fd_cv.h @@ -37,8 +37,8 @@ #include "src/core/lib/iomgr/ev_posix.h" -#define FD_TO_IDX(fd) (-(fd)-1) -#define IDX_TO_FD(idx) (-(idx)-1) +#define GRPC_FD_TO_IDX(fd) (-(fd)-1) +#define GRPC_IDX_TO_FD(idx) (-(idx)-1) typedef struct cv_node { gpr_cv* cv; -- cgit v1.2.3 From ebacb2f20d5b9fbf73d1f90d06eee7553c93aee6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:32:33 -0700 Subject: Fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index bf080fff0c..f7f6cec9ef 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -515,9 +515,9 @@ static grpc_error *pollset_kick_all(grpc_exec_ctx *exec_ctx, GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); break; case UNKICKED: - GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); SET_KICK_STATE(worker, KICKED); if (worker->initialized_cv) { + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->cv); } break; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index f1a3b97f4d..4e5bed5016 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -666,12 +666,8 @@ static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { -<<<<<<< HEAD - pollable *p = pollset->current_pollable; - GRPC_STATS_INC_POLLSET_KICK(exec_ctx); -======= pollable_t *p = pollset->current_pollable; ->>>>>>> d1fefe89dacf198a721053f2b1a3e173f1f24948 + GRPC_STATS_INC_POLLSET_KICK(exec_ctx); if (p != &pollset->pollable) { gpr_mu_lock(&p->po.mu); } -- cgit v1.2.3 From cf34fa58f476aae77c4e2dca0ff866542dd3af28 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 12:37:01 -0700 Subject: Document only epoll1 for these stats, add missing spot --- src/core/lib/debug/stats_data.c | 14 ++++++++------ src/core/lib/debug/stats_data.yaml | 6 ++++++ src/core/lib/iomgr/ev_epoll1_linux.c | 8 +++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index fc1ac223de..1a020a15c3 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -70,16 +70,18 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of server side calls created by this process", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", - "How many polling wakeups were performed by the process", - "How many times was a polling wakeup requested without an active poller", + "How many polling wakeups were performed by the process (only valid for " + "epoll1 right now)", + "How many times was a polling wakeup requested without an active poller " + "(only valid for epoll1 right now)", "How many times was the same polling worker awoken repeatedly before " - "waking up", + "waking up (only valid for epoll1 right now)", "How many times was an eventfd used as the wakeup vector for a polling " - "wakeup", + "wakeup (only valid for epoll1 right now)", "How many times was a condition variable used as the wakeup vector for a " - "polling wakeup", + "polling wakeup (only valid for epoll1 right now)", "How many times could a polling wakeup be satisfied by keeping the waking " - "thread awake?", + "thread awake? (only valid for epoll1 right now)", "Number of times histogram increments went through the slow (binary " "search) path", "Number of write syscalls (or equivalent - eg sendmsg) made by this " diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 26c8e1b11d..b43c466d4f 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -35,20 +35,26 @@ doc: How many events are called for each syscall_poll - counter: pollset_kick doc: How many polling wakeups were performed by the process + (only valid for epoll1 right now) - counter: pollset_kicked_without_poller doc: How many times was a polling wakeup requested without an active poller + (only valid for epoll1 right now) - counter: pollset_kicked_again doc: How many times was the same polling worker awoken repeatedly before waking up + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_fd doc: How many times was an eventfd used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_wakeup_cv doc: How many times was a condition variable used as the wakeup vector for a polling wakeup + (only valid for epoll1 right now) - counter: pollset_kick_own_thread doc: How many times could a polling wakeup be satisfied by keeping the waking thread awake? + (only valid for epoll1 right now) # stats system - counter: histogram_slow_lookups doc: Number of times histogram increments went through the slow diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index f7f6cec9ef..8f66158369 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -789,7 +789,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood) { + grpc_exec_ctx *exec_ctx, pollset_neighbourhood *neighbourhood) { GPR_TIMER_BEGIN("check_neighbourhood_for_available_poller", 0); bool found_worker = false; do { @@ -813,6 +813,7 @@ static bool check_neighbourhood_for_available_poller( SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER); if (inspect_worker->initialized_cv) { GPR_TIMER_MARK("signal worker", 0); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&inspect_worker->cv); } } else { @@ -891,7 +892,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; if (gpr_mu_trylock(&neighbourhood->mu)) { found_worker = - check_neighbourhood_for_available_poller(neighbourhood); + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); scan_state[i] = true; } else { @@ -904,7 +905,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, &g_neighbourhoods[(poller_neighbourhood_idx + i) % g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); - found_worker = check_neighbourhood_for_available_poller(neighbourhood); + found_worker = + check_neighbourhood_for_available_poller(exec_ctx, neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); -- cgit v1.2.3 From bc130daf5f78bac986cc5cdede8476896dc5f463 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 12 Sep 2017 22:44:08 -0700 Subject: Adding more pointer conversions and type renaming --- .../filters/client_channel/channel_connectivity.c | 10 +- .../ext/filters/client_channel/client_channel.c | 5 +- .../client_channel/client_channel_factory.c | 5 +- .../grpclb/client_load_reporting_filter.c | 3 +- .../client_channel/lb_policy/grpclb/grpclb.c | 67 ++++++++++---- .../lb_policy/grpclb/load_balancer_api.c | 6 +- .../lb_policy/pick_first/pick_first.c | 6 +- .../ext/filters/client_channel/lb_policy_factory.c | 9 +- .../dns/c_ares/grpc_ares_ev_driver_posix.c | 37 ++++---- .../resolver/dns/c_ares/grpc_ares_wrapper.c | 7 +- .../ext/filters/client_channel/retry_throttle.c | 9 +- .../ext/filters/client_channel/subchannel_index.c | 12 ++- .../ext/filters/http/server/http_server_filter.c | 6 +- .../ext/transport/chttp2/client/chttp2_connector.c | 2 +- .../ext/transport/chttp2/server/chttp2_server.c | 18 ++-- .../transport/chttp2/transport/chttp2_transport.c | 3 +- .../ext/transport/chttp2/transport/flow_control.c | 16 ++-- .../transport/chttp2/transport/frame_settings.c | 17 ++-- .../transport/chttp2/transport/incoming_metadata.c | 5 +- .../ext/transport/chttp2/transport/stream_map.c | 6 +- src/core/ext/transport/inproc/inproc_transport.c | 101 ++++++++++----------- src/core/lib/surface/completion_queue.c | 28 +++--- src/core/lib/surface/server.c | 60 ++++++------ src/core/lib/transport/metadata_batch.c | 32 +++---- src/core/lib/transport/transport.c | 5 +- test/core/end2end/tests/resource_quota_server.c | 12 +-- 26 files changed, 266 insertions(+), 221 deletions(-) diff --git a/src/core/ext/filters/client_channel/channel_connectivity.c b/src/core/ext/filters/client_channel/channel_connectivity.c index e5f6fa76ae..3844b98021 100644 --- a/src/core/ext/filters/client_channel/channel_connectivity.c +++ b/src/core/ext/filters/client_channel/channel_connectivity.c @@ -86,7 +86,7 @@ static void delete_state_watcher(grpc_exec_ctx *exec_ctx, state_watcher *w) { static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw, grpc_cq_completion *ignored) { - int delete = 0; + bool should_delete = false; state_watcher *w = (state_watcher *)pw; gpr_mu_lock(&w->mu); switch (w->phase) { @@ -94,12 +94,12 @@ static void finished_completion(grpc_exec_ctx *exec_ctx, void *pw, case READY_TO_CALL_BACK: GPR_UNREACHABLE_CODE(return ); case CALLING_BACK_AND_FINISHED: - delete = 1; + should_delete = true; break; } gpr_mu_unlock(&w->mu); - if (delete) { + if (should_delete) { delete_state_watcher(exec_ctx, w); } } @@ -161,12 +161,12 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w, static void watch_complete(grpc_exec_ctx *exec_ctx, void *pw, grpc_error *error) { - partly_done(exec_ctx, pw, true, GRPC_ERROR_REF(error)); + partly_done(exec_ctx, (state_watcher *)pw, true, GRPC_ERROR_REF(error)); } static void timeout_complete(grpc_exec_ctx *exec_ctx, void *pw, grpc_error *error) { - partly_done(exec_ctx, pw, false, GRPC_ERROR_REF(error)); + partly_done(exec_ctx, (state_watcher *)pw, false, GRPC_ERROR_REF(error)); } int grpc_channel_num_external_connectivity_watchers(grpc_channel *channel) { diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index e5f4a8a813..129d0f368b 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -85,7 +85,7 @@ static void method_parameters_unref(method_parameters *method_params) { } static void method_parameters_free(grpc_exec_ctx *exec_ctx, void *value) { - method_parameters_unref(value); + method_parameters_unref((method_parameters *)value); } static bool parse_wait_for_ready(grpc_json *field, @@ -717,7 +717,8 @@ static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx, return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "client channel factory arg must be a pointer"); } - grpc_client_channel_factory_ref(arg->value.pointer.p); + grpc_client_channel_factory_ref( + (grpc_client_channel_factory *)arg->value.pointer.p); chand->client_channel_factory = (grpc_client_channel_factory *)arg->value.pointer.p; // Get server name to resolve, using proxy mapper if needed. diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index 7220a8639e..a1ba31e5cf 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -43,14 +43,15 @@ grpc_channel* grpc_client_channel_factory_create_channel( } static void* factory_arg_copy(void* factory) { - grpc_client_channel_factory_ref(factory); + grpc_client_channel_factory_ref((grpc_client_channel_factory*)factory); return factory; } static void factory_arg_destroy(grpc_exec_ctx* exec_ctx, void* factory) { // TODO(roth): Remove local exec_ctx when // https://github.com/grpc/grpc/pull/8705 is merged. - grpc_client_channel_factory_unref(exec_ctx, factory); + grpc_client_channel_factory_unref(exec_ctx, + (grpc_client_channel_factory*)factory); } static int factory_arg_cmp(void* factory1, void* factory2) { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c index bd290464c8..7ad322902b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.c @@ -75,7 +75,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(args->context != NULL); GPR_ASSERT(args->context[GRPC_GRPCLB_CLIENT_STATS].value != NULL); calld->client_stats = grpc_grpclb_client_stats_ref( - args->context[GRPC_GRPCLB_CLIENT_STATS].value); + (grpc_grpclb_client_stats *)args->context[GRPC_GRPCLB_CLIENT_STATS] + .value); // Record call started. grpc_grpclb_client_stats_add_call_started(calld->client_stats); return GRPC_ERROR_NONE; diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 18979829bd..df4a00406e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ static grpc_error *initial_metadata_add_lb_token( } static void destroy_client_stats(void *arg) { - grpc_grpclb_client_stats_unref(arg); + grpc_grpclb_client_stats_unref((grpc_grpclb_client_stats *)arg); } typedef struct wrapped_rr_closure_arg { @@ -286,7 +286,49 @@ static void add_pending_ping(pending_ping **root, grpc_closure *notify) { * glb_lb_policy */ typedef struct rr_connectivity_data rr_connectivity_data; -static const grpc_lb_policy_vtable glb_lb_policy_vtable; + +/* Forward declare functions referred in glb_lb_policy_vtable */ +static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + const grpc_lb_policy_pick_args *pick_args, + grpc_connected_subchannel **target, + grpc_call_context_element *context, void **user_data, + grpc_closure *on_complete); +static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_connected_subchannel **target, + grpc_error *error); +static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, + grpc_lb_policy *pol, + uint32_t initial_metadata_flags_mask, + uint32_t initial_metadata_flags_eq, + grpc_error *error); +static void glb_ping_one_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_closure *closure); +static void glb_exit_idle_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); +static grpc_connectivity_state glb_check_connectivity_locked( + grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, + grpc_error **connectivity_error); +static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx, + grpc_lb_policy *pol, + grpc_connectivity_state *current, + grpc_closure *notify); +static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, + const grpc_lb_policy_args *args); + +/* Code wiring the policy with the rest of the core */ +static const grpc_lb_policy_vtable glb_lb_policy_vtable = { + glb_destroy, + glb_shutdown_locked, + glb_pick_locked, + glb_cancel_pick_locked, + glb_cancel_picks_locked, + glb_ping_one_locked, + glb_exit_idle_locked, + glb_check_connectivity_locked, + glb_notify_on_state_change_locked, + glb_update_locked}; + typedef struct glb_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -783,7 +825,7 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* Allocate the data for the tracking of the new RR policy's connectivity. * It'll be deallocated in glb_rr_connectivity_changed() */ rr_connectivity_data *rr_connectivity = - gpr_zalloc(sizeof(rr_connectivity_data)); + (rr_connectivity_data *)gpr_zalloc(sizeof(rr_connectivity_data)); GRPC_CLOSURE_INIT(&rr_connectivity->on_change, glb_rr_connectivity_changed_locked, rr_connectivity, grpc_combiner_scheduler(glb_policy->base.combiner)); @@ -923,7 +965,8 @@ static grpc_channel_args *build_lb_channel_args( grpc_lb_addresses *lb_addresses = grpc_lb_addresses_create(num_grpclb_addrs, NULL); grpc_slice_hash_table_entry *targets_info_entries = - gpr_zalloc(sizeof(*targets_info_entries) * num_grpclb_addrs); + (grpc_slice_hash_table_entry *)gpr_zalloc(sizeof(*targets_info_entries) * + num_grpclb_addrs); size_t lb_addresses_idx = 0; for (size_t i = 0; i < addresses->num_addresses; ++i) { @@ -1382,7 +1425,8 @@ static void do_send_client_load_report_locked(grpc_exec_ctx *exec_ctx, static bool load_report_counters_are_zero(grpc_grpclb_request *request) { grpc_grpclb_dropped_call_counts *drop_entries = - request->client_stats.calls_finished_with_drop.arg; + (grpc_grpclb_dropped_call_counts *) + request->client_stats.calls_finished_with_drop.arg; return request->client_stats.num_calls_started == 0 && request->client_stats.num_calls_finished == 0 && request->client_stats.num_calls_finished_with_client_failed_to_send == @@ -1969,19 +2013,6 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, } } -/* Code wiring the policy with the rest of the core */ -static const grpc_lb_policy_vtable glb_lb_policy_vtable = { - glb_destroy, - glb_shutdown_locked, - glb_pick_locked, - glb_cancel_pick_locked, - glb_cancel_picks_locked, - glb_ping_one_locked, - glb_exit_idle_locked, - glb_check_connectivity_locked, - glb_notify_on_state_change_locked, - glb_update_locked}; - static void glb_factory_ref(grpc_lb_policy_factory *factory) {} static void glb_factory_unref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c index 407bd18adb..8ef6dfc6f4 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c @@ -148,7 +148,8 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) { void grpc_grpclb_request_destroy(grpc_grpclb_request *request) { if (request->has_client_stats) { grpc_grpclb_dropped_call_counts *drop_entries = - request->client_stats.calls_finished_with_drop.arg; + (grpc_grpclb_dropped_call_counts *) + request->client_stats.calls_finished_with_drop.arg; grpc_grpclb_dropped_call_counts_destroy(drop_entries); } gpr_free(request); @@ -170,7 +171,8 @@ grpc_grpclb_initial_response *grpc_grpclb_initial_response_parse( if (!res.has_initial_response) return NULL; grpc_grpclb_initial_response *initial_res = - gpr_malloc(sizeof(grpc_grpclb_initial_response)); + (grpc_grpclb_initial_response *)gpr_malloc( + sizeof(grpc_grpclb_initial_response)); memcpy(initial_res, &res.initial_response, sizeof(grpc_grpclb_initial_response)); diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c index fab3073eb9..7029f71bf3 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c @@ -330,8 +330,8 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, gpr_log(GPR_INFO, "Pick First %p received update with %lu addresses", (void *)p, (unsigned long)addresses->num_addresses); } - grpc_subchannel_args *sc_args = - gpr_zalloc(sizeof(*sc_args) * addresses->num_addresses); + grpc_subchannel_args *sc_args = (grpc_subchannel_args *)gpr_zalloc( + sizeof(*sc_args) * addresses->num_addresses); /* We remove the following keys in order for subchannel keys belonging to * subchannels point to the same address to match. */ static const char *keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS, @@ -403,7 +403,7 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, } /* Create the subchannels for the new subchannel args/addresses. */ grpc_subchannel **new_subchannels = - gpr_zalloc(sizeof(*new_subchannels) * sc_args_count); + (grpc_subchannel **)gpr_zalloc(sizeof(*new_subchannels) * sc_args_count); size_t num_new_subchannels = 0; for (size_t i = 0; i < sc_args_count; i++) { grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index 918bab745c..a9a00f5b2b 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -126,13 +126,14 @@ void grpc_lb_addresses_destroy(grpc_exec_ctx* exec_ctx, } static void* lb_addresses_copy(void* addresses) { - return grpc_lb_addresses_copy(addresses); + return grpc_lb_addresses_copy((grpc_lb_addresses*)addresses); } static void lb_addresses_destroy(grpc_exec_ctx* exec_ctx, void* addresses) { - grpc_lb_addresses_destroy(exec_ctx, addresses); + grpc_lb_addresses_destroy(exec_ctx, (grpc_lb_addresses*)addresses); } static int lb_addresses_cmp(void* addresses1, void* addresses2) { - return grpc_lb_addresses_cmp(addresses1, addresses2); + return grpc_lb_addresses_cmp((grpc_lb_addresses*)addresses1, + (grpc_lb_addresses*)addresses2); } static const grpc_arg_pointer_vtable lb_addresses_arg_vtable = { lb_addresses_copy, lb_addresses_destroy, lb_addresses_cmp}; @@ -149,7 +150,7 @@ grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); if (lb_addresses_arg == NULL || lb_addresses_arg->type != GRPC_ARG_POINTER) return NULL; - return lb_addresses_arg->value.pointer.p; + return (grpc_lb_addresses*)lb_addresses_arg->value.pointer.p; } void grpc_lb_policy_factory_ref(grpc_lb_policy_factory* factory) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index 9747d39a16..7f1f57259a 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -38,7 +38,7 @@ typedef struct fd_node { /** the owner of this fd node */ grpc_ares_ev_driver *ev_driver; /** the grpc_fd owned by this fd node */ - grpc_fd *grpc_fd; + grpc_fd *fd; /** a closure wrapping on_readable_cb, which should be invoked when the grpc_fd in this node becomes readable. */ grpc_closure read_closure; @@ -96,15 +96,15 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver *ev_driver) { } static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { - gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "delete fd: %d", grpc_fd_wrapped_fd(fdn->fd)); GPR_ASSERT(!fdn->readable_registered); GPR_ASSERT(!fdn->writable_registered); gpr_mu_destroy(&fdn->mu); - grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->grpc_fd); + grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->fd); /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up immediately by another thread, and should not be closed by the following grpc_fd_orphan. */ - grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, NULL, true /* already_closed */, + grpc_fd_orphan(exec_ctx, fdn->fd, NULL, NULL, true /* already_closed */, "c-ares query finished"); gpr_free(fdn); } @@ -150,9 +150,8 @@ void grpc_ares_ev_driver_shutdown(grpc_exec_ctx *exec_ctx, ev_driver->shutting_down = true; fd_node *fn = ev_driver->fds; while (fn != NULL) { - grpc_fd_shutdown( - exec_ctx, fn->grpc_fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("grpc_ares_ev_driver_shutdown")); + grpc_fd_shutdown(exec_ctx, fn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "grpc_ares_ev_driver_shutdown")); fn = fn->next; } gpr_mu_unlock(&ev_driver->mu); @@ -165,7 +164,7 @@ static fd_node *pop_fd_node(fd_node **head, int fd) { dummy_head.next = *head; fd_node *node = &dummy_head; while (node->next != NULL) { - if (grpc_fd_wrapped_fd(node->next->grpc_fd) == fd) { + if (grpc_fd_wrapped_fd(node->next->fd) == fd) { fd_node *ret = node->next; node->next = node->next->next; *head = dummy_head.next; @@ -184,9 +183,9 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, fdn->readable_registered = false; gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->grpc_fd), + ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), ARES_SOCKET_BAD); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or @@ -211,10 +210,10 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, fdn->writable_registered = false; gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->grpc_fd)); + gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, - grpc_fd_wrapped_fd(fdn->grpc_fd)); + grpc_fd_wrapped_fd(fdn->fd)); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -253,7 +252,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, gpr_asprintf(&fd_name, "ares_ev_driver-%" PRIuPTR, i); fdn = (fd_node *)gpr_malloc(sizeof(fd_node)); gpr_log(GPR_DEBUG, "new fd: %d", socks[i]); - fdn->grpc_fd = grpc_fd_create(socks[i], fd_name); + fdn->fd = grpc_fd_create(socks[i], fd_name); fdn->ev_driver = ev_driver; fdn->readable_registered = false; fdn->writable_registered = false; @@ -262,8 +261,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); GRPC_CLOSURE_INIT(&fdn->write_closure, on_writable_cb, fdn, grpc_schedule_on_exec_ctx); - grpc_pollset_set_add_fd(exec_ctx, ev_driver->pollset_set, - fdn->grpc_fd); + grpc_pollset_set_add_fd(exec_ctx, ev_driver->pollset_set, fdn->fd); gpr_free(fd_name); } fdn->next = new_list; @@ -274,9 +272,8 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, if (ARES_GETSOCK_READABLE(socks_bitmask, i) && !fdn->readable_registered) { grpc_ares_ev_driver_ref(ev_driver); - gpr_log(GPR_DEBUG, "notify read on: %d", - grpc_fd_wrapped_fd(fdn->grpc_fd)); - grpc_fd_notify_on_read(exec_ctx, fdn->grpc_fd, &fdn->read_closure); + gpr_log(GPR_DEBUG, "notify read on: %d", grpc_fd_wrapped_fd(fdn->fd)); + grpc_fd_notify_on_read(exec_ctx, fdn->fd, &fdn->read_closure); fdn->readable_registered = true; } // Register write_closure if the socket is writable and write_closure @@ -284,9 +281,9 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, if (ARES_GETSOCK_WRITABLE(socks_bitmask, i) && !fdn->writable_registered) { gpr_log(GPR_DEBUG, "notify write on: %d", - grpc_fd_wrapped_fd(fdn->grpc_fd)); + grpc_fd_wrapped_fd(fdn->fd)); grpc_ares_ev_driver_ref(ev_driver); - grpc_fd_notify_on_write(exec_ctx, fdn->grpc_fd, &fdn->write_closure); + grpc_fd_notify_on_write(exec_ctx, fdn->fd, &fdn->write_closure); fdn->writable_registered = true; } gpr_mu_unlock(&fdn->mu); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 2e2b411ab8..0ffb38518a 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -123,8 +123,8 @@ static void grpc_ares_request_unref(grpc_exec_ctx *exec_ctx, static grpc_ares_hostbyname_request *create_hostbyname_request( grpc_ares_request *parent_request, char *host, uint16_t port, bool is_balancer) { - grpc_ares_hostbyname_request *hr = - gpr_zalloc(sizeof(grpc_ares_hostbyname_request)); + grpc_ares_hostbyname_request *hr = (grpc_ares_hostbyname_request *)gpr_zalloc( + sizeof(grpc_ares_hostbyname_request)); hr->parent_request = parent_request; hr->host = gpr_strdup(host); hr->port = port; @@ -527,7 +527,8 @@ static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_resolved_addresses **addrs) { grpc_resolve_address_ares_request *r = - gpr_zalloc(sizeof(grpc_resolve_address_ares_request)); + (grpc_resolve_address_ares_request *)gpr_zalloc( + sizeof(grpc_resolve_address_ares_request)); r->addrs_out = addrs; r->on_resolve_address_done = on_done; GRPC_CLOSURE_INIT(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r, diff --git a/src/core/ext/filters/client_channel/retry_throttle.c b/src/core/ext/filters/client_channel/retry_throttle.c index 6cd6654b6f..09dcade089 100644 --- a/src/core/ext/filters/client_channel/retry_throttle.c +++ b/src/core/ext/filters/client_channel/retry_throttle.c @@ -99,7 +99,7 @@ static grpc_server_retry_throttle_data* grpc_server_retry_throttle_data_create( int max_milli_tokens, int milli_token_ratio, grpc_server_retry_throttle_data* old_throttle_data) { grpc_server_retry_throttle_data* throttle_data = - gpr_malloc(sizeof(*throttle_data)); + (grpc_server_retry_throttle_data*)gpr_malloc(sizeof(*throttle_data)); memset(throttle_data, 0, sizeof(*throttle_data)); gpr_ref_init(&throttle_data->refs, 1); throttle_data->max_milli_tokens = max_milli_tokens; @@ -131,11 +131,11 @@ static grpc_server_retry_throttle_data* grpc_server_retry_throttle_data_create( // static void* copy_server_name(void* key, void* unused) { - return gpr_strdup(key); + return gpr_strdup((const char*)key); } static long compare_server_name(void* key1, void* key2, void* unused) { - return strcmp(key1, key2); + return strcmp((const char*)key1, (const char*)key2); } static void destroy_server_retry_throttle_data(void* value, void* unused) { @@ -177,7 +177,8 @@ grpc_server_retry_throttle_data* grpc_retry_throttle_map_get_data_for_server( const char* server_name, int max_milli_tokens, int milli_token_ratio) { gpr_mu_lock(&g_mu); grpc_server_retry_throttle_data* throttle_data = - gpr_avl_get(g_avl, (char*)server_name, NULL); + (grpc_server_retry_throttle_data*)gpr_avl_get(g_avl, (char*)server_name, + NULL); if (throttle_data == NULL) { // Entry not found. Create a new one. throttle_data = grpc_server_retry_throttle_data_create( diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c index f57b631c41..e05ab24622 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.c +++ b/src/core/ext/filters/client_channel/subchannel_index.c @@ -88,24 +88,26 @@ void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx, static void sck_avl_destroy(void *p, void *user_data) { grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data; - grpc_subchannel_key_destroy(exec_ctx, p); + grpc_subchannel_key_destroy(exec_ctx, (grpc_subchannel_key *)p); } static void *sck_avl_copy(void *p, void *unused) { - return subchannel_key_copy(p); + return subchannel_key_copy((grpc_subchannel_key *)p); } static long sck_avl_compare(void *a, void *b, void *unused) { - return grpc_subchannel_key_compare(a, b); + return grpc_subchannel_key_compare((grpc_subchannel_key *)a, + (grpc_subchannel_key *)b); } static void scv_avl_destroy(void *p, void *user_data) { grpc_exec_ctx *exec_ctx = (grpc_exec_ctx *)user_data; - GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, p, "subchannel_index"); + GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, (grpc_subchannel *)p, + "subchannel_index"); } static void *scv_avl_copy(void *p, void *unused) { - GRPC_SUBCHANNEL_WEAK_REF(p, "subchannel_index"); + GRPC_SUBCHANNEL_WEAK_REF((grpc_subchannel *)p, "subchannel_index"); return p; } diff --git a/src/core/ext/filters/http/server/http_server_filter.c b/src/core/ext/filters/http/server/http_server_filter.c index 554a7f530d..03958136b4 100644 --- a/src/core/ext/filters/http/server/http_server_filter.c +++ b/src/core/ext/filters/http/server/http_server_filter.c @@ -83,12 +83,12 @@ static grpc_error *server_filter_outgoing_metadata(grpc_exec_ctx *exec_ctx, } static void add_error(const char *error_name, grpc_error **cumulative, - grpc_error *new) { - if (new == GRPC_ERROR_NONE) return; + grpc_error *new_err) { + if (new_err == GRPC_ERROR_NONE) return; if (*cumulative == GRPC_ERROR_NONE) { *cumulative = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_name); } - *cumulative = grpc_error_add_child(*cumulative, new); + *cumulative = grpc_error_add_child(*cumulative, new_err); } static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, diff --git a/src/core/ext/transport/chttp2/client/chttp2_connector.c b/src/core/ext/transport/chttp2/client/chttp2_connector.c index 0ec9353c04..202bcd47f5 100644 --- a/src/core/ext/transport/chttp2/client/chttp2_connector.c +++ b/src/core/ext/transport/chttp2/client/chttp2_connector.c @@ -161,7 +161,7 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_endpoint_shutdown(exec_ctx, c->endpoint, GRPC_ERROR_REF(error)); } gpr_mu_unlock(&c->mu); - chttp2_connector_unref(exec_ctx, arg); + chttp2_connector_unref(exec_ctx, (grpc_connector *)arg); } else { GPR_ASSERT(c->endpoint != NULL); start_handshake_locked(exec_ctx, c); diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index d7add0538b..45e5b90823 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -49,10 +49,10 @@ typedef struct { grpc_closure tcp_server_shutdown_complete; grpc_closure *server_destroy_listener_done; grpc_handshake_manager *pending_handshake_mgrs; -} server_state; +} server_state_t; typedef struct { - server_state *server_state; + server_state_t *server_state; grpc_pollset *accepting_pollset; grpc_tcp_server_acceptor *acceptor; grpc_handshake_manager *handshake_mgr; @@ -109,7 +109,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *accepting_pollset, grpc_tcp_server_acceptor *acceptor) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); if (state->shutdown) { gpr_mu_unlock(&state->mu); @@ -124,7 +124,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_unlock(&state->mu); grpc_tcp_server_ref(state->tcp_server); server_connection_state *connection_state = - gpr_malloc(sizeof(*connection_state)); + (server_connection_state *)gpr_malloc(sizeof(*connection_state)); connection_state->server_state = state; connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; @@ -144,7 +144,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_pollset **pollsets, size_t pollset_count) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); state->shutdown = false; gpr_mu_unlock(&state->mu); @@ -154,7 +154,7 @@ static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; /* ensure all threads have unlocked */ gpr_mu_lock(&state->mu); grpc_closure *destroy_done = state->server_destroy_listener_done; @@ -179,7 +179,7 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, static void server_destroy_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_closure *destroy_done) { - server_state *state = (server_state *)arg; + server_state_t *state = (server_state_t *)arg; gpr_mu_lock(&state->mu); state->shutdown = true; state->server_destroy_listener_done = destroy_done; @@ -199,7 +199,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, size_t count = 0; int port_temp; grpc_error *err = GRPC_ERROR_NONE; - server_state *state = NULL; + server_state_t *state = NULL; grpc_error **errors = NULL; *port_num = -1; @@ -209,7 +209,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, if (err != GRPC_ERROR_NONE) { goto error; } - state = (server_state *)gpr_zalloc(sizeof(*state)); + state = (server_state_t *)gpr_zalloc(sizeof(*state)); GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, tcp_server_shutdown_complete, state, grpc_schedule_on_exec_ctx); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3fd701fe2f..930fa805a0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2912,7 +2912,8 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - gpr_malloc(sizeof(*incoming_byte_stream)); + (grpc_chttp2_incoming_byte_stream *)gpr_malloc( + sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; diff --git a/src/core/ext/transport/chttp2/transport/flow_control.c b/src/core/ext/transport/chttp2/transport/flow_control.c index 0f078e79e9..569a6349d3 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.c +++ b/src/core/ext/transport/chttp2/transport/flow_control.c @@ -60,24 +60,24 @@ static void pretrace(shadow_flow_control* shadow_fc, #define TRACE_PADDING 30 -static char* fmt_int64_diff_str(int64_t old, int64_t new) { +static char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { char* str; - if (old != new) { - gpr_asprintf(&str, "%" PRId64 " -> %" PRId64 "", old, new); + if (old_val != new_val) { + gpr_asprintf(&str, "%" PRId64 " -> %" PRId64 "", old_val, new_val); } else { - gpr_asprintf(&str, "%" PRId64 "", old); + gpr_asprintf(&str, "%" PRId64 "", old_val); } char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING); gpr_free(str); return str_lp; } -static char* fmt_uint32_diff_str(uint32_t old, uint32_t new) { +static char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) { char* str; - if (new > 0 && old != new) { - gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old, new); + if (new_val > 0 && old_val != new_val) { + gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old_val, new_val); } else { - gpr_asprintf(&str, "%" PRIu32 "", old); + gpr_asprintf(&str, "%" PRIu32 "", old_val); } char* str_lp = gpr_leftpad(str, ' ', TRACE_PADDING); gpr_free(str); diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index 806100adaa..2995bf7310 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -44,7 +44,8 @@ static uint8_t *fill_header(uint8_t *out, uint32_t length, uint8_t flags) { return out; } -grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, +grpc_slice grpc_chttp2_settings_create(uint32_t *old_settings, + const uint32_t *new_settings, uint32_t force_mask, size_t count) { size_t i; uint32_t n = 0; @@ -52,21 +53,21 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, uint8_t *p; for (i = 0; i < count; i++) { - n += (new[i] != old[i] || (force_mask & (1u << i)) != 0); + n += (new_settings[i] != old_settings[i] || (force_mask & (1u << i)) != 0); } output = GRPC_SLICE_MALLOC(9 + 6 * n); p = fill_header(GRPC_SLICE_START_PTR(output), 6 * n, 0); for (i = 0; i < count; i++) { - if (new[i] != old[i] || (force_mask & (1u << i)) != 0) { + if (new_settings[i] != old_settings[i] || (force_mask & (1u << i)) != 0) { *p++ = (uint8_t)(grpc_setting_id_to_wire_id[i] >> 8); *p++ = (uint8_t)(grpc_setting_id_to_wire_id[i]); - *p++ = (uint8_t)(new[i] >> 24); - *p++ = (uint8_t)(new[i] >> 16); - *p++ = (uint8_t)(new[i] >> 8); - *p++ = (uint8_t)(new[i]); - old[i] = new[i]; + *p++ = (uint8_t)(new_settings[i] >> 24); + *p++ = (uint8_t)(new_settings[i] >> 16); + *p++ = (uint8_t)(new_settings[i] >> 8); + *p++ = (uint8_t)(new_settings[i]); + old_settings[i] = new_settings[i]; } } diff --git a/src/core/ext/transport/chttp2/transport/incoming_metadata.c b/src/core/ext/transport/chttp2/transport/incoming_metadata.c index cf0a9ca920..ba680a89db 100644 --- a/src/core/ext/transport/chttp2/transport/incoming_metadata.c +++ b/src/core/ext/transport/chttp2/transport/incoming_metadata.c @@ -42,8 +42,9 @@ grpc_error *grpc_chttp2_incoming_metadata_buffer_add( grpc_mdelem elem) { buffer->size += GRPC_MDELEM_LENGTH(elem); return grpc_metadata_batch_add_tail( - exec_ctx, &buffer->batch, - gpr_arena_alloc(buffer->arena, sizeof(grpc_linked_mdelem)), elem); + exec_ctx, &buffer->batch, (grpc_linked_mdelem *)gpr_arena_alloc( + buffer->arena, sizeof(grpc_linked_mdelem)), + elem); } grpc_error *grpc_chttp2_incoming_metadata_buffer_replace_or_add( diff --git a/src/core/ext/transport/chttp2/transport/stream_map.c b/src/core/ext/transport/chttp2/transport/stream_map.c index 650090d8f0..d6079a9a33 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.c +++ b/src/core/ext/transport/chttp2/transport/stream_map.c @@ -72,8 +72,10 @@ void grpc_chttp2_stream_map_add(grpc_chttp2_stream_map *map, uint32_t key, /* resize when less than 25% of the table is free, because compaction won't help much */ map->capacity = capacity = 3 * capacity / 2; - map->keys = keys = gpr_realloc(keys, capacity * sizeof(uint32_t)); - map->values = values = gpr_realloc(values, capacity * sizeof(void *)); + map->keys = keys = + (uint32_t *)gpr_realloc(keys, capacity * sizeof(uint32_t)); + map->values = values = + (void **)gpr_realloc(values, capacity * sizeof(void *)); } } diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c index 036853a53b..cd3e76a0b5 100644 --- a/src/core/ext/transport/inproc/inproc_transport.c +++ b/src/core/ext/transport/inproc/inproc_transport.c @@ -37,7 +37,6 @@ if (GRPC_TRACER_ON(grpc_inproc_trace)) gpr_log(__VA_ARGS__); \ } while (0) -static const grpc_transport_vtable inproc_vtable; static grpc_slice g_empty_slice; static grpc_slice g_fake_path_key; static grpc_slice g_fake_path_value; @@ -1166,6 +1165,55 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { unref_transport(exec_ctx, t); } +/******************************************************************************* + * INTEGRATION GLUE + */ + +static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) { + // Nothing to do here +} + +static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset_set *pollset_set) { + // Nothing to do here +} + +static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { + return NULL; +} + +/******************************************************************************* + * GLOBAL INIT AND DESTROY + */ +static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void grpc_inproc_transport_init(void) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL, + grpc_schedule_on_exec_ctx); + g_empty_slice = grpc_slice_from_static_buffer(NULL, 0); + + grpc_slice key_tmp = grpc_slice_from_static_string(":path"); + g_fake_path_key = grpc_slice_intern(key_tmp); + grpc_slice_unref_internal(&exec_ctx, key_tmp); + + g_fake_path_value = grpc_slice_from_static_string("/"); + + grpc_slice auth_tmp = grpc_slice_from_static_string(":authority"); + g_fake_auth_key = grpc_slice_intern(auth_tmp); + grpc_slice_unref_internal(&exec_ctx, auth_tmp); + + g_fake_auth_value = grpc_slice_from_static_string("inproc-fail"); + grpc_exec_ctx_finish(&exec_ctx); +} + +static const grpc_transport_vtable inproc_vtable = { + sizeof(inproc_stream), "inproc", init_stream, + set_pollset, set_pollset_set, perform_stream_op, + perform_transport_op, destroy_stream, destroy_transport, + get_endpoint}; + /******************************************************************************* * Main inproc transport functions */ @@ -1178,7 +1226,7 @@ static void inproc_transports_create(grpc_exec_ctx *exec_ctx, inproc_transport *st = (inproc_transport *)gpr_zalloc(sizeof(*st)); inproc_transport *ct = (inproc_transport *)gpr_zalloc(sizeof(*ct)); // Share one lock between both sides since both sides get affected - st->mu = ct->mu = gpr_malloc(sizeof(*st->mu)); + st->mu = ct->mu = (shared_mu *)gpr_malloc(sizeof(*st->mu)); gpr_mu_init(&st->mu->mu); gpr_ref_init(&st->mu->refs, 2); st->base.vtable = &inproc_vtable; @@ -1240,55 +1288,6 @@ grpc_channel *grpc_inproc_channel_create(grpc_server *server, return channel; } -/******************************************************************************* - * INTEGRATION GLUE - */ - -static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) { - // Nothing to do here -} - -static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset_set *pollset_set) { - // Nothing to do here -} - -static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { - return NULL; -} - -static const grpc_transport_vtable inproc_vtable = { - sizeof(inproc_stream), "inproc", init_stream, - set_pollset, set_pollset_set, perform_stream_op, - perform_transport_op, destroy_stream, destroy_transport, - get_endpoint}; - -/******************************************************************************* - * GLOBAL INIT AND DESTROY - */ -static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} - -void grpc_inproc_transport_init(void) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL, - grpc_schedule_on_exec_ctx); - g_empty_slice = grpc_slice_from_static_buffer(NULL, 0); - - grpc_slice key_tmp = grpc_slice_from_static_string(":path"); - g_fake_path_key = grpc_slice_intern(key_tmp); - grpc_slice_unref_internal(&exec_ctx, key_tmp); - - g_fake_path_value = grpc_slice_from_static_string("/"); - - grpc_slice auth_tmp = grpc_slice_from_static_string(":authority"); - g_fake_auth_key = grpc_slice_intern(auth_tmp); - grpc_slice_unref_internal(&exec_ctx, auth_tmp); - - g_fake_auth_value = grpc_slice_from_static_string("inproc-fail"); - grpc_exec_ctx_finish(&exec_ctx); -} - void grpc_inproc_transport_shutdown(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_slice_unref_internal(&exec_ctx, g_empty_slice); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 4726503994..ab964c345c 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -575,12 +575,12 @@ static bool atm_inc_if_nonzero(gpr_atm *counter) { } static bool cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); return atm_inc_if_nonzero(&cqd->pending_events); } static bool cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); return atm_inc_if_nonzero(&cqd->pending_events); } @@ -625,7 +625,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, } } - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); int is_success = (error == GRPC_ERROR_NONE); storage->tag = tag; @@ -686,7 +686,7 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx, void *done_arg, grpc_cq_completion *storage), void *done_arg, grpc_cq_completion *storage) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); int is_success = (error == GRPC_ERROR_NONE); GPR_TIMER_BEGIN("cq_end_op_for_pluck", 0); @@ -769,7 +769,7 @@ typedef struct { static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) { cq_is_finished_arg *a = (cq_is_finished_arg *)arg; grpc_completion_queue *cq = a->cq; - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(a->stolen_completion == NULL); gpr_atm current_last_seen_things_queued_ever = @@ -820,7 +820,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, void *reserved) { grpc_event ret; gpr_timespec now; - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_TIMER_BEGIN("grpc_completion_queue_next", 0); @@ -953,7 +953,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, this function */ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); @@ -964,7 +964,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_next_data *cqd = DATA_FROM_CQ(cq); + cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); /* Need an extra ref for cq here because: * We call cq_finish_shutdown_next() below, that would call pollset shutdown. @@ -994,7 +994,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cq, static int add_plucker(grpc_completion_queue *cq, void *tag, grpc_pollset_worker **worker) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); if (cqd->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) { return 0; } @@ -1006,7 +1006,7 @@ static int add_plucker(grpc_completion_queue *cq, void *tag, static void del_plucker(grpc_completion_queue *cq, void *tag, grpc_pollset_worker **worker) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); for (int i = 0; i < cqd->num_pluckers; i++) { if (cqd->pluckers[i].tag == tag && cqd->pluckers[i].worker == worker) { cqd->num_pluckers--; @@ -1020,7 +1020,7 @@ static void del_plucker(grpc_completion_queue *cq, void *tag, static bool cq_is_pluck_finished(grpc_exec_ctx *exec_ctx, void *arg) { cq_is_finished_arg *a = (cq_is_finished_arg *)arg; grpc_completion_queue *cq = a->cq; - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_ASSERT(a->stolen_completion == NULL); gpr_atm current_last_seen_things_queued_ever = @@ -1057,7 +1057,7 @@ static grpc_event cq_pluck(grpc_completion_queue *cq, void *tag, grpc_cq_completion *prev; grpc_pollset_worker *worker = NULL; gpr_timespec now; - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0); @@ -1181,7 +1181,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cq, void *tag, static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); GPR_ASSERT(!gpr_atm_no_barrier_load(&cqd->shutdown)); @@ -1195,7 +1195,7 @@ static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx, * merging them is a bit tricky and probably not worth it */ static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq) { - cq_pluck_data *cqd = DATA_FROM_CQ(cq); + cq_pluck_data *cqd = (cq_pluck_data *)DATA_FROM_CQ(cq); /* Need an extra ref for cq here because: * We call cq_finish_shutdown_pluck() below, that would call pollset shutdown. diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 13ecc915ec..0bc2173713 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -55,7 +55,7 @@ typedef struct listener { typedef struct call_data call_data; typedef struct channel_data channel_data; -typedef struct registered_method registered_method; +typedef struct registered_method_t registered_method_t; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; @@ -76,7 +76,7 @@ typedef struct requested_call { grpc_call_details *details; } batch; struct { - registered_method *registered_method; + registered_method_t *registered_method; gpr_timespec *deadline; grpc_byte_buffer **optional_payload; } registered; @@ -84,7 +84,7 @@ typedef struct requested_call { } requested_call; typedef struct channel_registered_method { - registered_method *server_registered_method; + registered_method_t *server_registered_method; uint32_t flags; bool has_host; grpc_slice method; @@ -123,7 +123,7 @@ typedef enum { ZOMBIED } call_state; -typedef struct request_matcher request_matcher; +typedef struct request_matcher_t request_matcher_t; struct call_data { grpc_call *call; @@ -145,7 +145,7 @@ struct call_data { uint32_t recv_initial_metadata_flags; grpc_metadata_array initial_metadata; - request_matcher *request_matcher; + request_matcher_t *request_matcher; grpc_byte_buffer *payload; grpc_closure got_initial_metadata; @@ -158,21 +158,21 @@ struct call_data { call_data *pending_next; }; -struct request_matcher { +struct request_matcher_t { grpc_server *server; call_data *pending_head; call_data *pending_tail; gpr_stack_lockfree **requests_per_cq; }; -struct registered_method { +struct registered_method_t { char *method; char *host; grpc_server_register_method_payload_handling payload_handling; uint32_t flags; /* one request matcher per method */ - request_matcher request_matcher; - registered_method *next; + request_matcher_t request_matcher; + registered_method_t *next; }; typedef struct { @@ -204,9 +204,9 @@ struct grpc_server { bool starting; gpr_cv starting_cv; - registered_method *registered_methods; + registered_method_t *registered_methods; /** one request matcher for unregistered methods */ - request_matcher unregistered_request_matcher; + request_matcher_t unregistered_request_matcher; /** free list of available requested_calls_per_cq indices */ gpr_stack_lockfree **request_freelist_per_cq; /** requested call backing data */ @@ -313,7 +313,7 @@ static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx, * request_matcher */ -static void request_matcher_init(request_matcher *rm, size_t entries, +static void request_matcher_init(request_matcher_t *rm, size_t entries, grpc_server *server) { memset(rm, 0, sizeof(*rm)); rm->server = server; @@ -324,7 +324,7 @@ static void request_matcher_init(request_matcher *rm, size_t entries, } } -static void request_matcher_destroy(request_matcher *rm) { +static void request_matcher_destroy(request_matcher_t *rm) { for (size_t i = 0; i < rm->server->cq_count; i++) { GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1); gpr_stack_lockfree_destroy(rm->requests_per_cq[i]); @@ -334,11 +334,11 @@ static void request_matcher_destroy(request_matcher *rm) { static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, grpc_error *error) { - grpc_call_unref(grpc_call_from_top_element(elem)); + grpc_call_unref(grpc_call_from_top_element((grpc_call_element *)elem)); } static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, - request_matcher *rm) { + request_matcher_t *rm) { while (rm->pending_head) { call_data *calld = rm->pending_head; rm->pending_head = calld->pending_next; @@ -355,7 +355,7 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx, grpc_server *server, - request_matcher *rm, + request_matcher_t *rm, grpc_error *error) { int request_id; for (size_t i = 0; i < server->cq_count; i++) { @@ -378,7 +378,7 @@ static void server_ref(grpc_server *server) { } static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { - registered_method *rm; + registered_method_t *rm; size_t i; grpc_channel_args_destroy(exec_ctx, server->channel_args); gpr_mu_destroy(&server->mu_global); @@ -519,7 +519,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, grpc_call_element *call_elem = (grpc_call_element *)arg; call_data *calld = (call_data *)call_elem->call_data; channel_data *chand = (channel_data *)call_elem->channel_data; - request_matcher *rm = calld->request_matcher; + request_matcher_t *rm = calld->request_matcher; grpc_server *server = rm->server; if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) { @@ -569,7 +569,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, static void finish_start_new_rpc( grpc_exec_ctx *exec_ctx, grpc_server *server, grpc_call_element *elem, - request_matcher *rm, + request_matcher_t *rm, grpc_server_register_method_payload_handling payload_handling) { call_data *calld = (call_data *)elem->call_data; @@ -668,7 +668,7 @@ static int num_listeners(grpc_server *server) { static void done_shutdown_event(grpc_exec_ctx *exec_ctx, void *server, grpc_cq_completion *completion) { - server_unref(exec_ctx, server); + server_unref(exec_ctx, (grpc_server *)server); } static int num_channels(grpc_server *server) { @@ -689,7 +689,7 @@ static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_REF(error)); request_matcher_zombify_all_pending_calls( exec_ctx, &server->unregistered_request_matcher); - for (registered_method *rm = server->registered_methods; rm; + for (registered_method_t *rm = server->registered_methods; rm; rm = rm->next) { request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher, GRPC_ERROR_REF(error)); @@ -1036,7 +1036,7 @@ void *grpc_server_register_method( grpc_server *server, const char *method, const char *host, grpc_server_register_method_payload_handling payload_handling, uint32_t flags) { - registered_method *m; + registered_method_t *m; GRPC_API_TRACE( "grpc_server_register_method(server=%p, method=%s, host=%s, " "flags=0x%08x)", @@ -1058,7 +1058,7 @@ void *grpc_server_register_method( flags); return NULL; } - m = (registered_method *)gpr_zalloc(sizeof(registered_method)); + m = (registered_method_t *)gpr_zalloc(sizeof(registered_method_t)); m->method = gpr_strdup(method); m->host = gpr_strdup(host); m->next = server->registered_methods; @@ -1113,7 +1113,8 @@ void grpc_server_start(grpc_server *server) { } request_matcher_init(&server->unregistered_request_matcher, (size_t)server->max_requested_calls_per_cq, server); - for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { + for (registered_method_t *rm = server->registered_methods; rm; + rm = rm->next) { request_matcher_init(&rm->request_matcher, (size_t)server->max_requested_calls_per_cq, server); } @@ -1141,7 +1142,7 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, const grpc_channel_args *args) { size_t num_registered_methods; size_t alloc; - registered_method *rm; + registered_method_t *rm; channel_registered_method *crm; grpc_channel *channel; channel_data *chand; @@ -1267,8 +1268,9 @@ void grpc_server_shutdown_and_notify(grpc_server *server, /* stay locked, and gather up some stuff to do */ GPR_ASSERT(grpc_cq_begin_op(cq, tag)); if (server->shutdown_published) { - grpc_cq_end_op(&exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown, - NULL, gpr_malloc(sizeof(grpc_cq_completion))); + grpc_cq_end_op( + &exec_ctx, cq, tag, GRPC_ERROR_NONE, done_published_shutdown, NULL, + (grpc_cq_completion *)gpr_malloc(sizeof(grpc_cq_completion))); gpr_mu_unlock(&server->mu_global); goto done; } @@ -1369,7 +1371,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, grpc_server *server, size_t cq_idx, requested_call *rc) { call_data *calld = NULL; - request_matcher *rm = NULL; + request_matcher_t *rm = NULL; int request_id; if (gpr_atm_acq_load(&server->shutdown_flag)) { fail_call(exec_ctx, server, cq_idx, rc, @@ -1481,7 +1483,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc)); - registered_method *rm = (registered_method *)rmp; + registered_method_t *rm = (registered_method_t *)rmp; GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" diff --git a/src/core/lib/transport/metadata_batch.c b/src/core/lib/transport/metadata_batch.c index a077052561..54388bdcda 100644 --- a/src/core/lib/transport/metadata_batch.c +++ b/src/core/lib/transport/metadata_batch.c @@ -233,32 +233,32 @@ void grpc_metadata_batch_remove(grpc_exec_ctx *exec_ctx, void grpc_metadata_batch_set_value(grpc_exec_ctx *exec_ctx, grpc_linked_mdelem *storage, grpc_slice value) { - grpc_mdelem old = storage->md; - grpc_mdelem new = grpc_mdelem_from_slices( - exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(old)), value); - storage->md = new; - GRPC_MDELEM_UNREF(exec_ctx, old); + grpc_mdelem old_mdelem = storage->md; + grpc_mdelem new_mdelem = grpc_mdelem_from_slices( + exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(old_mdelem)), value); + storage->md = new_mdelem; + GRPC_MDELEM_UNREF(exec_ctx, old_mdelem); } grpc_error *grpc_metadata_batch_substitute(grpc_exec_ctx *exec_ctx, grpc_metadata_batch *batch, grpc_linked_mdelem *storage, - grpc_mdelem new) { + grpc_mdelem new_mdelem) { assert_valid_callouts(exec_ctx, batch); grpc_error *error = GRPC_ERROR_NONE; - grpc_mdelem old = storage->md; - if (!grpc_slice_eq(GRPC_MDKEY(new), GRPC_MDKEY(old))) { + grpc_mdelem old_mdelem = storage->md; + if (!grpc_slice_eq(GRPC_MDKEY(new_mdelem), GRPC_MDKEY(old_mdelem))) { maybe_unlink_callout(batch, storage); - storage->md = new; + storage->md = new_mdelem; error = maybe_link_callout(batch, storage); if (error != GRPC_ERROR_NONE) { unlink_storage(&batch->list, storage); GRPC_MDELEM_UNREF(exec_ctx, storage->md); } } else { - storage->md = new; + storage->md = new_mdelem; } - GRPC_MDELEM_UNREF(exec_ctx, old); + GRPC_MDELEM_UNREF(exec_ctx, old_mdelem); assert_valid_callouts(exec_ctx, batch); return error; } @@ -302,12 +302,12 @@ grpc_error *grpc_metadata_batch_filter(grpc_exec_ctx *exec_ctx, grpc_error *error = GRPC_ERROR_NONE; while (l) { grpc_linked_mdelem *next = l->next; - grpc_filtered_mdelem new = func(exec_ctx, user_data, l->md); - add_error(&error, new.error, composite_error_string); - if (GRPC_MDISNULL(new.md)) { + grpc_filtered_mdelem new_mdelem = func(exec_ctx, user_data, l->md); + add_error(&error, new_mdelem.error, composite_error_string); + if (GRPC_MDISNULL(new_mdelem.md)) { grpc_metadata_batch_remove(exec_ctx, batch, l); - } else if (new.md.payload != l->md.payload) { - grpc_metadata_batch_substitute(exec_ctx, batch, l, new.md); + } else if (new_mdelem.md.payload != l->md.payload) { + grpc_metadata_batch_substitute(exec_ctx, batch, l, new_mdelem.md); } l = next; } diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index caa11a956e..ae705195f3 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -102,8 +102,9 @@ static void slice_stream_unref(grpc_exec_ctx *exec_ctx, void *p) { grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount *refcount, void *buffer, size_t length) { slice_stream_ref(&refcount->slice_refcount); - return (grpc_slice){.refcount = &refcount->slice_refcount, - .data.refcounted = {.bytes = buffer, .length = length}}; + return (grpc_slice){ + .refcount = &refcount->slice_refcount, + .data.refcounted = {.bytes = (uint8_t *)buffer, .length = length}}; } static const grpc_slice_refcount_vtable stream_ref_slice_vtable = { diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index 0316920762..93befeee0c 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -136,20 +136,20 @@ void resource_quota_server(grpc_end2end_test_config config) { grpc_call **server_calls = (grpc_call **)malloc(sizeof(grpc_call *) * NUM_CALLS); grpc_metadata_array *initial_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_metadata_array *trailing_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_metadata_array *request_metadata_recv = - malloc(sizeof(grpc_metadata_array) * NUM_CALLS); + (grpc_metadata_array *)malloc(sizeof(grpc_metadata_array) * NUM_CALLS); grpc_call_details *call_details = - malloc(sizeof(grpc_call_details) * NUM_CALLS); + (grpc_call_details *)malloc(sizeof(grpc_call_details) * NUM_CALLS); grpc_status_code *status = (grpc_status_code *)malloc(sizeof(grpc_status_code) * NUM_CALLS); grpc_slice *details = (grpc_slice *)malloc(sizeof(grpc_slice) * NUM_CALLS); grpc_byte_buffer **request_payload = - malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); + (grpc_byte_buffer **)malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); grpc_byte_buffer **request_payload_recv = - malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); + (grpc_byte_buffer **)malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS); int *was_cancelled = (int *)malloc(sizeof(int) * NUM_CALLS); grpc_call_error error; int pending_client_calls = 0; -- cgit v1.2.3 From 6502ecc812cba929453115a0475804304db243a6 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Wed, 13 Sep 2017 13:10:54 -0700 Subject: Add refcount to subchannel_index. --- .../client_channel/lb_policy/grpclb/grpclb.c | 3 +++ .../lb_policy/pick_first/pick_first.c | 2 ++ .../lb_policy/round_robin/round_robin.c | 3 +++ .../ext/filters/client_channel/subchannel_index.c | 22 ++++++++++++++++++---- .../ext/filters/client_channel/subchannel_index.h | 7 +++++++ src/core/lib/surface/init.c | 9 ++++++--- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 5aafed1374..734a328446 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -101,6 +101,7 @@ #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h" +#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/iomgr/combiner.h" @@ -988,6 +989,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, gpr_free(glb_policy); return NULL; } + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, glb_lb_channel_on_connectivity_changed_cb, glb_policy, grpc_combiner_scheduler(args->combiner)); @@ -1011,6 +1013,7 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); + grpc_subchannel_index_unref(); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); gpr_free(glb_policy->pending_update_args); diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c index fab3073eb9..50b02016a1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c @@ -89,6 +89,7 @@ static void pf_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { "picked_first_destroy"); } grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker); + grpc_subchannel_index_unref(); if (p->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, p->pending_update_args->args); gpr_free(p->pending_update_args); @@ -686,6 +687,7 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, } pf_update_locked(exec_ctx, &p->base, args); grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable, args->combiner); + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&p->connectivity_changed, pf_connectivity_changed_locked, p, grpc_combiner_scheduler(args->combiner)); return &p->base; diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index be91d3d651..8ac1a46abd 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -30,6 +30,7 @@ #include "src/core/ext/filters/client_channel/lb_policy_registry.h" #include "src/core/ext/filters/client_channel/subchannel.h" +#include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/combiner.h" @@ -310,6 +311,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { (void *)pol, (void *)pol); } grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker); + grpc_subchannel_index_unref(); gpr_free(p); } @@ -890,6 +892,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(args->client_channel_factory != NULL); round_robin_lb_policy *p = (round_robin_lb_policy *)gpr_zalloc(sizeof(*p)); grpc_lb_policy_init(&p->base, &round_robin_lb_policy_vtable, args->combiner); + grpc_subchannel_index_ref(); grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE, "round_robin"); rr_update_locked(exec_ctx, &p->base, args); diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c index f57b631c41..000ee70be7 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.c +++ b/src/core/ext/filters/client_channel/subchannel_index.c @@ -34,6 +34,8 @@ static gpr_avl g_subchannel_index; static gpr_mu g_mu; +static gpr_refcount g_refcount; + struct grpc_subchannel_key { grpc_subchannel_args args; }; @@ -119,15 +121,27 @@ static const gpr_avl_vtable subchannel_avl_vtable = { void grpc_subchannel_index_init(void) { g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable); gpr_mu_init(&g_mu); + gpr_ref_init(&g_refcount, 1); } void grpc_subchannel_index_shutdown(void) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - gpr_mu_destroy(&g_mu); - gpr_avl_unref(g_subchannel_index, &exec_ctx); - grpc_exec_ctx_finish(&exec_ctx); + // TODO(juanlishen): This refcounting mechanism may lead to memory leackage. + // To solve that, we should force polling to flush any pending callbacks, then + // shutdown safely. + grpc_subchannel_index_unref(); +} + +void grpc_subchannel_index_unref(void) { + if (gpr_unref(&g_refcount)) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_destroy(&g_mu); + gpr_avl_unref(g_subchannel_index, &exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); + } } +void grpc_subchannel_index_ref(void) { gpr_ref_non_zero(&g_refcount); } + grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx, grpc_subchannel_key *key) { // Lock, and take a reference to the subchannel index. diff --git a/src/core/ext/filters/client_channel/subchannel_index.h b/src/core/ext/filters/client_channel/subchannel_index.h index 98d882a453..92e36d5283 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.h +++ b/src/core/ext/filters/client_channel/subchannel_index.h @@ -59,6 +59,13 @@ void grpc_subchannel_index_init(void); /** Shutdown the subchannel index (global) */ void grpc_subchannel_index_shutdown(void); +/** Increment the refcount (non-zero) of subchannel index (global). */ +void grpc_subchannel_index_ref(void); + +/** Decrement the refcount of subchannel index (global). If the refcount drops + to zero, unref the subchannel index and destroy its mutex. */ +void grpc_subchannel_index_unref(void); + /** \em TEST ONLY. * If \a force_creation is true, all key comparisons will be false, resulting in * new subchannels always being created. Otherwise, the keys will be compared as diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 280315036f..b089da2c54 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/resource_quota.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/alarm_internal.h" @@ -179,14 +180,16 @@ void grpc_shutdown(void) { GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); gpr_mu_lock(&g_init_mu); if (--g_initializations == 0) { - grpc_iomgr_shutdown(&exec_ctx); - gpr_timers_global_destroy(); - grpc_tracer_shutdown(); + grpc_executor_shutdown(&exec_ctx); + grpc_timer_manager_set_threading(false); // shutdown timer_manager thread for (i = g_number_of_plugins; i >= 0; i--) { if (g_all_of_the_plugins[i].destroy != NULL) { g_all_of_the_plugins[i].destroy(); } } + grpc_iomgr_shutdown(&exec_ctx); + gpr_timers_global_destroy(); + grpc_tracer_shutdown(); grpc_mdctx_global_shutdown(&exec_ctx); grpc_handshaker_factory_registry_shutdown(&exec_ctx); grpc_slice_intern_shutdown(); -- cgit v1.2.3 From 09a02ec04038e9eb45aae249d65bdf66f305d9d4 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 13 Sep 2017 20:05:34 +0000 Subject: Kill (grpcio_tests/)commands.BuildProtoModules In 1ff429da2a94bc79300ebce3f8aae7efb10e9a75 it appears to have been copied out of src/python/grpcio/commands.py and not used, and it seems to have remained without use since. --- src/python/grpcio_tests/commands.py | 49 ------------------------------------- 1 file changed, 49 deletions(-) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 162200112a..93f84572b7 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -67,55 +67,6 @@ class GatherProto(setuptools.Command): open(path, 'a').close() -class BuildProtoModules(setuptools.Command): - """Command to generate project *_pb2.py modules from proto files.""" - - description = 'build protobuf modules' - user_options = [ - ('include=', None, 'path patterns to include in protobuf generation'), - ('exclude=', None, 'path patterns to exclude from protobuf generation') - ] - - def initialize_options(self): - self.exclude = None - self.include = r'.*\.proto$' - - def finalize_options(self): - pass - - def run(self): - import grpc_tools.protoc as protoc - - include_regex = re.compile(self.include) - exclude_regex = re.compile(self.exclude) if self.exclude else None - paths = [] - for walk_root, directories, filenames in os.walk(PROTO_STEM): - for filename in filenames: - path = os.path.join(walk_root, filename) - if include_regex.match(path) and not ( - exclude_regex and exclude_regex.match(path)): - paths.append(path) - - # TODO(kpayson): It would be nice to do this in a batch command, - # but we currently have name conflicts in src/proto - for path in paths: - command = [ - 'grpc_tools.protoc', - '-I {}'.format(PROTO_STEM), - '--python_out={}'.format(PROTO_STEM), - '--grpc_python_out={}'.format(PROTO_STEM), - ] + [path] - if protoc.main(command) != 0: - sys.stderr.write( - 'warning: Command:\n{}\nFailed'.format(command)) - - # Generated proto directories dont include __init__.py, but - # these are needed for python package resolution - for walk_root, _, _ in os.walk(PROTO_STEM): - path = os.path.join(walk_root, '__init__.py') - open(path, 'a').close() - - class BuildPy(build_py.build_py): """Custom project build command.""" -- cgit v1.2.3 From d01fd422ece32c875abc9ad3cd022b3727cce2f4 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Wed, 13 Sep 2017 20:13:32 +0000 Subject: grpc_1_0 flag for gRPC Python Protoc Plug-In We'll soon be flipping the default behavior of gRPC Python Protoc Plug-In to the later-than-gRPC-1.0 "code elements only in a separate _pb2_grpc.py file and no Beta-API-using generated code elements" behavior; this flag will be the supported (supported for a while, though not exactly sure how long) means of using the older "also put gRPC-using code elements in the generated _pb2.py file" behavior that has been in place since gRPC 1.0. --- src/compiler/python_generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 31b177c28e..a60b528e3b 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -769,7 +769,7 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file, PrivateGenerator generator(config_, &pbfile); if (parameter == "grpc_2_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true); - } else if (parameter == "") { + } else if (parameter == "grpc_1_0" || parameter == "") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true) && GenerateGrpc(context, generator, pb2_file_name, false); } else { -- cgit v1.2.3 From a495220f3128c94cf2f59a258de6d0ba8f4b2871 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 10:53:28 -0700 Subject: removing a completed TODO, removing forward declarations and rearranging code instead, removing _t suffixes with other recommendations --- .../client_channel/client_channel_factory.c | 2 - .../client_channel/lb_policy/grpclb/grpclb.c | 235 +++++++++------------ .../ext/transport/chttp2/server/chttp2_server.c | 30 +-- src/core/lib/surface/server.c | 69 +++--- 4 files changed, 145 insertions(+), 191 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index a1ba31e5cf..e8aa4cda29 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -48,8 +48,6 @@ static void* factory_arg_copy(void* factory) { } static void factory_arg_destroy(grpc_exec_ctx* exec_ctx, void* factory) { - // TODO(roth): Remove local exec_ctx when - // https://github.com/grpc/grpc/pull/8705 is merged. grpc_client_channel_factory_unref(exec_ctx, (grpc_client_channel_factory*)factory); } diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index df4a00406e..8bc6095712 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -287,48 +287,6 @@ static void add_pending_ping(pending_ping **root, grpc_closure *notify) { */ typedef struct rr_connectivity_data rr_connectivity_data; -/* Forward declare functions referred in glb_lb_policy_vtable */ -static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static void glb_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, - grpc_call_context_element *context, void **user_data, - grpc_closure *on_complete); -static void glb_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_connected_subchannel **target, - grpc_error *error); -static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, - grpc_lb_policy *pol, - uint32_t initial_metadata_flags_mask, - uint32_t initial_metadata_flags_eq, - grpc_error *error); -static void glb_ping_one_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_closure *closure); -static void glb_exit_idle_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol); -static grpc_connectivity_state glb_check_connectivity_locked( - grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_error **connectivity_error); -static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx, - grpc_lb_policy *pol, - grpc_connectivity_state *current, - grpc_closure *notify); -static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - const grpc_lb_policy_args *args); - -/* Code wiring the policy with the rest of the core */ -static const grpc_lb_policy_vtable glb_lb_policy_vtable = { - glb_destroy, - glb_shutdown_locked, - glb_pick_locked, - glb_cancel_pick_locked, - glb_cancel_picks_locked, - glb_ping_one_locked, - glb_exit_idle_locked, - glb_check_connectivity_locked, - glb_notify_on_state_change_locked, - glb_update_locked}; - typedef struct glb_lb_policy { /** base policy: must be first */ grpc_lb_policy base; @@ -1011,92 +969,6 @@ static grpc_channel_args *build_lb_channel_args( static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); -static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, - grpc_lb_policy_factory *factory, - grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. */ - const grpc_arg *arg = - grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); - if (arg == NULL || arg->type != GRPC_ARG_POINTER) { - return NULL; - } - grpc_lb_addresses *addresses = (grpc_lb_addresses *)arg->value.pointer.p; - size_t num_grpclb_addrs = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; - } - if (num_grpclb_addrs == 0) return NULL; - - glb_lb_policy *glb_policy = (glb_lb_policy *)gpr_zalloc(sizeof(*glb_policy)); - - /* Get server name. */ - arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); - GPR_ASSERT(arg != NULL); - GPR_ASSERT(arg->type == GRPC_ARG_STRING); - grpc_uri *uri = grpc_uri_parse(exec_ctx, arg->value.string, true); - GPR_ASSERT(uri->path[0] != '\0'); - glb_policy->server_name = - gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", - glb_policy->server_name); - } - grpc_uri_destroy(uri); - - glb_policy->cc_factory = args->client_channel_factory; - GPR_ASSERT(glb_policy->cc_factory != NULL); - - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS); - glb_policy->lb_call_timeout_ms = - grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); - - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); - glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( - arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, - INT_MAX}); - - // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, - // since we use this to trigger the client_load_reporting filter. - grpc_arg new_arg = - grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); - static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; - glb_policy->args = grpc_channel_args_copy_and_add_and_remove( - args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); - - /* Extract the backend addresses (may be empty) from the resolver for - * fallback. */ - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - - /* Create a client channel over them to communicate with a LB service */ - glb_policy->response_generator = - grpc_fake_resolver_response_generator_create(); - grpc_channel_args *lb_channel_args = build_lb_channel_args( - exec_ctx, addresses, glb_policy->response_generator, args->args); - char *uri_str; - gpr_asprintf(&uri_str, "fake:///%s", glb_policy->server_name); - glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( - exec_ctx, uri_str, args->client_channel_factory, lb_channel_args); - - /* Propagate initial resolution */ - grpc_fake_resolver_response_generator_set_response( - exec_ctx, glb_policy->response_generator, lb_channel_args); - grpc_channel_args_destroy(exec_ctx, lb_channel_args); - gpr_free(uri_str); - if (glb_policy->lb_channel == NULL) { - gpr_free((void *)glb_policy->server_name); - grpc_channel_args_destroy(exec_ctx, glb_policy->args); - gpr_free(glb_policy); - return NULL; - } - GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, - glb_lb_channel_on_connectivity_changed_cb, glb_policy, - grpc_combiner_scheduler(args->combiner)); - grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable, args->combiner); - grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, - "grpclb"); - return &glb_policy->base; -} static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; @@ -1936,17 +1808,6 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } - - // Propagate update to fallback_backend_addresses if a non-empty serverlist - // hasn't been received from the balancer. - if (glb_policy->serverlist == NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - if (glb_policy->rr_policy != NULL) { - rr_handover_locked(exec_ctx, glb_policy); - } - } } // Invoked as part of the update process. It continues watching the LB channel @@ -2013,6 +1874,102 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, } } +/* Code wiring the policy with the rest of the core */ +static const grpc_lb_policy_vtable glb_lb_policy_vtable = { + glb_destroy, + glb_shutdown_locked, + glb_pick_locked, + glb_cancel_pick_locked, + glb_cancel_picks_locked, + glb_ping_one_locked, + glb_exit_idle_locked, + glb_check_connectivity_locked, + glb_notify_on_state_change_locked, + glb_update_locked}; + +static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, + grpc_lb_policy_factory *factory, + grpc_lb_policy_args *args) { + /* Count the number of gRPC-LB addresses. There must be at least one. + * TODO(roth): For now, we ignore non-balancer addresses, but in the + * future, we may change the behavior such that we fall back to using + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ + const grpc_arg *arg = + grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); + if (arg == NULL || arg->type != GRPC_ARG_POINTER) { + return NULL; + } + grpc_lb_addresses *addresses = (grpc_lb_addresses *)arg->value.pointer.p; + size_t num_grpclb_addrs = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) ++num_grpclb_addrs; + } + if (num_grpclb_addrs == 0) return NULL; + + glb_lb_policy *glb_policy = (glb_lb_policy *)gpr_zalloc(sizeof(*glb_policy)); + + /* Get server name. */ + arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); + GPR_ASSERT(arg != NULL); + GPR_ASSERT(arg->type == GRPC_ARG_STRING); + grpc_uri *uri = grpc_uri_parse(exec_ctx, arg->value.string, true); + GPR_ASSERT(uri->path[0] != '\0'); + glb_policy->server_name = + gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", + glb_policy->server_name); + } + grpc_uri_destroy(uri); + + glb_policy->cc_factory = args->client_channel_factory; + GPR_ASSERT(glb_policy->cc_factory != NULL); + + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS); + glb_policy->lb_call_timeout_ms = + grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, + // since we use this to trigger the client_load_reporting filter. + grpc_arg new_arg = + grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); + static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; + glb_policy->args = grpc_channel_args_copy_and_add_and_remove( + args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + + /* Create a client channel over them to communicate with a LB service */ + glb_policy->response_generator = + grpc_fake_resolver_response_generator_create(); + grpc_channel_args *lb_channel_args = build_lb_channel_args( + exec_ctx, addresses, glb_policy->response_generator, args->args); + char *uri_str; + gpr_asprintf(&uri_str, "fake:///%s", glb_policy->server_name); + glb_policy->lb_channel = grpc_lb_policy_grpclb_create_lb_channel( + exec_ctx, uri_str, args->client_channel_factory, lb_channel_args); + + /* Propagate initial resolution */ + grpc_fake_resolver_response_generator_set_response( + exec_ctx, glb_policy->response_generator, lb_channel_args); + grpc_channel_args_destroy(exec_ctx, lb_channel_args); + gpr_free(uri_str); + if (glb_policy->lb_channel == NULL) { + gpr_free((void *)glb_policy->server_name); + grpc_channel_args_destroy(exec_ctx, glb_policy->args); + gpr_free(glb_policy); + return NULL; + } + GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, + glb_lb_channel_on_connectivity_changed_cb, glb_policy, + grpc_combiner_scheduler(args->combiner)); + grpc_lb_policy_init(&glb_policy->base, &glb_lb_policy_vtable, args->combiner); + grpc_connectivity_state_init(&glb_policy->state_tracker, GRPC_CHANNEL_IDLE, + "grpclb"); + return &glb_policy->base; +} + static void glb_factory_ref(grpc_lb_policy_factory *factory) {} static void glb_factory_unref(grpc_lb_policy_factory *factory) {} diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index 45e5b90823..f5a409a403 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -49,10 +49,10 @@ typedef struct { grpc_closure tcp_server_shutdown_complete; grpc_closure *server_destroy_listener_done; grpc_handshake_manager *pending_handshake_mgrs; -} server_state_t; +} server_state; typedef struct { - server_state_t *server_state; + server_state *svr_state; grpc_pollset *accepting_pollset; grpc_tcp_server_acceptor *acceptor; grpc_handshake_manager *handshake_mgr; @@ -63,8 +63,8 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_handshaker_args *args = (grpc_handshaker_args *)arg; server_connection_state *connection_state = (server_connection_state *)args->user_data; - gpr_mu_lock(&connection_state->server_state->mu); - if (error != GRPC_ERROR_NONE || connection_state->server_state->shutdown) { + gpr_mu_lock(&connection_state->svr_state->mu); + if (error != GRPC_ERROR_NONE || connection_state->svr_state->shutdown) { const char *error_str = grpc_error_string(error); gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str); @@ -89,7 +89,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_transport *transport = grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 0); grpc_server_setup_transport( - exec_ctx, connection_state->server_state->server, transport, + exec_ctx, connection_state->svr_state->server, transport, connection_state->accepting_pollset, args->args); grpc_chttp2_transport_start_reading(exec_ctx, transport, args->read_buffer); @@ -97,11 +97,11 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, } } grpc_handshake_manager_pending_list_remove( - &connection_state->server_state->pending_handshake_mgrs, + &connection_state->svr_state->pending_handshake_mgrs, connection_state->handshake_mgr); - gpr_mu_unlock(&connection_state->server_state->mu); + gpr_mu_unlock(&connection_state->svr_state->mu); grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr); - grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp_server); + grpc_tcp_server_unref(exec_ctx, connection_state->svr_state->tcp_server); gpr_free(connection_state->acceptor); gpr_free(connection_state); } @@ -109,7 +109,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_pollset *accepting_pollset, grpc_tcp_server_acceptor *acceptor) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); if (state->shutdown) { gpr_mu_unlock(&state->mu); @@ -125,7 +125,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, grpc_tcp_server_ref(state->tcp_server); server_connection_state *connection_state = (server_connection_state *)gpr_malloc(sizeof(*connection_state)); - connection_state->server_state = state; + connection_state->svr_state = state; connection_state->accepting_pollset = accepting_pollset; connection_state->acceptor = acceptor; connection_state->handshake_mgr = handshake_mgr; @@ -144,7 +144,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_pollset **pollsets, size_t pollset_count) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); state->shutdown = false; gpr_mu_unlock(&state->mu); @@ -154,7 +154,7 @@ static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; /* ensure all threads have unlocked */ gpr_mu_lock(&state->mu); grpc_closure *destroy_done = state->server_destroy_listener_done; @@ -179,7 +179,7 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg, static void server_destroy_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, void *arg, grpc_closure *destroy_done) { - server_state_t *state = (server_state_t *)arg; + server_state *state = (server_state *)arg; gpr_mu_lock(&state->mu); state->shutdown = true; state->server_destroy_listener_done = destroy_done; @@ -199,7 +199,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, size_t count = 0; int port_temp; grpc_error *err = GRPC_ERROR_NONE; - server_state_t *state = NULL; + server_state *state = NULL; grpc_error **errors = NULL; *port_num = -1; @@ -209,7 +209,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, if (err != GRPC_ERROR_NONE) { goto error; } - state = (server_state_t *)gpr_zalloc(sizeof(*state)); + state = (server_state *)gpr_zalloc(sizeof(*state)); GRPC_CLOSURE_INIT(&state->tcp_server_shutdown_complete, tcp_server_shutdown_complete, state, grpc_schedule_on_exec_ctx); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 0bc2173713..1d0fd472d0 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -55,7 +55,7 @@ typedef struct listener { typedef struct call_data call_data; typedef struct channel_data channel_data; -typedef struct registered_method_t registered_method_t; +typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; @@ -76,7 +76,7 @@ typedef struct requested_call { grpc_call_details *details; } batch; struct { - registered_method_t *registered_method; + registered_method *method; gpr_timespec *deadline; grpc_byte_buffer **optional_payload; } registered; @@ -84,7 +84,7 @@ typedef struct requested_call { } requested_call; typedef struct channel_registered_method { - registered_method_t *server_registered_method; + registered_method *server_registered_method; uint32_t flags; bool has_host; grpc_slice method; @@ -123,7 +123,7 @@ typedef enum { ZOMBIED } call_state; -typedef struct request_matcher_t request_matcher_t; +typedef struct request_matcher request_matcher; struct call_data { grpc_call *call; @@ -145,7 +145,7 @@ struct call_data { uint32_t recv_initial_metadata_flags; grpc_metadata_array initial_metadata; - request_matcher_t *request_matcher; + request_matcher *matcher; grpc_byte_buffer *payload; grpc_closure got_initial_metadata; @@ -158,21 +158,21 @@ struct call_data { call_data *pending_next; }; -struct request_matcher_t { +struct request_matcher { grpc_server *server; call_data *pending_head; call_data *pending_tail; gpr_stack_lockfree **requests_per_cq; }; -struct registered_method_t { +struct registered_method { char *method; char *host; grpc_server_register_method_payload_handling payload_handling; uint32_t flags; /* one request matcher per method */ - request_matcher_t request_matcher; - registered_method_t *next; + request_matcher matcher; + registered_method *next; }; typedef struct { @@ -204,9 +204,9 @@ struct grpc_server { bool starting; gpr_cv starting_cv; - registered_method_t *registered_methods; + registered_method *registered_methods; /** one request matcher for unregistered methods */ - request_matcher_t unregistered_request_matcher; + request_matcher unregistered_request_matcher; /** free list of available requested_calls_per_cq indices */ gpr_stack_lockfree **request_freelist_per_cq; /** requested call backing data */ @@ -313,7 +313,7 @@ static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx, * request_matcher */ -static void request_matcher_init(request_matcher_t *rm, size_t entries, +static void request_matcher_init(request_matcher *rm, size_t entries, grpc_server *server) { memset(rm, 0, sizeof(*rm)); rm->server = server; @@ -324,7 +324,7 @@ static void request_matcher_init(request_matcher_t *rm, size_t entries, } } -static void request_matcher_destroy(request_matcher_t *rm) { +static void request_matcher_destroy(request_matcher *rm) { for (size_t i = 0; i < rm->server->cq_count; i++) { GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1); gpr_stack_lockfree_destroy(rm->requests_per_cq[i]); @@ -338,7 +338,7 @@ static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, } static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, - request_matcher_t *rm) { + request_matcher *rm) { while (rm->pending_head) { call_data *calld = rm->pending_head; rm->pending_head = calld->pending_next; @@ -355,7 +355,7 @@ static void request_matcher_zombify_all_pending_calls(grpc_exec_ctx *exec_ctx, static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx, grpc_server *server, - request_matcher_t *rm, + request_matcher *rm, grpc_error *error) { int request_id; for (size_t i = 0; i < server->cq_count; i++) { @@ -378,7 +378,7 @@ static void server_ref(grpc_server *server) { } static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { - registered_method_t *rm; + registered_method *rm; size_t i; grpc_channel_args_destroy(exec_ctx, server->channel_args); gpr_mu_destroy(&server->mu_global); @@ -387,7 +387,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { while ((rm = server->registered_methods) != NULL) { server->registered_methods = rm->next; if (server->started) { - request_matcher_destroy(&rm->request_matcher); + request_matcher_destroy(&rm->matcher); } gpr_free(rm->method); gpr_free(rm->host); @@ -519,7 +519,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, grpc_call_element *call_elem = (grpc_call_element *)arg; call_data *calld = (call_data *)call_elem->call_data; channel_data *chand = (channel_data *)call_elem->channel_data; - request_matcher_t *rm = calld->request_matcher; + request_matcher *rm = calld->matcher; grpc_server *server = rm->server; if (error != GRPC_ERROR_NONE || gpr_atm_acq_load(&server->shutdown_flag)) { @@ -569,7 +569,7 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, static void finish_start_new_rpc( grpc_exec_ctx *exec_ctx, grpc_server *server, grpc_call_element *elem, - request_matcher_t *rm, + request_matcher *rm, grpc_server_register_method_payload_handling payload_handling) { call_data *calld = (call_data *)elem->call_data; @@ -583,7 +583,7 @@ static void finish_start_new_rpc( return; } - calld->request_matcher = rm; + calld->matcher = rm; switch (payload_handling) { case GRPC_SRM_PAYLOAD_NONE: @@ -629,7 +629,7 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { continue; } finish_start_new_rpc(exec_ctx, server, elem, - &rm->server_registered_method->request_matcher, + &rm->server_registered_method->matcher, rm->server_registered_method->payload_handling); return; } @@ -647,7 +647,7 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { continue; } finish_start_new_rpc(exec_ctx, server, elem, - &rm->server_registered_method->request_matcher, + &rm->server_registered_method->matcher, rm->server_registered_method->payload_handling); return; } @@ -689,11 +689,11 @@ static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_REF(error)); request_matcher_zombify_all_pending_calls( exec_ctx, &server->unregistered_request_matcher); - for (registered_method_t *rm = server->registered_methods; rm; + for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { - request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher, + request_matcher_kill_requests(exec_ctx, server, &rm->matcher, GRPC_ERROR_REF(error)); - request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher); + request_matcher_zombify_all_pending_calls(exec_ctx, &rm->matcher); } } GRPC_ERROR_UNREF(error); @@ -1036,7 +1036,7 @@ void *grpc_server_register_method( grpc_server *server, const char *method, const char *host, grpc_server_register_method_payload_handling payload_handling, uint32_t flags) { - registered_method_t *m; + registered_method *m; GRPC_API_TRACE( "grpc_server_register_method(server=%p, method=%s, host=%s, " "flags=0x%08x)", @@ -1058,7 +1058,7 @@ void *grpc_server_register_method( flags); return NULL; } - m = (registered_method_t *)gpr_zalloc(sizeof(registered_method_t)); + m = (registered_method *)gpr_zalloc(sizeof(registered_method)); m->method = gpr_strdup(method); m->host = gpr_strdup(host); m->next = server->registered_methods; @@ -1113,9 +1113,8 @@ void grpc_server_start(grpc_server *server) { } request_matcher_init(&server->unregistered_request_matcher, (size_t)server->max_requested_calls_per_cq, server); - for (registered_method_t *rm = server->registered_methods; rm; - rm = rm->next) { - request_matcher_init(&rm->request_matcher, + for (registered_method *rm = server->registered_methods; rm; rm = rm->next) { + request_matcher_init(&rm->matcher, (size_t)server->max_requested_calls_per_cq, server); } @@ -1142,7 +1141,7 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, const grpc_channel_args *args) { size_t num_registered_methods; size_t alloc; - registered_method_t *rm; + registered_method *rm; channel_registered_method *crm; grpc_channel *channel; channel_data *chand; @@ -1371,7 +1370,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, grpc_server *server, size_t cq_idx, requested_call *rc) { call_data *calld = NULL; - request_matcher_t *rm = NULL; + request_matcher *rm = NULL; int request_id; if (gpr_atm_acq_load(&server->shutdown_flag)) { fail_call(exec_ctx, server, cq_idx, rc, @@ -1392,7 +1391,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx, rm = &server->unregistered_request_matcher; break; case REGISTERED_CALL: - rm = &rc->data.registered.registered_method->request_matcher; + rm = &rc->data.registered.method->matcher; break; } server->requested_calls_per_cq[cq_idx][request_id] = *rc; @@ -1483,7 +1482,7 @@ grpc_call_error grpc_server_request_registered_call( grpc_call_error error; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; requested_call *rc = (requested_call *)gpr_malloc(sizeof(*rc)); - registered_method_t *rm = (registered_method_t *)rmp; + registered_method *rm = (registered_method *)rmp; GRPC_STATS_INC_SERVER_REQUESTED_CALLS(&exec_ctx); GRPC_API_TRACE( "grpc_server_request_registered_call(" @@ -1521,7 +1520,7 @@ grpc_call_error grpc_server_request_registered_call( rc->tag = tag; rc->cq_bound_to_call = cq_bound_to_call; rc->call = call; - rc->data.registered.registered_method = rm; + rc->data.registered.method = rm; rc->data.registered.deadline = deadline; rc->initial_metadata = initial_metadata; rc->data.registered.optional_payload = optional_payload; -- cgit v1.2.3 From 2ed5d80ddac6b233ca4b7b4b6caa8a9115ea0560 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 13:07:58 -0700 Subject: Removing forward declaration --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 8bc6095712..ef16151b01 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -966,10 +966,6 @@ static grpc_channel_args *build_lb_channel_args( return result; } -static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, - void *arg, - grpc_error *error); - static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; GPR_ASSERT(glb_policy->pending_picks == NULL); -- cgit v1.2.3 From 2e2bf250fbf72ea22172cae7db2d3c985363958b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 13:32:48 -0700 Subject: Merging with master --- .../client_channel/lb_policy/grpclb/grpclb.c | 31 ++++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index ef16151b01..7e4f1b3487 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1666,7 +1666,7 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - glb_lb_policy *glb_policy = arg; + glb_lb_policy *glb_policy = (glb_lb_policy *)arg; /* If we receive a serverlist after the timer fires but before this callback * actually runs, don't do anything. */ if (glb_policy->serverlist != NULL) return; @@ -1804,6 +1804,17 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } + + // Propagate update to fallback_backend_addresses if a non-empty serverlist + // hasn't been received from the balancer. + if (glb_policy->serverlist == NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + if (glb_policy->rr_policy != NULL) { + rr_handover_locked(exec_ctx, glb_policy); + } + } } // Invoked as part of the update process. It continues watching the LB channel @@ -1886,13 +1897,7 @@ static const grpc_lb_policy_vtable glb_lb_policy_vtable = { static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. - * TODO(roth): For now, we ignore non-balancer addresses, but in the - * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. In the - * fallback case, we should use the LB policy indicated by - * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is - * unset, we should default to pick_first). */ + /* Count the number of gRPC-LB addresses. There must be at least one. */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1928,6 +1933,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); + glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( + arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, + INT_MAX}); + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -1936,6 +1946,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + /* Extract the backend addresses (may be empty) from the resolver for + * fallback. */ + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); -- cgit v1.2.3 From 6b84005651517c9136a32cec49db8333a5b40ee9 Mon Sep 17 00:00:00 2001 From: Alok Kumar Date: Wed, 13 Sep 2017 11:49:18 -0700 Subject: Add independent library targets for newly added .h files for microbenchmarks. Also fix includes in fullstack_fixtures.h --- test/cpp/microbenchmarks/BUILD | 34 ++++++++++++++++++---- test/cpp/microbenchmarks/bm_fullstack_trickle.cc | 1 - test/cpp/microbenchmarks/fullstack_fixtures.h | 1 + .../fullstack_streaming_ping_pong.h | 1 - .../cpp/microbenchmarks/fullstack_streaming_pump.h | 1 - .../microbenchmarks/fullstack_unary_ping_pong.h | 1 - 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 985a335f1b..0b69e9ba9a 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -73,12 +73,29 @@ grpc_cc_binary( deps = [":helpers"], ) +grpc_cc_library( + name = "fullstack_streaming_ping_pong_h", + testonly = 1, + hdrs = [ + "fullstack_streaming_ping_pong.h", + ], + deps = [":helpers"], +) + grpc_cc_binary( name = "bm_fullstack_streaming_ping_pong", testonly = 1, srcs = [ "bm_fullstack_streaming_ping_pong.cc", - "fullstack_streaming_ping_pong.h", + ], + deps = [":fullstack_streaming_ping_pong_h"], +) + +grpc_cc_library( + name = "fullstack_streaming_pump_h", + testonly = 1, + hdrs = [ + "fullstack_streaming_pump.h", ], deps = [":helpers"], ) @@ -88,9 +105,8 @@ grpc_cc_binary( testonly = 1, srcs = [ "bm_fullstack_streaming_pump.cc", - "fullstack_streaming_pump.h", ], - deps = [":helpers"], + deps = [":fullstack_streaming_pump_h"], ) grpc_cc_binary( @@ -103,14 +119,22 @@ grpc_cc_binary( ], ) +grpc_cc_library( + name = "fullstack_unary_ping_pong_h", + testonly = 1, + hdrs = [ + "fullstack_unary_ping_pong.h", + ], + deps = [":helpers"], +) + grpc_cc_binary( name = "bm_fullstack_unary_ping_pong", testonly = 1, srcs = [ "bm_fullstack_unary_ping_pong.cc", - "fullstack_unary_ping_pong.h", ], - deps = [":helpers"], + deps = [":fullstack_unary_ping_pong_h"], ) grpc_cc_binary( diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index 59fb29dd60..2656566a50 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -22,7 +22,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index 5477b860b4..ecd28c3f8a 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -41,6 +41,7 @@ extern "C" { #include "test/core/util/port.h" } +#include "src/cpp/client/create_channel_internal.h" #include "test/cpp/microbenchmarks/helpers.h" namespace grpc { diff --git a/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h b/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h index ff1f966753..6df044f344 100644 --- a/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h +++ b/test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_streaming_pump.h b/test/cpp/microbenchmarks/fullstack_streaming_pump.h index f9db212b02..9e826091ec 100644 --- a/test/cpp/microbenchmarks/fullstack_streaming_pump.h +++ b/test/cpp/microbenchmarks/fullstack_streaming_pump.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" diff --git a/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h b/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h index 76d278b2a0..d448938295 100644 --- a/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h +++ b/test/cpp/microbenchmarks/fullstack_unary_ping_pong.h @@ -24,7 +24,6 @@ #include #include #include "src/core/lib/profiling/timers.h" -#include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/cpp/microbenchmarks/fullstack_context_mutators.h" #include "test/cpp/microbenchmarks/fullstack_fixtures.h" -- cgit v1.2.3 From f1dc9c36a71ce11e0ebfda8553b5deb383d25107 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 14:21:27 -0700 Subject: Count api object creations --- src/core/ext/filters/client_channel/subchannel.c | 2 ++ src/core/lib/debug/stats_data.c | 6 ++++++ src/core/lib/debug/stats_data.h | 13 +++++++++++++ src/core/lib/debug/stats_data.yaml | 8 ++++++++ src/core/lib/surface/channel.c | 6 ++++++ src/core/lib/surface/completion_queue.c | 5 +++++ 6 files changed, 40 insertions(+) diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 05c55aaa89..bc9c3cc782 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -32,6 +32,7 @@ #include "src/core/ext/filters/client_channel/uri_parser.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/connected_channel.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" @@ -290,6 +291,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, return c; } + GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED(exec_ctx); c = (grpc_subchannel *)gpr_zalloc(sizeof(*c)); c->key = key; gpr_atm_no_barrier_store(&c->ref_pair, 1 << INTERNAL_REF_BITS); diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 3fd8ee38ef..1e60a33358 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -25,6 +25,10 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "client_calls_created", "server_calls_created", + "cqs_created", + "client_channels_created", + "client_subchannels_created", + "server_channels_created", "syscall_poll", "syscall_wait", "histogram_slow_lookups", @@ -60,6 +64,8 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client side calls created by this process", "Number of server side calls created by this process", + "Number of completion queues created", "Number of client channels created", + "Number of client subchannels created", "Number of server channels created", "Number of polling syscalls (epoll_wait, poll, etc) made by this process", "Number of sleeping syscalls made by this process", "Number of times histogram increments went through the slow (binary " diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index b7c15c08a5..ec4cbcce6c 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -27,6 +27,10 @@ typedef enum { GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED, GRPC_STATS_COUNTER_SERVER_CALLS_CREATED, + GRPC_STATS_COUNTER_CQS_CREATED, + GRPC_STATS_COUNTER_CLIENT_CHANNELS_CREATED, + GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED, + GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED, GRPC_STATS_COUNTER_SYSCALL_POLL, GRPC_STATS_COUNTER_SYSCALL_WAIT, GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS, @@ -104,6 +108,15 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) #define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED) +#define GRPC_STATS_INC_CQS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CQS_CREATED) +#define GRPC_STATS_INC_CLIENT_CHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CHANNELS_CREATED) +#define GRPC_STATS_INC_CLIENT_SUBCHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_CLIENT_SUBCHANNELS_CREATED) +#define GRPC_STATS_INC_SERVER_CHANNELS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CHANNELS_CREATED) #define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL) #define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a9d71f4fcb..7bb1d2e2ed 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -20,6 +20,14 @@ doc: Number of client side calls created by this process - counter: server_calls_created doc: Number of server side calls created by this process +- counter: cqs_created + doc: Number of completion queues created +- counter: client_channels_created + doc: Number of client channels created +- counter: client_subchannels_created + doc: Number of client subchannels created +- counter: server_channels_created + doc: Number of server channels created # polling - counter: syscall_poll doc: Number of polling syscalls (epoll_wait, poll, etc) made by this process diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 34548dac26..2f9b677c85 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -27,6 +27,7 @@ #include #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" @@ -77,6 +78,11 @@ grpc_channel *grpc_channel_create_with_builder( grpc_channel_args *args = grpc_channel_args_copy( grpc_channel_stack_builder_get_channel_arguments(builder)); grpc_channel *channel; + if (channel_stack_type == GRPC_SERVER_CHANNEL) { + GRPC_STATS_INC_SERVER_CHANNELS_CREATED(exec_ctx); + } else { + GRPC_STATS_INC_CLIENT_CHANNELS_CREATED(exec_ctx); + } grpc_error *error = grpc_channel_stack_builder_finish( exec_ctx, builder, sizeof(grpc_channel), 1, destroy_channel, NULL, (void **)&channel); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 4726503994..3205ddeadf 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -26,6 +26,7 @@ #include #include +#include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" @@ -420,6 +421,10 @@ grpc_completion_queue *grpc_completion_queue_create_internal( const cq_poller_vtable *poller_vtable = &g_poller_vtable_by_poller_type[polling_type]; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_STATS_INC_CQS_CREATED(&exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); + cq = (grpc_completion_queue *)gpr_zalloc(sizeof(grpc_completion_queue) + vtable->data_size + poller_vtable->size()); -- cgit v1.2.3 From 1a012bbe8fceacfff9a2fc49528ad07de75743a2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 14:29:00 -0700 Subject: Review feedback --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8f855a7b6e..aa861d3cc1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -873,6 +873,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); SET_KICK_STATE(worker->next, DESIGNATED_POLLER); + GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV(exec_ctx); gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { gpr_mu_unlock(&pollset->mu); @@ -1098,7 +1099,7 @@ static grpc_error *pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, goto done; } } else { - GRPC_STATS_INC_POLLSET_KICKED_AGAIN(exec_ctx); + GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD(exec_ctx); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked while waking up"); } -- cgit v1.2.3 From 324ad6a6c239ebb6dfafbe00a825a90a409daa11 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Wed, 13 Sep 2017 15:10:08 -0700 Subject: Update service_config doc The current grpclb fallback implementation is to continue using the internal round_robin policy but feed it with the backends from the resolver. We didn't fall back to the policy indicated in service config (or client API) because that's too complex and has more different behavior from the normal situation where we do connect to the balancer. --- doc/service_config.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/service_config.md b/doc/service_config.md index 99d985f3bf..0abbd7f324 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -24,10 +24,7 @@ The service config is a JSON string of the following form: // opposed to backend addresses), gRPC will use grpclb (see // https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), // regardless of what LB policy is requested either here or via the - // client API. However, if the resolver returns at least one backend - // address in addition to the balancer address(es), the client may fall - // back to the requested policy if it is unable to reach any of the - // grpclb load balancers. + // client API. 'loadBalancingPolicy': string, // Per-method configuration. Optional. -- cgit v1.2.3 From fb0262b0fda6b3f605ef8f6a65ed628f22180d54 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:19:19 -0700 Subject: Eliminate lock in self-scheduling path in executor --- src/core/lib/iomgr/executor.c | 57 ++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index 892385d7d7..edc391331f 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -38,10 +38,10 @@ typedef struct { gpr_mu mu; gpr_cv cv; grpc_closure_list elems; - size_t depth; bool shutdown; bool queued_long_job; gpr_thd_id id; + grpc_closure_list local_elems; } thread_state; static thread_state *g_thread_state; @@ -56,32 +56,30 @@ static grpc_tracer_flag executor_trace = static void executor_thread(void *arg); -static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { - size_t n = 0; - - grpc_closure *c = list.head; - while (c != NULL) { - grpc_closure *next = c->next_data.next; - grpc_error *error = c->error_data.error; - if (GRPC_TRACER_ON(executor_trace)) { +static void run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { + while (!grpc_closure_list_empty(*list)) { + grpc_closure *c = list->head; + grpc_closure_list_init(list); + while (c != NULL) { + grpc_closure *next = c->next_data.next; + grpc_error *error = c->error_data.error; + if (GRPC_TRACER_ON(executor_trace)) { #ifndef NDEBUG - gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, - c->file_created, c->line_created); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, + c->file_created, c->line_created); #else - gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); #endif - } + } #ifndef NDEBUG - c->scheduled = false; + c->scheduled = false; #endif - c->cb(exec_ctx, c->cb_arg, error); - GRPC_ERROR_UNREF(error); - c = next; - n++; - grpc_exec_ctx_flush(exec_ctx); + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + c = next; + grpc_exec_ctx_flush(exec_ctx); + } } - - return n; } bool grpc_executor_is_threaded() { @@ -126,7 +124,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) { for (size_t i = 0; i < g_max_threads; i++) { gpr_mu_destroy(&g_thread_state[i].mu); gpr_cv_destroy(&g_thread_state[i].cv); - run_closures(exec_ctx, g_thread_state[i].elems); + run_closures(exec_ctx, &g_thread_state[i].elems); } gpr_free(g_thread_state); gpr_tls_destroy(&g_this_thread_state); @@ -150,14 +148,11 @@ static void executor_thread(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); - size_t subtract_depth = 0; for (;;) { if (GRPC_TRACER_ON(executor_trace)) { - gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", - (int)(ts - g_thread_state), subtract_depth); + gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step", (int)(ts - g_thread_state)); } gpr_mu_lock(&ts->mu); - ts->depth -= subtract_depth; while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { ts->queued_long_job = false; gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); @@ -171,14 +166,15 @@ static void executor_thread(void *arg) { break; } GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx); - grpc_closure_list exec = ts->elems; + GPR_ASSERT(grpc_closure_list_empty(ts->local_elems)); + ts->local_elems = ts->elems; ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; gpr_mu_unlock(&ts->mu); if (GRPC_TRACER_ON(executor_trace)) { gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state)); } - subtract_depth = run_closures(&exec_ctx, exec); + run_closures(&exec_ctx, &ts->local_elems); } grpc_exec_ctx_finish(&exec_ctx); } @@ -211,6 +207,8 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); + grpc_closure_list_append(&ts->local_elems, closure, error); + return; } thread_state *orig_ts = ts; @@ -250,8 +248,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, gpr_cv_signal(&ts->cv); } grpc_closure_list_append(&ts->elems, closure, error); - ts->depth++; - try_new_thread = ts->depth > MAX_DEPTH && + try_new_thread = ts->elems.head != closure && cur_thread_count < g_max_threads && !ts->shutdown; if (!is_short) ts->queued_long_job = true; gpr_mu_unlock(&ts->mu); -- cgit v1.2.3 From 21b2c189d2b09b72c1afd9e5ff3cebb0b9b70a08 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:21:10 -0700 Subject: Remove old constant --- src/core/lib/iomgr/executor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index edc391331f..fa8c5bfaab 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -32,8 +32,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/spinlock.h" -#define MAX_DEPTH 2 - typedef struct { gpr_mu mu; gpr_cv cv; -- cgit v1.2.3 From 213d2744b543cf8593838a5eb995bf9c7924c7d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:22:14 -0700 Subject: Only use fast path for short items --- src/core/lib/iomgr/executor.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index fa8c5bfaab..e5372c7b08 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -205,8 +205,10 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); - grpc_closure_list_append(&ts->local_elems, closure, error); - return; + if (!is_short) { + grpc_closure_list_append(&ts->local_elems, closure, error); + return; + } } thread_state *orig_ts = ts; -- cgit v1.2.3 From 7f10c299c7eb5d6688c20ae43c63b0548ca8acd1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 15:26:56 -0700 Subject: Windows --- src/core/lib/iomgr/pollset_windows.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index ea017a6054..eb295d3eeb 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -98,7 +98,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { pollset->shutting_down = 1; - grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + grpc_pollset_kick(exec_ctx, pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_NONE); } else { @@ -181,7 +181,7 @@ done: return GRPC_ERROR_NONE; } -grpc_error *grpc_pollset_kick(grpc_pollset *p, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { @@ -209,7 +209,7 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, specific_worker = pop_front_worker(&p->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); if (specific_worker != NULL) { - grpc_pollset_kick(p, specific_worker); + grpc_pollset_kick(exec_ctx, p, specific_worker); } else if (p->is_iocp_worker) { grpc_iocp_kick(); } else { -- cgit v1.2.3 From c7cebe8d8c17df2d0d2f97e402294b2717e2ff73 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 13 Sep 2017 17:51:26 -0700 Subject: Consistent acq/rel semantics for pending_events atomic --- src/core/lib/surface/completion_queue.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 4726503994..c4eff350da 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -444,7 +444,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal( static void cq_init_next(void *ptr) { cq_next_data *cqd = (cq_next_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_no_barrier_store(&cqd->pending_events, 1); + gpr_atm_rel_store(&cqd->pending_events, 1); cqd->shutdown_called = false; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); cq_event_queue_init(&cqd->queue); @@ -459,7 +459,7 @@ static void cq_destroy_next(void *ptr) { static void cq_init_pluck(void *ptr) { cq_pluck_data *cqd = (cq_pluck_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_no_barrier_store(&cqd->pending_events, 1); + gpr_atm_rel_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; cqd->completed_head.next = (uintptr_t)cqd->completed_tail; gpr_atm_no_barrier_store(&cqd->shutdown, 0); @@ -560,13 +560,13 @@ static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {} * true if the increment was successful; false if the counter is zero */ static bool atm_inc_if_nonzero(gpr_atm *counter) { while (true) { - gpr_atm count = gpr_atm_no_barrier_load(counter); + gpr_atm count = gpr_atm_acq_load(counter); /* If zero, we are done. If not, we must to a CAS (instead of an atomic * increment) to maintain the contract: do not increment the counter if it * is zero. */ if (count == 0) { return false; - } else if (gpr_atm_no_barrier_cas(counter, count, count + 1)) { + } else if (gpr_atm_full_cas(counter, count, count + 1)) { break; } } @@ -638,8 +638,7 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, /* Add the completion to the queue */ bool is_first = cq_event_queue_push(&cqd->queue, storage); gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); - bool will_definitely_shutdown = - gpr_atm_no_barrier_load(&cqd->pending_events) == 1; + bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1; if (!will_definitely_shutdown) { /* Only kick if this is the first item queued */ @@ -883,7 +882,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } } - if (gpr_atm_no_barrier_load(&cqd->pending_events) == 0) { + if (gpr_atm_acq_load(&cqd->pending_events) == 0) { /* Before returning, check if the queue has any items left over (since gpr_mpscq_pop() can sometimes return NULL even if the queue is not empty. If so, keep retrying but do not return GRPC_QUEUE_SHUTDOWN */ @@ -929,7 +928,7 @@ static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline, } if (cq_event_queue_num_items(&cqd->queue) > 0 && - gpr_atm_no_barrier_load(&cqd->pending_events) > 0) { + gpr_atm_acq_load(&cqd->pending_events) > 0) { gpr_mu_lock(cq->mu); cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL); gpr_mu_unlock(cq->mu); @@ -956,7 +955,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, cq_next_data *cqd = DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); - GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); + GPR_ASSERT(gpr_atm_acq_load(&cqd->pending_events) == 0); cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); -- cgit v1.2.3 From 319652fa85c01543d6695ba721d8fc82b9860726 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 13 Sep 2017 18:13:48 -0700 Subject: Track pings in bmdiff --- tools/profiling/microbenchmarks/bm_diff/bm_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py index ad79a0a197..f8989b17fc 100644 --- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -28,4 +28,4 @@ _INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration', 'atm_cas_per_iteration', 'atm_add_per_iteration', 'nows_per_iteration', 'cli_transport_stalls_per_iteration', 'cli_stream_stalls_per_iteration', 'svr_transport_stalls_per_iteration', - 'svr_stream_stalls_per_iteration',) + 'svr_stream_stalls_per_iteration', 'http2_pings_sent_per_iteration') -- cgit v1.2.3 From 438304960f95169a7a0272ff684b80805ced9cba Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 13 Sep 2017 18:27:55 -0700 Subject: Add allow-ssh tags to newly created GCE VMs --- tools/gce/create_interop_worker.sh | 3 ++- tools/gce/create_linux_performance_worker.sh | 3 ++- tools/gce/create_linux_worker.sh | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh index 76e4905272..3e59dc501a 100755 --- a/tools/gce/create_interop_worker.sh +++ b/tools/gce/create_interop_worker.sh @@ -33,7 +33,8 @@ gcloud compute instances create $INSTANCE_NAME \ --machine-type n1-standard-16 \ --image ubuntu-15-10 \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo + --scopes https://www.googleapis.com/auth/xapi.zoo \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh index 7f53732c05..4270f5cbfb 100755 --- a/tools/gce/create_linux_performance_worker.sh +++ b/tools/gce/create_linux_performance_worker.sh @@ -36,7 +36,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image-project ubuntu-os-cloud \ --image-family ubuntu-1704 \ --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index c96f3281fc..8bf44aa729 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -31,7 +31,8 @@ gcloud compute instances create $INSTANCE_NAME \ --image=ubuntu-1510 \ --image-project=grpc-testing \ --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/bigquery + --scopes https://www.googleapis.com/auth/bigquery \ + --tags=allow-ssh echo 'Created GCE instance, waiting 60 seconds for it to come online.' sleep 60 -- cgit v1.2.3 From 01e83b55e4700459ae82efeac088b512d5dfabd7 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 13 Sep 2017 19:45:07 -0700 Subject: Revert "Add fallback (use backends from resolver if can't reach balancer) to grpclb." This reverts commit aba0a0a54412fe59bc2090334ae26c99bb148097. --- include/grpc++/support/channel_arguments.h | 6 - include/grpc/impl/codegen/grpc_types.h | 6 +- .../client_channel/lb_policy/grpclb/grpclb.c | 197 +++++---------------- .../ext/filters/client_channel/lb_policy_factory.c | 2 +- .../ext/filters/client_channel/lb_policy_factory.h | 2 +- src/cpp/common/channel_arguments.cc | 4 - test/cpp/end2end/grpclb_end2end_test.cc | 74 +------- 7 files changed, 47 insertions(+), 244 deletions(-) diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 9dc505f008..7b6befeaf1 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,12 +64,6 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); - /// Set the grpclb fallback timeout (in ms) for the channel. If this amount - /// of time has passed but we have not gotten any non-empty \a serverlist from - /// the balancer, we will fall back to use the backend address(es) returned by - /// the resolver. - void SetGrpclbFallbackTimeout(int fallback_timeout); - /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index beb6080c94..748dc717a3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -287,11 +287,7 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" -/* Timeout in milliseconds to wait for the serverlist from the grpclb load - balancer before using fallback backend addresses from the resolver. - If 0, fallback will never be used. */ -#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 18979829bd..5aafed1374 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -122,7 +122,6 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -299,10 +298,6 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; - /** timeout in milliseconds for before using fallback backend addresses. - * 0 means not using fallback. */ - int lb_fallback_timeout_ms; - /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -329,9 +324,6 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; - /** stores the backend addresses from the resolver */ - grpc_lb_addresses *fallback_backend_addresses; - /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -352,9 +344,6 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; - /** is \a lb_fallback_timer active? */ - bool fallback_timer_active; - /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -377,9 +366,6 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; - /* LB fallback timer callback. */ - grpc_closure lb_on_fallback; - grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -403,9 +389,6 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; - /** LB fallback timer */ - grpc_timer lb_fallback_timer; - bool initial_request_sent; bool seen_initial_response; @@ -552,32 +535,6 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } -/* Returns the backend addresses extracted from the given addresses */ -static grpc_lb_addresses *extract_backend_addresses_locked( - grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { - /* first pass: count the number of backend addresses */ - size_t num_backends = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (!addresses->addresses[i].is_balancer) { - ++num_backends; - } - } - /* second pass: actually populate the addresses and (empty) LB tokens */ - grpc_lb_addresses *backend_addresses = - grpc_lb_addresses_create(num_backends, &lb_token_vtable); - size_t num_copied = 0; - for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (addresses->addresses[i].is_balancer) continue; - const grpc_resolved_address *addr = &addresses->addresses[i].address; - grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, - addr->len, false /* is_balancer */, - NULL /* balancer_name */, - (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); - ++num_copied; - } - return backend_addresses; -} - static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -645,38 +602,35 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Check for drops if we are not using fallback backend addresses. - if (glb_policy->serverlist != NULL) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. + } + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); - } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked( - server->load_balance_token, wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); - gpr_free(wc_arg->free_when_done); - return false; - } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, + wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); gpr_free(wc_arg->free_when_done); - return true; + return false; } + gpr_free(wc_arg->free_when_done); + return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -714,18 +668,8 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses; - if (glb_policy->serverlist != NULL) { - GPR_ASSERT(glb_policy->serverlist->num_servers > 0); - addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); - } else { - // If rr_handover_locked() is invoked when we haven't received any - // serverlist from the balancer, we use the fallback backends returned by - // the resolver. Note that the fallback backend list may be empty, in which - // case the new round_robin policy will keep the requested picks pending. - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); - } + grpc_lb_addresses *addresses = + process_serverlist_locked(exec_ctx, glb_policy->serverlist); GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -831,6 +775,8 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + GPR_ASSERT(glb_policy->serverlist != NULL && + glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -971,7 +917,13 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx, static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. */ + /* Count the number of gRPC-LB addresses. There must be at least one. + * TODO(roth): For now, we ignore non-balancer addresses, but in the + * future, we may change the behavior such that we fall back to using + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1007,11 +959,6 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); - arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); - glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( - arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, - INT_MAX}); - // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = @@ -1020,11 +967,6 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); - /* Extract the backend addresses (may be empty) from the resolver for - * fallback. */ - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); @@ -1068,9 +1010,6 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } - if (glb_policy->fallback_backend_addresses != NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); if (glb_policy->pending_update_args != NULL) { grpc_channel_args_destroy(exec_ctx, glb_policy->pending_update_args->args); @@ -1211,28 +1150,10 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - /* start a timer to fall back */ - if (glb_policy->lb_fallback_timeout_ms > 0 && - glb_policy->serverlist == NULL) { - gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec deadline = gpr_time_add( - now, - gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); - GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, - glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->fallback_timer_active = true; - grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, - &glb_policy->lb_on_fallback, now); - } - glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1686,15 +1607,6 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); - } else { - /* or dispose of the fallback */ - grpc_lb_addresses_destroy(exec_ctx, - glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = NULL; - if (glb_policy->fallback_timer_active) { - grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); - glb_policy->fallback_timer_active = false; - } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1705,7 +1617,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Received empty server list, ignoring."); + gpr_log(GPR_INFO, + "Received empty server list. Picks will stay pending until " + "a response with > 0 servers is received"); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1752,26 +1666,6 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); } -static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - glb_lb_policy *glb_policy = arg; - /* If we receive a serverlist after the timer fires but before this callback - * actually runs, don't do anything. */ - if (glb_policy->serverlist != NULL) return; - glb_policy->fallback_timer_active = false; - if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Falling back to use backends from resolver (grpclb %p)", - (void *)glb_policy); - } - GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); - rr_handover_locked(exec_ctx, glb_policy); - } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, - "grpclb_fallback_timer"); -} - static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1892,17 +1786,6 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, &glb_policy->lb_channel_connectivity, &glb_policy->lb_channel_on_connectivity_changed, NULL); } - - // Propagate update to fallback_backend_addresses if a non-empty serverlist - // hasn't been received from the balancer. - if (glb_policy->serverlist == NULL) { - grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); - glb_policy->fallback_backend_addresses = - extract_backend_addresses_locked(exec_ctx, addresses); - if (glb_policy->rr_policy != NULL) { - rr_handover_locked(exec_ctx, glb_policy); - } - } } // Invoked as part of the update process. It continues watching the LB channel diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index 918bab745c..cdcaf17544 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - const void* address, size_t address_len, + void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index cf0f8cb615..9d9fb143df 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - const void *address, size_t address_len, + void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f89f5f1f03..f130aecd4b 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,10 +86,6 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } -void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { - SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); -} - void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 17a094f7a2..570a3d1067 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,9 +368,8 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub(int fallback_timeout = 0) { + void ResetStub() { ChannelArguments args; - args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -442,10 +441,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector GetBackendPorts(const size_t start_index = 0) const { + const std::vector GetBackendPorts() const { std::vector backend_ports; - for (size_t i = start_index; i < backend_servers_.size(); ++i) { - backend_ports.push_back(backend_servers_[i].port_); + for (const auto& bs : backend_servers_) { + backend_ports.push_back(bs.port_); } return backend_ports; } @@ -616,71 +615,6 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } -TEST_F(SingleBalancerTest, Fallback) { - const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); - const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); - const size_t kNumBackendInResolution = backends_.size() / 2; - - ResetStub(kFallbackTimeoutMs); - std::vector addresses; - addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); - } - SetNextResolution(addresses); - - // Send non-empty serverlist only after kServerlistDelayMs - ScheduleResponseForBalancer( - 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(kNumBackendInResolution /* start_index */), {}), - kServerlistDelayMs); - - // The first request. The client will block while it's still trying to - // contact the balancer. - gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); - CheckRpcSendOk(kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); - - // Fallback is used: each backend returned by the resolver should have - // gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - - // Wait until update has been processed, as signaled by the backend returned - // by the balancer receiving a request. - do { - CheckRpcSendOk(1); - } while ( - backend_servers_[kNumBackendInResolution].service_->request_count() == 0); - for (size_t i = 0; i < backends_.size(); ++i) { - backend_servers_[i].service_->ResetCounters(); - } - - // Send out the second request. - gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); - CheckRpcSendOk(backends_.size() - kNumBackendInResolution); - gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); - - // Serverlist is used: each backend returned by the balancer should - // have gotten one request. - for (size_t i = 0; i < kNumBackendInResolution; ++i) { - EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); - } - for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); - } - - balancers_[0]->NotifyDoneWithServerlists(); - // The balancer got a single request. - EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); - // and sent a single response. - EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); -} - TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( -- cgit v1.2.3 From 759c7029b18f8bbcf6a6399d512abed8b6e12436 Mon Sep 17 00:00:00 2001 From: Vladimir Moskva Date: Thu, 14 Sep 2017 15:40:27 +0200 Subject: Rename set to depset --- bazel/generate_cc.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index 7fffb58218..f88ee2f56f 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -55,7 +55,7 @@ def generate_cc_impl(ctx): arguments = arguments, ) - return struct(files=set(out_files)) + return struct(files=depset(out_files)) _generate_cc = rule( attrs = { -- cgit v1.2.3 From 5432dd887430984713f9fc3ad595c85e995d6451 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 14 Sep 2017 17:55:42 +0200 Subject: Revert "Allow SerializationTraits to use grpc::ByteBuffer rather than only grpc_byte_buffer" --- BUILD | 2 - CMakeLists.txt | 15 --- Makefile | 15 --- build.yaml | 2 - gRPC-Core.podspec | 1 - grpc.def | 20 +-- grpc.gemspec | 1 - include/grpc++/impl/codegen/byte_buffer.h | 141 --------------------- include/grpc++/impl/codegen/call.h | 114 ++++------------- include/grpc++/impl/codegen/method_handler_impl.h | 9 +- include/grpc++/impl/codegen/rpc_service_method.h | 14 +- include/grpc++/impl/codegen/serialization_traits.h | 25 ++-- include/grpc++/impl/codegen/slice.h | 78 ------------ include/grpc++/support/byte_buffer.h | 68 +++++++++- include/grpc++/support/slice.h | 80 +++++++++++- include/grpc/byte_buffer.h | 64 +++++++++- include/grpc/impl/codegen/byte_buffer.h | 86 ------------- package.xml | 1 - src/core/lib/iomgr/resource_quota.c | 1 - .../server/health/default_health_check_service.cc | 1 - src/cpp/util/byte_buffer_cc.cc | 37 ++---- src/cpp/util/slice_cc.cc | 2 - src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 +++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++----- test/core/surface/public_headers_must_be_c89.c | 1 - test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 - tools/doxygen/Doxyfile.c++.internal | 2 - tools/doxygen/Doxyfile.core | 1 - tools/doxygen/Doxyfile.core.internal | 1 - tools/run_tests/generated/sources_and_headers.json | 4 - 31 files changed, 330 insertions(+), 565 deletions(-) delete mode 100644 include/grpc++/impl/codegen/byte_buffer.h delete mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index d76b821be9..281375fc1d 100644 --- a/BUILD +++ b/BUILD @@ -989,7 +989,6 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1488,7 +1487,6 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index eac63a9fe3..fa1d7d87dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1238,7 +1238,6 @@ target_link_libraries(grpc ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1545,7 +1544,6 @@ target_link_libraries(grpc_cronet ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1822,7 +1820,6 @@ target_link_libraries(grpc_test_util ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2083,7 +2080,6 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2379,7 +2375,6 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2689,7 +2684,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2700,7 +2694,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3183,7 +3176,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3194,7 +3186,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3555,7 +3546,6 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3583,7 +3573,6 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3695,7 +3684,6 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3723,7 +3711,6 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3926,7 +3913,6 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h - include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3937,7 +3923,6 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h - include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index 1ed1877f77..0c3648566a 100644 --- a/Makefile +++ b/Makefile @@ -3190,7 +3190,6 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3497,7 +3496,6 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3775,7 +3773,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4027,7 +4024,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4300,7 +4296,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4589,7 +4584,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4600,7 +4594,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5084,7 +5077,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5095,7 +5087,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5449,7 +5440,6 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5477,7 +5467,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5566,7 +5555,6 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5594,7 +5582,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5802,7 +5789,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ - include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5813,7 +5799,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ - include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index 92ca2b6237..d15cf0339c 100644 --- a/build.yaml +++ b/build.yaml @@ -502,7 +502,6 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: - - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -970,7 +969,6 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h - - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 6f84ef235d..2f1f415866 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,7 +141,6 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', - 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index 558be60c3c..a7a11601ed 100644 --- a/grpc.def +++ b/grpc.def @@ -1,4 +1,14 @@ EXPORTS + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -135,16 +145,6 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index a226e9bc31..2d0f9fd450 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -145,7 +145,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) - s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h deleted file mode 100644 index 87d390c688..0000000000 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H -#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H - -#include - -#include -#include -#include -#include -#include - -#include - -namespace grpc { - -template -class CallOpRecvMessage; -class MethodHandler; -namespace internal { -template -class MessageDeserializer; -} - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer() { - if (buffer_) { - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); - } - } - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear() { - if (buffer_) { - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } - } - - /// Make a duplicate copy of the internals of this byte - /// buffer so that we have our own owned version of it. - /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable - void Duplicate() { - buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); - } - - /// Forget underlying byte buffer without destroying - /// Use this only for un-owned byte buffers - void Release() { buffer_ = nullptr; } - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - /// Is this ByteBuffer valid? - bool Valid() const { return (buffer_ != nullptr); } - - private: - friend class SerializationTraits; - friend class CallOpSendMessage; - template - friend class CallOpRecvMessage; - friend class CallOpGenericRecvMessage; - friend class MethodHandler; - template - friend class internal::MessageDeserializer; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - grpc_byte_buffer* c_buffer() { return buffer_; } - grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } - - // DEPRECATED: Implicit conversion to transparently - // support deprecated SerializationTraits API - // No need to inline since deprecated - operator grpc_byte_buffer*(); - operator const grpc_byte_buffer*() const; - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer.buffer_); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, - bool* own_buffer) { - *buffer = source; - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - -#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 3c418c7ae2..74ed5cbfb9 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -40,6 +39,8 @@ #include #include +struct grpc_byte_buffer; + namespace grpc { class ByteBuffer; @@ -280,7 +281,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_() {} + CallOpSendMessage() : send_buf_(nullptr) {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -293,67 +294,33 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (!send_buf_.Valid()) return; + if (send_buf_ == nullptr) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_.c_buffer(); + op->data.send_message.send_message = send_buf_; // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { send_buf_.Clear(); } + void FinishOp(bool* status) { + g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); + send_buf_ = nullptr; + } private: - template - class MessageSerializer; - - ByteBuffer send_buf_; + grpc_byte_buffer* send_buf_; WriteOptions write_options_; }; -namespace internal { -template -T Example(); -} // namespace internal - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), - own_buf); - } -}; - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example<::grpc::ByteBuffer*>(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf, own_buf); - } -}; - template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); + SerializationTraits::Serialize(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_.Duplicate(); + send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); } return result; } @@ -363,36 +330,6 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } -namespace internal { -template -class MessageDeserializer; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize(bbuf, message); - } -}; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize( - const_cast(bbuf).c_buffer(), message); - } -}; -} // namespace internal - template class CallOpRecvMessage { public: @@ -415,20 +352,18 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); + op->data.recv_message.recv_message = &recv_buf_; } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_.Valid()) { + if (recv_buf_) { if (*status) { got_message = *status = - internal::MessageDeserializer::Deserialize(recv_buf_, message_) - .ok(); - recv_buf_.Release(); + SerializationTraits::Deserialize(recv_buf_, message_).ok(); } else { got_message = false; - recv_buf_.Clear(); + g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); } } else { got_message = false; @@ -441,14 +376,14 @@ class CallOpRecvMessage { private: R* message_; - ByteBuffer recv_buf_; + grpc_byte_buffer* recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(const ByteBuffer& buf) = 0; + virtual Status Deserialize(grpc_byte_buffer* buf) = 0; virtual ~DeserializeFunc() {} }; @@ -456,8 +391,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(const ByteBuffer& buf) override { - return grpc::internal::MessageDeserializer::Deserialize(buf, message_); + Status Deserialize(grpc_byte_buffer* buf) override { + return SerializationTraits::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -493,19 +428,18 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); + op->data.recv_message.recv_message = &recv_buf_; } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_.Valid()) { + if (recv_buf_) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); - recv_buf_.Release(); } else { got_message = false; - recv_buf_.Clear(); + g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); } } else { got_message = false; @@ -518,7 +452,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - ByteBuffer recv_buf_; + grpc_byte_buffer* recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 8125e0a651..15e24bdcdc 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,7 +19,6 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H -#include #include #include #include @@ -38,8 +37,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = + SerializationTraits::Deserialize(param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -124,8 +123,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = + SerializationTraits::Deserialize(param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index d356012ad6..7165774172 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,11 +25,14 @@ #include #include -#include #include #include #include +extern "C" { +struct grpc_byte_buffer; +} + namespace grpc { class ServerContext; class StreamContextInterface; @@ -40,14 +43,11 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context) { - request.set_buffer(req); - } - ~HandlerParameter() { request.Release(); } + : call(c), server_context(context), request(req) {} Call* call; ServerContext* server_context; - // Handler required to destroy these contents - ByteBuffer request; + // Handler required to grpc_byte_buffer_destroy this + grpc_byte_buffer* request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index 7fc4e24172..b72d474126 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,24 +26,15 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// 1. static Status Serialize(const Message& msg, -/// ByteBuffer* buffer, -/// bool* own_buffer); -/// AND/OR -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// The former is preferred; the latter is deprecated +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg, +/// int max_receive_message_size); /// -/// 2. static Status Deserialize(const ByteBuffer& buffer, -/// Message* msg); -/// AND/OR -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg); -/// The former is preferred; the latter is deprecated -/// -/// Serialize is required to convert message to a ByteBuffer, and -/// return that byte buffer through *buffer. *own_buffer should +/// Serialize is required to convert message to a grpc_byte_buffer, and +/// to store a pointer to that byte buffer at *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index c185bf4fd0..e682bdef6a 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,89 +19,11 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H -#include #include #include -#include - namespace grpc { -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const; - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index 81fa3b0a18..df16c6a75e 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,7 +19,6 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H -#include #include #include #include @@ -28,4 +27,71 @@ #include #include +#include + +namespace grpc { + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer(); + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear(); + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + private: + friend class SerializationTraits; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + // For \a SerializationTraits's usage. + grpc_byte_buffer* buffer() const { return buffer_; } + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, + bool* own_buffer) { + *buffer = grpc_byte_buffer_copy(source.buffer()); + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index 10db10d79c..bbf97f280e 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,8 +19,86 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H -#include #include #include +namespace grpc { + +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const { return grpc_slice_ref(slice_); } + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + +} // namespace grpc + #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 7669582af2..55e191da31 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,7 +19,69 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h deleted file mode 100644 index fc33305713..0000000000 --- a/include/grpc/impl/codegen/byte_buffer.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H -#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - -#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 4f67e46673..b7c0e679e5 100644 --- a/package.xml +++ b/package.xml @@ -155,7 +155,6 @@ - diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 4d69986fbc..4895e0d1c9 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,7 +22,6 @@ #include #include -#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index d2cba6d662..815b607032 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,7 +20,6 @@ #include #include -#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index 1ebe22b6a4..b1ff25252a 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,15 +16,11 @@ * */ -#include #include -#include #include namespace grpc { -static internal::GrpcLibraryInitializer g_gli_initializer; - ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -33,7 +29,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); - g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -42,6 +37,19 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } +ByteBuffer::~ByteBuffer() { + if (buffer_) { + grpc_byte_buffer_destroy(buffer_); + } +} + +void ByteBuffer::Clear() { + if (buffer_) { + grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } +} + Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -72,9 +80,7 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - if (this != &buf) { - Clear(); // first remove existing data - } + Clear(); // first remove existing data if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -87,19 +93,4 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } -ByteBuffer::operator grpc_byte_buffer*() { - // The following assertions check that the representation of a ByteBuffer is - // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, - // and nothing else. - static_assert(std::is_same::value, - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - return buffer_; -} - -ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } - } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 3ae17e8052..486d0cdf0e 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,6 +50,4 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} -grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } - } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 70831494fa..57b543967e 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,6 +22,16 @@ #include "rb_grpc_imports.generated.h" +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -158,16 +168,6 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -330,6 +330,16 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -466,16 +476,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 263057f309..c5c848ae44 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include +#include #include #include #include #include #include -#include #include #include #include @@ -47,6 +47,36 @@ #include #include +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -455,36 +485,6 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index d36d116afb..0d7f68c0ad 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index 8fb51bc663..cac01a7307 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - ByteBuffer send_buffer; + grpc_byte_buffer* send_buffer = nullptr; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,7 +101,8 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer.Valid()); + EXPECT_TRUE(send_buffer != nullptr); + grpc_byte_buffer_destroy(send_buffer); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index d81b7b4d11..62f113907d 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,7 +811,6 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ -include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,7 +881,6 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 263c6c1afb..49919415bd 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,7 +811,6 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ -include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -883,7 +882,6 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index b8514fe311..632735342b 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,7 +811,6 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 77cd28e9df..e352cb78f1 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,7 +811,6 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ -include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 5581240a9a..2f6e34bfb3 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8395,7 +8395,6 @@ "gpr_codegen" ], "headers": [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8409,7 +8408,6 @@ "language": "c", "name": "grpc_codegen", "src": [ - "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9288,7 +9286,6 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9323,7 +9320,6 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", - "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", -- cgit v1.2.3 From f68978c41f13197171325d944f971db7d9d85a54 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Thu, 14 Sep 2017 09:18:24 -0700 Subject: Pass exec_ctx in TSI zero_copy_protector create --- src/core/lib/security/transport/security_handshaker.c | 2 +- src/core/tsi/fake_transport_security.c | 3 ++- src/core/tsi/transport_security.h | 10 ++++++++-- src/core/tsi/transport_security_grpc.c | 8 +++++--- src/core/tsi/transport_security_grpc.h | 3 ++- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index 975d599523..3d19605617 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -137,7 +137,7 @@ static void on_peer_checked_inner(grpc_exec_ctx *exec_ctx, // Create zero-copy frame protector, if implemented. tsi_zero_copy_grpc_protector *zero_copy_protector = NULL; tsi_result result = tsi_handshaker_result_create_zero_copy_grpc_protector( - h->handshaker_result, NULL, &zero_copy_protector); + exec_ctx, h->handshaker_result, NULL, &zero_copy_protector); if (result != TSI_OK && result != TSI_UNIMPLEMENTED) { error = grpc_set_tsi_error_result( GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index e7b3be3d86..64043fea08 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -493,7 +493,8 @@ static tsi_result fake_handshaker_result_extract_peer( } static tsi_result fake_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + void *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { *protector = tsi_create_fake_zero_copy_grpc_protector(max_output_protected_frame_size); diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index b0d7039850..3bba38149c 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -84,11 +84,17 @@ struct tsi_handshaker { }; /* Base for tsi_handshaker_result implementations. - See transport_security_interface.h for documentation. */ + See transport_security_interface.h for documentation. + The exec_ctx parameter in create_zero_copy_grpc_protector is supposed to be + of type grpc_exec_ctx*, but we're using void* instead to avoid making the TSI + API depend on grpc. The create_zero_copy_grpc_protector() method is only used + in grpc, where we do need the exec_ctx passed through, but the API still + needs to compile in other applications, where grpc_exec_ctx is not defined. +*/ typedef struct { tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer); tsi_result (*create_zero_copy_grpc_protector)( - const tsi_handshaker_result *self, + void *exec_ctx, const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); tsi_result (*create_frame_protector)(const tsi_handshaker_result *self, diff --git a/src/core/tsi/transport_security_grpc.c b/src/core/tsi/transport_security_grpc.c index 773b35e717..affd995230 100644 --- a/src/core/tsi/transport_security_grpc.c +++ b/src/core/tsi/transport_security_grpc.c @@ -20,16 +20,18 @@ /* This method creates a tsi_zero_copy_grpc_protector object. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector) { - if (self == NULL || self->vtable == NULL || protector == NULL) { + if (exec_ctx == NULL || self == NULL || self->vtable == NULL || + protector == NULL) { return TSI_INVALID_ARGUMENT; } if (self->vtable->create_zero_copy_grpc_protector == NULL) { return TSI_UNIMPLEMENTED; } return self->vtable->create_zero_copy_grpc_protector( - self, max_output_protected_frame_size, protector); + exec_ctx, self, max_output_protected_frame_size, protector); } /* --- tsi_zero_copy_grpc_protector common implementation. --- diff --git a/src/core/tsi/transport_security_grpc.h b/src/core/tsi/transport_security_grpc.h index 375a758888..ca6755c12f 100644 --- a/src/core/tsi/transport_security_grpc.h +++ b/src/core/tsi/transport_security_grpc.h @@ -30,7 +30,8 @@ extern "C" { assuming there is no fatal error. The caller is responsible for destroying the protector. */ tsi_result tsi_handshaker_result_create_zero_copy_grpc_protector( - const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + grpc_exec_ctx *exec_ctx, const tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, tsi_zero_copy_grpc_protector **protector); /* -- tsi_zero_copy_grpc_protector object -- */ -- cgit v1.2.3 From 3218e2b975a71cc32aedad71d00267e07244c109 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 14 Sep 2017 10:29:42 -0700 Subject: minor build fix --- src/core/lib/compression/stream_compression_gzip.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/compression/stream_compression_gzip.c b/src/core/lib/compression/stream_compression_gzip.c index 587bf8379e..abcbdb3a91 100644 --- a/src/core/lib/compression/stream_compression_gzip.c +++ b/src/core/lib/compression/stream_compression_gzip.c @@ -183,7 +183,7 @@ grpc_stream_compression_context_create_gzip( GPR_ASSERT(method == GRPC_STREAM_COMPRESSION_GZIP_COMPRESS || method == GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS); grpc_stream_compression_context_gzip *gzip_ctx = - (grpc_stream_compression_context *)gpr_zalloc( + (grpc_stream_compression_context_gzip *)gpr_zalloc( sizeof(grpc_stream_compression_context_gzip)); int r; if (gzip_ctx == NULL) { -- cgit v1.2.3 From 3ff1fa6e05726fb40cc60271a8e6a189bf901eb8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 11:07:48 -0700 Subject: Collect stats on why writes are initiated --- .../transport/chttp2/transport/chttp2_transport.c | 239 ++++++++++++++++---- .../ext/transport/chttp2/transport/frame_ping.c | 3 +- .../chttp2/transport/frame_window_update.c | 10 +- .../ext/transport/chttp2/transport/hpack_parser.c | 3 +- src/core/ext/transport/chttp2/transport/internal.h | 37 ++- src/core/ext/transport/chttp2/transport/writing.c | 5 +- src/core/lib/debug/stats_data.c | 45 ++++ src/core/lib/debug/stats_data.h | 110 +++++++++ src/core/lib/debug/stats_data.yaml | 42 ++++ src/core/lib/debug/stats_data_bq_schema.sql | 25 +++ tools/run_tests/performance/massage_qps_stats.py | 25 +++ .../performance/scenario_result_schema.json | 250 +++++++++++++++++++++ 12 files changed, 733 insertions(+), 61 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 930fa805a0..7e392b6d63 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -144,10 +144,11 @@ static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error); -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, - grpc_closure *on_complete); +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_complete, + grpc_chttp2_initiate_write_reason initiate_write_reason); static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); @@ -346,7 +347,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (is_client) { grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( GRPC_CHTTP2_CLIENT_CONNECT_STRING)); - grpc_chttp2_initiate_write(exec_ctx, t, "initial_write"); } /* configure http2 the way we like it */ @@ -578,7 +578,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DISABLED; } - grpc_chttp2_initiate_write(exec_ctx, t, "init"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE); post_benign_reclaimer(exec_ctx, t); } @@ -846,13 +847,91 @@ static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } +static void inc_initiate_write_reason( + grpc_exec_ctx *exec_ctx, grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( + exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx); + break; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx); + break; + } +} + void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason) { + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason) { GPR_TIMER_BEGIN("grpc_chttp2_initiate_write", 0); switch (t->write_state) { case GRPC_CHTTP2_WRITE_STATE_IDLE: - set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, reason); + inc_initiate_write_reason(exec_ctx, reason); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + grpc_chttp2_initiate_write_reason_string(reason)); t->is_first_write_in_batch = true; GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); GRPC_CLOSURE_SCHED( @@ -864,7 +943,7 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, break; case GRPC_CHTTP2_WRITE_STATE_WRITING: set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE, - reason); + grpc_chttp2_initiate_write_reason_string(reason)); break; case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: break; @@ -872,16 +951,12 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, GPR_TIMER_END("grpc_chttp2_initiate_write", 0); } -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason) { +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:become"); } - if (also_initiate_write) { - grpc_chttp2_initiate_write(exec_ctx, t, reason); - } } static grpc_closure_scheduler *write_scheduler(grpc_chttp2_transport *t, @@ -1105,7 +1180,9 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); post_destructive_reclaimer(exec_ctx, t); - grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM); } /* cancel out streams that will never be started */ while (t->next_stream_id >= MAX_CLIENT_STREAM_ID && @@ -1202,7 +1279,9 @@ static void maybe_become_writable_due_to_send_msg(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { if (s->id != 0 && (!s->write_buffering || s->flow_controlled_buffer.length > t->write_buffer_size)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE); } } @@ -1404,14 +1483,13 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } } else { GPR_ASSERT(s->id != 0); - bool initiate_write = true; - if (op->send_message && - (op->payload->send_message.send_message->flags & - GRPC_WRITE_BUFFER_HINT)) { - initiate_write = false; + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + if (!(op->send_message && + (op->payload->send_message.send_message->flags & + GRPC_WRITE_BUFFER_HINT))) { + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA); } - grpc_chttp2_become_writable(exec_ctx, t, s, initiate_write, - "op.send_initial_metadata"); } } else { s->send_initial_metadata = NULL; @@ -1519,8 +1597,9 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } else if (s->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "op.send_trailing_metadata"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA); } } } @@ -1632,15 +1711,17 @@ static void cancel_pings(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } -static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_ping_type ping_type, - grpc_closure *on_initiate, grpc_closure *on_ack) { +static void send_ping_locked( + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_ping_type ping_type, grpc_closure *on_initiate, + grpc_closure *on_ack, + grpc_chttp2_initiate_write_reason initiate_write_reason) { grpc_chttp2_ping_queue *pq = &t->ping_queues[ping_type]; grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_INITIATE], on_initiate, GRPC_ERROR_NONE); if (grpc_closure_list_append(&pq->lists[GRPC_CHTTP2_PCL_NEXT], on_ack, GRPC_ERROR_NONE)) { - grpc_chttp2_initiate_write(exec_ctx, t, "send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, initiate_write_reason); } } @@ -1648,7 +1729,8 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)tp; t->ping_state.is_delayed_ping_timer_set = false; - grpc_chttp2_initiate_write(exec_ctx, t, "retry_send_ping"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -1663,7 +1745,8 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } GRPC_CLOSURE_LIST_SCHED(exec_ctx, &pq->lists[GRPC_CHTTP2_PCL_INFLIGHT]); if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_NEXT])) { - grpc_chttp2_initiate_write(exec_ctx, t, "continue_pings"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS); } } @@ -1676,7 +1759,8 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &slice, &http_error); grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)http_error, grpc_slice_ref_internal(slice), &t->qbuf); - grpc_chttp2_initiate_write(exec_ctx, t, "goaway_sent"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT); GRPC_ERROR_UNREF(error); } @@ -1723,7 +1807,8 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, if (op->send_ping) { send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, NULL, - op->send_ping); + op->send_ping, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING); } if (op->on_connectivity_state_change != NULL) { @@ -1968,7 +2053,8 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM); } } if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { @@ -2289,7 +2375,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, &s->stats.outgoing)); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); - grpc_chttp2_initiate_write(exec_ctx, t, "close_from_api"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API); } typedef struct { @@ -2324,19 +2411,20 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "immediate stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL); break; case GRPC_CHTTP2_FLOWCTL_QUEUE_UPDATE: - grpc_chttp2_become_writable(exec_ctx, t, s, false, - "queue stream flowctl"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); break; } switch (action.send_transport_update) { case GRPC_CHTTP2_FLOWCTL_NO_ACTION_NEEDED: break; case GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY: - grpc_chttp2_initiate_write(exec_ctx, t, "immediate transport flowctl"); + grpc_chttp2_initiate_write( + exec_ctx, t, GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL); break; // this is the same as no action b/c every time the transport enters the // writing path it will maybe do an update @@ -2354,7 +2442,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, (uint32_t)action.max_frame_size); } if (action.send_setting_update == GRPC_CHTTP2_FLOWCTL_UPDATE_IMMEDIATELY) { - grpc_chttp2_initiate_write(exec_ctx, t, "immediate setting update"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS); } } if (action.need_ping) { @@ -2362,7 +2451,8 @@ void grpc_chttp2_act_on_flowctl_action(grpc_exec_ctx *exec_ctx, grpc_bdp_estimator_schedule_ping(&t->flow_control.bdp_estimator); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_BEFORE_TRANSPORT_WINDOW_UPDATE, - &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked); + &t->start_bdp_ping_locked, &t->finish_bdp_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING); } } @@ -2441,7 +2531,10 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, if (t->flow_control.initial_window_update > 0) { grpc_chttp2_stream *s; while (grpc_chttp2_list_pop_stalled_by_stream(t, &s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, "unstalled"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING); } } t->flow_control.initial_window_update = 0; @@ -2556,7 +2649,8 @@ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive ping end"); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, &t->start_keepalive_ping_locked, - &t->finish_keepalive_ping_locked); + &t->finish_keepalive_ping_locked, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING); } else { GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); grpc_timer_init( @@ -2912,8 +3006,7 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - (grpc_chttp2_incoming_byte_stream *)gpr_malloc( - sizeof(*incoming_byte_stream)); + gpr_malloc(sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; @@ -3017,6 +3110,56 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, /******************************************************************************* * MONITORING */ + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason) { + switch (reason) { + case GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE: + return "INITIAL_WRITE"; + case GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM: + return "START_NEW_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE: + return "SEND_MESSAGE"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA: + return "SEND_INITIAL_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA: + return "SEND_TRAILING_METADATA"; + case GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING: + return "RETRY_SEND_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS: + return "CONTINUE_PINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT: + return "GOAWAY_SENT"; + case GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM: + return "RST_STREAM"; + case GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API: + return "CLOSE_FROM_API"; + case GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL: + return "STREAM_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL: + return "TRANSPORT_FLOW_CONTROL"; + case GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS: + return "SEND_SETTINGS"; + case GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING: + return "BDP_ESTIMATOR_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING: + return "FLOW_CONTROL_UNSTALLED_BY_SETTING"; + case GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE: + return "FLOW_CONTROL_UNSTALLED_BY_UPDATE"; + case GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING: + return "APPLICATION_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING: + return "KEEPALIVE_PING"; + case GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED: + return "TRANSPORT_FLOW_CONTROL_UNSTALLED"; + case GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE: + return "PING_RESPONSE"; + case GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM: + return "FORCE_RST_STREAM"; + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + static grpc_endpoint *chttp2_get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) { return ((grpc_chttp2_transport *)t)->ep; diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 582fd7bfaa..81bd02ae70 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -117,7 +117,8 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, t->ping_acks, t->ping_ack_capacity * sizeof(*t->ping_acks)); } t->ping_acks[t->ping_ack_count++] = p->opaque_8bytes; - grpc_chttp2_initiate_write(exec_ctx, t, "ping response"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE); } } } diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index c94f7725bf..c9ab8d1b50 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -99,8 +99,10 @@ grpc_error *grpc_chttp2_window_update_parser_parse( grpc_chttp2_flowctl_recv_stream_update( &t->flow_control, &s->flow_control, received_update); if (grpc_chttp2_list_remove_stalled_by_stream(t, s)) { - grpc_chttp2_become_writable(exec_ctx, t, s, true, - "stream.read_flow_control"); + grpc_chttp2_mark_stream_writable(exec_ctx, t, s); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE); } } } else { @@ -109,7 +111,9 @@ grpc_error *grpc_chttp2_window_update_parser_parse( received_update); bool is_zero = t->flow_control.remote_window <= 0; if (was_zero && !is_zero) { - grpc_chttp2_initiate_write(exec_ctx, t, "new_global_flow_control"); + grpc_chttp2_initiate_write( + exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED); } } } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 82ff2c8e2c..901e7413c0 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -1649,7 +1649,8 @@ static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_add( &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_HTTP2_NO_ERROR, &s->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, t, "force_rst_stream"); + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM); grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); } GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 0fbedd1e56..c2dfce7c9c 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -79,6 +79,33 @@ typedef enum { GRPC_CHTTP2_PCL_COUNT /* must be last */ } grpc_chttp2_ping_closure_list; +typedef enum { + GRPC_CHTTP2_INITIATE_WRITE_INITIAL_WRITE, + GRPC_CHTTP2_INITIATE_WRITE_START_NEW_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_SEND_MESSAGE, + GRPC_CHTTP2_INITIATE_WRITE_SEND_INITIAL_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_SEND_TRAILING_METADATA, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING, + GRPC_CHTTP2_INITIATE_WRITE_CONTINUE_PINGS, + GRPC_CHTTP2_INITIATE_WRITE_GOAWAY_SENT, + GRPC_CHTTP2_INITIATE_WRITE_RST_STREAM, + GRPC_CHTTP2_INITIATE_WRITE_CLOSE_FROM_API, + GRPC_CHTTP2_INITIATE_WRITE_STREAM_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL, + GRPC_CHTTP2_INITIATE_WRITE_SEND_SETTINGS, + GRPC_CHTTP2_INITIATE_WRITE_BDP_ESTIMATOR_PING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_CHTTP2_INITIATE_WRITE_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_CHTTP2_INITIATE_WRITE_APPLICATION_PING, + GRPC_CHTTP2_INITIATE_WRITE_KEEPALIVE_PING, + GRPC_CHTTP2_INITIATE_WRITE_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_CHTTP2_INITIATE_WRITE_PING_RESPONSE, + GRPC_CHTTP2_INITIATE_WRITE_FORCE_RST_STREAM, +} grpc_chttp2_initiate_write_reason; + +const char *grpc_chttp2_initiate_write_reason_string( + grpc_chttp2_initiate_write_reason reason); + typedef struct { grpc_closure_list lists[GRPC_CHTTP2_PCL_COUNT]; uint64_t inflight_id; @@ -599,7 +626,8 @@ struct grpc_chttp2_stream { The actual call chain is documented in the implementation of this function. */ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, const char *reason); + grpc_chttp2_transport *t, + grpc_chttp2_initiate_write_reason reason); typedef struct { /** are we writing? */ @@ -851,10 +879,9 @@ void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, /** add a ref to the stream and add it to the writable list; ref will be dropped in writing.c */ -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s, - bool also_initiate_write, const char *reason); +void grpc_chttp2_mark_stream_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 3fa38db4db..3ded801985 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -201,9 +201,8 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( if (t->flow_control.remote_window > 0) { while (grpc_chttp2_list_pop_stalled_by_transport(t, &s)) { - if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s) && - stream_ref_if_not_destroyed(&s->refcount->refs)) { - grpc_chttp2_initiate_write(exec_ctx, t, "transport.read_flow_control"); + if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { + stream_ref_if_not_destroyed(&s->refcount->refs); } } } diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 2c56765343..b3e1ee9b4e 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -50,6 +50,27 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "http2_writes_offloaded", "http2_writes_continued", "http2_partial_writes", + "http2_initiate_write_due_to_initial_write", + "http2_initiate_write_due_to_start_new_stream", + "http2_initiate_write_due_to_send_message", + "http2_initiate_write_due_to_send_initial_metadata", + "http2_initiate_write_due_to_send_trailing_metadata", + "http2_initiate_write_due_to_retry_send_ping", + "http2_initiate_write_due_to_continue_pings", + "http2_initiate_write_due_to_goaway_sent", + "http2_initiate_write_due_to_rst_stream", + "http2_initiate_write_due_to_close_from_api", + "http2_initiate_write_due_to_stream_flow_control", + "http2_initiate_write_due_to_transport_flow_control", + "http2_initiate_write_due_to_send_settings", + "http2_initiate_write_due_to_bdp_estimator_ping", + "http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "http2_initiate_write_due_to_flow_control_unstalled_by_update", + "http2_initiate_write_due_to_application_ping", + "http2_initiate_write_due_to_keepalive_ping", + "http2_initiate_write_due_to_transport_flow_control_unstalled", + "http2_initiate_write_due_to_ping_response", + "http2_initiate_write_due_to_force_rst_stream", "combiner_locks_initiated", "combiner_locks_scheduled_items", "combiner_locks_scheduled_final_items", @@ -92,6 +113,30 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "written", "Number of HTTP2 writes that were made knowing there was still more data " "to be written (we cap maximum write size to syscall_write)", + "Number of HTTP2 writes initiated due to 'initial_write'", + "Number of HTTP2 writes initiated due to 'start_new_stream'", + "Number of HTTP2 writes initiated due to 'send_message'", + "Number of HTTP2 writes initiated due to 'send_initial_metadata'", + "Number of HTTP2 writes initiated due to 'send_trailing_metadata'", + "Number of HTTP2 writes initiated due to 'retry_send_ping'", + "Number of HTTP2 writes initiated due to 'continue_pings'", + "Number of HTTP2 writes initiated due to 'goaway_sent'", + "Number of HTTP2 writes initiated due to 'rst_stream'", + "Number of HTTP2 writes initiated due to 'close_from_api'", + "Number of HTTP2 writes initiated due to 'stream_flow_control'", + "Number of HTTP2 writes initiated due to 'transport_flow_control'", + "Number of HTTP2 writes initiated due to 'send_settings'", + "Number of HTTP2 writes initiated due to 'bdp_estimator_ping'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_setting'", + "Number of HTTP2 writes initiated due to " + "'flow_control_unstalled_by_update'", + "Number of HTTP2 writes initiated due to 'application_ping'", + "Number of HTTP2 writes initiated due to 'keepalive_ping'", + "Number of HTTP2 writes initiated due to " + "'transport_flow_control_unstalled'", + "Number of HTTP2 writes initiated due to 'ping_response'", + "Number of HTTP2 writes initiated due to 'force_rst_stream'", "Number of combiner lock entries by process (first items queued to a " "combiner)", "Number of items scheduled against combiner locks", diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index e93db32c6a..c9871c4a56 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -52,6 +52,27 @@ typedef enum { GRPC_STATS_COUNTER_HTTP2_WRITES_OFFLOADED, GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED, GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE, + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM, GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS, @@ -169,6 +190,95 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_WRITES_CONTINUED) #define GRPC_STATS_INC_HTTP2_PARTIAL_WRITES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HTTP2_PARTIAL_WRITES) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_INITIAL_WRITE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_START_NEW_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_MESSAGE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_INITIAL_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_TRAILING_METADATA) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RETRY_SEND_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CONTINUE_PINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_GOAWAY_SENT) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_RST_STREAM) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_CLOSE_FROM_API) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_APPLICATION_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_KEEPALIVE_PING) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED( \ + exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE) +#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM(exec_ctx) \ + GRPC_STATS_INC_COUNTER( \ + (exec_ctx), \ + GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM) #define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED) diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 37dff9c5f4..84727fe6c4 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -117,6 +117,48 @@ - counter: http2_partial_writes doc: Number of HTTP2 writes that were made knowing there was still more data to be written (we cap maximum write size to syscall_write) +- counter: http2_initiate_write_due_to_initial_write + doc: Number of HTTP2 writes initiated due to 'initial_write' +- counter: http2_initiate_write_due_to_start_new_stream + doc: Number of HTTP2 writes initiated due to 'start_new_stream' +- counter: http2_initiate_write_due_to_send_message + doc: Number of HTTP2 writes initiated due to 'send_message' +- counter: http2_initiate_write_due_to_send_initial_metadata + doc: Number of HTTP2 writes initiated due to 'send_initial_metadata' +- counter: http2_initiate_write_due_to_send_trailing_metadata + doc: Number of HTTP2 writes initiated due to 'send_trailing_metadata' +- counter: http2_initiate_write_due_to_retry_send_ping + doc: Number of HTTP2 writes initiated due to 'retry_send_ping' +- counter: http2_initiate_write_due_to_continue_pings + doc: Number of HTTP2 writes initiated due to 'continue_pings' +- counter: http2_initiate_write_due_to_goaway_sent + doc: Number of HTTP2 writes initiated due to 'goaway_sent' +- counter: http2_initiate_write_due_to_rst_stream + doc: Number of HTTP2 writes initiated due to 'rst_stream' +- counter: http2_initiate_write_due_to_close_from_api + doc: Number of HTTP2 writes initiated due to 'close_from_api' +- counter: http2_initiate_write_due_to_stream_flow_control + doc: Number of HTTP2 writes initiated due to 'stream_flow_control' +- counter: http2_initiate_write_due_to_transport_flow_control + doc: Number of HTTP2 writes initiated due to 'transport_flow_control' +- counter: http2_initiate_write_due_to_send_settings + doc: Number of HTTP2 writes initiated due to 'send_settings' +- counter: http2_initiate_write_due_to_bdp_estimator_ping + doc: Number of HTTP2 writes initiated due to 'bdp_estimator_ping' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_setting + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_setting' +- counter: http2_initiate_write_due_to_flow_control_unstalled_by_update + doc: Number of HTTP2 writes initiated due to 'flow_control_unstalled_by_update' +- counter: http2_initiate_write_due_to_application_ping + doc: Number of HTTP2 writes initiated due to 'application_ping' +- counter: http2_initiate_write_due_to_keepalive_ping + doc: Number of HTTP2 writes initiated due to 'keepalive_ping' +- counter: http2_initiate_write_due_to_transport_flow_control_unstalled + doc: Number of HTTP2 writes initiated due to 'transport_flow_control_unstalled' +- counter: http2_initiate_write_due_to_ping_response + doc: Number of HTTP2 writes initiated due to 'ping_response' +- counter: http2_initiate_write_due_to_force_rst_stream + doc: Number of HTTP2 writes initiated due to 'force_rst_stream' # combiner locks - counter: combiner_locks_initiated doc: Number of combiner lock entries by process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 5354769291..d21afbbfe4 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -1,5 +1,9 @@ client_calls_created_per_iteration:FLOAT, server_calls_created_per_iteration:FLOAT, +cqs_created_per_iteration:FLOAT, +client_channels_created_per_iteration:FLOAT, +client_subchannels_created_per_iteration:FLOAT, +server_channels_created_per_iteration:FLOAT, syscall_poll_per_iteration:FLOAT, syscall_wait_per_iteration:FLOAT, histogram_slow_lookups_per_iteration:FLOAT, @@ -21,6 +25,27 @@ http2_writes_begun_per_iteration:FLOAT, http2_writes_offloaded_per_iteration:FLOAT, http2_writes_continued_per_iteration:FLOAT, http2_partial_writes_per_iteration:FLOAT, +http2_initiate_write_due_to_initial_write_per_iteration:FLOAT, +http2_initiate_write_due_to_start_new_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_send_message_per_iteration:FLOAT, +http2_initiate_write_due_to_send_initial_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_send_trailing_metadata_per_iteration:FLOAT, +http2_initiate_write_due_to_retry_send_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_continue_pings_per_iteration:FLOAT, +http2_initiate_write_due_to_goaway_sent_per_iteration:FLOAT, +http2_initiate_write_due_to_rst_stream_per_iteration:FLOAT, +http2_initiate_write_due_to_close_from_api_per_iteration:FLOAT, +http2_initiate_write_due_to_stream_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_per_iteration:FLOAT, +http2_initiate_write_due_to_send_settings_per_iteration:FLOAT, +http2_initiate_write_due_to_bdp_estimator_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_setting_per_iteration:FLOAT, +http2_initiate_write_due_to_flow_control_unstalled_by_update_per_iteration:FLOAT, +http2_initiate_write_due_to_application_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_keepalive_ping_per_iteration:FLOAT, +http2_initiate_write_due_to_transport_flow_control_unstalled_per_iteration:FLOAT, +http2_initiate_write_due_to_ping_response_per_iteration:FLOAT, +http2_initiate_write_due_to_force_rst_stream_per_iteration:FLOAT, combiner_locks_initiated_per_iteration:FLOAT, combiner_locks_scheduled_items_per_iteration:FLOAT, combiner_locks_scheduled_final_items_per_iteration:FLOAT, diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index 30a94ca7cd..7994b544ac 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -22,6 +22,10 @@ def massage_qps_stats(scenario_result): del stats["coreStats"] stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created") stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created") + stats["core_cqs_created"] = massage_qps_stats_helpers.counter(core_stats, "cqs_created") + stats["core_client_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_channels_created") + stats["core_client_subchannels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_subchannels_created") + stats["core_server_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "server_channels_created") stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll") stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait") stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups") @@ -43,6 +47,27 @@ def massage_qps_stats(scenario_result): stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded") stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued") stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes") + stats["core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_initial_write") + stats["core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_start_new_stream") + stats["core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_message") + stats["core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_initial_metadata") + stats["core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_trailing_metadata") + stats["core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_retry_send_ping") + stats["core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_continue_pings") + stats["core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_goaway_sent") + stats["core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_rst_stream") + stats["core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_close_from_api") + stats["core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_stream_flow_control") + stats["core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control") + stats["core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_settings") + stats["core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_bdp_estimator_ping") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_setting") + stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_update") + stats["core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_application_ping") + stats["core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_keepalive_ping") + stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled") + stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response") + stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream") stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index 84b3cb7c0a..12a56dbec0 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -120,6 +120,26 @@ "name": "core_server_calls_created", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_syscall_poll", @@ -225,6 +245,111 @@ "name": "core_http2_partial_writes", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", @@ -597,6 +722,26 @@ "name": "core_server_calls_created", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_cqs_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_channels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_client_subchannels_created", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_server_channels_created", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_syscall_poll", @@ -702,6 +847,111 @@ "name": "core_http2_partial_writes", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_initial_write", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_start_new_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_message", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_initial_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_trailing_metadata", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_retry_send_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_continue_pings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_goaway_sent", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_rst_stream", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_close_from_api", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_stream_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_send_settings", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_bdp_estimator_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_flow_control_unstalled_by_update", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_application_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_keepalive_ping", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_transport_flow_control_unstalled", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_ping_response", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_http2_initiate_write_due_to_force_rst_stream", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", -- cgit v1.2.3 From 1c87515f8da87dd96182dc3070029599b0a7165f Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Mon, 11 Sep 2017 16:11:46 -0700 Subject: add php extension --- src/ruby/qps/proxy-worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/qps/proxy-worker.rb b/src/ruby/qps/proxy-worker.rb index d7a9f114bd..488610ae74 100755 --- a/src/ruby/qps/proxy-worker.rb +++ b/src/ruby/qps/proxy-worker.rb @@ -41,7 +41,7 @@ class ProxyBenchmarkClientServiceImpl < Grpc::Testing::ProxyClientService::Servi @histogram = Histogram.new(@histres, @histmax) @start_time = Time.now # TODO(vjpai): Support multiple client channels by spawning off a PHP client per channel - command = "php " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget + command = "php -d extension=" + File.expand_path(File.dirname(__FILE__)) + "/../../php/ext/grpc/modules/grpc.so " + File.expand_path(File.dirname(__FILE__)) + "/../../php/tests/qps/client.php " + @mytarget puts "Starting command: " + command @php_pid = spawn(command) end -- cgit v1.2.3 From 3ad5c74311b6f603ed590c5330ac71d3387c7710 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 11:29:30 -0700 Subject: Fix errant change --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 7e392b6d63..79a9ed827f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -3006,7 +3006,8 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = - gpr_malloc(sizeof(*incoming_byte_stream)); + (grpc_chttp2_incoming_byte_stream *)gpr_malloc( + sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; -- cgit v1.2.3 From 9fa10cce9ffe2689174af7b69940c8c2f2b51d0f Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 14 Sep 2017 11:49:52 -0700 Subject: Fix merge issue --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 707fc293ad..a776a07d99 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1857,6 +1857,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, gpr_free(glb_policy); return NULL; } + grpc_subchannel_index_ref(); GRPC_CLOSURE_INIT(&glb_policy->lb_channel_on_connectivity_changed, glb_lb_channel_on_connectivity_changed_cb, glb_policy, grpc_combiner_scheduler(args->combiner)); -- cgit v1.2.3 From f66ed288538a7e28cdbdc694bef80bb86b696b90 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 12:30:27 -0700 Subject: Instrument executor a little better --- src/core/lib/debug/stats_data.c | 48 ++++++++++++++++++++++++++++++++------ src/core/lib/debug/stats_data.h | 24 +++++++++++++++---- src/core/lib/debug/stats_data.yaml | 8 +++++++ src/core/lib/iomgr/executor.c | 9 +++++++ 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 3fd8ee38ef..b52852d000 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -56,6 +56,8 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_wakeup_initiated", "executor_queue_drained", "executor_push_retries", + "executor_threads_created", + "executor_threads_used", }; const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of client side calls created by this process", @@ -98,6 +100,8 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of times an executor queue was drained", "Number of times we raced and were forced to retry pushing a closure to " "the executor", + "Size of the backing thread pool for overflow gRPC Core work", + "How many executor threads actually got used", }; const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "tcp_write_size", @@ -110,6 +114,7 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "http2_send_message_per_write", "http2_send_trailing_metadata_per_write", "http2_send_flowctl_per_write", + "executor_closures_per_wakeup", }; const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Number of bytes offered to each syscall_write", @@ -122,6 +127,7 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Number of streams whose payload was written per TCP write", "Number of streams terminated per TCP write", "Number of flow control updates written per TCP write", + "Number of closures executed each time an executor wakes up", }; const int grpc_stats_table_0[65] = { 0, 1, 2, 3, 4, 6, 8, 11, @@ -418,16 +424,43 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_2, 64)); } -const int grpc_stats_histo_buckets[10] = {64, 64, 64, 64, 64, +void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, + int value) { + value = GPR_CLAMP(value, 0, 1024); + if (value < 13) { + GRPC_STATS_INC_HISTOGRAM( + (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, value); + return; + } + union { + double dbl; + uint64_t uint; + } _val, _bkt; + _val.dbl = value; + if (_val.uint < 4637863191261478912ull) { + int bucket = + grpc_stats_table_3[((_val.uint - 4623507967449235456ull) >> 48)] + 13; + _bkt.dbl = grpc_stats_table_2[bucket]; + bucket -= (_val.uint < _bkt.uint); + GRPC_STATS_INC_HISTOGRAM( + (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, bucket); + return; + } + GRPC_STATS_INC_HISTOGRAM((exec_ctx), + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, + grpc_stats_histo_find_bucket_slow( + (exec_ctx), value, grpc_stats_table_2, 64)); +} +const int grpc_stats_histo_buckets[11] = {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64}; -const int grpc_stats_histo_start[10] = {0, 64, 128, 192, 256, - 320, 384, 448, 512, 576}; -const int *const grpc_stats_histo_bucket_boundaries[10] = { +const int grpc_stats_histo_start[11] = {0, 64, 128, 192, 256, 320, + 384, 448, 512, 576, 640}; +const int *const grpc_stats_histo_bucket_boundaries[11] = { grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_2, grpc_stats_table_2, - grpc_stats_table_2}; -void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_table_2, grpc_stats_table_2}; +void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_tcp_write_size, grpc_stats_inc_tcp_write_iov_size, grpc_stats_inc_tcp_read_size, @@ -437,4 +470,5 @@ void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_http2_send_initial_metadata_per_write, grpc_stats_inc_http2_send_message_per_write, grpc_stats_inc_http2_send_trailing_metadata_per_write, - grpc_stats_inc_http2_send_flowctl_per_write}; + grpc_stats_inc_http2_send_flowctl_per_write, + grpc_stats_inc_executor_closures_per_wakeup}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index b7c15c08a5..cf7d3b60e2 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -58,6 +58,8 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED, GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED, GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES, + GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED, + GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED, GRPC_STATS_COUNTER_COUNT } grpc_stats_counters; extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT]; @@ -73,6 +75,7 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE, + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, GRPC_STATS_HISTOGRAM_COUNT } grpc_stats_histograms; extern const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT]; @@ -98,7 +101,9 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_BUCKETS = 64, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 576, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_BUCKETS = 640 + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_FIRST_SLOT = 640, + GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_BUCKETS = 64, + GRPC_STATS_HISTOGRAM_BUCKETS = 704 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -179,6 +184,11 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) #define GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES) +#define GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED) +#define GRPC_STATS_INC_EXECUTOR_THREADS_USED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED) #define GRPC_STATS_INC_TCP_WRITE_SIZE(exec_ctx, value) \ grpc_stats_inc_tcp_write_size((exec_ctx), (int)(value)) void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int x); @@ -214,10 +224,14 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_stats_inc_http2_send_flowctl_per_write((exec_ctx), (int)(value)) void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[10]; -extern const int grpc_stats_histo_start[10]; -extern const int *const grpc_stats_histo_bucket_boundaries[10]; -extern void (*const grpc_stats_inc_histogram[10])(grpc_exec_ctx *exec_ctx, +#define GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, value) \ + grpc_stats_inc_executor_closures_per_wakeup((exec_ctx), (int)(value)) +void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, + int x); +extern const int grpc_stats_histo_buckets[11]; +extern const int grpc_stats_histo_start[11]; +extern const int *const grpc_stats_histo_bucket_boundaries[11]; +extern void (*const grpc_stats_inc_histogram[11])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index a9d71f4fcb..08f49d1150 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -135,3 +135,11 @@ - counter: executor_push_retries doc: Number of times we raced and were forced to retry pushing a closure to the executor +- counter: executor_threads_created + doc: Size of the backing thread pool for overflow gRPC Core work +- counter: executor_threads_used + doc: How many executor threads actually got used +- histogram: executor_closures_per_wakeup + max: 1024 + buckets: 64 + doc: Number of closures executed each time an executor wakes up diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index 892385d7d7..41c7ff959e 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -81,6 +81,8 @@ static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { grpc_exec_ctx_flush(exec_ctx); } + GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, n); + return n; } @@ -150,7 +152,10 @@ static void executor_thread(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(&exec_ctx); + size_t subtract_depth = 0; + bool used = false; for (;;) { if (GRPC_TRACER_ON(executor_trace)) { gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", @@ -170,6 +175,10 @@ static void executor_thread(void *arg) { gpr_mu_unlock(&ts->mu); break; } + if (!used) { + GRPC_STATS_INC_EXECUTOR_THREADS_USED(&exec_ctx); + used = true; + } GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx); grpc_closure_list exec = ts->elems; ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; -- cgit v1.2.3 From b2b4a85c86b6e1e8f9b4e278355e77b4a4b5c682 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 14 Sep 2017 12:38:38 -0700 Subject: Correct comparison --- src/core/lib/iomgr/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index cf2c051c92..2439f15a8a 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -217,7 +217,7 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); - if (!is_short) { + if (is_short) { grpc_closure_list_append(&ts->local_elems, closure, error); return; } -- cgit v1.2.3 From 27bf05d003f48e6f95b836a5e5450e4bb787f9ef Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 7 Aug 2017 18:09:11 -0700 Subject: Add tests for c-ares wrapper using a local DNS server. --- CMakeLists.txt | 184 ++++++++++++ Makefile | 188 ++++++++++++ bazel/grpc_build_system.bzl | 16 + .../resolver/dns/c_ares/dns_resolver_ares.c | 2 +- .../naming/resolver_component_tests_defs.include | 92 ++++++ .../resolver_component_tests_runner.sh.template | 4 + templates/tools/dockerfile/apt_get_basic.include | 1 + templates/tools/dockerfile/python_deps.include | 2 +- test/cpp/naming/BUILD | 49 ++++ test/cpp/naming/gen_build_yaml.py | 99 +++++++ .../naming/generate_resolver_component_tests.bzl | 64 ++++ test/cpp/naming/resolver_component_test.cc | 323 +++++++++++++++++++++ test/cpp/naming/resolver_component_tests_runner.sh | 173 +++++++++++ .../resolver_component_tests_runner_invoker.cc | 189 ++++++++++++ test/cpp/naming/resolver_test_record_groups.yaml | 155 ++++++++++ test/cpp/naming/test_dns_server.py | 134 +++++++++ tools/buildgen/generate_build_additions.sh | 1 + .../interoptest/grpc_interop_csharp/Dockerfile | 3 +- .../interoptest/grpc_interop_cxx/Dockerfile | 3 +- .../interoptest/grpc_interop_go/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.7/Dockerfile | 2 +- .../interoptest/grpc_interop_go1.8/Dockerfile | 2 +- .../interoptest/grpc_interop_http2/Dockerfile | 2 +- .../interoptest/grpc_interop_java/Dockerfile | 2 +- .../grpc_interop_java_oracle8/Dockerfile | 2 +- .../interoptest/grpc_interop_node/Dockerfile | 3 +- .../interoptest/grpc_interop_php/Dockerfile | 1 + .../interoptest/grpc_interop_python/Dockerfile | 3 +- .../interoptest/grpc_interop_ruby/Dockerfile | 3 +- tools/dockerfile/test/csharp_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/cxx_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/cxx_jessie_x86/Dockerfile | 3 +- .../dockerfile/test/cxx_ubuntu1404_x64/Dockerfile | 3 +- .../dockerfile/test/cxx_ubuntu1604_x64/Dockerfile | 3 +- tools/dockerfile/test/fuzzer/Dockerfile | 3 +- .../test/multilang_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/node_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/php7_jessie_x64/Dockerfile | 2 +- tools/dockerfile/test/php_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/python_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/python_pyenv_x64/Dockerfile | 3 +- tools/dockerfile/test/ruby_jessie_x64/Dockerfile | 3 +- tools/dockerfile/test/sanity/Dockerfile | 3 +- .../helper_scripts/prepare_build_macos_rc | 2 +- tools/run_tests/generated/sources_and_headers.json | 80 +++++ tools/run_tests/generated/tests.json | 46 +++ 46 files changed, 1845 insertions(+), 28 deletions(-) create mode 100644 templates/test/cpp/naming/resolver_component_tests_defs.include create mode 100644 templates/test/cpp/naming/resolver_component_tests_runner.sh.template create mode 100644 test/cpp/naming/BUILD create mode 100755 test/cpp/naming/gen_build_yaml.py create mode 100755 test/cpp/naming/generate_resolver_component_tests.bzl create mode 100644 test/cpp/naming/resolver_component_test.cc create mode 100755 test/cpp/naming/resolver_component_tests_runner.sh create mode 100644 test/cpp/naming/resolver_component_tests_runner_invoker.cc create mode 100644 test/cpp/naming/resolver_test_record_groups.yaml create mode 100755 test/cpp/naming/test_dns_server.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dd68016be8..1186e2b503 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,18 @@ add_dependencies(buildtests_cxx thread_stress_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_test) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) +endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -14113,6 +14125,178 @@ target_link_libraries(inproc_nosec_test gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test_unsecure + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util_unsecure + grpc_test_util_unsecure + gpr_test_util + grpc++_unsecure + grpc_unsecure + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_test + test/cpp/naming/resolver_component_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker_unsecure + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker_unsecure + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker_unsecure + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + +add_executable(resolver_component_tests_runner_invoker + test/cpp/naming/resolver_component_tests_runner_invoker.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(resolver_component_tests_runner_invoker + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(resolver_component_tests_runner_invoker + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + gpr_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 0c3648566a..61d6cae6ab 100644 --- a/Makefile +++ b/Makefile @@ -1266,6 +1266,10 @@ h2_sockpair+trace_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair+trace_nosec_test h2_sockpair_1byte_nosec_test: $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_nosec_test h2_uds_nosec_test: $(BINDIR)/$(CONFIG)/h2_uds_nosec_test inproc_nosec_test: $(BINDIR)/$(CONFIG)/inproc_nosec_test +resolver_component_test_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure +resolver_component_test: $(BINDIR)/$(CONFIG)/resolver_component_test +resolver_component_tests_runner_invoker_unsecure: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure +resolver_component_tests_runner_invoker: $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker api_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/api_fuzzer_one_entry client_fuzzer_one_entry: $(BINDIR)/$(CONFIG)/client_fuzzer_one_entry hpack_parser_fuzzer_test_one_entry: $(BINDIR)/$(CONFIG)/hpack_parser_fuzzer_test_one_entry @@ -1652,6 +1656,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/boringssl_x509_test \ $(BINDIR)/$(CONFIG)/boringssl_tab_test \ $(BINDIR)/$(CONFIG)/boringssl_v3name_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ else buildtests_cxx: privatelibs_cxx \ @@ -1730,6 +1738,10 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/thread_manager_test \ $(BINDIR)/$(CONFIG)/thread_stress_test \ $(BINDIR)/$(CONFIG)/writes_per_rpc_test \ + $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_test \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure \ + $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker \ endif @@ -2141,6 +2153,10 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 ) $(E) "[RUN] Testing writes_per_rpc_test" $(Q) $(BINDIR)/$(CONFIG)/writes_per_rpc_test || ( echo test writes_per_rpc_test failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker_unsecure" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure || ( echo test resolver_component_tests_runner_invoker_unsecure failed ; exit 1 ) + $(E) "[RUN] Testing resolver_component_tests_runner_invoker" + $(Q) $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker || ( echo test resolver_component_tests_runner_invoker failed ; exit 1 ) flaky_test_cxx: buildtests_cxx @@ -19480,6 +19496,178 @@ ifneq ($(NO_DEPS),true) endif +RESOLVER_COMPONENT_TEST_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TEST_SRC = \ + test/cpp/naming/resolver_component_test.cc \ + +RESOLVER_COMPONENT_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker_unsecure + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker_unsecure: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_UNSECURE_OBJS:.o=.dep) +endif +endif + + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC = \ + test/cpp/naming/resolver_component_tests_runner_invoker.cc \ + +RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_tests_runner_invoker + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_tests_runner_invoker.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_resolver_component_tests_runner_invoker: $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(RESOLVER_COMPONENT_TESTS_RUNNER_INVOKER_OBJS:.o=.dep) +endif +endif + + API_FUZZER_ONE_ENTRY_SRC = \ test/core/end2end/fuzzers/api_fuzzer.c \ test/core/util/one_corpus_entry_fuzzer.c \ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index ba7ffcc9be..6cfed6b0e7 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -106,6 +106,22 @@ def grpc_sh_test(name, srcs, args = [], data = []): args = args, data = data) +def grpc_sh_binary(name, srcs, data = []): + native.sh_test( + name = name, + srcs = srcs, + data = data) + +def grpc_py_binary(name, srcs, data = [], deps = []): + if name == "test_dns_server": + # TODO: allow running test_dns_server in oss bazel test suite + deps = [] + native.py_binary( + name = name, + srcs = srcs, + data = data, + deps = deps) + def grpc_package(name, visibility = "private", features = []): if visibility == "tests": visibility = ["//test:__subpackages__"] diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index b87a3b7082..371c59b8cf 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -204,7 +204,7 @@ static char *choose_service_config(char *service_config_choice_json) { int random_pct = rand() % 100; int percentage; if (sscanf(field->value, "%d", &percentage) != 1 || - random_pct > percentage) { + random_pct > percentage || percentage == 0) { service_config_json = NULL; break; } diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include new file mode 100644 index 0000000000..6fa91c741a --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -0,0 +1,92 @@ +<%def name="resolver_component_tests(tests)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +% for test in tests: +$FLAGS_test_bin_path \\ + + --target_name='${test['target_name']}' \\ + + --expected_addrs='${test['expected_addrs']}' \\ + + --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + + --expected_lb_policy='${test['expected_lb_policy']}' \\ + + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +% endfor +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/templates/test/cpp/naming/resolver_component_tests_runner.sh.template b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template new file mode 100644 index 0000000000..86772dd141 --- /dev/null +++ b/templates/test/cpp/naming/resolver_component_tests_runner.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="resolver_component_tests_defs.include" import="*"/>\ + ${resolver_component_tests(resolver_component_test_cases)} diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include index 9237e7dace..6e19e65157 100644 --- a/templates/tools/dockerfile/apt_get_basic.include +++ b/templates/tools/dockerfile/apt_get_basic.include @@ -6,6 +6,7 @@ RUN apt-get update && apt-get install -y ${'\\'} bzip2 ${'\\'} ccache ${'\\'} curl ${'\\'} + dnsutils ${'\\'} gcc ${'\\'} gcc-multilib ${'\\'} git ${'\\'} diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include index 2c12981418..94b854ad21 100644 --- a/templates/tools/dockerfile/python_deps.include +++ b/templates/tools/dockerfile/python_deps.include @@ -11,4 +11,4 @@ RUN apt-get update && apt-get install -y ${'\\'} # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 diff --git a/test/cpp/naming/BUILD b/test/cpp/naming/BUILD new file mode 100644 index 0000000000..24c3d1a443 --- /dev/null +++ b/test/cpp/naming/BUILD @@ -0,0 +1,49 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], + features = [ + "-layering_check", + "-parse_headers", + ], +) + +licenses(["notice"]) # Apache v2 + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_py_binary") + +load(":generate_resolver_component_tests.bzl", "generate_resolver_component_tests") + +# Meant to be invoked only through the top-level shell script driver. +grpc_sh_binary( + name = "resolver_component_tests_runner", + srcs = [ + "resolver_component_tests_runner.sh", + ], +) + +grpc_py_binary( + name = "test_dns_server", + srcs = ["test_dns_server.py"], + data = [ + "resolver_test_record_groups.yaml", + ], + deps = [ + "twisted", + "yaml", + ] +) + +generate_resolver_component_tests() diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py new file mode 100755 index 0000000000..3a51fef7a0 --- /dev/null +++ b/test/cpp/naming/gen_build_yaml.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Generates the appropriate build.json data for all the naming tests.""" + + +import yaml +import collections +import hashlib +import json + +_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353' + +def _append_zone_name(name, zone_name): + return '%s.%s' % (name, zone_name) + +def _build_expected_addrs_cmd_arg(expected_addrs): + out = [] + for addr in expected_addrs: + out.append('%s,%s' % (addr['address'], str(addr['is_balancer']))) + return ';'.join(out) + +def main(): + resolver_component_data = '' + with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: + resolver_component_data = yaml.load(f) + + json = { + 'resolver_component_test_cases': [ + { + 'target_name': _append_zone_name(test_case['record_to_resolve'], + resolver_component_data['resolver_component_tests_common_zone_name']), + 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), + 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), + 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), + } for test_case in resolver_component_data['resolver_component_tests'] + ], + 'targets': [ + { + 'name': 'resolver_component_test' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': False, + 'src': ['test/cpp/naming/resolver_component_test.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util' + unsecure_build_config_suffix, + 'grpc_test_util' + unsecure_build_config_suffix, + 'gpr_test_util', + 'grpc++' + unsecure_build_config_suffix, + 'grpc' + unsecure_build_config_suffix, + 'gpr', + 'grpc++_test_config', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + [ + { + 'name': 'resolver_component_tests_runner_invoker' + unsecure_build_config_suffix, + 'build': 'test', + 'language': 'c++', + 'gtest': False, + 'run': True, + 'src': ['test/cpp/naming/resolver_component_tests_runner_invoker.cc'], + 'platforms': ['linux', 'posix', 'mac'], + 'deps': [ + 'grpc++_test_util', + 'grpc_test_util', + 'gpr_test_util', + 'grpc++', + 'grpc', + 'gpr', + 'grpc++_test_config', + ], + 'args': [ + '--test_bin_name=resolver_component_test%s' % unsecure_build_config_suffix, + '--running_under_bazel=false', + ], + } for unsecure_build_config_suffix in ['_unsecure', ''] + ] + } + + print(yaml.dump(json)) + +if __name__ == '__main__': + main() diff --git a/test/cpp/naming/generate_resolver_component_tests.bzl b/test/cpp/naming/generate_resolver_component_tests.bzl new file mode 100755 index 0000000000..118d9452d9 --- /dev/null +++ b/test/cpp/naming/generate_resolver_component_tests.bzl @@ -0,0 +1,64 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_sh_binary", "grpc_cc_test", "grpc_cc_binary") + +def generate_resolver_component_tests(): + for unsecure_build_config_suffix in ['_unsecure', '']: + # meant to be invoked only through the top-level shell script driver + grpc_cc_binary( + name = "resolver_component_test%s" % unsecure_build_config_suffix, + testonly = 1, + srcs = [ + "resolver_component_test.cc", + ], + external_deps = [ + "gmock", + ], + deps = [ + "//test/cpp/util:test_util%s" % unsecure_build_config_suffix, + "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix, + "//test/core/util:gpr_test_util", + "//:grpc++%s" % unsecure_build_config_suffix, + "//:grpc%s" % unsecure_build_config_suffix, + "//:gpr", + "//test/cpp/util:test_config", + ], + ) + grpc_cc_test( + name = "resolver_component_tests_runner_invoker%s" % unsecure_build_config_suffix, + srcs = [ + "resolver_component_tests_runner_invoker.cc", + ], + deps = [ + "//test/cpp/util:test_util", + "//test/core/util:grpc_test_util", + "//test/core/util:gpr_test_util", + "//:grpc++", + "//:grpc", + "//:gpr", + "//test/cpp/util:test_config", + ], + data = [ + ":resolver_component_tests_runner", + ":resolver_component_test%s" % unsecure_build_config_suffix, + ":test_dns_server", + "resolver_test_record_groups.yaml", # include the transitive dependency so that the dns sever py binary can locate this + ], + args = [ + "--test_bin_name=resolver_component_test%s" % unsecure_build_config_suffix, + "--running_under_bazel=true", + ] + ) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc new file mode 100644 index 0000000000..0857fb6a32 --- /dev/null +++ b/test/cpp/naming/resolver_component_test.cc @@ -0,0 +1,323 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/ext/filters/client_channel/resolver_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/support/env.h" +#include "src/core/lib/support/string.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +} + +using std::vector; +using grpc::SubProcess; +using testing::UnorderedElementsAreArray; + +// Hack copied from "test/cpp/end2end/server_crash_test_client.cc"! +// In some distros, gflags is in the namespace google, and in some others, +// in gflags. This hack is enabling us to find both. +namespace google {} +namespace gflags {} +using namespace google; +using namespace gflags; + +DEFINE_string(target_name, "", "Target name to resolve."); +DEFINE_string(expected_addrs, "", + "Comma-separated list of expected " + "',;,;...' " + "addresses of " + "backend and/or balancers. 'is_balancer' should be bool, i.e. " + "true or false."); +DEFINE_string(expected_chosen_service_config, "", + "Expected service config json string that gets chosen (no " + "whitespace). Empty for none."); +DEFINE_string( + local_dns_server_address, "", + "Optional. This address is placed as the uri authority if present."); +DEFINE_string(expected_lb_policy, "", + "Expected lb policy name that appears in resolver result channel " + "arg. Empty for none."); + +namespace { + +class GrpcLBAddress final { + public: + GrpcLBAddress(std::string address, bool is_balancer) + : is_balancer(is_balancer), address(address) {} + + bool operator==(const GrpcLBAddress &other) const { + return this->is_balancer == other.is_balancer && + this->address == other.address; + } + + bool operator!=(const GrpcLBAddress &other) const { + return !(*this == other); + } + + bool is_balancer; + std::string address; +}; + +vector ParseExpectedAddrs(std::string expected_addrs) { + std::vector out; + while (expected_addrs.size() != 0) { + // get the next , (v4 or v6) + size_t next_comma = expected_addrs.find(","); + if (next_comma == std::string::npos) { + gpr_log( + GPR_ERROR, + "Missing ','. Expected_addrs arg should be a semi-colon-separated " + "list of " + ", pairs. Left-to-be-parsed arg is |%s|", + expected_addrs.c_str()); + abort(); + } + std::string next_addr = expected_addrs.substr(0, next_comma); + expected_addrs = expected_addrs.substr(next_comma + 1, std::string::npos); + // get the next is_balancer 'bool' associated with this address + size_t next_semicolon = expected_addrs.find(";"); + bool is_balancer = + gpr_is_true(expected_addrs.substr(0, next_semicolon).c_str()); + out.emplace_back(GrpcLBAddress(next_addr, is_balancer)); + if (next_semicolon == std::string::npos) { + break; + } + expected_addrs = + expected_addrs.substr(next_semicolon + 1, std::string::npos); + } + if (out.size() == 0) { + gpr_log(GPR_ERROR, + "expected_addrs arg should be a comma-separated list of " + ", pairs"); + abort(); + } + return out; +} + +gpr_timespec TestDeadline(void) { + return grpc_timeout_seconds_to_deadline(100); +} + +struct ArgsStruct { + gpr_event ev; + gpr_atm done_atm; + gpr_mu *mu; + grpc_pollset *pollset; + grpc_pollset_set *pollset_set; + grpc_combiner *lock; + grpc_channel_args *channel_args; + vector expected_addrs; + std::string expected_service_config_string; + std::string expected_lb_policy; +}; + +void ArgsInit(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + gpr_event_init(&args->ev); + args->pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); + grpc_pollset_init(args->pollset, &args->mu); + args->pollset_set = grpc_pollset_set_create(); + grpc_pollset_set_add_pollset(exec_ctx, args->pollset_set, args->pollset); + args->lock = grpc_combiner_create(); + gpr_atm_rel_store(&args->done_atm, 0); + args->channel_args = NULL; +} + +void DoNothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} + +void ArgsFinish(grpc_exec_ctx *exec_ctx, ArgsStruct *args) { + GPR_ASSERT(gpr_event_wait(&args->ev, TestDeadline())); + grpc_pollset_set_del_pollset(exec_ctx, args->pollset_set, args->pollset); + grpc_pollset_set_destroy(exec_ctx, args->pollset_set); + grpc_closure DoNothing_cb; + GRPC_CLOSURE_INIT(&DoNothing_cb, DoNothing, NULL, grpc_schedule_on_exec_ctx); + grpc_pollset_shutdown(exec_ctx, args->pollset, &DoNothing_cb); + // exec_ctx needs to be flushed before calling grpc_pollset_destroy() + grpc_channel_args_destroy(exec_ctx, args->channel_args); + grpc_exec_ctx_flush(exec_ctx); + grpc_pollset_destroy(exec_ctx, args->pollset); + gpr_free(args->pollset); + GRPC_COMBINER_UNREF(exec_ctx, args->lock, NULL); +} + +gpr_timespec NSecondDeadline(int seconds) { + return gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(seconds, GPR_TIMESPAN)); +} + +void PollPollsetUntilRequestDone(ArgsStruct *args) { + gpr_timespec deadline = NSecondDeadline(10); + while (true) { + bool done = gpr_atm_acq_load(&args->done_atm) != 0; + if (done) { + break; + } + gpr_timespec time_left = + gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME)); + gpr_log(GPR_DEBUG, "done=%d, time_left=%" PRId64 ".%09d", done, + time_left.tv_sec, time_left.tv_nsec); + GPR_ASSERT(gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) >= 0); + grpc_pollset_worker *worker = NULL; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(&exec_ctx, args->pollset, &worker, + gpr_now(GPR_CLOCK_REALTIME), NSecondDeadline(1))); + gpr_mu_unlock(args->mu); + grpc_exec_ctx_finish(&exec_ctx); + } + gpr_event_set(&args->ev, (void *)1); +} + +void CheckServiceConfigResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *service_config_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_SERVICE_CONFIG); + if (args->expected_service_config_string != "") { + GPR_ASSERT(service_config_arg != NULL); + GPR_ASSERT(service_config_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(service_config_arg->value.string, + args->expected_service_config_string); + } else { + GPR_ASSERT(service_config_arg == NULL); + } +} + +void CheckLBPolicyResultLocked(grpc_channel_args *channel_args, + ArgsStruct *args) { + const grpc_arg *lb_policy_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_POLICY_NAME); + if (args->expected_lb_policy != "") { + GPR_ASSERT(lb_policy_arg != NULL); + GPR_ASSERT(lb_policy_arg->type == GRPC_ARG_STRING); + EXPECT_EQ(lb_policy_arg->value.string, args->expected_lb_policy); + } else { + GPR_ASSERT(lb_policy_arg == NULL); + } +} + +void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, + grpc_error *err) { + ArgsStruct *args = (ArgsStruct *)argsp; + grpc_channel_args *channel_args = args->channel_args; + const grpc_arg *channel_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); + GPR_ASSERT(channel_arg != NULL); + GPR_ASSERT(channel_arg->type == GRPC_ARG_POINTER); + grpc_lb_addresses *addresses = + (grpc_lb_addresses *)channel_arg->value.pointer.p; + gpr_log(GPR_INFO, "num addrs found: %" PRIdPTR ". expected %" PRIdPTR, + addresses->num_addresses, args->expected_addrs.size()); + GPR_ASSERT(addresses->num_addresses == args->expected_addrs.size()); + std::vector found_lb_addrs; + for (size_t i = 0; i < addresses->num_addresses; i++) { + grpc_lb_address addr = addresses->addresses[i]; + char *str; + grpc_sockaddr_to_string(&str, &addr.address, 1 /* normalize */); + gpr_log(GPR_INFO, "%s", str); + found_lb_addrs.emplace_back( + GrpcLBAddress(std::string(str), addr.is_balancer)); + gpr_free(str); + } + if (args->expected_addrs.size() != found_lb_addrs.size()) { + gpr_log(GPR_DEBUG, "found lb addrs size is: %" PRIdPTR + ". expected addrs size is %" PRIdPTR, + found_lb_addrs.size(), args->expected_addrs.size()); + abort(); + } + EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); + CheckServiceConfigResultLocked(channel_args, args); + CheckLBPolicyResultLocked(channel_args, args); + gpr_atm_rel_store(&args->done_atm, 1); + gpr_mu_lock(args->mu); + GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + gpr_mu_unlock(args->mu); +} + +TEST(ResolverComponentTest, TestResolvesRelevantRecords) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + ArgsStruct args; + ArgsInit(&exec_ctx, &args); + args.expected_addrs = ParseExpectedAddrs(FLAGS_expected_addrs); + args.expected_service_config_string = FLAGS_expected_chosen_service_config; + args.expected_lb_policy = FLAGS_expected_lb_policy; + // maybe build the address with an authority + char *whole_uri = NULL; + GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); + // create resolver and resolve + grpc_resolver *resolver = grpc_resolver_create(&exec_ctx, whole_uri, NULL, + args.pollset_set, args.lock); + gpr_free(whole_uri); + grpc_closure on_resolver_result_changed; + GRPC_CLOSURE_INIT(&on_resolver_result_changed, CheckResolverResultLocked, + (void *)&args, grpc_combiner_scheduler(args.lock)); + grpc_resolver_next_locked(&exec_ctx, resolver, &args.channel_args, + &on_resolver_result_changed); + grpc_exec_ctx_flush(&exec_ctx); + PollPollsetUntilRequestDone(&args); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, NULL); + ArgsFinish(&exec_ctx, &args); + grpc_exec_ctx_finish(&exec_ctx); +} + +} // namespace + +int main(int argc, char **argv) { + grpc_init(); + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_target_name == "") { + gpr_log(GPR_ERROR, "Missing target_name param."); + abort(); + } + if (FLAGS_local_dns_server_address != "") { + gpr_log(GPR_INFO, "Specifying authority in uris to: %s", + FLAGS_local_dns_server_address.c_str()); + } + auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; +} diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh new file mode 100755 index 0000000000..83b03b67a3 --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +# all command args required in this set order +FLAGS_test_bin_path=`echo "$1" | grep '\--test_bin_path=' | cut -d "=" -f 2` +FLAGS_dns_server_bin_path=`echo "$2" | grep '\--dns_server_bin_path=' | cut -d "=" -f 2` +FLAGS_records_config_path=`echo "$3" | grep '\--records_config_path=' | cut -d "=" -f 2` +FLAGS_test_dns_server_port=`echo "$4" | grep '\--test_dns_server_port=' | cut -d "=" -f 2` + +for cmd_arg in "$FLAGS_test_bin_path" "$FLAGS_dns_server_bin_path" "$FLAGS_records_config_path" "$FLAGS_test_dns_server_port"; do + if [[ "$cmd_arg" == "" ]]; then + echo "Missing a CMD arg" && exit 1 + fi +done + +if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "This test only works under GRPC_DNS_RESOLVER=ares. Have GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER" && exit 1 +fi +export GRPC_DNS_RESOLVER=ares + +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_test_dns_server_port" 2>&1 > /dev/null & +DNS_SERVER_PID=$! +echo "Local DNS server started. PID: $DNS_SERVER_PID" + +# Health check local DNS server TCP and UDP ports +for ((i=0;i<30;i++)); +do + echo "Retry health-check DNS query to local DNS server over tcp and udp" + RETRY=0 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 | grep '123.123.123.123' || RETRY=1 + dig A health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp. @localhost -p "$FLAGS_test_dns_server_port" +tries=1 +timeout=1 +tcp | grep '123.123.123.123' || RETRY=1 + if [[ "$RETRY" == 0 ]]; then + break + fi; + sleep 0.1 +done + +if [[ $RETRY == 1 ]]; then + echo "FAILED TO START LOCAL DNS SERVER" + kill -SIGTERM $DNS_SERVER_PID + wait + exit 1 +fi + +function terminate_all { + echo "Received signal. Terminating $! and $DNS_SERVER_PID" + kill -SIGTERM $! || true + kill -SIGTERM $DNS_SERVER_PID || true + wait + exit 1 +} + +trap terminate_all SIGTERM SIGINT + +EXIT_CODE=0 +# TODO: this test should check for GCE residency and skip tests using _grpclb._tcp.* SRV records once GCE residency checks are made +# in the resolver. + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +$FLAGS_test_bin_path \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + +kill -SIGTERM $DNS_SERVER_PID || true +wait +exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_component_tests_runner_invoker.cc b/test/cpp/naming/resolver_component_tests_runner_invoker.cc new file mode 100644 index 0000000000..b14391284d --- /dev/null +++ b/test/cpp/naming/resolver_component_tests_runner_invoker.cc @@ -0,0 +1,189 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "test/cpp/util/subprocess.h" +#include "test/cpp/util/test_config.h" + +extern "C" { +#include "src/core/lib/support/env.h" +#include "test/core/util/port.h" +} + +DEFINE_bool( + running_under_bazel, false, + "True if this test is running under bazel. " + "False indicates that this test is running under run_tests.py. " + "Child process test binaries are located differently based on this flag. "); + +DEFINE_string(test_bin_name, "", + "Name, without the preceding path, of the test binary"); + +DEFINE_string(grpc_test_directory_relative_to_test_srcdir, "/__main__", + "This flag only applies if runner_under_bazel is true. This " + "flag is ignored if runner_under_bazel is false. " + "Directory of the /test directory relative to bazel's " + "TEST_SRCDIR environment variable"); + +using grpc::SubProcess; + +static volatile sig_atomic_t abort_wait_for_child = 0; + +static void sighandler(int sig) { abort_wait_for_child = 1; } + +static void register_sighandler() { + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = sighandler; + sigaction(SIGINT, &act, NULL); + sigaction(SIGTERM, &act, NULL); +} + +namespace { + +const int kTestTimeoutSeconds = 60 * 2; + +void RunSigHandlingThread(SubProcess *test_driver, gpr_mu *test_driver_mu, + gpr_cv *test_driver_cv, int *test_driver_done) { + gpr_timespec overall_deadline = + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(kTestTimeoutSeconds, GPR_TIMESPAN)); + while (true) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (gpr_time_cmp(now, overall_deadline) > 0 || abort_wait_for_child) break; + gpr_mu_lock(test_driver_mu); + if (*test_driver_done) { + gpr_mu_unlock(test_driver_mu); + return; + } + gpr_timespec wait_deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)); + gpr_cv_wait(test_driver_cv, test_driver_mu, wait_deadline); + gpr_mu_unlock(test_driver_mu); + } + gpr_log(GPR_DEBUG, + "Test timeout reached or received signal. Interrupting test driver " + "child process."); + test_driver->Interrupt(); + return; +} +} + +namespace grpc { + +namespace testing { + +void InvokeResolverComponentTestsRunner(std::string test_runner_bin_path, + std::string test_bin_path, + std::string dns_server_bin_path, + std::string records_config_path) { + int test_dns_server_port = grpc_pick_unused_port_or_die(); + + SubProcess *test_driver = new SubProcess( + {test_runner_bin_path, "--test_bin_path=" + test_bin_path, + "--dns_server_bin_path=" + dns_server_bin_path, + "--records_config_path=" + records_config_path, + "--test_dns_server_port=" + std::to_string(test_dns_server_port)}); + gpr_mu test_driver_mu; + gpr_mu_init(&test_driver_mu); + gpr_cv test_driver_cv; + gpr_cv_init(&test_driver_cv); + int test_driver_done = 0; + register_sighandler(); + std::thread sig_handling_thread(RunSigHandlingThread, test_driver, + &test_driver_mu, &test_driver_cv, + &test_driver_done); + int status = test_driver->Join(); + if (WIFEXITED(status)) { + if (WEXITSTATUS(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner exited with code %d", + WEXITSTATUS(status)); + abort(); + } + } else if (WIFSIGNALED(status)) { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended from signal %d", + WTERMSIG(status)); + abort(); + } else { + gpr_log(GPR_INFO, + "Resolver component test test-runner ended with unknown status %d", + status); + abort(); + } + gpr_mu_lock(&test_driver_mu); + test_driver_done = 1; + gpr_cv_signal(&test_driver_cv); + gpr_mu_unlock(&test_driver_mu); + sig_handling_thread.join(); + delete test_driver; + gpr_mu_destroy(&test_driver_mu); + gpr_cv_destroy(&test_driver_cv); +} + +} // namespace testing + +} // namespace grpc + +int main(int argc, char **argv) { + grpc::testing::InitTest(&argc, &argv, true); + grpc_init(); + GPR_ASSERT(FLAGS_test_bin_name != ""); + std::string my_bin = argv[0]; + if (FLAGS_running_under_bazel) { + GPR_ASSERT(FLAGS_grpc_test_directory_relative_to_test_srcdir != ""); + // Use bazel's TEST_SRCDIR environment variable to locate the "test data" + // binaries. + std::string const bin_dir = + gpr_getenv("TEST_SRCDIR") + + FLAGS_grpc_test_directory_relative_to_test_srcdir + + std::string("/test/cpp/naming"); + // Invoke bazel's executeable links to the .sh and .py scripts (don't use + // the .sh and .py suffixes) to make + // sure that we're using bazel's test environment. + grpc::testing::InvokeResolverComponentTestsRunner( + bin_dir + "/resolver_component_tests_runner", + bin_dir + "/" + FLAGS_test_bin_name, bin_dir + "/test_dns_server", + bin_dir + "/resolver_test_record_groups.yaml"); + } else { + // Get the current binary's directory relative to repo root to invoke the + // correct build config (asan/tsan/dbg, etc.). + std::string const bin_dir = my_bin.substr(0, my_bin.rfind('/')); + // Invoke the .sh and .py scripts directly where they are in source code. + grpc::testing::InvokeResolverComponentTestsRunner( + "test/cpp/naming/resolver_component_tests_runner.sh", + bin_dir + "/" + FLAGS_test_bin_name, + "test/cpp/naming/test_dns_server.py", + "test/cpp/naming/resolver_test_record_groups.yaml"); + } + grpc_shutdown(); + return 0; +} diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml new file mode 100644 index 0000000000..67c611d831 --- /dev/null +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -0,0 +1,155 @@ +resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp. +resolver_component_tests: +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-single-target + records: + _grpclb._tcp.srv-ipv4-single-target: + - {TTL: '2100', data: 0 0 1234 ipv4-single-target, type: SRV} + ipv4-single-target: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '1.2.3.5:1234', is_balancer: true} + - {address: '1.2.3.6:1234', is_balancer: true} + - {address: '1.2.3.7:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-multi-target + records: + _grpclb._tcp.srv-ipv4-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv4-multi-target, type: SRV} + ipv4-multi-target: + - {TTL: '2100', data: 1.2.3.5, type: A} + - {TTL: '2100', data: 1.2.3.6, type: A} + - {TTL: '2100', data: 1.2.3.7, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1001]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-single-target + records: + _grpclb._tcp.srv-ipv6-single-target: + - {TTL: '2100', data: 0 0 1234 ipv6-single-target, type: SRV} + ipv6-single-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1001', type: AAAA} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1003]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1004]:1234', is_balancer: true} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-multi-target + records: + _grpclb._tcp.srv-ipv6-multi-target: + - {TTL: '2100', data: 0 0 1234 ipv6-multi-target, type: SRV} + ipv6-multi-target: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1003', type: AAAA} + - {TTL: '2100', data: '2607:f8b0:400a:801::1004', type: AAAA} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: srv-ipv4-simple-service-config + records: + _grpclb._tcp.srv-ipv4-simple-service-config: + - {TTL: '2100', data: 0 0 1234 ipv4-simple-service-config, type: SRV} + ipv4-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-simple-service-config: + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-no-srv-simple-service-config + records: + ipv4-no-srv-simple-service-config: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-no-config-for-cpp + records: + ipv4-no-config-for-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["python"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"PythonService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: ipv4-cpp-config-has-zero-percentage + records: + ipv4-cpp-config-has-zero-percentage: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-second-language-is-cpp + records: + ipv4-second-language-is-cpp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"clientLanguage":["go"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"GoService","waitForReady":true}]}]}},{"clientLanguage":["c++"],"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' + expected_lb_policy: round_robin + record_to_resolve: ipv4-config-with-percentages + records: + ipv4-config-with-percentages: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"percentage":0,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NeverPickedService","waitForReady":true}]}]}},{"percentage":100,"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}}]', + type: TXT} +- expected_addrs: + - {address: '1.2.3.4:1234', is_balancer: true} + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv4-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv4-has-backend-and-balancer, type: SRV} + balancer-for-ipv4-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} + srv-ipv4-target-has-backend-and-balancer: + - {TTL: '2100', data: 1.2.3.4, type: A} +- expected_addrs: + - {address: '[2607:f8b0:400a:801::1002]:1234', is_balancer: true} + - {address: '[2607:f8b0:400a:801::1002]:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + records: + _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: 0 0 1234 balancer-for-ipv6-has-backend-and-balancer, type: SRV} + balancer-for-ipv6-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + srv-ipv6-target-has-backend-and-balancer: + - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} + +resolver_component_tests_TODO: +- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 + bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, + e.g. via notes in RFC 2671)' +- expected_addrs: + - {address: '1.2.3.4:443', is_balancer: false} + expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' + expected_lb_policy: null + record_to_resolve: srv-ipv6-target-has-backend-and-balancer + record_to_resolve: ipv4-config-causing-fallback-to-tcp + records: + ipv4-config-causing-fallback-to-tcp: + - {TTL: '2100', data: 1.2.3.4, type: A} + - {TTL: '2100', data: 'grpc_config=[{"serviceConfig":{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}}]', + type: TXT} diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py new file mode 100755 index 0000000000..9d4b89cffb --- /dev/null +++ b/test/cpp/naming/test_dns_server.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Starts a local DNS server for use in tests""" + +import argparse +import sys +import yaml +import signal +import os + +import twisted +import twisted.internet +import twisted.internet.reactor +import twisted.internet.threads +import twisted.internet.defer +import twisted.internet.protocol +import twisted.names +import twisted.names.client +import twisted.names.dns +import twisted.names.server +from twisted.names import client, server, common, authority, dns +import argparse + +_SERVER_HEALTH_CHECK_RECORD_NAME = 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp' # missing end '.' for twisted syntax +_SERVER_HEALTH_CHECK_RECORD_DATA = '123.123.123.123' + +class NoFileAuthority(authority.FileAuthority): + def __init__(self, soa, records): + # skip FileAuthority + common.ResolverBase.__init__(self) + self.soa = soa + self.records = records + +def start_local_dns_server(args): + all_records = {} + def _push_record(name, r): + print('pushing record: |%s|' % name) + if all_records.get(name) is not None: + all_records[name].append(r) + return + all_records[name] = [r] + + def _maybe_split_up_txt_data(name, txt_data, r_ttl): + start = 0 + txt_data_list = [] + while len(txt_data[start:]) > 0: + next_read = len(txt_data[start:]) + if next_read > 255: + next_read = 255 + txt_data_list.append(txt_data[start:start+next_read]) + start += next_read + _push_record(name, dns.Record_TXT(*txt_data_list, ttl=r_ttl)) + + with open(args.records_config_path) as config: + test_records_config = yaml.load(config) + common_zone_name = test_records_config['resolver_component_tests_common_zone_name'] + for group in test_records_config['resolver_component_tests']: + for name in group['records'].keys(): + for record in group['records'][name]: + r_type = record['type'] + r_data = record['data'] + r_ttl = int(record['TTL']) + record_full_name = '%s.%s' % (name, common_zone_name) + assert record_full_name[-1] == '.' + record_full_name = record_full_name[:-1] + if r_type == 'A': + _push_record(record_full_name, dns.Record_A(r_data, ttl=r_ttl)) + if r_type == 'AAAA': + _push_record(record_full_name, dns.Record_AAAA(r_data, ttl=r_ttl)) + if r_type == 'SRV': + p, w, port, target = r_data.split(' ') + p = int(p) + w = int(w) + port = int(port) + target_full_name = '%s.%s' % (target, common_zone_name) + r_data = '%s %s %s %s' % (p, w, port, target_full_name) + _push_record(record_full_name, dns.Record_SRV(p, w, port, target_full_name, ttl=r_ttl)) + if r_type == 'TXT': + _maybe_split_up_txt_data(record_full_name, r_data, r_ttl) + # Server health check record + _push_record(_SERVER_HEALTH_CHECK_RECORD_NAME, dns.Record_A(_SERVER_HEALTH_CHECK_RECORD_DATA, ttl=0)) + soa_record = dns.Record_SOA(mname = common_zone_name) + test_domain_com = NoFileAuthority( + soa = (common_zone_name, soa_record), + records = all_records, + ) + server = twisted.names.server.DNSServerFactory( + authorities=[test_domain_com], verbose=2) + server.noisy = 2 + twisted.internet.reactor.listenTCP(args.port, server) + dns_proto = twisted.names.dns.DNSDatagramProtocol(server) + dns_proto.noisy = 2 + twisted.internet.reactor.listenUDP(args.port, dns_proto) + print('starting local dns server on 127.0.0.1:%s' % args.port) + print('starting twisted.internet.reactor') + twisted.internet.reactor.suggestThreadPoolSize(1) + twisted.internet.reactor.run() + +def _quit_on_signal(signum, _frame): + print('Received SIGNAL %d. Quitting with exit code 0' % signum) + twisted.internet.reactor.stop() + sys.stdout.flush() + sys.exit(0) + +def main(): + argp = argparse.ArgumentParser(description='Local DNS Server for resolver tests') + argp.add_argument('-p', '--port', default=None, type=int, + help='Port for DNS server to listen on for TCP and UDP.') + argp.add_argument('-r', '--records_config_path', default=None, type=str, + help=('Directory of resolver_test_record_groups.yaml file. ' + 'Defauls to path needed when the test is invoked as part of run_tests.py.')) + args = argp.parse_args() + signal.signal(signal.SIGALRM, _quit_on_signal) + signal.signal(signal.SIGTERM, _quit_on_signal) + signal.signal(signal.SIGINT, _quit_on_signal) + # Prevent zombies. Tests that use this server are short-lived. + signal.alarm(2 * 60) + start_local_dns_server(args) + +if __name__ == '__main__': + main() diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 7649145f86..693c02fdb2 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -24,6 +24,7 @@ gen_build_yaml_dirs=" \ test/core/bad_client \ test/core/bad_ssl \ test/core/end2end \ + test/cpp/naming \ test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 4ccfbc43c3..dbf58023c5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index 824ecf4604..7cfe98cbc8 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index 5f7bedde73..febe2fa251 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index 2bfa87c4ea..3a516cbb62 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index 96391372ca..acb640a81d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 27a924343f..354b7bfdb5 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 RUN pip install twisted h2==2.6.1 hyper diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 264a0f9271..92a542ff76 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Trigger download of as many Gradle artifacts as possible. diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index c59b12d155..4343d56cb4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index f859a46514..d6e229f84a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 42b1107c11..271c6e75e9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 3c744b434e..7bcada68e0 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -61,7 +62,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 60dc7eb21a..40d46fcf58 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================ # C# dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index f9622dd757..888a37baca 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index 2d396ce2ff..319f1e1889 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 349a6eff83..61f005d9da 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index bccfdba7cd..f35247eccb 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 94c37b3f63..ce1badfeb1 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # C++ dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index b7373b5d9c..59fe4d8f93 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -122,7 +123,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install coverage for Python test coverage reporting RUN pip install coverage diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 31602e594e..103be8412b 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -76,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Node dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index e79305bb27..f6d426bcd6 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -77,7 +77,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 245a642367..ae82a8d99f 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================= # PHP dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 5093b793b0..d5d781cd1a 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index ff8f0231cb..3b4ad12b6d 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 # Install dependencies for pyenv RUN apt-get update && apt-get install -y \ diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d77917c87f..3d879bb0c9 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #================== # Ruby dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 0f2f9ede58..44732a5ae4 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -22,6 +22,7 @@ RUN apt-get update && apt-get install -y \ bzip2 \ ccache \ curl \ + dnsutils \ gcc \ gcc-multilib \ git \ @@ -65,7 +66,7 @@ RUN apt-get update && apt-get install -y \ # Install Python packages from PyPI RUN pip install pip --upgrade RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 twisted==17.5.0 #======================== # Sanity test dependencies diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index dd2741e595..3319aa4394 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -46,7 +46,7 @@ pod repo update # needed by python # python brew install coreutils # we need grealpath pip install virtualenv --user python -pip install -U six tox setuptools --user python +pip install -U six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # set xcode version for Obj-C tests diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 2f6e34bfb3..8d42c3e432 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -5627,6 +5627,86 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc++_test_config", + "grpc++_test_util_unsecure", + "grpc++_unsecure", + "grpc_test_util_unsecure", + "grpc_unsecure" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test_unsecure", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_test", + "src": [ + "test/cpp/naming/resolver_component_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "src": [ + "test/cpp/naming/resolver_component_tests_runner_invoker.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 45d7145e8e..72dfbc0b98 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4404,6 +4404,52 @@ "posix" ] }, + { + "args": [ + "--test_bin_name=resolver_component_test_unsecure", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker_unsecure", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ + "--test_bin_name=resolver_component_test", + "--running_under_bazel=false" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "resolver_component_tests_runner_invoker", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "third_party/boringssl/crypto/aes/aes_tests.txt" -- cgit v1.2.3 From b2a54ac4cb95742a11325c923f63a8aa1dd145e0 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 10:18:07 -0700 Subject: Reverting suffixes and changing field names instead --- src/core/lib/iomgr/ev_epoll1_linux.c | 54 +++++----- src/core/lib/iomgr/ev_epollex_linux.c | 186 +++++++++++++++++----------------- src/core/lib/surface/call.c | 66 ++++++------ src/core/lib/surface/call.h | 2 +- src/core/lib/surface/channel.c | 2 +- 5 files changed, 157 insertions(+), 153 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 6946a2cbf5..0d57833d85 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -130,9 +130,9 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state_t; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; -static const char *kick_state_string(kick_state_t st) { +static const char *kick_state_string(kick_state st) { switch (st) { case UNKICKED: return "UNKICKED"; @@ -145,7 +145,7 @@ static const char *kick_state_string(kick_state_t st) { } struct grpc_pollset_worker { - kick_state_t kick_state; + kick_state state; int kick_state_mutator; // which line of code last changed kick state bool initialized_cv; grpc_pollset_worker *next; @@ -154,9 +154,9 @@ struct grpc_pollset_worker { grpc_closure_list schedule_on_end_work; }; -#define SET_KICK_STATE(worker, state) \ +#define SET_KICK_STATE(worker, kick_state) \ do { \ - (worker)->kick_state = (state); \ + (worker)->state = (kick_state); \ (worker)->kick_state_mutator = __LINE__; \ } while (false) @@ -508,7 +508,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - switch (worker->kick_state) { + switch (worker->state) { case KICKED: break; case UNKICKED: @@ -688,7 +688,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_mu_lock(&pollset->mu); if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), is_reassigning); } if (pollset->seen_inactive) { @@ -708,12 +708,12 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, at this point is if it were "kicked specifically". Since the worker has not added itself to the pollset yet (by calling worker_insert()), it is not visible in the "kick any" path yet */ - if (worker->kick_state == UNKICKED) { + if (worker->state == UNKICKED) { pollset->seen_inactive = false; if (neighborhood->active_root == NULL) { neighborhood->active_root = pollset->next = pollset->prev = pollset; /* Make this the designated poller if there isn't one already */ - if (worker->kick_state == UNKICKED && + if (worker->state == UNKICKED && gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { SET_KICK_STATE(worker, DESIGNATED_POLLER); } @@ -733,19 +733,19 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker_insert(pollset, worker); pollset->begin_refs--; - if (worker->kick_state == UNKICKED && !pollset->kicked_without_poller) { + if (worker->state == UNKICKED && !pollset->kicked_without_poller) { GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (worker->kick_state == UNKICKED && !pollset->shutting_down) { + while (worker->state == UNKICKED && !pollset->shutting_down) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down); } if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && - worker->kick_state == UNKICKED) { + worker->state == UNKICKED) { /* If gpr_cv_wait returns true (i.e a timeout), pretend that the worker received a kick */ SET_KICK_STATE(worker, KICKED); @@ -758,7 +758,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_log(GPR_ERROR, "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d " "kicked_without_poller: %d", - pollset, worker, kick_state_string(worker->kick_state), + pollset, worker, kick_state_string(worker->state), pollset->shutting_down, pollset->kicked_without_poller); } @@ -778,7 +778,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } GPR_TIMER_END("begin_worker", 0); - return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down; + return worker->state == DESIGNATED_POLLER && !pollset->shutting_down; } static bool check_neighborhood_for_available_poller( @@ -795,7 +795,7 @@ static bool check_neighborhood_for_available_poller( grpc_pollset_worker *inspect_worker = inspect->root_worker; if (inspect_worker != NULL) { do { - switch (inspect_worker->kick_state) { + switch (inspect_worker->state) { case UNKICKED: if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { @@ -858,7 +858,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_list_move(&worker->schedule_on_end_work, &exec_ctx->closure_list); if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { - if (worker->next != worker && worker->next->kick_state == UNKICKED) { + if (worker->next != worker && worker->next->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, " .. choose next poller to be peer %p", worker); } @@ -993,14 +993,14 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_strvec_add(&log, tmp); if (pollset->root_worker != NULL) { gpr_asprintf(&tmp, " {kick_state=%s next=%p {kick_state=%s}}", - kick_state_string(pollset->root_worker->kick_state), + kick_state_string(pollset->root_worker->state), pollset->root_worker->next, - kick_state_string(pollset->root_worker->next->kick_state)); + kick_state_string(pollset->root_worker->next->state)); gpr_strvec_add(&log, tmp); } if (specific_worker != NULL) { gpr_asprintf(&tmp, " worker_kick_state=%s", - kick_state_string(specific_worker->kick_state)); + kick_state_string(specific_worker->state)); gpr_strvec_add(&log, tmp); } tmp = gpr_strvec_flatten(&log, NULL); @@ -1020,13 +1020,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } grpc_pollset_worker *next_worker = root_worker->next; - if (root_worker->kick_state == KICKED) { + if (root_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", root_worker); } SET_KICK_STATE(root_worker, KICKED); goto done; - } else if (next_worker->kick_state == KICKED) { + } else if (next_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. already kicked %p", next_worker); } @@ -1043,7 +1043,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(root_worker, KICKED); ret_err = grpc_wakeup_fd_wakeup(&global_wakeup_fd); goto done; - } else if (next_worker->kick_state == UNKICKED) { + } else if (next_worker->state == UNKICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. kicked %p", next_worker); } @@ -1051,8 +1051,8 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, SET_KICK_STATE(next_worker, KICKED); gpr_cv_signal(&next_worker->cv); goto done; - } else if (next_worker->kick_state == DESIGNATED_POLLER) { - if (root_worker->kick_state != DESIGNATED_POLLER) { + } else if (next_worker->state == DESIGNATED_POLLER) { + if (root_worker->state != DESIGNATED_POLLER) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log( GPR_ERROR, @@ -1074,7 +1074,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, goto done; } } else { - GPR_ASSERT(next_worker->kick_state == KICKED); + GPR_ASSERT(next_worker->state == KICKED); SET_KICK_STATE(next_worker, KICKED); goto done; } @@ -1088,7 +1088,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, GPR_UNREACHABLE_CODE(goto done); } - if (specific_worker->kick_state == KICKED) { + if (specific_worker->state == KICKED) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_ERROR, " .. specific worker already kicked"); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index df69025f1a..9934f8d329 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -97,12 +97,12 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, * pollable Declarations */ -typedef struct pollable_t { +typedef struct pollable { polling_obj po; int epfd; grpc_wakeup_fd wakeup; grpc_pollset_worker *root_worker; -} pollable_t; +} pollable; static const char *polling_obj_type_string(polling_obj_type t) { switch (t) { @@ -122,7 +122,7 @@ static const char *polling_obj_type_string(polling_obj_type t) { return ""; } -static char *pollable_desc(pollable_t *p) { +static char *pollable_desc(pollable *p) { char *out; gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d", polling_obj_type_string(p->po.type), p->po.group, p->epfd, @@ -130,19 +130,19 @@ static char *pollable_desc(pollable_t *p) { return out; } -static pollable_t g_empty_pollable; +static pollable g_empty_pollable; -static void pollable_init(pollable_t *p, polling_obj_type type); -static void pollable_destroy(pollable_t *p); +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p); +static grpc_error *pollable_materialize(pollable *p); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pollable_t pollable; + pollable pollable_obj; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -193,15 +193,15 @@ struct grpc_pollset_worker { pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; grpc_pollset *pollset; - pollable_t *pollable; + pollable *pollable_obj; }; #define MAX_EPOLL_EVENTS 100 #define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 struct grpc_pollset { - pollable_t pollable; - pollable_t *current_pollable; + pollable pollable_obj; + pollable *current_pollable; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -282,7 +282,7 @@ static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_fd *fd = (grpc_fd *)arg; /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); - pollable_destroy(&fd->pollable); + pollable_destroy(&fd->pollable_obj); gpr_mu_destroy(&fd->orphaned_mu); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; @@ -343,7 +343,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = (grpc_fd *)gpr_malloc(sizeof(grpc_fd)); } - pollable_init(&new_fd->pollable, PO_FD); + pollable_init(&new_fd->pollable_obj, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -385,7 +385,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = already_closed; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->pollable_obj.po.mu); gpr_mu_lock(&fd->orphaned_mu); fd->on_done_closure = on_done; @@ -411,7 +411,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_CLOSURE_SCHED(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->orphaned_mu); - gpr_mu_unlock(&fd->pollable.po.mu); + gpr_mu_unlock(&fd->pollable_obj.po.mu); UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -451,13 +451,13 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, * Pollable Definitions */ -static void pollable_init(pollable_t *p, polling_obj_type type) { +static void pollable_init(pollable *p, polling_obj_type type) { po_init(&p->po, type); p->root_worker = NULL; p->epfd = -1; } -static void pollable_destroy(pollable_t *p) { +static void pollable_destroy(pollable *p) { po_destroy(&p->po); if (p->epfd != -1) { close(p->epfd); @@ -466,7 +466,7 @@ static void pollable_destroy(pollable_t *p) { } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ -static grpc_error *pollable_materialize(pollable_t *p) { +static grpc_error *pollable_materialize(pollable *p) { if (p->epfd == -1) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { @@ -492,7 +492,7 @@ static grpc_error *pollable_materialize(pollable_t *p) { } /* pollable must be materialized */ -static grpc_error *pollable_add_fd(pollable_t *p, grpc_fd *fd) { +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; @@ -557,30 +557,33 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error_unused) { grpc_error *error = GRPC_ERROR_NONE; grpc_pollset *pollset = (grpc_pollset *)arg; - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - if (worker->pollable != &pollset->pollable) { - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker->pollable_obj->po.mu); } if (worker->initialized_cv && worker != pollset->root_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_cv %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } worker->kicked = true; gpr_cv_signal(&worker->cv); } else { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kickall_via_wakeup %p (pollable %p vs %p)", - pollset, worker, &pollset->pollable, worker->pollable); + pollset, worker, &pollset->pollable_obj, + worker->pollable_obj); } - append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + append_error(&error, + grpc_wakeup_fd_wakeup(&worker->pollable_obj->wakeup), "pollset_shutdown"); } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); } worker = worker->links[PWL_POLLSET].next; @@ -588,7 +591,7 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg, } pollset->kick_alls_pending--; pollset_maybe_finish_shutdown(exec_ctx, pollset); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("kick_all", error); } @@ -599,7 +602,7 @@ static void pollset_kick_all(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GRPC_ERROR_NONE); } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -664,24 +667,24 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable_t *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable_t *p = pollset->current_pollable; - if (p != &pollset->pollable) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable_obj) { gpr_mu_lock(&p->po.mu); } grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); - if (p != &pollset->pollable) { + if (p != &pollset->pollable_obj) { gpr_mu_unlock(&p->po.mu); } return error; } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - pollable_init(&pollset->pollable, PO_POLLSET); + pollable_init(&pollset->pollable_obj, PO_POLLSET); pollset->current_pollable = &g_empty_pollable; pollset->kicked_without_poller = false; pollset->shutdown_closure = NULL; pollset->root_worker = NULL; - *mu = &pollset->pollable.po.mu; + *mu = &pollset->pollable_obj.po.mu; } /* Convert a timespec to milliseconds: @@ -729,8 +732,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "fd_become_pollable"; - if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { - append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + if (append_error(&error, pollable_materialize(&fd->pollable_obj), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable_obj, fd), err_desc); } return error; } @@ -744,8 +747,8 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable_t *p) { - return p != &g_empty_pollable && p != &pollset->pollable; +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable_obj; } static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, @@ -791,7 +794,7 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - pollable_destroy(&pollset->pollable); + pollable_destroy(&pollset->pollable_obj); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); @@ -801,7 +804,7 @@ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { } static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - pollable_t *p, gpr_timespec now, + pollable *p, gpr_timespec now, gpr_timespec deadline) { int timeout = poll_deadline_to_millis_timeout(deadline, now); @@ -883,68 +886,69 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = false; worker->kicked = false; worker->pollset = pollset; - worker->pollable = pollset->current_pollable; + worker->pollable_obj = pollset->current_pollable; - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + REF_BY((grpc_fd *)worker->pollable_obj, 2, "one_poll"); } worker_insert(&pollset->root_worker, PWL_POLLSET, worker); - if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + if (!worker_insert(&worker->pollable_obj->root_worker, PWL_POLLABLE, + worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&pollset->pollable.po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&pollset->pollable_obj.po.mu); } if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, - worker->pollable, worker, + worker->pollable_obj, worker, poll_deadline_to_millis_timeout(deadline, *now)); } - while (do_poll && worker->pollable->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + while (do_poll && worker->pollable_obj->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable_obj->po.mu, deadline)) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } do_poll = false; } else if (worker->kicked) { if (GRPC_TRACER_ON(grpc_polling_trace)) { - gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, - worker); + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, + worker->pollable_obj, worker); } do_poll = false; } else if (GRPC_TRACER_ON(grpc_polling_trace) && - worker->pollable->root_worker != worker) { + worker->pollable_obj->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, - worker->pollable, worker); + worker->pollable_obj, worker); } } - if (worker->pollable != &pollset->pollable) { - gpr_mu_unlock(&worker->pollable->po.mu); - gpr_mu_lock(&pollset->pollable.po.mu); - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker->pollable_obj->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + gpr_mu_lock(&worker->pollable_obj->po.mu); } *now = gpr_now(now->clock_type); } return do_poll && pollset->shutdown_closure == NULL && - pollset->current_pollable == worker->pollable; + pollset->current_pollable == worker->pollable_obj; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { if (NEW_ROOT == - worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { - gpr_cv_signal(&worker->pollable->root_worker->cv); + worker_remove(&worker->pollable_obj->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable_obj->root_worker->cv); } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } - if (pollset_is_pollable_fd(pollset, worker->pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); + if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable_obj, 2, "one_poll"); } if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); @@ -972,41 +976,41 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } - if (pollset->current_pollable != &pollset->pollable) { + if (pollset->current_pollable != &pollset->pollable_obj) { gpr_mu_lock(&pollset->current_pollable->po.mu); } if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - append_error(&error, pollable_materialize(worker.pollable), err_desc); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + append_error(&error, pollable_materialize(worker.pollable_obj), err_desc); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); if (pollset->event_cursor == pollset->event_count) { - append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable_obj, now, deadline), err_desc); } append_error(&error, pollset_process_events(exec_ctx, pollset, false), err_desc); - gpr_mu_lock(&pollset->pollable.po.mu); - if (worker.pollable != &pollset->pollable) { - gpr_mu_lock(&worker.pollable->po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&worker.pollable_obj->po.mu); } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(exec_ctx, pollset, &worker, worker_hdl); - if (worker.pollable != &pollset->pollable) { - gpr_mu_unlock(&worker.pollable->po.mu); + if (worker.pollable_obj != &pollset->pollable_obj) { + gpr_mu_unlock(&worker.pollable_obj->po.mu); } if (grpc_exec_ctx_has_work(exec_ctx)) { - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); } return error; } @@ -1029,20 +1033,20 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, "PS:%p add fd %p; transition pollable from empty to fd", pollset, fd); } - /* empty pollable --> single fd pollable_t */ + /* empty pollable --> single fd pollable */ pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &fd->pollable; - if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + pollset->current_pollable = &fd->pollable_obj; + if (!fd_locked) gpr_mu_lock(&fd->pollable_obj.po.mu); append_error(&error, fd_become_pollable_locked(fd), err_desc); - if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); + if (!fd_locked) gpr_mu_unlock(&fd->pollable_obj.po.mu); REF_BY(fd, 2, "pollset_pollable"); - } else if (pollset->current_pollable == &pollset->pollable) { + } else if (pollset->current_pollable == &pollset->pollable_obj) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); } append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); - } else if (pollset->current_pollable != &fd->pollable) { + } else if (pollset->current_pollable != &fd->pollable_obj) { grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, @@ -1055,11 +1059,11 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read"); grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write"); pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &pollset->pollable; - if (append_error(&error, pollable_materialize(&pollset->pollable), + pollset->current_pollable = &pollset->pollable_obj; + if (append_error(&error, pollable_materialize(&pollset->pollable_obj), err_desc)) { - pollable_add_fd(&pollset->pollable, had_fd); - pollable_add_fd(&pollset->pollable, fd); + pollable_add_fd(&pollset->pollable_obj, had_fd); + pollable_add_fd(&pollset->pollable_obj, fd); } GRPC_CLOSURE_SCHED(exec_ctx, GRPC_CLOSURE_CREATE(unref_fd_no_longer_poller, had_fd, @@ -1071,9 +1075,9 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&pollset->pollable_obj.po.mu); grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); - gpr_mu_unlock(&pollset->pollable.po.mu); + gpr_mu_unlock(&pollset->pollable_obj.po.mu); GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -1095,7 +1099,7 @@ static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - po_join(exec_ctx, &pss->po, &fd->pollable.po); + po_join(exec_ctx, &pss->po, &fd->pollable_obj.po); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, @@ -1103,7 +1107,7 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - po_join(exec_ctx, &pss->po, &ps->pollable.po); + po_join(exec_ctx, &pss->po, &ps->pollable_obj.po); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index de5aa7b6be..e258719c4d 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -135,7 +135,7 @@ typedef struct batch_control { typedef struct { gpr_mu child_list_mu; grpc_call *first_child; -} parent_call_t; +} parent_call; typedef struct { grpc_call *parent; @@ -144,7 +144,7 @@ typedef struct { parent->mu */ grpc_call *sibling_next; grpc_call *sibling_prev; -} child_call_t; +} child_call; #define RECV_NONE ((gpr_atm)0) #define RECV_INITIAL_METADATA_FIRST ((gpr_atm)1) @@ -157,8 +157,8 @@ struct grpc_call { grpc_polling_entity pollent; grpc_channel *channel; gpr_timespec start_time; - /* parent_call_t* */ gpr_atm parent_call_atm; - child_call_t *child_call; + /* parent_call* */ gpr_atm parent_call_atm; + child_call *child; /* client or server call */ bool is_client; @@ -304,21 +304,21 @@ void *grpc_call_arena_alloc(grpc_call *call, size_t size) { return gpr_arena_alloc(call->arena, size); } -static parent_call_t *get_or_create_parent_call(grpc_call *call) { - parent_call_t *p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_or_create_parent_call(grpc_call *call) { + parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); if (p == NULL) { - p = (parent_call_t *)gpr_arena_alloc(call->arena, sizeof(*p)); + p = (parent_call *)gpr_arena_alloc(call->arena, sizeof(*p)); gpr_mu_init(&p->child_list_mu); if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { gpr_mu_destroy(&p->child_list_mu); - p = (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); + p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } } return p; } -static parent_call_t *get_parent_call(grpc_call *call) { - return (parent_call_t *)gpr_atm_acq_load(&call->parent_call_atm); +static parent_call *get_parent_call(grpc_call *call) { + return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); } grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, @@ -377,24 +377,24 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, bool immediately_cancel = false; - if (args->parent_call != NULL) { - child_call_t *cc = call->child_call = - (child_call_t *)gpr_arena_alloc(arena, sizeof(child_call_t)); - call->child_call->parent = args->parent_call; + if (args->parent != NULL) { + child_call *cc = call->child = + (child_call *)gpr_arena_alloc(arena, sizeof(child_call)); + call->child->parent = args->parent; - GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); + GRPC_CALL_INTERNAL_REF(args->parent, "child"); GPR_ASSERT(call->is_client); - GPR_ASSERT(!args->parent_call->is_client); + GPR_ASSERT(!args->parent->is_client); - parent_call_t *pc = get_or_create_parent_call(args->parent_call); + parent_call *pc = get_or_create_parent_call(args->parent); gpr_mu_lock(&pc->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( gpr_convert_clock_type(send_deadline, - args->parent_call->send_deadline.clock_type), - args->parent_call->send_deadline); + args->parent->send_deadline.clock_type), + args->parent->send_deadline); } /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with * GRPC_PROPAGATE_STATS_CONTEXT */ @@ -406,9 +406,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, "Census tracing propagation requested " "without Census context propagation")); } - grpc_call_context_set( - call, GRPC_CONTEXT_TRACING, - args->parent_call->context[GRPC_CONTEXT_TRACING].value, NULL); + grpc_call_context_set(call, GRPC_CONTEXT_TRACING, + args->parent->context[GRPC_CONTEXT_TRACING].value, + NULL); } else if (args->propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) { add_init_error(&error, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Census context propagation requested " @@ -416,7 +416,7 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) { call->cancellation_is_inherited = 1; - if (gpr_atm_acq_load(&args->parent_call->received_final_op_atm)) { + if (gpr_atm_acq_load(&args->parent->received_final_op_atm)) { immediately_cancel = true; } } @@ -426,9 +426,9 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, cc->sibling_next = cc->sibling_prev = call; } else { cc->sibling_next = pc->first_child; - cc->sibling_prev = pc->first_child->child_call->sibling_prev; - cc->sibling_next->child_call->sibling_prev = - cc->sibling_prev->child_call->sibling_next = call; + cc->sibling_prev = pc->first_child->child->sibling_prev; + cc->sibling_next->child->sibling_prev = + cc->sibling_prev->child->sibling_next = call; } gpr_mu_unlock(&pc->child_list_mu); @@ -533,7 +533,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - parent_call_t *pc = get_parent_call(c); + parent_call *pc = get_parent_call(c); if (pc != NULL) { gpr_mu_destroy(&pc->child_list_mu); } @@ -570,14 +570,14 @@ void grpc_call_ref(grpc_call *c) { gpr_ref(&c->ext_ref); } void grpc_call_unref(grpc_call *c) { if (!gpr_unref(&c->ext_ref)) return; - child_call_t *cc = c->child_call; + child_call *cc = c->child; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_unref", 0); GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (c)); if (cc) { - parent_call_t *pc = get_parent_call(cc->parent); + parent_call *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; @@ -585,8 +585,8 @@ void grpc_call_unref(grpc_call *c) { pc->first_child = NULL; } } - cc->sibling_prev->child_call->sibling_next = cc->sibling_next; - cc->sibling_next->child_call->sibling_prev = cc->sibling_prev; + cc->sibling_prev->child->sibling_next = cc->sibling_next; + cc->sibling_next->child->sibling_prev = cc->sibling_prev; gpr_mu_unlock(&pc->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, cc->parent, "child"); } @@ -1309,14 +1309,14 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - parent_call_t *pc = get_parent_call(call); + parent_call *pc = get_parent_call(call); if (pc != NULL) { grpc_call *child; gpr_mu_lock(&pc->child_list_mu); child = pc->first_child; if (child != NULL) { do { - next_child_call = child->child_call->sibling_next; + next_child_call = child->child->sibling_next; if (child->cancellation_is_inherited) { GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); cancel_with_error(exec_ctx, child, STATUS_FROM_API_OVERRIDE, diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index d537637cbb..c680139cf6 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -37,7 +37,7 @@ typedef void (*grpc_ioreq_completion_func)(grpc_exec_ctx *exec_ctx, typedef struct grpc_call_create_args { grpc_channel *channel; - grpc_call *parent_call; + grpc_call *parent; uint32_t propagation_mask; grpc_completion_queue *cq; diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 2f9b677c85..48962e5e45 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -282,7 +282,7 @@ static grpc_call *grpc_channel_create_call_internal( grpc_call_create_args args; memset(&args, 0, sizeof(args)); args.channel = channel; - args.parent_call = parent_call; + args.parent = parent_call; args.propagation_mask = propagation_mask; args.cq = cq; args.pollset_set_alternative = pollset_set_alternative; -- cgit v1.2.3 From 8f798e9ca153f844033ce49f261eeb345bd09c4f Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 13 Sep 2017 15:45:47 -0700 Subject: %s/current_pollable/current_pollable_obj --- src/core/lib/iomgr/ev_epollex_linux.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 9934f8d329..51200fc064 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -201,7 +201,7 @@ struct grpc_pollset_worker { struct grpc_pollset { pollable pollable_obj; - pollable *current_pollable; + pollable *current_pollable_obj; int kick_alls_pending; bool kicked_without_poller; grpc_closure *shutdown_closure; @@ -667,7 +667,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; + pollable *p = pollset->current_pollable_obj; if (p != &pollset->pollable_obj) { gpr_mu_lock(&p->po.mu); } @@ -680,7 +680,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollable_init(&pollset->pollable_obj, PO_POLLSET); - pollset->current_pollable = &g_empty_pollable; + pollset->current_pollable_obj = &g_empty_pollable; pollset->kicked_without_poller = false; pollset->shutdown_closure = NULL; pollset->root_worker = NULL; @@ -795,8 +795,8 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable_obj); - if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + if (pollset_is_pollable_fd(pollset, pollset->current_pollable_obj)) { + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable_obj, 2, "pollset_pollable"); } GRPC_LOG_IF_ERROR("pollset_process_events", @@ -886,7 +886,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = false; worker->kicked = false; worker->pollset = pollset; - worker->pollable_obj = pollset->current_pollable; + worker->pollable_obj = pollset->current_pollable_obj; if (pollset_is_pollable_fd(pollset, worker->pollable_obj)) { REF_BY((grpc_fd *)worker->pollable_obj, 2, "one_poll"); @@ -934,7 +934,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } return do_poll && pollset->shutdown_closure == NULL && - pollset->current_pollable == worker->pollable_obj; + pollset->current_pollable_obj == worker->pollable_obj; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -976,8 +976,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } - if (pollset->current_pollable != &pollset->pollable_obj) { - gpr_mu_lock(&pollset->current_pollable->po.mu); + if (pollset->current_pollable_obj != &pollset->pollable_obj) { + gpr_mu_lock(&pollset->current_pollable_obj->po.mu); } if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); @@ -1027,7 +1027,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, bool fd_locked) { static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; - if (pollset->current_pollable == &g_empty_pollable) { + if (pollset->current_pollable_obj == &g_empty_pollable) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from empty to fd", pollset, @@ -1035,19 +1035,19 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, } /* empty pollable --> single fd pollable */ pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &fd->pollable_obj; + pollset->current_pollable_obj = &fd->pollable_obj; if (!fd_locked) gpr_mu_lock(&fd->pollable_obj.po.mu); append_error(&error, fd_become_pollable_locked(fd), err_desc); if (!fd_locked) gpr_mu_unlock(&fd->pollable_obj.po.mu); REF_BY(fd, 2, "pollset_pollable"); - } else if (pollset->current_pollable == &pollset->pollable_obj) { + } else if (pollset->current_pollable_obj == &pollset->pollable_obj) { if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); } - append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + append_error(&error, pollable_add_fd(pollset->current_pollable_obj, fd), err_desc); - } else if (pollset->current_pollable != &fd->pollable_obj) { - grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + } else if (pollset->current_pollable_obj != &fd->pollable_obj) { + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable_obj; if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p add fd %p; transition pollable from fd %p to multipoller", @@ -1059,7 +1059,7 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read"); grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write"); pollset_kick_all(exec_ctx, pollset); - pollset->current_pollable = &pollset->pollable_obj; + pollset->current_pollable_obj = &pollset->pollable_obj; if (append_error(&error, pollable_materialize(&pollset->pollable_obj), err_desc)) { pollable_add_fd(&pollset->pollable_obj, had_fd); -- cgit v1.2.3 From 989466171296e296867a3009f615a90d46acf28d Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 14 Sep 2017 15:21:16 -0700 Subject: Polish comment --- src/objective-c/GRPCClient/GRPCCall.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 3b0ef56c3a..df563ca36c 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -170,9 +170,9 @@ extern id const kGRPCTrailersKey; @property (atomic, copy, readwrite) NSString *serverName; /** - * The timeout for the RPC call in milliseconds. If set to 0, the call will not timeout. If set to - * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded. A negative value is - * not allowed. + * The timeout for the RPC call in seconds. If set to 0, the call will not timeout. If set to + * positive, the gRPC call returns with status GRPCErrorCodeDeadlineExceeded if it is not completed + * within \a timeout seconds. A negative value is not allowed. */ @property NSTimeInterval timeout; -- cgit v1.2.3 From 4b047a3bff92ff377205be38d9044b9d99e06fbb Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 21 Aug 2017 22:34:37 -0700 Subject: Construction of streams shouldn't require triggering async ops --- examples/cpp/helloworld/greeter_async_client.cc | 12 +- examples/cpp/helloworld/greeter_async_client2.cc | 14 +- include/grpc++/generic/generic_stub.h | 7 + include/grpc++/impl/codegen/async_stream.h | 136 ++++-- include/grpc++/impl/codegen/async_unary_call.h | 41 +- src/compiler/cpp_generator.cc | 578 ++++++++++++++--------- src/cpp/client/generic_stub.cc | 23 +- test/cpp/codegen/compiler_test_golden | 40 ++ test/cpp/codegen/compiler_test_mock_golden | 5 + test/cpp/qps/client_async.cc | 153 +++--- 10 files changed, 651 insertions(+), 358 deletions(-) diff --git a/examples/cpp/helloworld/greeter_async_client.cc b/examples/cpp/helloworld/greeter_async_client.cc index 64da044ea6..ddf6c1aaf3 100644 --- a/examples/cpp/helloworld/greeter_async_client.cc +++ b/examples/cpp/helloworld/greeter_async_client.cc @@ -60,11 +60,15 @@ class GreeterClient { // Storage for the status of the RPC upon completion. Status status; - // stub_->AsyncSayHello() performs the RPC call, returning an instance we - // store in "rpc". Because we are using the asynchronous API, we need to - // hold on to the "rpc" instance in order to get updates on the ongoing RPC. + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. std::unique_ptr > rpc( - stub_->AsyncSayHello(&context, request, &cq)); + stub_->PrepareAsyncSayHello(&context, request, &cq)); + + // StartCall initiates the RPC call + rpc->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/examples/cpp/helloworld/greeter_async_client2.cc b/examples/cpp/helloworld/greeter_async_client2.cc index 4192546bef..3154e84d85 100644 --- a/examples/cpp/helloworld/greeter_async_client2.cc +++ b/examples/cpp/helloworld/greeter_async_client2.cc @@ -49,11 +49,15 @@ class GreeterClient { // Call object to store rpc data AsyncClientCall* call = new AsyncClientCall; - // stub_->AsyncSayHello() performs the RPC call, returning an instance to - // store in "call". Because we are using the asynchronous API, we need to - // hold on to the "call" instance in order to get updates on the ongoing RPC. - call->response_reader = stub_->AsyncSayHello(&call->context, request, &cq_); - + // stub_->PrepareAsyncSayHello() creates an RPC object, returning + // an instance to store in "call" but does not actually start the RPC + // Because we are using the asynchronous API, we need to hold on to + // the "call" instance in order to get updates on the ongoing RPC. + call->response_reader = + stub_->PrepareAsyncSayHello(&call->context, request, &cq_); + + // StartCall initiates the RPC call + call->response_reader->StartCall(); // Request that, upon completion of the RPC, "reply" be updated with the // server's response; "status" with the indication of whether the operation diff --git a/include/grpc++/generic/generic_stub.h b/include/grpc++/generic/generic_stub.h index 7b269d0887..2b3ff59ea2 100644 --- a/include/grpc++/generic/generic_stub.h +++ b/include/grpc++/generic/generic_stub.h @@ -44,6 +44,13 @@ class GenericStub final { ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag); + /// Setup a call to a named method \a method using \a context, but don't + /// start it. Let it be started explicitly with StartCall and a tag. + /// The return value only indicates whether or not registration of the call + /// succeeded (i.e. the call won't proceed if the return value is nullptr). + std::unique_ptr PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq); + private: std::shared_ptr channel_; }; diff --git a/include/grpc++/impl/codegen/async_stream.h b/include/grpc++/impl/codegen/async_stream.h index 9cf7ac30dd..e60572fc93 100644 --- a/include/grpc++/impl/codegen/async_stream.h +++ b/include/grpc++/impl/codegen/async_stream.h @@ -35,6 +35,11 @@ class ClientAsyncStreamingInterface { public: virtual ~ClientAsyncStreamingInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall(void* tag) = 0; + /// Request notification of the reading of the initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -156,20 +161,22 @@ class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface, template class ClientAsyncReader final : public ClientAsyncReaderInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Write the first request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started and - /// \a request has been written out. + /// \a request has been written out. If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template static ClientAsyncReader* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, const W& request, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReader))) - ClientAsyncReader(call, context, request, tag); + ClientAsyncReader(call, context, request, start, tag); } // always allocated against a call arena, no memory free required @@ -177,6 +184,12 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { assert(size == sizeof(ClientAsyncReader)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata /// method for semantics. /// @@ -186,6 +199,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { /// calling code can access the received metadata through the /// \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -194,6 +208,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -208,6 +223,7 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata received from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -219,19 +235,28 @@ class ClientAsyncReader final : public ClientAsyncReaderInterface { private: template ClientAsyncReader(Call call, ClientContext* context, const W& request, - void* tag) - : context_(context), call_(call) { - init_ops_.set_output_tag(tag); - init_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + bool start, void* tag) + : context_(context), call_(call), started_(start) { // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_ops_.SendMessage(request).ok()); init_ops_.ClientSendClose(); + if (start) { + StartCallInternal(tag); + } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + init_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + init_ops_.set_output_tag(tag); call_.PerformOps(&init_ops_); } ClientContext* context_; Call call_; + bool started_; CallOpSet init_ops_; CallOpSet meta_ops_; @@ -257,9 +282,12 @@ class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface, template class ClientAsyncWriter final : public ClientAsyncWriterInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC if \a start is set /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, \a tag must be nullptr and the actual call + /// must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. /// \a response will be filled in with the single expected response @@ -269,11 +297,11 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { static ClientAsyncWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, ClientContext* context, R* response, - void* tag) { + bool start, void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncWriter))) - ClientAsyncWriter(call, context, response, tag); + ClientAsyncWriter(call, context, response, start, tag); } // always allocated against a call arena, no memory free required @@ -281,6 +309,12 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { assert(size == sizeof(ClientAsyncWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method for /// semantics. /// @@ -289,6 +323,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { /// associated with this call is updated, and the calling code can access /// the received metadata through the \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -297,6 +332,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -304,6 +340,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -315,6 +352,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -328,6 +366,7 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { /// - attempts to fill in the \a response parameter passed to this class's /// constructor with the server's response message. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -338,25 +377,32 @@ class ClientAsyncWriter final : public ClientAsyncWriterInterface { private: template - ClientAsyncWriter(Call call, ClientContext* context, R* response, void* tag) - : context_(context), call_(call) { + ClientAsyncWriter(Call call, ClientContext* context, R* response, bool start, + void* tag) + : context_(context), call_(call), started_(start) { finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - if (context_->initial_metadata_corked_) { - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet meta_ops_; CallOpSet write_ops_; @@ -388,20 +434,23 @@ template class ClientAsyncReaderWriter final : public ClientAsyncReaderWriterInterface { public: - /// Create a stream and write the first request out. + /// Create a stream object. + /// Start the RPC request if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. - /// intitial metadata sent). + /// intitial metadata sent). If \a start is not set, \a tag must be + /// nullptr and the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. static ClientAsyncReaderWriter* Create(ChannelInterface* channel, CompletionQueue* cq, const RpcMethod& method, - ClientContext* context, void* tag) { + ClientContext* context, bool start, + void* tag) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncReaderWriter))) - ClientAsyncReaderWriter(call, context, tag); + ClientAsyncReaderWriter(call, context, start, tag); } // always allocated against a call arena, no memory free required @@ -409,6 +458,12 @@ class ClientAsyncReaderWriter final assert(size == sizeof(ClientAsyncReaderWriter)); } + void StartCall(void* tag) override { + assert(!started_); + started_ = true; + StartCallInternal(tag); + } + /// See the \a ClientAsyncStreamingInterface.ReadInitialMetadata method /// for semantics of this method. /// @@ -417,6 +472,7 @@ class ClientAsyncReaderWriter final /// is updated with it, and then the receiving initial metadata can /// be accessed through this \a ClientContext. void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_ops_.set_output_tag(tag); @@ -425,6 +481,7 @@ class ClientAsyncReaderWriter final } void Read(R* msg, void* tag) override { + assert(started_); read_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { read_ops_.RecvInitialMetadata(context_); @@ -434,6 +491,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(write_ops_.SendMessage(msg).ok()); @@ -441,6 +499,7 @@ class ClientAsyncReaderWriter final } void Write(const W& msg, WriteOptions options, void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); if (options.is_last_message()) { options.set_buffer_hint(); @@ -452,6 +511,7 @@ class ClientAsyncReaderWriter final } void WritesDone(void* tag) override { + assert(started_); write_ops_.set_output_tag(tag); write_ops_.ClientSendClose(); call_.PerformOps(&write_ops_); @@ -462,6 +522,7 @@ class ClientAsyncReaderWriter final /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. void Finish(Status* status, void* tag) override { + assert(started_); finish_ops_.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_ops_.RecvInitialMetadata(context_); @@ -471,23 +532,30 @@ class ClientAsyncReaderWriter final } private: - ClientAsyncReaderWriter(Call call, ClientContext* context, void* tag) - : context_(context), call_(call) { - if (context_->initial_metadata_corked_) { - // if corked bit is set in context, we buffer up the initial metadata to - // coalesce with later message to be sent. No op is performed. - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncReaderWriter(Call call, ClientContext* context, bool start, + void* tag) + : context_(context), call_(call), started_(start) { + if (start) { + StartCallInternal(tag); } else { + assert(tag == nullptr); + } + } + + void StartCallInternal(void* tag) { + write_ops_.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); + // if corked bit is set in context, we just keep the initial metadata + // buffered up to coalesce with later message send. No op is performed. + if (!context_->initial_metadata_corked_) { write_ops_.set_output_tag(tag); - write_ops_.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); call_.PerformOps(&write_ops_); } } ClientContext* context_; Call call_; + bool started_; CallOpSet meta_ops_; CallOpSet> read_ops_; CallOpSet diff --git a/include/grpc++/impl/codegen/async_unary_call.h b/include/grpc++/impl/codegen/async_unary_call.h index f0f909686b..e472f04f56 100644 --- a/include/grpc++/impl/codegen/async_unary_call.h +++ b/include/grpc++/impl/codegen/async_unary_call.h @@ -32,13 +32,18 @@ namespace grpc { class CompletionQueue; extern CoreCodegenInterface* g_core_codegen_interface; -/// An interface relevant for async client side unary RPCS (which send +/// An interface relevant for async client side unary RPCs (which send /// one request message to a server and receive one response message). template class ClientAsyncResponseReaderInterface { public: virtual ~ClientAsyncResponseReaderInterface() {} + /// Start the call that was set up by the constructor, but only if the + /// constructor was invoked through the "Prepare" API which doesn't actually + /// start the call + virtual void StartCall() = 0; + /// Request notification of the reading of initial metadata. Completion /// will be notified by \a tag on the associated completion queue. /// This call is optional, but if it is used, it cannot be used concurrently @@ -70,9 +75,10 @@ template class ClientAsyncResponseReader final : public ClientAsyncResponseReaderInterface { public: - /// Start a call and write the request out. + /// Start a call and write the request out if \a start is set. /// \a tag will be notified on \a cq when the call has been started (i.e. /// intitial metadata sent) and \a request has been written out. + /// If \a start is not set, the actual call must be initiated by StartCall /// Note that \a context will be used to fill in custom initial metadata /// used to send to the server when starting the call. template @@ -80,11 +86,11 @@ class ClientAsyncResponseReader final CompletionQueue* cq, const RpcMethod& method, ClientContext* context, - const W& request) { + const W& request, bool start) { Call call = channel->CreateCall(method, context, cq); return new (g_core_codegen_interface->grpc_call_arena_alloc( call.call(), sizeof(ClientAsyncResponseReader))) - ClientAsyncResponseReader(call, context, request); + ClientAsyncResponseReader(call, context, request, start); } // always allocated against a call arena, no memory free required @@ -92,13 +98,20 @@ class ClientAsyncResponseReader final assert(size == sizeof(ClientAsyncResponseReader)); } + void StartCall() override { + assert(!started_); + started_ = true; + StartCallInternal(); + } + /// See \a ClientAsyncResponseReaderInterface::ReadInitialMetadata for /// semantics. /// /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void ReadInitialMetadata(void* tag) { + void ReadInitialMetadata(void* tag) override { + assert(started_); GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_); meta_buf.set_output_tag(tag); @@ -111,7 +124,8 @@ class ClientAsyncResponseReader final /// Side effect: /// - the \a ClientContext associated with this call is updated with /// possible initial and trailing metadata sent from the server. - void Finish(R* msg, Status* status, void* tag) { + void Finish(R* msg, Status* status, void* tag) override { + assert(started_); finish_buf.set_output_tag(tag); if (!context_->initial_metadata_received_) { finish_buf.RecvInitialMetadata(context_); @@ -125,15 +139,22 @@ class ClientAsyncResponseReader final private: ClientContext* const context_; Call call_; + bool started_; template - ClientAsyncResponseReader(Call call, ClientContext* context, const W& request) - : context_(context), call_(call) { - init_buf.SendInitialMetadata(context->send_initial_metadata_, - context->initial_metadata_flags()); + ClientAsyncResponseReader(Call call, ClientContext* context, const W& request, + bool start) + : context_(context), call_(call), started_(start) { + // Bind the metadata at time of StartCallInternal but set up the rest here // TODO(ctiller): don't assert GPR_CODEGEN_ASSERT(init_buf.SendMessage(request).ok()); init_buf.ClientSendClose(); + if (start) StartCallInternal(); + } + + void StartCallInternal() { + init_buf.SendInitialMetadata(context_->send_initial_metadata_, + context_->initial_metadata_flags()); call_.PerformOps(&init_buf); } diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index b09bf99677..c2db8eff71 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -165,25 +165,37 @@ void PrintHeaderClientMethodInterfaces( (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "virtual ::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) = 0;\n"); - printer->Print(*vars, - "std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -197,19 +209,26 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, $Response$* " - "response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncWriterInterface< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* " + "response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncWriterInterface< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -223,19 +242,25 @@ void PrintHeaderClientMethodInterfaces( "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderInterface< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderInterface< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderInterface< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "std::unique_ptr< ::grpc::ClientReaderWriterInterface< " @@ -249,61 +274,83 @@ void PrintHeaderClientMethodInterfaces( "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriterInterface< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncResponseReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) = 0;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientWriterInterface< $Request$>*" " $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" - " Async$Method$Raw(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*" + " $AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "virtual ::grpc::ClientReaderInterface< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request) = 0;\n"); - printer->Print( - *vars, - "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " - "Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderInterface< $Response$>* " + "$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "virtual ::grpc::ClientReaderWriterInterface< $Request$, " "$Response$>* " "$Method$Raw(::grpc::ClientContext* context) = 0;\n"); - printer->Print(*vars, - "virtual ::grpc::ClientAsyncReaderWriterInterface< " - "$Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) = 0;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + printer->Print( + *vars, + "virtual ::grpc::ClientAsyncReaderWriterInterface< " + "$Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) = 0;\n"); + } } } } @@ -315,25 +362,35 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + grpc::string raw_args; // extra arguments to raw version of method + } async_prefixes[] = {{"Async", ", void* tag", ", tag"}, + {"PrepareAsync", "", ""}}; + if (is_public) { if (method->NoStreaming()) { printer->Print( *vars, "::grpc::Status $Method$(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response) override;\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncResponseReader< $Response$>>(" - "Async$Method$Raw(context, request, cq));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncResponseReader< $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Indent(); + printer->Print(*vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncResponseReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, @@ -346,18 +403,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, response));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" - " Async$Method$(::grpc::ClientContext* context, " - "$Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" - "Async$Method$Raw(context, response, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>" + " $AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>(" + "$AsyncPrefix$$Method$Raw(context, response, " + "cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -371,19 +434,24 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "($Method$Raw(context, request));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print( - *vars, - "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " - "Async$Method$(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print( - *vars, - "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" - "Async$Method$Raw(context, request, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>> " + "$AsyncPrefix$$Method$(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< ::grpc::ClientAsyncReader< $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, request, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -396,53 +464,80 @@ void PrintHeaderClientMethod(grpc_generator::Printer *printer, "$Method$Raw(context));\n"); printer->Outdent(); printer->Print("}\n"); - printer->Print(*vars, - "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " - "$Request$, $Response$>> " - "Async$Method$(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Indent(); - printer->Print(*vars, - "return std::unique_ptr< " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" - "Async$Method$Raw(context, cq, tag));\n"); - printer->Outdent(); - printer->Print("}\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print(*vars, + "std::unique_ptr< ::grpc::ClientAsyncReaderWriter< " + "$Request$, $Response$>> " + "$AsyncPrefix$$Method$(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Indent(); + printer->Print( + *vars, + "return std::unique_ptr< " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>(" + "$AsyncPrefix$$Method$Raw(context, cq$AsyncRawArgs$));\n"); + printer->Outdent(); + printer->Print("}\n"); + } } } else { if (method->NoStreaming()) { - printer->Print(*vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) override;\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* $Method$Raw(" "::grpc::ClientContext* context, $Response$* response) " "override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncWriter< $Request$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientReader< $Response$>* $Method$Raw(" "::grpc::ClientContext* context, const $Request$& request)" " override;\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReader< $Response$>* Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* $AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } else if (method->BidiStreaming()) { printer->Print(*vars, "::grpc::ClientReaderWriter< $Request$, $Response$>* " "$Method$Raw(::grpc::ClientContext* context) override;\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) override;\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncRawArgs"] = async_prefix.raw_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$AsyncPrefix$$Method$Raw(::grpc::ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) override;\n"); + } } } } @@ -1077,6 +1172,13 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, (*vars)["Method"] = method->name(); (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string start; // bool literal expressed as string + grpc::string method_params; // extra arguments to method + grpc::string create_args; // extra arguments to creator + } async_prefixes[] = {{"Async", "true", ", void* tag", ", tag"}, + {"PrepareAsync", "false", "", ", nullptr"}}; if (method->NoStreaming()) { printer->Print(*vars, "::grpc::Status $ns$$Service$::Stub::$Method$(" @@ -1087,19 +1189,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request, response);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncResponseReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "const $Request$& request, " - "::grpc::CompletionQueue* cq) {\n"); - printer->Print(*vars, - " return " - "::grpc::ClientAsyncResponseReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + printer->Print(*vars, + "::grpc::ClientAsyncResponseReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "const $Request$& request, " + "::grpc::CompletionQueue* cq) {\n"); + printer->Print(*vars, + " return " + "::grpc::ClientAsyncResponseReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$);\n" + "}\n\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print(*vars, "::grpc::ClientWriter< $Request$>* " @@ -1111,17 +1217,23 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, response);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncWriter< $Request$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncWriter< $Request$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, response, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncWriter< $Request$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncWriter< $Request$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, response, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, @@ -1134,17 +1246,24 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context, request);\n" "}\n\n"); - printer->Print(*vars, - "::grpc::ClientAsyncReader< $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(" - "::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print(*vars, - " return ::grpc::ClientAsyncReader< $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, request, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print( + *vars, + "::grpc::ClientAsyncReader< $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(" + "::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print(*vars, + " return ::grpc::ClientAsyncReader< $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, request, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, @@ -1157,19 +1276,25 @@ void PrintSourceClientMethod(grpc_generator::Printer *printer, "rpcmethod_$Method$_, " "context);\n" "}\n\n"); - printer->Print( - *vars, - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " - "$ns$$Service$::Stub::Async$Method$Raw(::grpc::ClientContext* context, " - "::grpc::CompletionQueue* cq, void* tag) {\n"); - printer->Print( - *vars, - " return " - "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" - "channel_.get(), cq, " - "rpcmethod_$Method$_, " - "context, tag);\n" - "}\n\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncStart"] = async_prefix.start; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["AsyncCreateArgs"] = async_prefix.create_args; + printer->Print(*vars, + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* " + "$ns$$Service$::Stub::$AsyncPrefix$$Method$Raw(::grpc::" + "ClientContext* context, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$) {\n"); + printer->Print( + *vars, + " return " + "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>::Create(" + "channel_.get(), cq, " + "rpcmethod_$Method$_, " + "context, $AsyncStart$$AsyncCreateArgs$);\n" + "}\n\n"); + } } } @@ -1460,50 +1585,79 @@ void PrintMockClientMethods(grpc_generator::Printer *printer, (*vars)["Request"] = method->input_type_name(); (*vars)["Response"] = method->output_type_name(); + struct { + grpc::string prefix; + grpc::string method_params; // extra arguments to method + int extra_method_param_count; + } async_prefixes[] = {{"Async", ", void* tag", 1}, {"PrepareAsync", "", 0}}; + if (method->NoStreaming()) { printer->Print( *vars, "MOCK_METHOD3($Method$, ::grpc::Status(::grpc::ClientContext* context, " "const $Request$& request, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + printer->Print( + *vars, + "MOCK_METHOD3($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncResponseReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq));\n"); + } } else if (ClientOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientWriterInterface< $Request$>*" "(::grpc::ClientContext* context, $Response$* response));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncWriterInterface< $Request$>*" - "(::grpc::ClientContext* context, $Response$* response, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print(*vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncWriterInterface< $Request$>*" + "(::grpc::ClientContext* context, $Response$* response, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (ServerOnlyStreaming(method)) { printer->Print( *vars, "MOCK_METHOD2($Method$Raw, " "::grpc::ClientReaderInterface< $Response$>*" "(::grpc::ClientContext* context, const $Request$& request));\n"); - printer->Print(*vars, - "MOCK_METHOD4(Async$Method$Raw, " - "::grpc::ClientAsyncReaderInterface< $Response$>*" - "(::grpc::ClientContext* context, const $Request$& request, " - "::grpc::CompletionQueue* cq, void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(3 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderInterface< $Response$>*" + "(::grpc::ClientContext* context, const $Request$& request, " + "::grpc::CompletionQueue* cq$AsyncMethodParams$));\n"); + } } else if (method->BidiStreaming()) { printer->Print( *vars, "MOCK_METHOD1($Method$Raw, " "::grpc::ClientReaderWriterInterface< $Request$, $Response$>*" "(::grpc::ClientContext* context));\n"); - printer->Print( - *vars, - "MOCK_METHOD3(Async$Method$Raw, " - "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" - "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, " - "void* tag));\n"); + for (auto async_prefix : async_prefixes) { + (*vars)["AsyncPrefix"] = async_prefix.prefix; + (*vars)["AsyncMethodParams"] = async_prefix.method_params; + (*vars)["MockArgs"] = + std::to_string(2 + async_prefix.extra_method_param_count); + printer->Print( + *vars, + "MOCK_METHOD$MockArgs$($AsyncPrefix$$Method$Raw, " + "::grpc::ClientAsyncReaderWriterInterface<$Request$, $Response$>*" + "(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq" + "$AsyncMethodParams$));\n"); + } } } diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index 66b1ef0e39..de2e449fe8 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -22,14 +22,29 @@ namespace grpc { +namespace { +std::unique_ptr CallInternal( + ChannelInterface* channel, ClientContext* context, + const grpc::string& method, CompletionQueue* cq, bool start, void* tag) { + return std::unique_ptr( + GenericClientAsyncReaderWriter::Create( + channel, cq, RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), + context, start, tag)); +} + +} // namespace + // begin a call to a named method std::unique_ptr GenericStub::Call( ClientContext* context, const grpc::string& method, CompletionQueue* cq, void* tag) { - return std::unique_ptr( - GenericClientAsyncReaderWriter::Create( - channel_.get(), cq, - RpcMethod(method.c_str(), RpcMethod::BIDI_STREAMING), context, tag)); + return CallInternal(channel_.get(), context, method, cq, true, tag); +} + +// setup a call to a named method +std::unique_ptr GenericStub::PrepareCall( + ClientContext* context, const grpc::string& method, CompletionQueue* cq) { + return CallInternal(channel_.get(), context, method, cq, false, nullptr); } } // namespace grpc diff --git a/test/cpp/codegen/compiler_test_golden b/test/cpp/codegen/compiler_test_golden index b43c27f3f7..3d664e8825 100644 --- a/test/cpp/codegen/compiler_test_golden +++ b/test/cpp/codegen/compiler_test_golden @@ -65,6 +65,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } // MethodA1 trailing comment 1 // MethodA2 detached leading comment 1 // @@ -76,6 +79,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } // MethodA2 trailing comment 1 // Method A3 leading comment 1 std::unique_ptr< ::grpc::ClientReaderInterface< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { @@ -84,6 +90,9 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } // Method A3 trailing comment 1 // Method A4 leading comment 1 std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { @@ -92,15 +101,22 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } // Method A4 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientWriterInterface< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) = 0; virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderInterface< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) = 0; virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) = 0; virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) = 0; + virtual ::grpc::ClientAsyncReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -109,34 +125,50 @@ class ServiceA final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodA1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodA1Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>> MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response) { return std::unique_ptr< ::grpc::ClientWriter< ::grpc::testing::Request>>(MethodA2Raw(context, response)); } std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> AsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(AsyncMethodA2Raw(context, response, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>> PrepareAsyncMethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncWriter< ::grpc::testing::Request>>(PrepareAsyncMethodA2Raw(context, response, cq)); + } std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>> MethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request) { return std::unique_ptr< ::grpc::ClientReader< ::grpc::testing::Response>>(MethodA3Raw(context, request)); } std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> AsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(AsyncMethodA3Raw(context, request, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>> PrepareAsyncMethodA3(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReader< ::grpc::testing::Response>>(PrepareAsyncMethodA3Raw(context, request, cq)); + } std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> MethodA4(::grpc::ClientContext* context) { return std::unique_ptr< ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(MethodA4Raw(context)); } std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> AsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) { return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(AsyncMethodA4Raw(context, cq, tag)); } + std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>> PrepareAsyncMethodA4(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>>(PrepareAsyncMethodA4Raw(context, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodA1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientWriter< ::grpc::testing::Request>* MethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response) override; ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* AsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncWriter< ::grpc::testing::Request>* PrepareAsyncMethodA2Raw(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReader< ::grpc::testing::Response>* MethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request) override; ::grpc::ClientAsyncReader< ::grpc::testing::Response>* AsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReader< ::grpc::testing::Response>* PrepareAsyncMethodA3Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* MethodA4Raw(::grpc::ClientContext* context) override; ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* AsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag) override; + ::grpc::ClientAsyncReaderWriter< ::grpc::testing::Request, ::grpc::testing::Response>* PrepareAsyncMethodA4Raw(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodA1_; const ::grpc::RpcMethod rpcmethod_MethodA2_; const ::grpc::RpcMethod rpcmethod_MethodA3_; @@ -372,9 +404,13 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } // MethodB1 trailing comment 1 private: virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -383,10 +419,14 @@ class ServiceB final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> AsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(AsyncMethodB1Raw(context, request, cq)); } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>> PrepareAsyncMethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>>(PrepareAsyncMethodB1Raw(context, request, cq)); + } private: std::shared_ptr< ::grpc::ChannelInterface> channel_; ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* AsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::grpc::testing::Response>* PrepareAsyncMethodB1Raw(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq) override; const ::grpc::RpcMethod rpcmethod_MethodB1_; }; static std::unique_ptr NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions()); diff --git a/test/cpp/codegen/compiler_test_mock_golden b/test/cpp/codegen/compiler_test_mock_golden index 8e4b4d5911..f97c2dd83a 100644 --- a/test/cpp/codegen/compiler_test_mock_golden +++ b/test/cpp/codegen/compiler_test_mock_golden @@ -15,18 +15,23 @@ class MockServiceAStub : public ServiceA::StubInterface { public: MOCK_METHOD3(MethodA1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodA1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA2Raw, ::grpc::ClientWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response)); MOCK_METHOD4(AsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA2Raw, ::grpc::ClientAsyncWriterInterface< ::grpc::testing::Request>*(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::CompletionQueue* cq)); MOCK_METHOD2(MethodA3Raw, ::grpc::ClientReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request)); MOCK_METHOD4(AsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD3(PrepareAsyncMethodA3Raw, ::grpc::ClientAsyncReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); MOCK_METHOD1(MethodA4Raw, ::grpc::ClientReaderWriterInterface< ::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context)); MOCK_METHOD3(AsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq, void* tag)); + MOCK_METHOD2(PrepareAsyncMethodA4Raw, ::grpc::ClientAsyncReaderWriterInterface<::grpc::testing::Request, ::grpc::testing::Response>*(::grpc::ClientContext* context, ::grpc::CompletionQueue* cq)); }; class MockServiceBStub : public ServiceB::StubInterface { public: MOCK_METHOD3(MethodB1, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::testing::Response* response)); MOCK_METHOD3(AsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); + MOCK_METHOD3(PrepareAsyncMethodB1Raw, ::grpc::ClientAsyncResponseReaderInterface< ::grpc::testing::Response>*(::grpc::ClientContext* context, const ::grpc::testing::Request& request, ::grpc::CompletionQueue* cq)); }; } // namespace grpc diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 912c871482..1a4438047d 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -56,11 +56,6 @@ class ClientRpcContext { } virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0; - void lock() { mu_.lock(); } - void unlock() { mu_.unlock(); } - - private: - std::mutex mu_; }; template @@ -73,7 +68,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req, + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -83,7 +78,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { next_state_(State::READY), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextUnaryImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -92,7 +87,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { switch (next_state_) { case State::READY: start_ = UsageTimer::Now(); - response_reader_ = start_req_(stub_, &context_, req_, cq_); + response_reader_ = prepare_req_(stub_, &context_, req_, cq_); + response_reader_->StartCall(); next_state_ = State::RESP_DONE; response_reader_->Finish(&response_, &status_, ClientRpcContext::tag(this)); @@ -111,8 +107,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextUnaryImpl(stub_, req_, next_issue_, - start_req_, callback_); - std::lock_guard lclone(*clone); + prepare_req_, callback_); clone->StartInternal(cq); } @@ -130,7 +125,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, CompletionQueue*)> - start_req_; + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -252,29 +247,13 @@ class AsyncClient : public ClientImpl { // this thread isn't supposed to shut down std::lock_guard l(shutdown_state_[thread_idx]->mutex); if (shutdown_state_[thread_idx]->shutdown) { - // We want to delete the context. However, it is possible that - // another thread that just initiated an action on this - // context still has its lock even though the action on the - // context has completed. To delay for that, just grab the - // lock for serialization. Take a new scope. - { std::lock_guard lctx(*ctx); } delete ctx; return true; } - bool del = false; - - // Create a new scope for a lock_guard'ed region - { - std::lock_guard lctx(*ctx); - if (!ctx->RunNextState(ok, entry)) { - // The RPC and callback are done, so clone the ctx - // and kickstart the new one - ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); - // set the old version to delete - del = true; - } - } - if (del) { + if (!ctx->RunNextState(ok, entry)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + ctx->StartNewClone(cli_cqs_[cq_[thread_idx]].get()); delete ctx; } return true; @@ -311,15 +290,15 @@ class AsyncUnaryClient final entry->set_status(s.error_code()); } static std::unique_ptr> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& request, CompletionQueue* cq) { - return stub->AsyncUnaryCall(ctx, request, cq); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + const SimpleRequest& request, CompletionQueue* cq) { + return stub->PrepareAsyncUnaryCall(ctx, request, cq); }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, std::function next_issue, const SimpleRequest& req) { return new ClientRpcContextUnaryImpl( - stub, req, next_issue, AsyncUnaryClient::StartReq, + stub, req, next_issue, AsyncUnaryClient::PrepareReq, AsyncUnaryClient::CheckDone); } }; @@ -332,9 +311,8 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { std::function next_issue, std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> - start_req, + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -344,7 +322,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingPingPongImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -407,8 +385,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingPingPongImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -432,10 +409,10 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { State next_state_; std::function callback_; std::function next_issue_; - std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> - start_req_; + std::function< + std::unique_ptr>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -449,8 +426,9 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { cq_ = cq; messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -469,9 +447,9 @@ class AsyncStreamingPingPongClient final static void CheckDone(grpc::Status s, SimpleResponse* response) {} static std::unique_ptr< grpc::ClientAsyncReaderWriter> - StartReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingCall(ctx, cq, tag); + PrepareReq(BenchmarkService::Stub* stub, grpc::ClientContext* ctx, + CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingCall(ctx, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -479,7 +457,7 @@ class AsyncStreamingPingPongClient final const SimpleRequest& req) { return new ClientRpcContextStreamingPingPongImpl( - stub, req, next_issue, AsyncStreamingPingPongClient::StartReq, + stub, req, next_issue, AsyncStreamingPingPongClient::PrepareReq, AsyncStreamingPingPongClient::CheckDone); } }; @@ -492,8 +470,8 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function next_issue, std::function>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -503,7 +481,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromClientImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -546,8 +524,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromClientImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -570,17 +547,17 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, ResponseType*, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> stream_; void StartInternal(CompletionQueue* cq) { cq_ = cq; - stream_ = start_req_(stub_, &context_, &response_, cq, - ClientRpcContext::tag(this)); + stream_ = prepare_req_(stub_, &context_, &response_, cq); next_state_ = State::STREAM_IDLE; + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -597,10 +574,10 @@ class AsyncStreamingFromClientClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr> StartReq( + static std::unique_ptr> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - SimpleResponse* resp, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromClient(ctx, resp, cq, tag); + SimpleResponse* resp, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromClient(ctx, resp, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -608,7 +585,7 @@ class AsyncStreamingFromClientClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromClientImpl( - stub, req, next_issue, AsyncStreamingFromClientClient::StartReq, + stub, req, next_issue, AsyncStreamingFromClientClient::PrepareReq, AsyncStreamingFromClientClient::CheckDone); } }; @@ -621,8 +598,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function next_issue, std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req, + CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -632,7 +609,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextStreamingFromServerImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq); @@ -664,8 +641,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextStreamingFromServerImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } @@ -682,8 +658,8 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { std::function next_issue_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr> stream_; @@ -691,9 +667,9 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { void StartInternal(CompletionQueue* cq) { // TODO(vjpai): Add support to rate-pace this cq_ = cq; + stream_ = prepare_req_(stub_, &context_, req_, cq); next_state_ = State::STREAM_IDLE; - stream_ = - start_req_(stub_, &context_, req_, cq, ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -710,10 +686,10 @@ class AsyncStreamingFromServerClient final private: static void CheckDone(grpc::Status s, SimpleResponse* response) {} - static std::unique_ptr> StartReq( + static std::unique_ptr> PrepareReq( BenchmarkService::Stub* stub, grpc::ClientContext* ctx, - const SimpleRequest& req, CompletionQueue* cq, void* tag) { - auto stream = stub->AsyncStreamingFromServer(ctx, req, cq, tag); + const SimpleRequest& req, CompletionQueue* cq) { + auto stream = stub->PrepareAsyncStreamingFromServer(ctx, req, cq); return stream; }; static ClientRpcContext* SetupCtx(BenchmarkService::Stub* stub, @@ -721,7 +697,7 @@ class AsyncStreamingFromServerClient final const SimpleRequest& req) { return new ClientRpcContextStreamingFromServerImpl( - stub, req, next_issue, AsyncStreamingFromServerClient::StartReq, + stub, req, next_issue, AsyncStreamingFromServerClient::PrepareReq, AsyncStreamingFromServerClient::CheckDone); } }; @@ -733,8 +709,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> - start_req, + const grpc::string& method_name, CompletionQueue*)> + prepare_req, std::function on_done) : context_(), stub_(stub), @@ -744,7 +720,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { next_state_(State::INVALID), callback_(on_done), next_issue_(next_issue), - start_req_(start_req) {} + prepare_req_(prepare_req) {} ~ClientRpcContextGenericStreamingImpl() override {} void Start(CompletionQueue* cq, const ClientConfig& config) override { StartInternal(cq, config.messages_per_stream()); @@ -807,8 +783,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { } void StartNewClone(CompletionQueue* cq) override { auto* clone = new ClientRpcContextGenericStreamingImpl( - stub_, req_, next_issue_, start_req_, callback_); - std::lock_guard lclone(*clone); + stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } @@ -834,8 +809,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function next_issue_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*)> + prepare_req_; grpc::Status status_; double start_; std::unique_ptr stream_; @@ -850,9 +825,9 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { "/grpc.testing.BenchmarkService/StreamingCall"); messages_per_stream_ = messages_per_stream; messages_issued_ = 0; + stream_ = prepare_req_(stub_, &context_, kMethodName, cq); next_state_ = State::STREAM_IDLE; - stream_ = start_req_(stub_, &context_, kMethodName, cq, - ClientRpcContext::tag(this)); + stream_->StartCall(ClientRpcContext::tag(this)); } }; @@ -874,17 +849,17 @@ class GenericAsyncStreamingClient final private: static void CheckDone(grpc::Status s, ByteBuffer* response) {} - static std::unique_ptr StartReq( + static std::unique_ptr PrepareReq( grpc::GenericStub* stub, grpc::ClientContext* ctx, - const grpc::string& method_name, CompletionQueue* cq, void* tag) { - auto stream = stub->Call(ctx, method_name, cq, tag); + const grpc::string& method_name, CompletionQueue* cq) { + auto stream = stub->PrepareCall(ctx, method_name, cq); return stream; }; static ClientRpcContext* SetupCtx(grpc::GenericStub* stub, std::function next_issue, const ByteBuffer& req) { return new ClientRpcContextGenericStreamingImpl( - stub, req, next_issue, GenericAsyncStreamingClient::StartReq, + stub, req, next_issue, GenericAsyncStreamingClient::PrepareReq, GenericAsyncStreamingClient::CheckDone); } }; -- cgit v1.2.3 From 67fa513fbaaaffc9fc8b9bd1328c20cbcbc7746d Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Sun, 30 Jul 2017 00:12:23 -0700 Subject: make sure calls created in ruby unit tests are completed timely --- src/ruby/spec/client_server_spec.rb | 270 +++++++++++++++++++++--------- src/ruby/spec/generic/active_call_spec.rb | 57 ++++++- 2 files changed, 237 insertions(+), 90 deletions(-) diff --git a/src/ruby/spec/client_server_spec.rb b/src/ruby/spec/client_server_spec.rb index 1a9b47e2c3..adab8c9d14 100644 --- a/src/ruby/spec/client_server_spec.rb +++ b/src/ruby/spec/client_server_spec.rb @@ -29,14 +29,18 @@ shared_context 'setup: tags' do expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call ops = { CallOps::SEND_INITIAL_METADATA => metadata } - svr_batch = server_call.run_batch(ops) - expect(svr_batch.send_metadata).to be true + server_batch = server_call.run_batch(ops) + expect(server_batch.send_metadata).to be true server_call end def new_client_call @ch.create_call(nil, nil, '/method', nil, deadline) end + + def ok_status + Struct::Status.new(StatusCodes::OK, 'OK') + end end shared_examples 'basic GRPC message delivery is OK' do @@ -70,19 +74,32 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { - CallOps::RECV_MESSAGE => nil + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(sent_message) + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(sent_message) + expect(server_batch.send_close).to be true + expect(server_batch.send_status).to be true + + # finish the call + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'responses written by servers are received by the client' do @@ -95,21 +112,36 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { CallOps::RECV_MESSAGE => nil, - CallOps::SEND_MESSAGE => reply_text + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_MESSAGE => reply_text, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(sent_message) - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(sent_message) + expect(server_batch.send_close).to be true + expect(server_batch.send_message).to be true + expect(server_batch.send_status).to be true + + # finish the call + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.message).to eq(reply_text) + expect(final_client_batch.status.code).to eq(0) end it 'compressed messages can be sent and received' do @@ -125,30 +157,37 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => md, - CallOps::SEND_MESSAGE => long_request_str + CallOps::SEND_MESSAGE => long_request_str, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message server_thread.join server_ops = { CallOps::RECV_MESSAGE => nil, - CallOps::SEND_MESSAGE => long_response_str + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_MESSAGE => long_response_str, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq(long_request_str) - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq(long_request_str) + expect(server_batch.send_close).to be true + expect(server_batch.send_message).to be true + expect(server_batch.send_status).to be true client_ops = { - CallOps::SEND_CLOSE_FROM_CLIENT => nil, CallOps::RECV_INITIAL_METADATA => nil, - CallOps::RECV_MESSAGE => nil + CallOps::RECV_MESSAGE => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_close).to be true - expect(batch_result.message).to eq long_response_str + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.message).to eq long_response_str + expect(final_client_batch.status.code).to eq(0) end it 'servers can ignore a client write and send a status' do @@ -161,11 +200,13 @@ shared_examples 'basic GRPC message delivery is OK' do client_ops = { CallOps::SEND_INITIAL_METADATA => {}, - CallOps::SEND_MESSAGE => sent_message + CallOps::SEND_MESSAGE => sent_message, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true + expect(client_batch.send_close).to be true # confirm the server can read the inbound message the_status = Struct::Status.new(StatusCodes::OK, 'OK') @@ -173,9 +214,15 @@ shared_examples 'basic GRPC message delivery is OK' do server_ops = { CallOps::SEND_STATUS_FROM_SERVER => the_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq nil - expect(svr_batch.send_status).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq nil + expect(server_batch.send_status).to be true + + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'completes calls by sending status to client and server' do @@ -190,9 +237,9 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_INITIAL_METADATA => {}, CallOps::SEND_MESSAGE => sent_message } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.send_message).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_message).to be true # confirm the server can read the inbound message and respond the_status = Struct::Status.new(StatusCodes::OK, 'OK', {}) @@ -202,10 +249,10 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_MESSAGE => reply_text, CallOps::SEND_STATUS_FROM_SERVER => the_status } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.message).to eq sent_message - expect(svr_batch.send_status).to be true - expect(svr_batch.send_message).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.message).to eq sent_message + expect(server_batch.send_status).to be true + expect(server_batch.send_message).to be true # confirm the client can receive the server response and status. client_ops = { @@ -214,17 +261,17 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::RECV_MESSAGE => nil, CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_close).to be true - expect(batch_result.message).to eq reply_text - expect(batch_result.status).to eq the_status + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.send_close).to be true + expect(final_client_batch.message).to eq reply_text + expect(final_client_batch.status).to eq the_status # confirm the server can receive the client close. server_ops = { CallOps::RECV_CLOSE_ON_SERVER => nil } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.send_close).to be true + final_server_batch = server_call.run_batch(server_ops) + expect(final_server_batch.send_close).to be true end def client_cancel_test(cancel_proc, expected_code, @@ -240,9 +287,9 @@ shared_examples 'basic GRPC message delivery is OK' do CallOps::SEND_INITIAL_METADATA => {}, CallOps::RECV_INITIAL_METADATA => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true - expect(batch_result.metadata).to eq({}) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.metadata).to eq({}) cancel_proc.call(call) @@ -250,16 +297,16 @@ shared_examples 'basic GRPC message delivery is OK' do server_ops = { CallOps::RECV_CLOSE_ON_SERVER => nil } - svr_batch = server_call.run_batch(server_ops) - expect(svr_batch.send_close).to be true + server_batch = server_call.run_batch(server_ops) + expect(server_batch.send_close).to be true client_ops = { CallOps::RECV_STATUS_ON_CLIENT => {} } - batch_result = call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) - expect(batch_result.status.code).to be expected_code - expect(batch_result.status.details).to eq expected_details + expect(client_batch.status.code).to be expected_code + expect(client_batch.status.details).to eq expected_details end it 'clients can cancel a call on the server' do @@ -325,10 +372,11 @@ shared_examples 'GRPC metadata delivery works OK' do call = new_client_call client_ops = { - CallOps::SEND_INITIAL_METADATA => md + CallOps::SEND_INITIAL_METADATA => md, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true # confirm the server can receive the client metadata rcv_thread.join @@ -336,6 +384,21 @@ shared_examples 'GRPC metadata delivery works OK' do recvd_md = recvd_rpc.metadata replace_symbols = Hash[md.each_pair.collect { |x, y| [x.to_s, y] }] expect(recvd_md).to eq(recvd_md.merge(replace_symbols)) + + # finish the call + final_server_batch = recvd_rpc.call.run_batch( + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + expect(final_server_batch.send_close).to be(true) + expect(final_server_batch.send_metadata).to be(true) + expect(final_server_batch.send_status).to be(true) + + final_client_batch = call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end end end @@ -381,6 +444,9 @@ shared_examples 'GRPC metadata delivery works OK' do recvd_rpc.call.run_batch(server_ops) end expect(&blk).to raise_error + + # cancel the call so the server can shut down immediately + call.cancel end end @@ -394,25 +460,37 @@ shared_examples 'GRPC metadata delivery works OK' do # client signals that it's done sending metadata to allow server to # respond client_ops = { - CallOps::SEND_INITIAL_METADATA => nil + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # server gets the invocation but sends no metadata back rcv_thread.join expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call server_ops = { - CallOps::SEND_INITIAL_METADATA => nil + # receive close and send status to finish the call + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - server_call.run_batch(server_ops) + srv_batch = server_call.run_batch(server_ops) + expect(srv_batch.send_close).to be true + expect(srv_batch.send_metadata).to be true + expect(srv_batch.send_status).to be true # client receives nothing as expected client_ops = { - CallOps::RECV_INITIAL_METADATA => nil + CallOps::RECV_INITIAL_METADATA => nil, + # receive status to finish the call + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) - expect(batch_result.metadata).to eq({}) + final_client_batch = call.run_batch(client_ops) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'sends all the pairs when keys and values are valid' do @@ -426,26 +504,36 @@ shared_examples 'GRPC metadata delivery works OK' do # client signals that it's done sending metadata to allow server to # respond client_ops = { - CallOps::SEND_INITIAL_METADATA => nil + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_CLOSE_FROM_CLIENT => nil } - call.run_batch(client_ops) + client_batch = call.run_batch(client_ops) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # server gets the invocation but sends no metadata back rcv_thread.join expect(recvd_rpc).to_not eq nil server_call = recvd_rpc.call server_ops = { - CallOps::SEND_INITIAL_METADATA => md + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => md, + CallOps::SEND_STATUS_FROM_SERVER => ok_status } - server_call.run_batch(server_ops) + srv_batch = server_call.run_batch(server_ops) + expect(srv_batch.send_close).to be true + expect(srv_batch.send_metadata).to be true + expect(srv_batch.send_status).to be true # client receives nothing as expected client_ops = { - CallOps::RECV_INITIAL_METADATA => nil + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil } - batch_result = call.run_batch(client_ops) + final_client_batch = call.run_batch(client_ops) replace_symbols = Hash[md.each_pair.collect { |x, y| [x.to_s, y] }] - expect(batch_result.metadata).to eq(replace_symbols) + expect(final_client_batch.metadata).to eq(replace_symbols) + expect(final_client_batch.status.code).to eq(0) end end end @@ -510,8 +598,7 @@ describe 'the secure http client/server' do initial_md_key = 'k2' initial_md_val = 'v2' - initial_md = {} - initial_md[initial_md_key] = initial_md_val + initial_md = { initial_md_key => initial_md_val } expected_md = creds_update_md.clone fail 'bad test param' unless expected_md[initial_md_key].nil? expected_md[initial_md_key] = initial_md_val @@ -523,11 +610,12 @@ describe 'the secure http client/server' do call = new_client_call call.set_credentials! call_creds - client_ops = { - CallOps::SEND_INITIAL_METADATA => initial_md - } - batch_result = call.run_batch(client_ops) - expect(batch_result.send_metadata).to be true + + client_batch = call.run_batch( + CallOps::SEND_INITIAL_METADATA => initial_md, + CallOps::SEND_CLOSE_FROM_CLIENT => nil) + expect(client_batch.send_metadata).to be true + expect(client_batch.send_close).to be true # confirm the server can receive the client metadata rcv_thread.join @@ -535,6 +623,24 @@ describe 'the secure http client/server' do recvd_md = recvd_rpc.metadata replace_symbols = Hash[expected_md.each_pair.collect { |x, y| [x.to_s, y] }] expect(recvd_md).to eq(recvd_md.merge(replace_symbols)) + + credentials_update_test_finish_call(call, recvd_rpc.call) + end + + def credentials_update_test_finish_call(client_call, server_call) + final_server_batch = server_call.run_batch( + CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_INITIAL_METADATA => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + expect(final_server_batch.send_close).to be(true) + expect(final_server_batch.send_metadata).to be(true) + expect(final_server_batch.send_status).to be(true) + + final_client_batch = client_call.run_batch( + CallOps::RECV_INITIAL_METADATA => nil, + CallOps::RECV_STATUS_ON_CLIENT => nil) + expect(final_client_batch.metadata).to eq({}) + expect(final_client_batch.status.code).to eq(0) end it 'modifies metadata with CallCredentials' do diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index ec0c294174..a00df9236d 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -22,6 +22,21 @@ describe GRPC::ActiveCall do CallOps = GRPC::Core::CallOps WriteFlags = GRPC::Core::WriteFlags + def ok_status + Struct::Status.new(OK, 'OK') + end + + def send_and_receive_close_and_status(client_call, server_call) + client_call.run_batch(CallOps::SEND_CLOSE_FROM_CLIENT => nil) + server_call.run_batch(CallOps::RECV_CLOSE_ON_SERVER => nil, + CallOps::SEND_STATUS_FROM_SERVER => ok_status) + client_call.run_batch(CallOps::RECV_STATUS_ON_CLIENT => nil) + end + + def inner_call_of_active_call(active_call) + active_call.instance_variable_get(:@call) + end + before(:each) do @pass_through = proc { |x| x } host = '0.0.0.0:0' @@ -70,13 +85,13 @@ describe GRPC::ActiveCall do end describe '#remote_send' do - it 'allows a client to send a payload to the server' do + it 'allows a client to send a payload to the server', test: true do call = make_test_call ActiveCall.client_invoke(call) - @client_call = ActiveCall.new(call, @pass_through, - @pass_through, deadline) + client_call = ActiveCall.new(call, @pass_through, + @pass_through, deadline) msg = 'message is a string' - @client_call.remote_send(msg) + client_call.remote_send(msg) # check that server rpc new was received recvd_rpc = @server.request_call @@ -86,8 +101,13 @@ describe GRPC::ActiveCall do # Accept the call, and verify that the server reads the response ok. server_call = ActiveCall.new(recvd_call, @pass_through, @pass_through, deadline, - metadata_received: true) + metadata_received: true, + started: false) expect(server_call.remote_read).to eq(msg) + # finish the call + server_call.send_initial_metadata + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end it 'marshals the payload using the marshal func' do @@ -109,6 +129,9 @@ describe GRPC::ActiveCall do @pass_through, deadline, metadata_received: true) expect(server_call.remote_read).to eq('marshalled:' + msg) + # finish the call + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end TEST_WRITE_FLAGS = [WriteFlags::BUFFER_HINT, WriteFlags::NO_COMPRESS] @@ -136,6 +159,9 @@ describe GRPC::ActiveCall do @pass_through, deadline, metadata_received: true) expect(server_call.remote_read).to eq('marshalled:' + msg) + # finish the call + server_call.send_status(OK, '', true) + client_call.receive_and_check_status end end end @@ -177,7 +203,6 @@ describe GRPC::ActiveCall do @pass_through, @pass_through, deadline) - expect(@client_call.metadata_sent).to eql(true) expect(call).to( receive(:run_batch).with(hash_including( @@ -291,6 +316,10 @@ describe GRPC::ActiveCall do expect(recvd_rpc.metadata).to_not be_nil expect(recvd_rpc.metadata['k1']).to eq('v1') expect(recvd_rpc.metadata['k2']).to eq('v2') + # finish the call + recvd_call.run_batch(CallOps::SEND_INITIAL_METADATA => {}) + call.run_batch(CallOps::RECV_INITIAL_METADATA => nil) + send_and_receive_close_and_status(call, recvd_call) end end @@ -324,6 +353,8 @@ describe GRPC::ActiveCall do server_call = expect_server_to_receive(msg) server_call.remote_send('server_response') expect(client_call.remote_read).to eq('server_response') + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'saves no metadata when the server adds no metadata' do @@ -338,6 +369,8 @@ describe GRPC::ActiveCall do expect(client_call.metadata).to be_nil client_call.remote_read expect(client_call.metadata).to eq({}) + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'saves metadata add by the server' do @@ -353,6 +386,8 @@ describe GRPC::ActiveCall do client_call.remote_read expected = { 'k1' => 'v1', 'k2' => 'v2' } expect(client_call.metadata).to eq(expected) + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'get a status from server when nothing else sent from server' do @@ -409,6 +444,8 @@ describe GRPC::ActiveCall do server_call = expect_server_to_receive(msg) server_call.remote_send('server_response') expect(client_call.remote_read).to eq('unmarshalled:server_response') + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end end @@ -418,9 +455,11 @@ describe GRPC::ActiveCall do client_call = ActiveCall.new(call, @pass_through, @pass_through, deadline) expect(client_call.each_remote_read).to be_a(Enumerator) + # finish the call + client_call.cancel end - it 'the returns an enumerator that can read n responses' do + it 'the returned enumerator can read n responses' do call = make_test_call ActiveCall.client_invoke(call) client_call = ActiveCall.new(call, @pass_through, @@ -435,6 +474,8 @@ describe GRPC::ActiveCall do server_call.remote_send(reply) expect(e.next).to eq(reply) end + send_and_receive_close_and_status( + call, inner_call_of_active_call(server_call)) end it 'the returns an enumerator that stops after an OK Status' do @@ -453,7 +494,7 @@ describe GRPC::ActiveCall do server_call.remote_send(reply) expect(e.next).to eq(reply) end - server_call.send_status(OK, 'OK') + server_call.send_status(OK, 'OK', true) expect { e.next }.to raise_error(StopIteration) end end -- cgit v1.2.3 From 05eadf21e05d521d73a05babf7c8f45218728355 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 11 Sep 2017 11:46:59 +0200 Subject: reveal leftover processes --- tools/internal_ci/linux/grpc_run_tests_matrix.sh | 12 +++++++++++- tools/internal_ci/macos/grpc_run_tests_matrix.sh | 5 +++++ tools/internal_ci/windows/grpc_run_tests_matrix.bat | 10 ++++++---- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tools/internal_ci/linux/grpc_run_tests_matrix.sh b/tools/internal_ci/linux/grpc_run_tests_matrix.sh index 028704b3ff..bd1430b741 100755 --- a/tools/internal_ci/linux/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/linux/grpc_run_tests_matrix.sh @@ -20,4 +20,14 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS +tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" + +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_run_tests_matrix.sh b/tools/internal_ci/macos/grpc_run_tests_matrix.sh index 9a43e4869b..8e7fd54a62 100755 --- a/tools/internal_ci/macos/grpc_run_tests_matrix.sh +++ b/tools/internal_ci/macos/grpc_run_tests_matrix.sh @@ -25,6 +25,11 @@ tools/run_tests/run_tests_matrix.py $RUN_TESTS_FLAGS || FAILED="true" # kill port_server.py to prevent the build from hanging ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 +# Reveal leftover processes that might be left behind by the build +ps aux | grep -i kbuilder + +echo 'Exiting gRPC main test script.' + if [ "$FAILED" != "" ] then exit 1 diff --git a/tools/internal_ci/windows/grpc_run_tests_matrix.bat b/tools/internal_ci/windows/grpc_run_tests_matrix.bat index 08d834f8b0..10627add25 100644 --- a/tools/internal_ci/windows/grpc_run_tests_matrix.bat +++ b/tools/internal_ci/windows/grpc_run_tests_matrix.bat @@ -17,8 +17,10 @@ cd /d %~dp0\..\..\.. call tools/internal_ci/helper_scripts/prepare_build_windows.bat -python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% || goto :error -goto :EOF +python tools/run_tests/run_tests_matrix.py %RUN_TESTS_FLAGS% +set RUNTESTS_EXITCODE=%errorlevel% -:error -exit /b %errorlevel% +@rem Reveal leftover processes that might be left behind by the build +tasklist /V + +exit /b %RUNTESTS_EXITCODE% -- cgit v1.2.3 From 8d3cc5ff0c2b86bd6766944891c1706153135a47 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Fri, 15 Sep 2017 16:43:33 +0000 Subject: Clean up a few more uses of gRPC in _pb2.py files Were this not done these would break when the default behavior of gRPC Python Protoc Plug-In is changed to be the put-gRPC-code-elements-only- in-_pb2_grpc.py-files behavior that currently happens only when the grpc_2_0 flag is passed. --- .../tests/protoc_plugin/_python_plugin_test.py | 23 ++++++++++++---------- .../grpcio_tests/tests/qps/benchmark_server.py | 6 +++--- .../grpcio_tests/tests/stress/metrics_server.py | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py index 71493bfec6..5b84001aab 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py @@ -33,7 +33,7 @@ from tests.unit.framework.common import test_constants import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +import tests.protoc_plugin.protos.service.test_service_pb2_grpc as service_pb2_grpc # Identifiers of entities we expect to find in the generated module. STUB_IDENTIFIER = 'TestServiceStub' @@ -138,7 +138,7 @@ def _CreateService(): """ servicer_methods = _ServicerMethods() - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): def UnaryCall(self, request, context): return servicer_methods.UnaryCall(request, context) @@ -157,11 +157,12 @@ def _CreateService(): server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(servicer_methods, server, stub) @@ -173,16 +174,17 @@ def _CreateIncompleteService(): servicer_methods implements none of the methods required of it. """ - class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): + class Servicer(getattr(service_pb2_grpc, SERVICER_IDENTIFIER)): pass server = grpc.server( futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), server) + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER)(Servicer(), + server) port = server.add_insecure_port('[::]:0') server.start() channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = getattr(service_pb2, STUB_IDENTIFIER)(channel) + stub = getattr(service_pb2_grpc, STUB_IDENTIFIER)(channel) return _Service(None, server, stub) @@ -223,10 +225,11 @@ class PythonPluginTest(unittest.TestCase): def testImportAttributes(self): # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(service_pb2_grpc, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) + getattr(service_pb2_grpc, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(service_pb2_grpc, ADD_SERVICER_TO_SERVER_IDENTIFIER, None)) def testUpDown(self): service = _CreateService() diff --git a/src/python/grpcio_tests/tests/qps/benchmark_server.py b/src/python/grpcio_tests/tests/qps/benchmark_server.py index 05101fdc6d..bb07844491 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_server.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_server.py @@ -13,10 +13,10 @@ # limitations under the License. from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import services_pb2 +from src.proto.grpc.testing import services_pb2_grpc -class BenchmarkServer(services_pb2.BenchmarkServiceServicer): +class BenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Synchronous Server implementation for the Benchmark service.""" def UnaryCall(self, request, context): @@ -29,7 +29,7 @@ class BenchmarkServer(services_pb2.BenchmarkServiceServicer): yield messages_pb2.SimpleResponse(payload=payload) -class GenericBenchmarkServer(services_pb2.BenchmarkServiceServicer): +class GenericBenchmarkServer(services_pb2_grpc.BenchmarkServiceServicer): """Generic Server implementation for the Benchmark service.""" def __init__(self, resp_size): diff --git a/src/python/grpcio_tests/tests/stress/metrics_server.py b/src/python/grpcio_tests/tests/stress/metrics_server.py index 11ab6c3f4e..33a74b4a38 100644 --- a/src/python/grpcio_tests/tests/stress/metrics_server.py +++ b/src/python/grpcio_tests/tests/stress/metrics_server.py @@ -16,11 +16,12 @@ import time from src.proto.grpc.testing import metrics_pb2 +from src.proto.grpc.testing import metrics_pb2_grpc GAUGE_NAME = 'python_overall_qps' -class MetricsServer(metrics_pb2.MetricsServiceServicer): +class MetricsServer(metrics_pb2_grpc.MetricsServiceServicer): def __init__(self, histogram): self._start_time = time.time() -- cgit v1.2.3 From 4984135a78594215b3fbf04e33262b36a99bdb14 Mon Sep 17 00:00:00 2001 From: Justin Burke Date: Thu, 31 Aug 2017 17:42:54 -0700 Subject: Add reference counting to tsi_ssl_handshaker_factories --- src/core/lib/http/httpcli_security_connector.c | 3 +- .../lib/security/transport/security_connector.c | 22 ++-- src/core/tsi/ssl_transport_security.c | 113 +++++++++++++++++-- src/core/tsi/ssl_transport_security.h | 37 +++++-- test/core/tsi/ssl_transport_security_test.c | 121 ++++++++++++++++++++- 5 files changed, 265 insertions(+), 31 deletions(-) diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 97c2886525..c553fa3981 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -43,7 +43,8 @@ static void httpcli_ssl_destroy(grpc_exec_ctx *exec_ctx, grpc_httpcli_ssl_channel_security_connector *c = (grpc_httpcli_ssl_channel_security_connector *)sc; if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); + tsi_ssl_client_handshaker_factory_unref(c->handshaker_factory); + c->handshaker_factory = NULL; } if (c->secure_peer_name != NULL) gpr_free(c->secure_peer_name); gpr_free(sc); diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index a7568b995f..2a9e939d40 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -455,14 +455,14 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( typedef struct { grpc_channel_security_connector base; - tsi_ssl_client_handshaker_factory *handshaker_factory; + tsi_ssl_client_handshaker_factory *client_handshaker_factory; char *target_name; char *overridden_target_name; } grpc_ssl_channel_security_connector; typedef struct { grpc_server_security_connector base; - tsi_ssl_server_handshaker_factory *handshaker_factory; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; } grpc_ssl_server_security_connector; static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, @@ -470,9 +470,8 @@ static void ssl_channel_destroy(grpc_exec_ctx *exec_ctx, grpc_ssl_channel_security_connector *c = (grpc_ssl_channel_security_connector *)sc; grpc_call_credentials_unref(exec_ctx, c->base.request_metadata_creds); - if (c->handshaker_factory != NULL) { - tsi_ssl_client_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_client_handshaker_factory_unref(c->client_handshaker_factory); + c->client_handshaker_factory = NULL; if (c->target_name != NULL) gpr_free(c->target_name); if (c->overridden_target_name != NULL) gpr_free(c->overridden_target_name); gpr_free(sc); @@ -482,9 +481,8 @@ static void ssl_server_destroy(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc) { grpc_ssl_server_security_connector *c = (grpc_ssl_server_security_connector *)sc; - if (c->handshaker_factory != NULL) { - tsi_ssl_server_handshaker_factory_destroy(c->handshaker_factory); - } + tsi_ssl_server_handshaker_factory_unref(c->server_handshaker_factory); + c->server_handshaker_factory = NULL; gpr_free(sc); } @@ -496,7 +494,7 @@ static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker( - c->handshaker_factory, + c->client_handshaker_factory, c->overridden_target_name != NULL ? c->overridden_target_name : c->target_name, &tsi_hs); @@ -521,7 +519,7 @@ static void ssl_server_add_handshakers(grpc_exec_ctx *exec_ctx, // Instantiate TSI handshaker. tsi_handshaker *tsi_hs = NULL; tsi_result result = tsi_ssl_server_handshaker_factory_create_handshaker( - c->handshaker_factory, &tsi_hs); + c->server_handshaker_factory, &tsi_hs); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.", tsi_result_to_string(result)); @@ -852,7 +850,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create( result = tsi_create_ssl_client_handshaker_factory( has_key_cert_pair ? &config->pem_key_cert_pair : NULL, pem_root_certs, ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->client_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -897,7 +895,7 @@ grpc_security_status grpc_ssl_server_security_connector_create( config->pem_root_certs, get_tsi_client_certificate_request_type( config->client_certificate_request), ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, - &c->handshaker_factory); + &c->server_handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 1fd65928f9..7ebf9dd96f 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -67,7 +67,13 @@ /* --- Structure definitions. ---*/ +struct tsi_ssl_handshaker_factory { + const tsi_ssl_handshaker_factory_vtable *vtable; + gpr_refcount refcount; +}; + struct tsi_ssl_client_handshaker_factory { + tsi_ssl_handshaker_factory base; SSL_CTX *ssl_context; unsigned char *alpn_protocol_list; size_t alpn_protocol_list_length; @@ -77,6 +83,7 @@ struct tsi_ssl_server_handshaker_factory { /* Several contexts to support SNI. The tsi_peer array contains the subject names of the server certificates associated with the contexts at the same index. */ + tsi_ssl_handshaker_factory base; SSL_CTX **ssl_contexts; tsi_peer *ssl_context_x509_subject_names; size_t ssl_context_count; @@ -90,6 +97,7 @@ typedef struct { BIO *into_ssl; BIO *from_ssl; tsi_result result; + tsi_ssl_handshaker_factory *factory_ref; } tsi_ssl_handshaker; typedef struct { @@ -846,6 +854,47 @@ static const tsi_frame_protector_vtable frame_protector_vtable = { ssl_protector_destroy, }; +/* --- tsi_server_handshaker_factory methods implementation. --- */ + +static void tsi_ssl_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (self->vtable != NULL && self->vtable->destroy != NULL) { + self->vtable->destroy(self); + } + /* Note, we don't free(self) here because this object is always directly + * embedded in another object. If tsi_ssl_handshaker_factory_init allocates + * any memory, it should be free'd here. */ +} + +static tsi_ssl_handshaker_factory *tsi_ssl_handshaker_factory_ref( + tsi_ssl_handshaker_factory *self) { + if (self == NULL) return NULL; + gpr_refn(&self->refcount, 1); + return self; +} + +static void tsi_ssl_handshaker_factory_unref(tsi_ssl_handshaker_factory *self) { + if (self == NULL) return; + + if (gpr_unref(&self->refcount)) { + tsi_ssl_handshaker_factory_destroy(self); + } +} + +static tsi_ssl_handshaker_factory_vtable handshaker_factory_vtable = {NULL}; + +/* Initializes a tsi_ssl_handshaker_factory object. Caller is responsible for + * allocating memory for the factory. */ +static void tsi_ssl_handshaker_factory_init( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + + factory->vtable = &handshaker_factory_vtable; + gpr_ref_init(&factory->refcount, 1); +} + /* --- tsi_handshaker methods implementation. ---*/ static tsi_result ssl_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, @@ -1013,6 +1062,7 @@ static tsi_result ssl_handshaker_create_frame_protector( static void ssl_handshaker_destroy(tsi_handshaker *self) { tsi_ssl_handshaker *impl = (tsi_ssl_handshaker *)self; SSL_free(impl->ssl); /* The BIO objects are owned by ssl */ + tsi_ssl_handshaker_factory_unref(impl->factory_ref); gpr_free(impl); } @@ -1030,6 +1080,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, const char *server_name_indication, + tsi_ssl_handshaker_factory *factory, tsi_handshaker **handshaker) { SSL *ssl = SSL_new(ctx); BIO *into_ssl = NULL; @@ -1085,6 +1136,8 @@ static tsi_result create_tsi_ssl_handshaker(SSL_CTX *ctx, int is_client, impl->from_ssl = from_ssl; impl->result = TSI_HANDSHAKE_IN_PROGRESS; impl->base.vtable = &handshaker_vtable; + impl->factory_ref = tsi_ssl_handshaker_factory_ref(factory); + *handshaker = &impl->base; return TSI_OK; } @@ -1121,11 +1174,20 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker) { return create_tsi_ssl_handshaker(self->ssl_context, 1, server_name_indication, - handshaker); + &self->base, handshaker); } -void tsi_ssl_client_handshaker_factory_destroy( +void tsi_ssl_client_handshaker_factory_unref( tsi_ssl_client_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_client_handshaker_factory *self = + (tsi_ssl_client_handshaker_factory *)factory; if (self->ssl_context != NULL) SSL_CTX_free(self->ssl_context); if (self->alpn_protocol_list != NULL) gpr_free(self->alpn_protocol_list); gpr_free(self); @@ -1150,11 +1212,21 @@ tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( if (self->ssl_context_count == 0) return TSI_INVALID_ARGUMENT; /* Create the handshaker with the first context. We will switch if needed because of SNI in ssl_server_handshaker_factory_servername_callback. */ - return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, handshaker); + return create_tsi_ssl_handshaker(self->ssl_contexts[0], 0, NULL, &self->base, + handshaker); } -void tsi_ssl_server_handshaker_factory_destroy( +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self) { + if (self == NULL) return; + tsi_ssl_handshaker_factory_unref(&self->base); +} + +static void tsi_ssl_server_handshaker_factory_destroy( + tsi_ssl_handshaker_factory *factory) { + if (factory == NULL) return; + tsi_ssl_server_handshaker_factory *self = + (tsi_ssl_server_handshaker_factory *)factory; size_t i; for (i = 0; i < self->ssl_context_count; i++) { if (self->ssl_contexts[i] != NULL) { @@ -1263,6 +1335,9 @@ static int server_handshaker_factory_npn_advertised_callback( /* --- tsi_ssl_handshaker_factory constructors. --- */ +static tsi_ssl_handshaker_factory_vtable client_handshaker_factory_vtable = { + tsi_ssl_client_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_client_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pair, const char *pem_root_certs, const char *cipher_suites, @@ -1285,6 +1360,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &client_handshaker_factory_vtable; + impl->ssl_context = ssl_context; do { @@ -1322,7 +1400,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } } while (0); if (result != TSI_OK) { - tsi_ssl_client_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } SSL_CTX_set_verify(ssl_context, SSL_VERIFY_PEER, NULL); @@ -1332,6 +1410,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory( return TSI_OK; } +static tsi_ssl_handshaker_factory_vtable server_handshaker_factory_vtable = { + tsi_ssl_server_handshaker_factory_destroy}; + tsi_result tsi_create_ssl_server_handshaker_factory( const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, size_t num_key_cert_pairs, const char *pem_client_root_certs, @@ -1364,12 +1445,15 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } impl = gpr_zalloc(sizeof(*impl)); + tsi_ssl_handshaker_factory_init(&impl->base); + impl->base.vtable = &server_handshaker_factory_vtable; + impl->ssl_contexts = gpr_zalloc(num_key_cert_pairs * sizeof(SSL_CTX *)); impl->ssl_context_x509_subject_names = gpr_zalloc(num_key_cert_pairs * sizeof(tsi_peer)); if (impl->ssl_contexts == NULL || impl->ssl_context_x509_subject_names == NULL) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return TSI_OUT_OF_RESOURCES; } impl->ssl_context_count = num_key_cert_pairs; @@ -1379,7 +1463,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( &impl->alpn_protocol_list, &impl->alpn_protocol_list_length); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } @@ -1451,10 +1535,11 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } while (0); if (result != TSI_OK) { - tsi_ssl_server_handshaker_factory_destroy(impl); + tsi_ssl_handshaker_factory_unref(&impl->base); return result; } } + *factory = impl; return TSI_OK; } @@ -1501,3 +1586,15 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name) { return 0; /* Not found. */ } + +/* --- Testing support. --- */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable) { + GPR_ASSERT(factory != NULL); + GPR_ASSERT(factory->vtable != NULL); + + const tsi_ssl_handshaker_factory_vtable *orig_vtable = factory->vtable; + factory->vtable = new_vtable; + return orig_vtable; +} diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 177599930b..3abfdf5ed8 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -96,10 +96,10 @@ tsi_result tsi_ssl_client_handshaker_factory_create_handshaker( tsi_ssl_client_handshaker_factory *self, const char *server_name_indication, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_client_handshaker_factory_destroy( - tsi_ssl_client_handshaker_factory *self); +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_client_handshaker_factory_unref( + tsi_ssl_client_handshaker_factory *factory); /* --- tsi_ssl_server_handshaker_factory object --- @@ -158,9 +158,9 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( tsi_result tsi_ssl_server_handshaker_factory_create_handshaker( tsi_ssl_server_handshaker_factory *self, tsi_handshaker **handshaker); -/* Destroys the handshaker factory. WARNING: it is unsafe to destroy a factory - while handshakers created with this factory are still in use. */ -void tsi_ssl_server_handshaker_factory_destroy( +/* Decrements reference count of the handshaker factory. Handshaker factory will + * be destroyed once no references exist. */ +void tsi_ssl_server_handshaker_factory_unref( tsi_ssl_server_handshaker_factory *self); /* Util that checks that an ssl peer matches a specific name. @@ -170,6 +170,29 @@ void tsi_ssl_server_handshaker_factory_destroy( - handle public suffix wildchar more strictly (e.g. *.co.uk) */ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); +/* --- Testing support. --- + + These functions and typedefs are not intended to be used outside of testing. + */ + +/* Base type of client and server handshaker factories. */ +typedef struct tsi_ssl_handshaker_factory tsi_ssl_handshaker_factory; + +/* Function pointer to handshaker_factory destructor. */ +typedef void (*tsi_ssl_handshaker_factory_destructor)( + tsi_ssl_handshaker_factory *factory); + +/* Virtual table for tsi_ssl_handshaker_factory. */ +typedef struct { + tsi_ssl_handshaker_factory_destructor destroy; +} tsi_ssl_handshaker_factory_vtable; + +/* Set destructor of handshaker_factory to new_destructor, returns previous + destructor. */ +const tsi_ssl_handshaker_factory_vtable *tsi_ssl_handshaker_factory_swap_vtable( + tsi_ssl_handshaker_factory *factory, + tsi_ssl_handshaker_factory_vtable *new_vtable); + #ifdef __cplusplus } #endif diff --git a/test/core/tsi/ssl_transport_security_test.c b/test/core/tsi/ssl_transport_security_test.c index 364dfa1b73..2399b054b1 100644 --- a/test/core/tsi/ssl_transport_security_test.c +++ b/test/core/tsi/ssl_transport_security_test.c @@ -23,7 +23,9 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/transport/security_connector.h" #include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" #include "src/core/tsi/transport_security_adapter.h" +#include "src/core/tsi/transport_security_interface.h" #include "test/core/tsi/transport_security_test_lib.h" #include "test/core/util/test_config.h" @@ -312,10 +314,10 @@ static void ssl_test_destruct(tsi_test_fixture *fixture) { key_cert_lib->bad_client_pem_key_cert_pair); gpr_free(key_cert_lib->root_cert); gpr_free(key_cert_lib); - /* Destroy others. */ - tsi_ssl_server_handshaker_factory_destroy( + /* Unreference others. */ + tsi_ssl_server_handshaker_factory_unref( ssl_fixture->server_handshaker_factory); - tsi_ssl_client_handshaker_factory_destroy( + tsi_ssl_client_handshaker_factory_unref( ssl_fixture->client_handshaker_factory); } @@ -536,6 +538,118 @@ void ssl_tsi_test_do_round_trip_odd_buffer_size() { } } +static const tsi_ssl_handshaker_factory_vtable *original_vtable; +static bool handshaker_factory_destructor_called; + +static void ssl_tsi_test_handshaker_factory_destructor( + tsi_ssl_handshaker_factory *factory) { + GPR_ASSERT(factory != NULL); + handshaker_factory_destructor_called = true; + if (original_vtable != NULL && original_vtable->destroy != NULL) { + original_vtable->destroy(factory); + } +} + +static tsi_ssl_handshaker_factory_vtable test_handshaker_factory_vtable = { + ssl_tsi_test_handshaker_factory_destructor}; + +void test_tsi_ssl_client_handshaker_factory_refcounting() { + int i; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "client.pem"); + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)client_handshaker_factory, + &test_handshaker_factory_vtable); + + tsi_handshaker *handshaker[3]; + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_client_handshaker_factory_create_handshaker( + client_handshaker_factory, "google.com", &handshaker[i]) == + TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + gpr_free((void *)cert_chain); +} + +void test_tsi_ssl_server_handshaker_factory_refcounting() { + int i; + tsi_ssl_server_handshaker_factory *server_handshaker_factory; + tsi_handshaker *handshaker[3]; + const char *cert_chain = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.pem"); + tsi_ssl_pem_key_cert_pair cert_pair; + + cert_pair.cert_chain = cert_chain; + cert_pair.private_key = + load_file(SSL_TSI_TEST_CREDENTIALS_DIR, "server0.key"); + + GPR_ASSERT(tsi_create_ssl_server_handshaker_factory( + &cert_pair, 1, cert_chain, 0, NULL, NULL, 0, + &server_handshaker_factory) == TSI_OK); + + handshaker_factory_destructor_called = false; + original_vtable = tsi_ssl_handshaker_factory_swap_vtable( + (tsi_ssl_handshaker_factory *)server_handshaker_factory, + &test_handshaker_factory_vtable); + + for (i = 0; i < 3; ++i) { + GPR_ASSERT(tsi_ssl_server_handshaker_factory_create_handshaker( + server_handshaker_factory, &handshaker[i]) == TSI_OK); + } + + tsi_handshaker_destroy(handshaker[1]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[0]); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_ssl_server_handshaker_factory_unref(server_handshaker_factory); + GPR_ASSERT(!handshaker_factory_destructor_called); + + tsi_handshaker_destroy(handshaker[2]); + GPR_ASSERT(handshaker_factory_destructor_called); + + ssl_test_pem_key_cert_pair_destroy(cert_pair); +} + +/* Attempting to create a handshaker factory with invalid parameters should fail + * but not crash. */ +void test_tsi_ssl_client_handshaker_factory_bad_params() { + const char *cert_chain = "This is not a valid PEM file."; + + tsi_ssl_client_handshaker_factory *client_handshaker_factory; + GPR_ASSERT(tsi_create_ssl_client_handshaker_factory( + NULL, cert_chain, NULL, NULL, 0, &client_handshaker_factory) == + TSI_INVALID_ARGUMENT); + tsi_ssl_client_handshaker_factory_unref(client_handshaker_factory); +} + +void ssl_tsi_test_handshaker_factory_internals() { + test_tsi_ssl_client_handshaker_factory_refcounting(); + test_tsi_ssl_server_handshaker_factory_refcounting(); + test_tsi_ssl_client_handshaker_factory_bad_params(); +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_init(); @@ -553,6 +667,7 @@ int main(int argc, char **argv) { ssl_tsi_test_do_handshake_alpn_client_server_ok(); ssl_tsi_test_do_round_trip_for_all_configs(); ssl_tsi_test_do_round_trip_odd_buffer_size(); + ssl_tsi_test_handshaker_factory_internals(); grpc_shutdown(); return 0; } -- cgit v1.2.3 From 3fb250bdb16dda206596f28442502523e19bd2c4 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 15 Sep 2017 11:47:50 -0700 Subject: Polish output more --- src/objective-c/tests/run_tests.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 5b7a2d104a..608ae6884b 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -49,7 +49,8 @@ xcodebuild \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ | egrep -v "$XCODEBUILD_FILTER" \ - | egrep -v '^$' - + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -57,7 +58,8 @@ xcodebuild \ -scheme CoreCronetEnd2EndTests \ -destination name="iPhone 6" \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" @@ -65,7 +67,10 @@ xcodebuild \ -workspace Tests.xcworkspace \ -scheme CronetUnitTests \ -destination name="iPhone 6" \ - test | xcpretty + test \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ + | egrep -v "(GPBDictionary|GPBArray)" - echo "TIME: $(date)" xcodebuild \ @@ -74,5 +79,6 @@ xcodebuild \ -destination name="iPhone 6" \ HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test \ - | egrep "$XCODEBUILD_FILTER" \ + | egrep -v "$XCODEBUILD_FILTER" \ + | egrep -v '^$' \ | egrep -v "(GPBDictionary|GPBArray)" - -- cgit v1.2.3 From 4478a88ad92562a315267cfea8e6f3a29d1da654 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 15 Sep 2017 14:25:35 -0700 Subject: More detailed error when double-scheduling a closure --- src/core/lib/iomgr/closure.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 7236e23cf7..2253604342 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -167,7 +167,15 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, GPR_TIMER_BEGIN("grpc_closure_sched", 0); if (c != NULL) { #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + closure, closure->file_created, closure->line_created, + closure->file_initiated, closure->line_initiated, + closure->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; -- cgit v1.2.3 From f68858464afd15402e09f480938e5efc6c7ff02f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 21 Aug 2017 08:09:06 -0700 Subject: Add Kokoro perf PR jobs --- .../helper_scripts/prepare_build_linux_perf_rc | 15 ++++++++ .../internal_ci/linux/grpc_microbenchmark_diff.sh | 38 ++++++++++++++++++++ tools/internal_ci/linux/grpc_trickle_diff.sh | 42 ++++++++++++++++++++++ .../pull_request/grpc_microbenchmark_diff.cfg | 25 +++++++++++++ .../linux/pull_request/grpc_trickle_diff.cfg | 25 +++++++++++++ 5 files changed, 145 insertions(+) create mode 100755 tools/internal_ci/linux/grpc_microbenchmark_diff.sh create mode 100755 tools/internal_ci/linux/grpc_trickle_diff.sh create mode 100644 tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc index 1b7779caa8..dc3c75a6a1 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc @@ -19,4 +19,19 @@ ulimit -n 32768 ulimit -c unlimited +# Performance PR testing needs GH API key and PR metadata to comment results +if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then + set +x + sudo apt-get install -y jq + export ghprbTargetBranch=$(curl -s https://api.github.com/repos/grpc/grpc/pulls/$KOKORO_GITHUB_PULL_REQUEST_NUMBER | jq -r .base.ref) + + gsutil cp gs://grpc-testing-secrets/github_credentials/oauth_token.txt ~/ + # TODO(matt-kwong): rename this to GITHUB_OAUTH_TOKEN after Jenkins deprecation + export JENKINS_OAUTH_TOKEN=$(cat ~/oauth_token.txt) + export ghprbPullId=$KOKORO_GITHUB_PULL_REQUEST_NUMBER + set -x +fi + +sudo pip install tabulate + git submodule update --init diff --git a/tools/internal_ci/linux/grpc_microbenchmark_diff.sh b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh new file mode 100755 index 0000000000..58ffcf336b --- /dev/null +++ b/tools/internal_ci/linux/grpc_microbenchmark_diff.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b $BENCHMARKS_TO_RUN || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/grpc_trickle_diff.sh b/tools/internal_ci/linux/grpc_trickle_diff.sh new file mode 100755 index 0000000000..624031afa7 --- /dev/null +++ b/tools/internal_ci/linux/grpc_trickle_diff.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="cli_transport_stalls_per_iteration cli_stream_stalls_per_iteration svr_transport_stalls_per_iteration svr_stream_stalls_per_iteration" + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc + +tools/run_tests/start_port_server.py +tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ + -d origin/$ghprbTargetBranch \ + -b bm_fullstack_trickle \ + -l 4 \ + -t $BENCHMARKS_TO_RUN \ + --no-counters \ + --pr_comment_name trickle || FAILED="true" + +# kill port_server.py to prevent the build from hanging +ps aux | grep port_server\\.py | awk '{print $2}' | xargs kill -9 + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg new file mode 100644 index 0000000000..9269c345f0 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg new file mode 100644 index 0000000000..e86b3ab475 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} -- cgit v1.2.3 From 49d75e3fc7022f7ff409fcb9025fe36e76d0f4ed Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 15 Sep 2017 15:25:50 -0700 Subject: Address some review comments --- src/core/lib/surface/completion_queue.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index b7109c4104..1329dec35d 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -449,7 +449,7 @@ grpc_completion_queue *grpc_completion_queue_create_internal( static void cq_init_next(void *ptr) { cq_next_data *cqd = (cq_next_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_rel_store(&cqd->pending_events, 1); + gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->shutdown_called = false; gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); cq_event_queue_init(&cqd->queue); @@ -464,7 +464,7 @@ static void cq_destroy_next(void *ptr) { static void cq_init_pluck(void *ptr) { cq_pluck_data *cqd = (cq_pluck_data *)ptr; /* Initial count is dropped by grpc_completion_queue_shutdown */ - gpr_atm_rel_store(&cqd->pending_events, 1); + gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; cqd->completed_head.next = (uintptr_t)cqd->completed_tail; gpr_atm_no_barrier_store(&cqd->shutdown, 0); @@ -643,6 +643,11 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx, /* Add the completion to the queue */ bool is_first = cq_event_queue_push(&cqd->queue, storage); gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); + + /* Since we do not hold the cq lock here, it is important to do an 'acquire' + load here (instead of a 'no_barrier' load) to match with the release store + (done via gpr_atm_full_fetch_add(pending_events, -1)) in cq_shutdown_next + */ bool will_definitely_shutdown = gpr_atm_acq_load(&cqd->pending_events) == 1; if (!will_definitely_shutdown) { -- cgit v1.2.3 From 2795eae06eca4f649682f06099520fdedc7e215a Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sat, 16 Sep 2017 09:59:20 -0700 Subject: Fix timeout test --- src/objective-c/tests/GRPCClientTests.m | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 4be3d1c5e8..82ac2600fa 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -28,6 +28,7 @@ #import #import #import +#import #define TEST_TIMEOUT 16 @@ -39,6 +40,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; static GRPCProtoMethod *kInexistentMethod; static GRPCProtoMethod *kEmptyCallMethod; static GRPCProtoMethod *kUnaryCallMethod; +static GRPCProtoMethod *kFullDuplexCallMethod; /** Observer class for testing that responseMetadata is KVO-compliant */ @interface PassthroughObserver : NSObject @@ -106,6 +108,9 @@ static GRPCProtoMethod *kUnaryCallMethod; kUnaryCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage service:kService method:@"UnaryCall"]; + kFullDuplexCallMethod = [[GRPCProtoMethod alloc] initWithPackage:kPackage + service:kService + method:@"FullDuplexCall"]; } - (void)testConnectionToRemoteServer { @@ -423,11 +428,12 @@ static GRPCProtoMethod *kUnaryCallMethod; } - (void)testTimeout { - __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + __weak XCTestExpectation *completion = [self expectationWithDescription:@"RPC completed."]; + GRXBufferedPipe *pipe = [GRXBufferedPipe pipe]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kEmptyCallMethod.HTTPPath - requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + path:kFullDuplexCallMethod.HTTPPath + requestsWriter:pipe]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { XCTAssert(0, @"Failure: response received; Expect: no response received."); -- cgit v1.2.3 From 44038eb9b630eafad57e30e09d021de8d68e94d5 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 16 Sep 2017 16:13:09 +0000 Subject: Tweak Python sanity test - Move it out of the "unit" package, as it's not itself a unit test. - Suffix the test class with "Test" as we do with every other subclass of unittest.TestCase. - Add a larger-than-we'll-need-any-time-soon maxDiff so that failures are fully described. - Relax the assertion from assertListEqual to assertSequenceEqual since we don't actually care whether or not the sequences being compared are list instances. - Change the order of the assertions arguments to match the ", " convention used in our assert*Equal calls elsewhere throughout the test corpus. - Internal implementation simplifications. --- src/python/grpcio_tests/tests/_sanity/__init__.py | 13 ++++++ .../grpcio_tests/tests/_sanity/_sanity_test.py | 46 ++++++++++++++++++++++ src/python/grpcio_tests/tests/tests.json | 2 +- .../grpcio_tests/tests/unit/_sanity/__init__.py | 13 ------ .../tests/unit/_sanity/_sanity_test.py | 45 --------------------- 5 files changed, 60 insertions(+), 59 deletions(-) create mode 100644 src/python/grpcio_tests/tests/_sanity/__init__.py create mode 100644 src/python/grpcio_tests/tests/_sanity/_sanity_test.py delete mode 100644 src/python/grpcio_tests/tests/unit/_sanity/__init__.py delete mode 100644 src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py diff --git a/src/python/grpcio_tests/tests/_sanity/__init__.py b/src/python/grpcio_tests/tests/_sanity/__init__.py new file mode 100644 index 0000000000..5772620b60 --- /dev/null +++ b/src/python/grpcio_tests/tests/_sanity/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/python/grpcio_tests/tests/_sanity/_sanity_test.py b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py new file mode 100644 index 0000000000..b4079850ff --- /dev/null +++ b/src/python/grpcio_tests/tests/_sanity/_sanity_test.py @@ -0,0 +1,46 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import unittest + +import pkg_resources +import six + +import tests + + +class SanityTest(unittest.TestCase): + + maxDiff = 32768 + + def testTestsJsonUpToDate(self): + """Autodiscovers all test suites and checks that tests.json is up to date""" + loader = tests.Loader() + loader.loadTestsFromNames(['tests']) + test_suite_names = sorted({ + test_case_class.id().rsplit('.', 1)[0] + for test_case_class in tests._loader.iterate_suite_cases( + loader.suite) + }) + + tests_json_string = pkg_resources.resource_string('tests', 'tests.json') + tests_json = json.loads(tests_json_string.decode() + if six.PY3 else tests_json_string) + + self.assertSequenceEqual(tests_json, test_suite_names) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index 4c078e6c22..a3f2a9d820 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -1,4 +1,5 @@ [ + "_sanity._sanity_test.SanityTest", "health_check._health_servicer_test.HealthServicerTest", "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", @@ -41,7 +42,6 @@ "unit._reconnect_test.ReconnectTest", "unit._resource_exhausted_test.ResourceExhaustedTest", "unit._rpc_test.RPCTest", - "unit._sanity._sanity_test.Sanity", "unit._thread_cleanup_test.CleanupThreadTest", "unit.beta._beta_features_test.BetaFeaturesTest", "unit.beta._beta_features_test.ContextManagementAndLifecycleTest", diff --git a/src/python/grpcio_tests/tests/unit/_sanity/__init__.py b/src/python/grpcio_tests/tests/unit/_sanity/__init__.py deleted file mode 100644 index 5772620b60..0000000000 --- a/src/python/grpcio_tests/tests/unit/_sanity/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py b/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py deleted file mode 100644 index 19bc8801eb..0000000000 --- a/src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import unittest - -import pkg_resources -import six - -import tests - - -class Sanity(unittest.TestCase): - - def testTestsJsonUpToDate(self): - """Autodiscovers all test suites and checks that tests.json is up to date""" - loader = tests.Loader() - loader.loadTestsFromNames(['tests']) - test_suite_names = [ - test_case_class.id().rsplit('.', 1)[0] - for test_case_class in tests._loader.iterate_suite_cases( - loader.suite) - ] - test_suite_names = sorted(set(test_suite_names)) - - tests_json_string = pkg_resources.resource_string('tests', 'tests.json') - if six.PY3: - tests_json_string = tests_json_string.decode() - tests_json = json.loads(tests_json_string) - self.assertListEqual(test_suite_names, tests_json) - - -if __name__ == '__main__': - unittest.main(verbosity=2) -- cgit v1.2.3 From 719ade3cdcb5e132087fa8a443f1735adfe05634 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sat, 16 Sep 2017 15:28:22 +0000 Subject: Overhaul protoc_plugins._split_definitions_test We were mistaken before when we were testing _pb2.py files being generated in one directory and _pb2_grpc.py files being generated in another directory. Sure, that was a thing our code generator could do, but because of the way paths and packages work in Python it wasn't a realistic use case for actual users. This test now tests _pb2.py files and _pb2_grpc.py files being generated either together or independently of one another, and if independently, in either order. Looking forward to an eventual py_grpc_library Bazel rule, that's what actually matters. --- .../tests/protoc_plugin/_split_definitions_test.py | 519 ++++++++++----------- .../invocation_testing/split_messages/__init__.py | 13 - .../split_messages/messages.proto | 20 - .../split_messages/sub/messages.proto | 20 + .../invocation_testing/split_services/__init__.py | 13 - src/python/grpcio_tests/tests/tests.json | 12 +- 6 files changed, 283 insertions(+), 314 deletions(-) delete mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py delete mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto create mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto delete mode 100644 src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py diff --git a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py index 1aeb62a7c5..7868cdbfb3 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/_split_definitions_test.py @@ -12,22 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +import abc from concurrent import futures import contextlib -import distutils.spawn -import errno import importlib import os -import os.path +from os import path import pkgutil +import platform import shutil -import subprocess import sys import tempfile -import threading import unittest -import platform + +import six import grpc from grpc_tools import protoc @@ -37,292 +35,285 @@ _MESSAGES_IMPORT = b'import "messages.proto";' _SPLIT_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing.split;' _COMMON_NAMESPACE = b'package grpc_protoc_plugin.invocation_testing;' +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + @contextlib.contextmanager -def _system_path(path): +def _system_path(path_insertion): old_system_path = sys.path[:] - sys.path = sys.path[0:1] + path + sys.path[1:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] yield sys.path = old_system_path -class DummySplitServicer(object): +# NOTE(nathaniel): https://twitter.com/exoplaneteer/status/677259364256747520 +# Life lesson "just always default to idempotence" reinforced. +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(proto_content, test_name_bytes, + messages_proto_relative_file_name_bytes): + package_substitution = (b'package grpc_protoc_plugin.invocation_testing.' + + test_name_bytes + b';') + common_namespace_substituted = proto_content.replace(_COMMON_NAMESPACE, + package_substitution) + split_namespace_substituted = common_namespace_substituted.replace( + _SPLIT_NAMESPACE, package_substitution) + message_import_replaced = split_namespace_substituted.replace( + _MESSAGES_IMPORT, + b'import "' + messages_proto_relative_file_name_bytes + b'";') + return message_import_replaced + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') - def __init__(self, request_class, response_class): - self.request_class = request_class - self.response_class = response_class + +class _Servicer(object): + + def __init__(self, response_class): + self._response_class = response_class def Call(self, request, context): - return self.response_class() + return self._response_class() -class SeparateTestMixin(object): +def _protoc(proto_path, python_out, grpc_python_out_flag, grpc_python_out, + absolute_proto_file_names): + args = [ + '', + '--proto_path={}'.format(proto_path), + ] + if python_out is not None: + args.append('--python_out={}'.format(python_out)) + if grpc_python_out is not None: + args.append('--grpc_python_out={}:{}'.format(grpc_python_out_flag, + grpc_python_out)) + args.extend(absolute_proto_file_names) + return protoc.main(args) - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -class CommonTestMixin(object): - - def testImportAttributes(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - pb2.Request - pb2.Response - if self.should_find_services_in_pb2: - pb2.TestServiceServicer - else: - with self.assertRaises(AttributeError): - pb2.TestServiceServicer - - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - pb2_grpc.TestServiceServicer - with self.assertRaises(AttributeError): - pb2_grpc.Request - with self.assertRaises(AttributeError): - pb2_grpc.Response - - def testCall(self): - with _system_path([self.python_out_directory]): - pb2 = importlib.import_module(self.pb2_import) - with _system_path([self.grpc_python_out_directory]): - pb2_grpc = importlib.import_module(self.pb2_grpc_import) - server = grpc.server( - futures.ThreadPoolExecutor(max_workers=test_constants.POOL_SIZE)) - pb2_grpc.add_TestServiceServicer_to_server( - DummySplitServicer(pb2.Request, pb2.Response), server) - port = server.add_insecure_port('[::]:0') - server.start() - channel = grpc.insecure_channel('localhost:{}'.format(port)) - stub = pb2_grpc.TestServiceStub(channel) - request = pb2.Request() - expected_response = pb2.Response() - response = stub.Call(request) - self.assertEqual(expected_response, response) - - -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameSeparateTest(unittest.TestCase, SeparateTestMixin): - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_separate.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_separate;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.grpc_python_out_directory, '__init__.py'), - 'w').write('') - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_separate_pb2' - self.pb2_grpc_import = 'same_separate_pb2_grpc' - self.should_find_services_in_pb2 = False +class _Mid2016ProtocStyle(object): - def tearDown(self): - shutil.rmtree(self.directory) + def name(self): + return 'Mid2016ProtocStyle' + def grpc_in_pb2_expected(self): + return True -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SameCommonTest(unittest.TestCase, CommonTestMixin): + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_1_0', python_out, + absolute_proto_file_names),) - def setUp(self): - same_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing', 'same.proto') - self.directory = tempfile.mkdtemp(suffix='same_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - same_proto_file = os.path.join(self.proto_directory, - 'same_common.proto') - open(same_proto_file, 'wb').write( - same_proto_contents.replace( - _COMMON_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.same_common;')) - - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - same_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'same_common_pb2' - self.pb2_grpc_import = 'same_common_pb2_grpc' - self.should_find_services_in_pb2 = True - def tearDown(self): - shutil.rmtree(self.directory) +class _SingleProtocExecutionProtocStyle(object): + def name(self): + return 'SingleProtocExecutionProtocStyle' -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitCommonTest(unittest.TestCase, CommonTestMixin): + def grpc_in_pb2_expected(self): + return False - def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_common', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = self.python_out_directory - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_common_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_common_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_common_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_common;')) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out={}'.format(self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_common_messages_pb2' - self.pb2_grpc_import = 'split_common_services_pb2_grpc' - self.should_find_services_in_pb2 = False + def protoc(self, proto_path, python_out, absolute_proto_file_names): + return (_protoc(proto_path, python_out, 'grpc_2_0', python_out, + absolute_proto_file_names),) + + +class _ProtoBeforeGrpcProtocStyle(object): + + def name(self): + return 'ProtoBeforeGrpcProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + return pb2_protoc_exit_code, pb2_grpc_protoc_exit_code, - def tearDown(self): - shutil.rmtree(self.directory) +class _GrpcBeforeProtoProtocStyle(object): -@unittest.skipIf(platform.python_implementation() == "PyPy", - "Skip test if run with PyPy") -class SplitSeparateTest(unittest.TestCase, SeparateTestMixin): + def name(self): + return 'GrpcBeforeProtoProtocStyle' + + def grpc_in_pb2_expected(self): + return False + + def protoc(self, proto_path, python_out, absolute_proto_file_names): + pb2_grpc_protoc_exit_code = _protoc( + proto_path, None, 'grpc_2_0', python_out, absolute_proto_file_names) + pb2_protoc_exit_code = _protoc(proto_path, python_out, None, None, + absolute_proto_file_names) + return pb2_grpc_protoc_exit_code, pb2_protoc_exit_code, + + +_PROTOC_STYLES = (_Mid2016ProtocStyle(), _SingleProtocExecutionProtocStyle(), + _ProtoBeforeGrpcProtocStyle(), _GrpcBeforeProtoProtocStyle(),) + + +@unittest.skipIf(platform.python_implementation() == 'PyPy', + 'Skip test if run with PyPy!') +class _Test(six.with_metaclass(abc.ABCMeta, unittest.TestCase)): def setUp(self): - services_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_services', - 'services.proto') - messages_proto_contents = pkgutil.get_data( - 'tests.protoc_plugin.protos.invocation_testing.split_messages', - 'messages.proto') - self.directory = tempfile.mkdtemp(suffix='split_separate', dir='.') - self.proto_directory = os.path.join(self.directory, 'proto_path') - self.python_out_directory = os.path.join(self.directory, 'python_out') - self.grpc_python_out_directory = os.path.join(self.directory, - 'grpc_python_out') - os.makedirs(self.proto_directory) - os.makedirs(self.python_out_directory) - os.makedirs(self.grpc_python_out_directory) - services_proto_file = os.path.join(self.proto_directory, - 'split_separate_services.proto') - messages_proto_file = os.path.join(self.proto_directory, - 'split_separate_messages.proto') - open(services_proto_file, 'wb').write( - services_proto_contents.replace( - _MESSAGES_IMPORT, b'import "split_separate_messages.proto";') - .replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - open(messages_proto_file, 'wb').write( - messages_proto_contents.replace( - _SPLIT_NAMESPACE, - b'package grpc_protoc_plugin.invocation_testing.split_separate;' - )) - protoc_result = protoc.main([ - '', - '--proto_path={}'.format(self.proto_directory), - '--python_out={}'.format(self.python_out_directory), - '--grpc_python_out=grpc_2_0:{}'.format( - self.grpc_python_out_directory), - services_proto_file, - messages_proto_file, - ]) - if protoc_result != 0: - raise Exception("unexpected protoc error") - open(os.path.join(self.python_out_directory, '__init__.py'), - 'w').write('') - self.pb2_import = 'split_separate_messages_pb2' - self.pb2_grpc_import = 'split_separate_services_pb2_grpc' - self.should_find_services_in_pb2 = False + self._directory = tempfile.mkdtemp(suffix=self.NAME, dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + proto_directories_and_names = { + (self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.MESSAGES_PROTO_FILE_NAME,), + (self.SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES, + self.SERVICES_PROTO_FILE_NAME,), + } + messages_proto_relative_file_name_forward_slashes = '/'.join( + self.MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES + ( + self.MESSAGES_PROTO_FILE_NAME,)) + _create_directory_tree(self._proto_path, ( + relative_proto_directory_names + for relative_proto_directory_names, _ in proto_directories_and_names + )) + self._absolute_proto_file_names = set() + for relative_directory_names, file_name in proto_directories_and_names: + absolute_proto_file_name = path.join( + self._proto_path, *relative_directory_names + (file_name,)) + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos.invocation_testing', + path.join(*relative_directory_names + (file_name,))) + massaged_proto_content = _massage_proto_content( + raw_proto_content, + self.NAME.encode(), + messages_proto_relative_file_name_forward_slashes.encode()) + with open(absolute_proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._absolute_proto_file_names.add(absolute_proto_file_name) def tearDown(self): - shutil.rmtree(self.directory) + shutil.rmtree(self._directory) + + def _protoc(self): + protoc_exit_codes = self.PROTOC_STYLE.protoc( + self._proto_path, self._python_out, self._absolute_proto_file_names) + for protoc_exit_code in protoc_exit_codes: + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + generated_modules = {} + expected_generated_full_module_names = { + self.EXPECTED_MESSAGES_PB2, + self.EXPECTED_SERVICES_PB2, + self.EXPECTED_SERVICES_PB2_GRPC, + } + with _system_path([self._python_out]): + for full_module_name in expected_generated_full_module_names: + module = importlib.import_module(full_module_name) + generated_modules[full_module_name] = module + + self._messages_pb2 = generated_modules[self.EXPECTED_MESSAGES_PB2] + self._services_pb2 = generated_modules[self.EXPECTED_SERVICES_PB2] + self._services_pb2_grpc = generated_modules[ + self.EXPECTED_SERVICES_PB2_GRPC] + + def _services_modules(self): + if self.PROTOC_STYLE.grpc_in_pb2_expected(): + return self._services_pb2, self._services_pb2_grpc, + else: + return self._services_pb2_grpc, + + def test_imported_attributes(self): + self._protoc() + + self._messages_pb2.Request + self._messages_pb2.Response + self._services_pb2.DESCRIPTOR.services_by_name['TestService'] + for services_module in self._services_modules(): + services_module.TestServiceStub + services_module.TestServiceServicer + services_module.add_TestServiceServicer_to_server + + def test_call(self): + self._protoc() + + for services_module in self._services_modules(): + server = grpc.server( + futures.ThreadPoolExecutor( + max_workers=test_constants.POOL_SIZE)) + services_module.add_TestServiceServicer_to_server( + _Servicer(self._messages_pb2.Response), server) + port = server.add_insecure_port('[::]:0') + server.start() + channel = grpc.insecure_channel('localhost:{}'.format(port)) + stub = services_module.TestServiceStub(channel) + response = stub.Call(self._messages_pb2.Request()) + self.assertEqual(self._messages_pb2.Response(), response) + + +def _create_test_case_class(split_proto, protoc_style): + attributes = {} + + name = '{}{}'.format('SplitProto' if split_proto else 'SameProto', + protoc_style.name()) + attributes['NAME'] = name + + if split_proto: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_messages', 'sub',) + attributes['MESSAGES_PROTO_FILE_NAME'] = 'messages.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = ( + 'split_services',) + attributes['SERVICES_PROTO_FILE_NAME'] = 'services.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'split_messages.sub.messages_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'split_services.services_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = ( + 'split_services.services_pb2_grpc') + else: + attributes['MESSAGES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['MESSAGES_PROTO_FILE_NAME'] = 'same.proto' + attributes['SERVICES_PROTO_RELATIVE_DIRECTORY_NAMES'] = () + attributes['SERVICES_PROTO_FILE_NAME'] = 'same.proto' + attributes['EXPECTED_MESSAGES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2'] = 'same_pb2' + attributes['EXPECTED_SERVICES_PB2_GRPC'] = 'same_pb2_grpc' + + attributes['PROTOC_STYLE'] = protoc_style + + attributes['__module__'] = _Test.__module__ + + return type('{}Test'.format(name), (_Test,), attributes) + + +def _create_test_case_classes(): + for split_proto in (False, True,): + for protoc_style in _PROTOC_STYLES: + yield _create_test_case_class(split_proto, protoc_style) + + +def load_tests(loader, tests, pattern): + tests = tuple( + loader.loadTestsFromTestCase(test_case_class) + for test_case_class in _create_test_case_classes()) + return unittest.TestSuite(tests=tests) if __name__ == '__main__': diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py deleted file mode 100644 index 5772620b60..0000000000 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto deleted file mode 100644 index 1b780c69ba..0000000000 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/messages.proto +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package grpc_protoc_plugin.invocation_testing.split; - -message Request {} -message Response {} diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto new file mode 100644 index 0000000000..1b780c69ba --- /dev/null +++ b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_messages/sub/messages.proto @@ -0,0 +1,20 @@ +// Copyright 2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package grpc_protoc_plugin.invocation_testing.split; + +message Request {} +message Response {} diff --git a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py b/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py deleted file mode 100644 index 5772620b60..0000000000 --- a/src/python/grpcio_tests/tests/protoc_plugin/protos/invocation_testing/split_services/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index d61297b918..bbd682b335 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -3,10 +3,14 @@ "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", "protoc_plugin._python_plugin_test.PythonPluginTest", - "protoc_plugin._split_definitions_test.SameCommonTest", - "protoc_plugin._split_definitions_test.SameSeparateTest", - "protoc_plugin._split_definitions_test.SplitCommonTest", - "protoc_plugin._split_definitions_test.SplitSeparateTest", + "protoc_plugin._split_definitions_test.SameProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SameProtoSingleProtocExecutionProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoGrpcBeforeProtoProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoMid2016ProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoProtoBeforeGrpcProtocStyleTest", + "protoc_plugin._split_definitions_test.SplitProtoSingleProtocExecutionProtocStyleTest", "protoc_plugin.beta_python_plugin_test.PythonPluginTest", "reflection._reflection_servicer_test.ReflectionServicerTest", "testing._client_test.ClientTest", -- cgit v1.2.3 From 9eb8672bbe02dc47f2f667b99670b215d7431ef0 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Sun, 17 Sep 2017 23:43:30 -0700 Subject: Removing deprecated string to char * conversions --- include/grpc/compression.h | 4 ++-- src/core/ext/census/base_resources.c | 28 +++++++++++----------- .../ext/filters/client_channel/client_channel.c | 13 +++++----- .../client_channel/client_channel_factory.c | 2 +- .../filters/client_channel/client_channel_plugin.c | 4 ++-- src/core/ext/filters/client_channel/http_proxy.c | 6 ++--- .../client_channel/lb_policy/grpclb/grpclb.c | 4 ++-- .../lb_policy/round_robin/round_robin.c | 2 +- .../ext/filters/client_channel/lb_policy_factory.c | 2 +- .../resolver/dns/c_ares/dns_resolver_ares.c | 4 ++-- .../client_channel/resolver/fake/fake_resolver.c | 2 +- src/core/ext/filters/client_channel/subchannel.c | 2 +- .../message_compress/message_compress_filter.c | 4 ++-- .../load_reporting/server_load_reporting_plugin.c | 3 ++- .../chttp2/client/insecure/channel_create.c | 2 +- .../chttp2/client/insecure/channel_create_posix.c | 2 +- src/core/ext/transport/inproc/inproc_transport.c | 4 ++-- src/core/lib/channel/channel_args.c | 12 +++++----- src/core/lib/channel/channel_stack.h | 2 +- src/core/lib/compression/compression.c | 4 ++-- src/core/lib/debug/trace.h | 2 +- src/core/lib/iomgr/error.c | 2 +- src/core/lib/iomgr/resolve_address_posix.c | 2 +- src/core/lib/support/log_linux.c | 2 +- src/core/lib/support/string.c | 2 +- src/core/lib/surface/call.c | 8 +++---- src/core/lib/transport/transport_op_string.c | 2 +- src/cpp/client/client_context.cc | 2 +- src/cpp/server/server_context.cc | 2 +- test/core/compression/algorithm_test.c | 2 +- test/core/compression/compression_test.c | 2 +- test/core/compression/message_compress_test.c | 2 +- test/core/end2end/tests/compressed_payload.c | 2 +- .../tests/stream_compression_compressed_payload.c | 2 +- 34 files changed, 71 insertions(+), 69 deletions(-) diff --git a/include/grpc/compression.h b/include/grpc/compression.h index 3a8de4b7b8..13a8dd66ad 100644 --- a/include/grpc/compression.h +++ b/include/grpc/compression.h @@ -44,13 +44,13 @@ int grpc_stream_compression_algorithm_parse( * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_compression_algorithm_name( - grpc_compression_algorithm algorithm, char **name); + grpc_compression_algorithm algorithm, const char **name); /** Updates \a name with the encoding name corresponding to a valid \a * algorithm. Note that \a name is statically allocated and must *not* be freed. * Returns 1 upon success, 0 otherwise. */ GRPCAPI int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name); + grpc_stream_compression_algorithm algorithm, const char **name); /** Returns the compression algorithm corresponding to \a level for the * compression algorithms encoded in the \a accepted_encodings bitset. diff --git a/src/core/ext/census/base_resources.c b/src/core/ext/census/base_resources.c index 2114bf04cd..1f2bb39fe0 100644 --- a/src/core/ext/census/base_resources.c +++ b/src/core/ext/census/base_resources.c @@ -37,20 +37,20 @@ void define_base_resources() { google_census_Resource_BasicUnit numerator = google_census_Resource_BasicUnit_SECS; - resource r = {"client_rpc_latency", // name - "Client RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + resource r = {(char *)"client_rpc_latency", // name + (char *)"Client RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); - r = (resource){"server_rpc_latency", // name - "Server RPC latency in seconds", // description - 0, // prefix - 1, // n_numerators - &numerator, // numerators - 0, // n_denominators - NULL}; // denominators + r = (resource){(char *)"server_rpc_latency", // name + (char *)"Server RPC latency in seconds", // description + 0, // prefix + 1, // n_numerators + &numerator, // numerators + 0, // n_denominators + NULL}; // denominators define_resource(&r); } diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 129d0f368b..016199b1f4 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -375,7 +375,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, } // Extract the following fields from the resolver result, if non-NULL. bool lb_policy_updated = false; - char *lb_policy_name = NULL; + char *lb_policy_name_dup = NULL; bool lb_policy_name_changed = false; grpc_lb_policy *new_lb_policy = NULL; char *service_config_json = NULL; @@ -383,6 +383,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, grpc_slice_hash_table *method_params_table = NULL; if (chand->resolver_result != NULL) { // Find LB policy name. + const char *lb_policy_name = NULL; const grpc_arg *channel_arg = grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_POLICY_NAME); if (channel_arg != NULL) { @@ -473,7 +474,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // Before we clean up, save a copy of lb_policy_name, since it might // be pointing to data inside chand->resolver_result. // The copy will be saved in chand->lb_policy_name below. - lb_policy_name = gpr_strdup(lb_policy_name); + lb_policy_name_dup = gpr_strdup(lb_policy_name); grpc_channel_args_destroy(exec_ctx, chand->resolver_result); chand->resolver_result = NULL; } @@ -481,8 +482,8 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, gpr_log(GPR_DEBUG, "chand=%p: resolver result: lb_policy_name=\"%s\"%s, " "service_config=\"%s\"", - chand, lb_policy_name, lb_policy_name_changed ? " (changed)" : "", - service_config_json); + chand, lb_policy_name_dup, + lb_policy_name_changed ? " (changed)" : "", service_config_json); } // Now swap out fields in chand. Note that the new values may still // be NULL if (e.g.) the resolver failed to return results or the @@ -490,9 +491,9 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, // // First, swap out the data used by cc_get_channel_info(). gpr_mu_lock(&chand->info_mu); - if (lb_policy_name != NULL) { + if (lb_policy_name_dup != NULL) { gpr_free(chand->info_lb_policy_name); - chand->info_lb_policy_name = lb_policy_name; + chand->info_lb_policy_name = lb_policy_name_dup; } if (service_config_json != NULL) { gpr_free(chand->info_service_config_json); diff --git a/src/core/ext/filters/client_channel/client_channel_factory.c b/src/core/ext/filters/client_channel/client_channel_factory.c index e8aa4cda29..57eac8f875 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.c +++ b/src/core/ext/filters/client_channel/client_channel_factory.c @@ -63,6 +63,6 @@ static const grpc_arg_pointer_vtable factory_arg_vtable = { grpc_arg grpc_client_channel_factory_create_channel_arg( grpc_client_channel_factory* factory) { - return grpc_channel_arg_pointer_create(GRPC_ARG_CLIENT_CHANNEL_FACTORY, + return grpc_channel_arg_pointer_create((char*)GRPC_ARG_CLIENT_CHANNEL_FACTORY, factory, &factory_arg_vtable); } diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.c b/src/core/ext/filters/client_channel/client_channel_plugin.c index c32e83d012..1f71c5a7f9 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.c +++ b/src/core/ext/filters/client_channel/client_channel_plugin.c @@ -54,8 +54,8 @@ static bool set_default_host_if_unset(grpc_exec_ctx *exec_ctx, char *default_authority = grpc_get_default_authority( exec_ctx, grpc_channel_stack_builder_get_target(builder)); if (default_authority != NULL) { - grpc_arg arg = grpc_channel_arg_string_create(GRPC_ARG_DEFAULT_AUTHORITY, - default_authority); + grpc_arg arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_DEFAULT_AUTHORITY, default_authority); grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1); grpc_channel_stack_builder_set_channel_arguments(exec_ctx, builder, new_args); diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index ef3512ed83..0c3b009e44 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -157,7 +157,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, } grpc_arg args_to_add[2]; args_to_add[0] = grpc_channel_arg_string_create( - GRPC_ARG_HTTP_CONNECT_SERVER, + (char*)GRPC_ARG_HTTP_CONNECT_SERVER, uri->path[0] == '/' ? uri->path + 1 : uri->path); if (user_cred != NULL) { /* Use base64 encoding for user credentials as stated in RFC 7617 */ @@ -166,8 +166,8 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, char* header; gpr_asprintf(&header, "Proxy-Authorization:Basic %s", encoded_user_cred); gpr_free(encoded_user_cred); - args_to_add[1] = - grpc_channel_arg_string_create(GRPC_ARG_HTTP_CONNECT_HEADERS, header); + args_to_add[1] = grpc_channel_arg_string_create( + (char*)GRPC_ARG_HTTP_CONNECT_HEADERS, header); *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 2); gpr_free(header); } else { diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a776a07d99..c4c4973b2d 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1830,8 +1830,8 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. - grpc_arg new_arg = - grpc_channel_arg_string_create(GRPC_ARG_LB_POLICY_NAME, "grpclb"); + grpc_arg new_arg = grpc_channel_arg_string_create( + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)"grpclb"); static const char *args_to_remove[] = {GRPC_ARG_LB_POLICY_NAME}; glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 8ac1a46abd..a3a62e9f3c 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -589,7 +589,7 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, // Dispose of outdated subchannel lists. if (sd->subchannel_list != p->subchannel_list && sd->subchannel_list != p->latest_pending_subchannel_list) { - char *reason = NULL; + const char *reason = NULL; if (sd->subchannel_list->shutting_down) { reason = "sl_outdated_straggler"; rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, reason); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index acf5929746..4d1405454c 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -141,7 +141,7 @@ static const grpc_arg_pointer_vtable lb_addresses_arg_vtable = { grpc_arg grpc_lb_addresses_create_channel_arg( const grpc_lb_addresses* addresses) { return grpc_channel_arg_pointer_create( - GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); + (char*)GRPC_ARG_LB_ADDRESSES, (void*)addresses, &lb_addresses_arg_vtable); } grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index 371c59b8cf..9bb229ad95 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -249,7 +249,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, service_config_string); args_to_remove[num_args_to_remove++] = GRPC_ARG_SERVICE_CONFIG; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_SERVICE_CONFIG, service_config_string); + (char *)GRPC_ARG_SERVICE_CONFIG, service_config_string); service_config = grpc_service_config_create(service_config_string); if (service_config != NULL) { const char *lb_policy_name = @@ -257,7 +257,7 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, if (lb_policy_name != NULL) { args_to_remove[num_args_to_remove++] = GRPC_ARG_LB_POLICY_NAME; new_args[num_args_to_add++] = grpc_channel_arg_string_create( - GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); + (char *)GRPC_ARG_LB_POLICY_NAME, (char *)lb_policy_name); } } } diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c index c4676e0299..69ea440ae6 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c @@ -210,7 +210,7 @@ grpc_arg grpc_fake_resolver_response_generator_arg( grpc_fake_resolver_response_generator* generator) { grpc_arg arg; arg.type = GRPC_ARG_POINTER; - arg.key = GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; + arg.key = (char*)GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; arg.value.pointer.p = generator; arg.value.pointer.vtable = &response_generator_arg_vtable; return arg; diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index bc9c3cc782..40a51c72d6 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -811,6 +811,6 @@ const char *grpc_get_subchannel_address_uri_arg(const grpc_channel_args *args) { grpc_arg grpc_create_subchannel_address_arg(const grpc_resolved_address *addr) { return grpc_channel_arg_string_create( - GRPC_ARG_SUBCHANNEL_ADDRESS, + (char *)GRPC_ARG_SUBCHANNEL_ADDRESS, addr->len > 0 ? grpc_sockaddr_to_uri(addr) : gpr_strdup("")); } diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 0145dffab0..f785e1355d 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -244,7 +244,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, &calld->slices, &tmp); if (did_compress) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; const size_t before_size = calld->slices.length; const size_t after_size = tmp.length; const float savings_ratio = 1.0f - (float)after_size / (float)before_size; @@ -258,7 +258,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name; + const char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); gpr_log(GPR_DEBUG, diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c index f56afd27d2..2486ead427 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c +++ b/src/core/ext/filters/load_reporting/server_load_reporting_plugin.c @@ -55,7 +55,8 @@ static bool maybe_add_server_load_reporting_filter( } grpc_arg grpc_load_reporting_enable_arg() { - return grpc_channel_arg_integer_create(GRPC_ARG_ENABLE_LOAD_REPORTING, 1); + return grpc_channel_arg_integer_create((char *)GRPC_ARG_ENABLE_LOAD_REPORTING, + 1); } /* Plugin registration */ diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index cccb347bf1..6410a6043d 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -55,7 +55,7 @@ static grpc_channel *client_channel_factory_create_channel( } // Add channel arg containing the server URI. grpc_arg arg = grpc_channel_arg_string_create( - GRPC_ARG_SERVER_URI, + (char *)GRPC_ARG_SERVER_URI, grpc_resolver_factory_add_default_prefix_if_needed(exec_ctx, target)); const char *to_remove[] = {GRPC_ARG_SERVER_URI}; grpc_channel_args *new_args = diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index 0346d50b6c..dd88136f7b 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -42,7 +42,7 @@ grpc_channel *grpc_insecure_channel_create_from_fd( (target, fd, args)); grpc_arg default_authority_arg = grpc_channel_arg_string_create( - GRPC_ARG_DEFAULT_AUTHORITY, "test.authority"); + (char *)GRPC_ARG_DEFAULT_AUTHORITY, (char *)"test.authority"); grpc_channel_args *final_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c index cd3e76a0b5..31739d07dd 100644 --- a/src/core/ext/transport/inproc/inproc_transport.c +++ b/src/core/ext/transport/inproc/inproc_transport.c @@ -1263,8 +1263,8 @@ grpc_channel *grpc_inproc_channel_create(grpc_server *server, grpc_arg default_authority_arg; default_authority_arg.type = GRPC_ARG_STRING; - default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY; - default_authority_arg.value.string = "inproc.authority"; + default_authority_arg.key = (char *)GRPC_ARG_DEFAULT_AUTHORITY; + default_authority_arg.value.string = (char *)"inproc.authority"; grpc_channel_args *client_args = grpc_channel_args_copy_and_add(args, &default_authority_arg, 1); diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 9215707f1c..30248b3c60 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -243,7 +243,7 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm( GPR_ASSERT(algorithm < GRPC_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -253,7 +253,7 @@ grpc_channel_args *grpc_channel_args_set_stream_compression_algorithm( GPR_ASSERT(algorithm < GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT); grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM; tmp.value.integer = algorithm; return grpc_channel_args_copy_and_add(a, &tmp, 1); } @@ -308,7 +308,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( if (grpc_channel_args_get_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, "Tried to disable default compression algorithm '%s'. The " @@ -324,7 +324,7 @@ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { @@ -349,7 +349,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( if (grpc_channel_args_get_stream_compression_algorithm(*a) == algorithm && state == 0) { - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_stream_compression_algorithm_name(algorithm, &algo_name) != 0); gpr_log(GPR_ERROR, @@ -366,7 +366,7 @@ grpc_channel_args *grpc_channel_args_stream_compression_algorithm_set_state( /* create a new arg */ grpc_arg tmp; tmp.type = GRPC_ARG_INTEGER; - tmp.key = GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; + tmp.key = (char *)GRPC_STREAM_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET; /* all enabled by default */ tmp.value.integer = (1u << GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT) - 1; if (state != 0) { diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index ae1cac31f7..f0de80f0c0 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -281,7 +281,7 @@ grpc_channel_stack *grpc_channel_stack_from_top_element( /* Given the top element of a call stack, get the call stack itself */ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem); -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op); diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c index ec84c01811..1cfac23129 100644 --- a/src/core/lib/compression/compression.c +++ b/src/core/lib/compression/compression.c @@ -60,7 +60,7 @@ int grpc_stream_compression_algorithm_parse( } int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, - char **name) { + const char **name) { GRPC_API_TRACE("grpc_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); switch (algorithm) { @@ -80,7 +80,7 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, } int grpc_stream_compression_algorithm_name( - grpc_stream_compression_algorithm algorithm, char **name) { + grpc_stream_compression_algorithm algorithm, const char **name) { GRPC_API_TRACE( "grpc_stream_compression_algorithm_parse(algorithm=%d, name=%p)", 2, ((int)algorithm, name)); diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index dd9e6a30fe..64f2e3fc33 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -35,7 +35,7 @@ typedef struct { #else bool value; #endif - char *name; + const char *name; } grpc_tracer_flag; #ifdef GRPC_THREADSAFE_TRACER diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index d5e00a8f64..aa05501537 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -641,7 +641,7 @@ static char *key_time(grpc_error_times which) { static char *fmt_time(gpr_timespec tm) { char *out; - char *pfx = "!!"; + const char *pfx = "!!"; switch (tm.clock_type) { case GPR_CLOCK_MONOTONIC: pfx = "@monotonic:"; diff --git a/src/core/lib/iomgr/resolve_address_posix.c b/src/core/lib/iomgr/resolve_address_posix.c index 082e3b7947..60cfeebd47 100644 --- a/src/core/lib/iomgr/resolve_address_posix.c +++ b/src/core/lib/iomgr/resolve_address_posix.c @@ -85,7 +85,7 @@ static grpc_error *blocking_resolve_address_impl( if (s != 0) { /* Retry if well-known service name is recognized */ - char *svc[][2] = {{"http", "80"}, {"https", "443"}}; + const char *svc[][2] = {{"http", "80"}, {"https", "443"}}; for (i = 0; i < GPR_ARRAY_SIZE(svc); i++) { if (strcmp(port, svc[i][0]) == 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c index 61d2346427..7755018693 100644 --- a/src/core/lib/support/log_linux.c +++ b/src/core/lib/support/log_linux.c @@ -57,7 +57,7 @@ void gpr_log(const char *file, int line, gpr_log_severity severity, } void gpr_default_log(gpr_log_func_args *args) { - char *final_slash; + const char *final_slash; char *prefix; const char *display_file; char time_buffer[64]; diff --git a/src/core/lib/support/string.c b/src/core/lib/support/string.c index 523e43445b..6b172df82f 100644 --- a/src/core/lib/support/string.c +++ b/src/core/lib/support/string.c @@ -276,7 +276,7 @@ static void add_string_to_split(const char *beg, const char *end, char ***strs, void gpr_string_split(const char *input, const char *sep, char ***strs, size_t *nstrs) { - char *next; + const char *next; *strs = NULL; *nstrs = 0; size_t capstrs = 0; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index e258719c4d..a41eb336d4 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1511,7 +1511,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_stream_compression_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Stream compression algorithm '%s' is disabled.", algo_name); @@ -1525,7 +1525,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->stream_encodings_accepted_by_peer, call->incoming_stream_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_stream_compression_algorithm_name( call->incoming_stream_compression_algorithm, &algo_name); gpr_log( @@ -1552,7 +1552,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, } else if (grpc_compression_options_is_algorithm_enabled( &compression_options, algo) == 0) { /* check if algorithm is supported by current channel config */ - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(algo, &algo_name); gpr_asprintf(&error_msg, "Compression algorithm '%s' is disabled.", algo_name); @@ -1568,7 +1568,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { if (GRPC_TRACER_ON(grpc_compression_trace)) { - char *algo_name = NULL; + const char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); gpr_log(GPR_ERROR, diff --git a/src/core/lib/transport/transport_op_string.c b/src/core/lib/transport/transport_op_string.c index 409a6c4103..858664715c 100644 --- a/src/core/lib/transport/transport_op_string.c +++ b/src/core/lib/transport/transport_op_string.c @@ -197,7 +197,7 @@ char *grpc_transport_op_string(grpc_transport_op *op) { return out; } -void grpc_call_log_op(char *file, int line, gpr_log_severity severity, +void grpc_call_log_op(const char *file, int line, gpr_log_severity severity, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { char *str = grpc_transport_stream_op_batch_string(op); diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc index 3af8bdc11a..40e95f3c05 100644 --- a/src/cpp/client/client_context.cc +++ b/src/cpp/client/client_context.cc @@ -96,7 +96,7 @@ void ClientContext::set_call(grpc_call* call, void ClientContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = nullptr; + const char* algorithm_name = nullptr; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 4913682f1d..d7876a000b 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -190,7 +190,7 @@ bool ServerContext::IsCancelled() const { void ServerContext::set_compression_algorithm( grpc_compression_algorithm algorithm) { - char* algorithm_name = NULL; + const char* algorithm_name = NULL; if (!grpc_compression_algorithm_name(algorithm, &algorithm_name)) { gpr_log(GPR_ERROR, "Name for compression algorithm '%d' unknown.", algorithm); diff --git a/test/core/compression/algorithm_test.c b/test/core/compression/algorithm_test.c index e37c8c3010..a11e6e90ac 100644 --- a/test/core/compression/algorithm_test.c +++ b/test/core/compression/algorithm_test.c @@ -35,7 +35,7 @@ static void test_algorithm_mesh(void) { gpr_log(GPR_DEBUG, "test_algorithm_mesh"); for (i = 0; i < GRPC_COMPRESS_ALGORITHMS_COUNT; i++) { - char *name; + const char *name; grpc_compression_algorithm parsed; grpc_slice mdstr; grpc_mdelem mdelem; diff --git a/test/core/compression/compression_test.c b/test/core/compression/compression_test.c index 4b6026ec77..326a800300 100644 --- a/test/core/compression/compression_test.c +++ b/test/core/compression/compression_test.c @@ -57,7 +57,7 @@ static void test_compression_algorithm_parse(void) { static void test_compression_algorithm_name(void) { int success; - char *name; + const char *name; size_t i; const char *valid_names[] = {"identity", "gzip", "deflate"}; const grpc_compression_algorithm valid_algorithms[] = { diff --git a/test/core/compression/message_compress_test.c b/test/core/compression/message_compress_test.c index fd692fe5a5..f7f4893dee 100644 --- a/test/core/compression/message_compress_test.c +++ b/test/core/compression/message_compress_test.c @@ -49,7 +49,7 @@ static void assert_passthrough(grpc_slice value, grpc_slice_buffer output; grpc_slice final; int was_compressed; - char *algorithm_name; + const char *algorithm_name; GPR_ASSERT(grpc_compression_algorithm_name(algorithm, &algorithm_name) != 0); gpr_log( diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 429717a7be..639af15454 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT(grpc_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; gpr_asprintf(&expected_details, "Compression algorithm '%s' is disabled.", diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 11e7999a1c..5a69091a57 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -228,7 +228,7 @@ static void request_for_disabled_algorithm( /* with a certain error */ GPR_ASSERT(status == expected_error); - char *algo_name = NULL; + const char *algo_name = NULL; GPR_ASSERT( grpc_stream_compression_algorithm_name(algorithm_to_disable, &algo_name)); char *expected_details = NULL; -- cgit v1.2.3 From 571fa0c5967a4e7a504e5ad5c495bb4a68953166 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 16:06:11 -0700 Subject: Add stream state tracer --- .../ext/transport/chttp2/transport/chttp2_plugin.c | 1 + .../transport/chttp2/transport/chttp2_transport.h | 1 + .../ext/transport/chttp2/transport/stream_lists.c | 52 +++++++++++++++------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c index 78551df9c3..6d09953830 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c @@ -23,6 +23,7 @@ void grpc_chttp2_plugin_init(void) { grpc_register_tracer(&grpc_http_trace); grpc_register_tracer(&grpc_flowctl_trace); + grpc_register_tracer(&grpc_trace_http2_stream_state); #ifndef NDEBUG grpc_register_tracer(&grpc_trace_chttp2_refcount); #endif diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 0c4e2a91c0..55fb1a8343 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -25,6 +25,7 @@ extern grpc_tracer_flag grpc_http_trace; extern grpc_tracer_flag grpc_flowctl_trace; +extern grpc_tracer_flag grpc_trace_http2_stream_state; #ifndef NDEBUG extern grpc_tracer_flag grpc_trace_chttp2_refcount; diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index 7cc85dea9c..ee23b77efc 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -20,6 +20,27 @@ #include +static char *stream_list_id_string(grpc_chttp2_stream_list_id id) { + switch (id) { + case GRPC_CHTTP2_LIST_WRITABLE: + return "writable"; + case GRPC_CHTTP2_LIST_WRITING: + return "writing"; + case GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT: + return "stalled_by_transport"; + case GRPC_CHTTP2_LIST_STALLED_BY_STREAM: + return "stalled_by_stream"; + case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: + return "waiting_for_concurrency"; + case STREAM_LIST_COUNT: + GPR_UNREACHABLE_CODE(return "unknown"); + } + GPR_UNREACHABLE_CODE(return "unknown"); +} + +grpc_tracer_flag grpc_trace_http2_stream_state = + GRPC_TRACER_INITIALIZER(false, "http2_stream_state"); + /* core list management */ static bool stream_list_empty(grpc_chttp2_transport *t, @@ -44,6 +65,10 @@ static bool stream_list_pop(grpc_chttp2_transport *t, s->included[id] = 0; } *stream = s; + if (s && GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: pop stream %d from %s", t, s->id, + stream_list_id_string(id)); + } return s != 0; } @@ -62,6 +87,10 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, } else { t->lists[id].tail = s->links[id].prev; } + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: remove stream %d from %s", t, s->id, + stream_list_id_string(id)); + } } static bool stream_list_maybe_remove(grpc_chttp2_transport *t, @@ -90,6 +119,10 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, } t->lists[id].tail = s; s->included[id] = 1; + if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { + gpr_log(GPR_DEBUG, "t[%p]: add stream %d from %s", t, s->id, + stream_list_id_string(id)); + } } static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, @@ -150,17 +183,12 @@ void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by transport", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } bool grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); - GRPC_FLOW_CONTROL_IF_TRACING(if (ret) gpr_log( - GPR_DEBUG, "stream %u un-stalled by transport", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, @@ -170,23 +198,15 @@ void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, void grpc_chttp2_list_add_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - GRPC_FLOW_CONTROL_IF_TRACING( - gpr_log(GPR_DEBUG, "stream %u stalled by stream", s->id)); stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream **s) { - bool ret = stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", (*s)->id)); - return ret; + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - bool ret = stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); - GRPC_FLOW_CONTROL_IF_TRACING( - if (ret) gpr_log(GPR_DEBUG, "stream %u un-stalled by stream", s->id)); - return ret; + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } -- cgit v1.2.3 From 0252252fdd5c468079b3d5e32cc4516b02dccbcc Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Sep 2017 18:01:23 -0700 Subject: Update env var documentation --- doc/environment_variables.md | 1 + src/core/ext/transport/chttp2/transport/stream_lists.c | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index b79cd97363..f90f1d5b10 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -50,6 +50,7 @@ some configuration as environment variables that can be set. - channel_stack_builder - traces information about channel stacks being built - executor - traces grpc's internal thread pool ('the executor') - http - traces state in the http2 transport engine + - http2_stream_state - traces all http2 stream state mutations. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - flowctl - traces http2 flow control diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index ee23b77efc..47cd22d177 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -66,8 +66,8 @@ static bool stream_list_pop(grpc_chttp2_transport *t, } *stream = s; if (s && GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: pop stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: pop from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } return s != 0; } @@ -88,8 +88,8 @@ static void stream_list_remove(grpc_chttp2_transport *t, grpc_chttp2_stream *s, t->lists[id].tail = s->links[id].prev; } if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: remove stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: remove from %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } } @@ -120,8 +120,8 @@ static void stream_list_add_tail(grpc_chttp2_transport *t, t->lists[id].tail = s; s->included[id] = 1; if (GRPC_TRACER_ON(grpc_trace_http2_stream_state)) { - gpr_log(GPR_DEBUG, "t[%p]: add stream %d from %s", t, s->id, - stream_list_id_string(id)); + gpr_log(GPR_DEBUG, "%p[%d][%s]: add to %s", t, s->id, + t->is_client ? "cli" : "svr", stream_list_id_string(id)); } } -- cgit v1.2.3 From 86eb4ec0ef68482faa960801899be964e545a8b2 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 18 Sep 2017 14:22:16 +0000 Subject: Fix a six.with_metaclass usage mistake I made this mistake in 2010985ab269c8df0443e4f3782cbdffb083e9d4 but only with yesterday's release of six 1.11.0 has it started failing ("TypeError: metaclass conflict: the metaclassof a derived class must be a (non-strict) subclass of the metaclasses of all its bases"). --- src/python/grpcio_testing/grpc_testing/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index 917e11808e..994274500c 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -213,7 +213,7 @@ class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): raise NotImplementedError() -class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): +class Channel(six.with_metaclass(abc.ABCMeta, grpc.Channel)): """A grpc.Channel double with which to test a system that invokes RPCs.""" @abc.abstractmethod -- cgit v1.2.3 From a29032fd32d1e275bd05afc011e7a7c8db61f644 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 18 Sep 2017 16:33:37 +0200 Subject: test everything if internal_ci scripts change --- tools/run_tests/python_utils/filter_pull_request_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py index 4ad981237b..f99143fdd7 100644 --- a/tools/run_tests/python_utils/filter_pull_request_tests.py +++ b/tools/run_tests/python_utils/filter_pull_request_tests.py @@ -86,7 +86,6 @@ _WHITELIST_DICT = { '^test/distrib/php/': [_PHP_TEST_SUITE], '^test/distrib/python/': [_PYTHON_TEST_SUITE], '^test/distrib/ruby/': [_RUBY_TEST_SUITE], - '^tools/internal_ci/': [], '^vsprojects/': [_WINDOWS_TEST_SUITE], 'binding\.gyp$': [_NODE_TEST_SUITE], 'composer\.json$': [_PHP_TEST_SUITE], -- cgit v1.2.3 From e4260e5b445ba01d141e2df2b642a846c5675ffa Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Mon, 11 Sep 2017 16:11:02 -0700 Subject: EMBED_OPENSSL before build php --- tools/run_tests/run_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 591a0bebbd..176c1097ee 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -533,6 +533,7 @@ class PhpLanguage(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -545,7 +546,7 @@ class PhpLanguage(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] @@ -569,6 +570,7 @@ class Php7Language(object): self.config = config self.args = args _check_compiler(self.args.compiler, ['default']) + self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] def test_specs(self): return [self.config.job_spec(['src/php/bin/run_tests.sh'], @@ -581,7 +583,7 @@ class Php7Language(object): return ['static_c', 'shared_c'] def make_options(self): - return [] + return self._make_options; def build_steps(self): return [['tools/run_tests/helper_scripts/build_php.sh']] -- cgit v1.2.3 From 3462fd1481e457a110b37c401f49b42bb31a0fb5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 18 Sep 2017 09:00:16 -0700 Subject: Fix test --- test/core/iomgr/pollset_set_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 5750ac0f4b..70efca8b16 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -433,8 +434,7 @@ int main(int argc, char **argv) { const char *poll_strategy = grpc_get_poll_strategy_name(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); - grpc_iomgr_init(&exec_ctx); - grpc_iomgr_start(&exec_ctx); + grpc_init(); if (poll_strategy != NULL && (strcmp(poll_strategy, "epoll") == 0 || @@ -449,8 +449,8 @@ int main(int argc, char **argv) { poll_strategy); } - grpc_iomgr_shutdown(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); + grpc_shutdown(); return 0; } #else /* defined(GRPC_LINUX_EPOLL) */ -- cgit v1.2.3 From 966bf49b9c7b916c4f4a2be1dc7a9e8dae07582d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 09:42:31 -0700 Subject: Add to closure_list_sched as well --- src/core/lib/iomgr/closure.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 2253604342..bb9e437b08 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -199,7 +199,15 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { while (c != NULL) { grpc_closure *next = c->next_data.next; #ifndef NDEBUG - GPR_ASSERT(!c->scheduled); + if (c->scheduled) { + gpr_log(GPR_ERROR, + "Closure already scheduled. (closure: %p, created: [%s:%d], " + "previously scheduled at: [%s: %d] run?: %s", + closure, closure->file_created, closure->line_created, + closure->file_initiated, closure->line_initiated, + closure->run ? "true" : "false"); + abort(); + } c->scheduled = true; c->file_initiated = file; c->line_initiated = line; -- cgit v1.2.3 From 0dbfde67507abf83d46bb24098419ce5523cc653 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 10:11:37 -0700 Subject: Restore no_barrier operations to some places where it is safe --- src/core/lib/surface/completion_queue.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 1329dec35d..468360fbde 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -965,7 +965,7 @@ static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx, cq_next_data *cqd = (cq_next_data *)DATA_FROM_CQ(cq); GPR_ASSERT(cqd->shutdown_called); - GPR_ASSERT(gpr_atm_acq_load(&cqd->pending_events) == 0); + GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0); cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); @@ -989,6 +989,9 @@ static void cq_shutdown_next(grpc_exec_ctx *exec_ctx, return; } cqd->shutdown_called = true; + /* Doing a full_fetch_add (i.e acq/release) here to match with + * cq_begin_op_for_next and and cq_end_op_for_next functions which read/write + * on this counter without necessarily holding a lock on cq */ if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { cq_finish_shutdown_next(exec_ctx, cq); } -- cgit v1.2.3 From 87c09525317add9d887cf0580f91feb8b1e970b7 Mon Sep 17 00:00:00 2001 From: Michael Bausor Date: Mon, 18 Sep 2017 10:45:04 -0700 Subject: Apply formatting tool --- src/compiler/php_generator.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/compiler/php_generator.cc b/src/compiler/php_generator.cc index 9fa14d1684..6d9ff3a29c 100644 --- a/src/compiler/php_generator.cc +++ b/src/compiler/php_generator.cc @@ -51,24 +51,25 @@ grpc::string PackageName(const FileDescriptor *file) { } } -grpc::string MessageIdentifierName(const grpc::string &name, const FileDescriptor *file) { +grpc::string MessageIdentifierName(const grpc::string &name, + const FileDescriptor *file) { std::vector tokens = grpc_generator::tokenize(name, "."); std::ostringstream oss; - oss << PackageName(file) - << "\\" + oss << PackageName(file) << "\\" << grpc_generator::CapitalizeFirstLetter(tokens[tokens.size() - 1]); return oss.str(); } - void PrintMethod(const MethodDescriptor *method, Printer *out) { const Descriptor *input_type = method->input_type(); const Descriptor *output_type = method->output_type(); map vars; vars["service_name"] = method->service()->full_name(); vars["name"] = method->name(); - vars["input_type_id"] = MessageIdentifierName(input_type->full_name(), input_type->file()); - vars["output_type_id"] = MessageIdentifierName(output_type->full_name(), output_type->file()); + vars["input_type_id"] = + MessageIdentifierName(input_type->full_name(), input_type->file()); + vars["output_type_id"] = + MessageIdentifierName(output_type->full_name(), output_type->file()); out->Print("/**\n"); out->Print(GetPHPComments(method, " *").c_str()); -- cgit v1.2.3 From dc1cd5437c81b57bad315a34dfe46c0eb6e666ad Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 10:51:22 -0700 Subject: Running generate_projects.sh --- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index c5c848ae44..55a7eed1a4 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -164,10 +164,10 @@ extern census_record_values_type census_record_values_import; typedef int(*grpc_compression_algorithm_parse_type)(grpc_slice value, grpc_compression_algorithm *algorithm); extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_import; #define grpc_compression_algorithm_parse grpc_compression_algorithm_parse_import -typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name); +typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, const char **name); extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import; #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import -typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, char **name); +typedef int(*grpc_stream_compression_algorithm_name_type)(grpc_stream_compression_algorithm algorithm, const char **name); extern grpc_stream_compression_algorithm_name_type grpc_stream_compression_algorithm_name_import; #define grpc_stream_compression_algorithm_name grpc_stream_compression_algorithm_name_import typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings); -- cgit v1.2.3 From 086e95ae657ab87b52de9675140b194c61874683 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Sun, 17 Sep 2017 23:59:11 +0000 Subject: Use grpc_1_0 flag in beta_python_plugin_test Beta code elements are not generated at all in _pb2_grpc.py files. This duplicates a lot of the in-test code generation done in _split_definitions_test. In a future clean-up we may want to deduplicate the common behavior, put it in a module available to all other tests, and do all of our testing of generated code with in-test code generation. --- .../tests/protoc_plugin/beta_python_plugin_test.py | 341 ++++++++++++++++----- 1 file changed, 256 insertions(+), 85 deletions(-) diff --git a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py index 83f21ecbbb..424b153ff8 100644 --- a/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py +++ b/src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py @@ -12,19 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import contextlib -import distutils.spawn -import errno -import itertools +import importlib import os -import pkg_resources +from os import path +import pkgutil import shutil -import subprocess import sys import tempfile import threading -import time import unittest from six import moves @@ -33,12 +29,22 @@ from grpc.beta import implementations from grpc.beta import interfaces from grpc.framework.foundation import future from grpc.framework.interfaces.face import face +from grpc_tools import protoc from tests.unit.framework.common import test_constants -import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2 -import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2 -import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2 -import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2 +_RELATIVE_PROTO_PATH = 'relative_proto_path' +_RELATIVE_PYTHON_OUT = 'relative_python_out' + +_PROTO_FILES_PATH_COMPONENTS = ( + ('beta_grpc_plugin_test', 'payload', 'test_payload.proto',), + ('beta_grpc_plugin_test', 'requests', 'r', 'test_requests.proto',), + ('beta_grpc_plugin_test', 'responses', 'test_responses.proto',), + ('beta_grpc_plugin_test', 'service', 'test_service.proto',),) + +_PAYLOAD_PB2 = 'beta_grpc_plugin_test.payload.test_payload_pb2' +_REQUESTS_PB2 = 'beta_grpc_plugin_test.requests.r.test_requests_pb2' +_RESPONSES_PB2 = 'beta_grpc_plugin_test.responses.test_responses_pb2' +_SERVICE_PB2 = 'beta_grpc_plugin_test.service.test_service_pb2' # Identifiers of entities we expect to find in the generated module. SERVICER_IDENTIFIER = 'BetaTestServiceServicer' @@ -47,12 +53,50 @@ SERVER_FACTORY_IDENTIFIER = 'beta_create_TestService_server' STUB_FACTORY_IDENTIFIER = 'beta_create_TestService_stub' +@contextlib.contextmanager +def _system_path(path_insertion): + old_system_path = sys.path[:] + sys.path = sys.path[0:1] + path_insertion + sys.path[1:] + yield + sys.path = old_system_path + + +def _create_directory_tree(root, path_components_sequence): + created = set() + for path_components in path_components_sequence: + thus_far = '' + for path_component in path_components: + relative_path = path.join(thus_far, path_component) + if relative_path not in created: + os.makedirs(path.join(root, relative_path)) + created.add(relative_path) + thus_far = path.join(thus_far, path_component) + + +def _massage_proto_content(raw_proto_content): + imports_substituted = raw_proto_content.replace( + b'import "tests/protoc_plugin/protos/', + b'import "beta_grpc_plugin_test/') + package_statement_substituted = imports_substituted.replace( + b'package grpc_protoc_plugin;', b'package beta_grpc_protoc_plugin;') + return package_statement_substituted + + +def _packagify(directory): + for subdirectory, _, _ in os.walk(directory): + init_file_name = path.join(subdirectory, '__init__.py') + with open(init_file_name, 'wb') as init_file: + init_file.write(b'') + + class _ServicerMethods(object): - def __init__(self): + def __init__(self, payload_pb2, responses_pb2): self._condition = threading.Condition() self._paused = False self._fail = False + self._payload_pb2 = payload_pb2 + self._responses_pb2 = responses_pb2 @contextlib.contextmanager def pause(self): # pylint: disable=invalid-name @@ -79,22 +123,22 @@ class _ServicerMethods(object): self._condition.wait() def UnaryCall(self, request, unused_rpc_context): - response = response_pb2.SimpleResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.SimpleResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * request.response_size self._control() return response def StreamingOutputCall(self, request, unused_rpc_context): for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response def StreamingInputCall(self, request_iter, unused_rpc_context): - response = response_pb2.StreamingInputCallResponse() + response = self._responses_pb2.StreamingInputCallResponse() aggregated_payload_size = 0 for request in request_iter: aggregated_payload_size += len(request.payload.payload_compressable) @@ -105,8 +149,8 @@ class _ServicerMethods(object): def FullDuplexCall(self, request_iter, unused_rpc_context): for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() yield response @@ -115,8 +159,8 @@ class _ServicerMethods(object): responses = [] for request in request_iter: for parameter in request.response_parameters: - response = response_pb2.StreamingOutputCallResponse() - response.payload.payload_type = payload_pb2.COMPRESSABLE + response = self._responses_pb2.StreamingOutputCallResponse() + response.payload.payload_type = self._payload_pb2.COMPRESSABLE response.payload.payload_compressable = 'a' * parameter.size self._control() responses.append(response) @@ -125,7 +169,7 @@ class _ServicerMethods(object): @contextlib.contextmanager -def _CreateService(): +def _CreateService(payload_pb2, responses_pb2, service_pb2): """Provides a servicer backend and a stub. The servicer is just the implementation of the actual servicer passed to the @@ -136,7 +180,7 @@ def _CreateService(): the service bound to the stub and and stub is the stub on which to invoke RPCs. """ - servicer_methods = _ServicerMethods() + servicer_methods = _ServicerMethods(payload_pb2, responses_pb2) class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)): @@ -161,12 +205,12 @@ def _CreateService(): server.start() channel = implementations.insecure_channel('localhost', port) stub = getattr(service_pb2, STUB_FACTORY_IDENTIFIER)(channel) - yield (servicer_methods, stub) + yield servicer_methods, stub, server.stop(0) @contextlib.contextmanager -def _CreateIncompleteService(): +def _CreateIncompleteService(service_pb2): """Provides a servicer backend that fails to implement methods and its stub. The servicer is just the implementation of the actual servicer passed to the @@ -192,16 +236,16 @@ def _CreateIncompleteService(): server.stop(0) -def _streaming_input_request_iterator(): +def _streaming_input_request_iterator(payload_pb2, requests_pb2): for _ in range(3): - request = request_pb2.StreamingInputCallRequest() + request = requests_pb2.StreamingInputCallRequest() request.payload.payload_type = payload_pb2.COMPRESSABLE request.payload.payload_compressable = 'a' yield request -def _streaming_output_request(): - request = request_pb2.StreamingOutputCallRequest() +def _streaming_output_request(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() sizes = [1, 2, 3] request.response_parameters.add(size=sizes[0], interval_us=0) request.response_parameters.add(size=sizes[1], interval_us=0) @@ -209,11 +253,11 @@ def _streaming_output_request(): return request -def _full_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() +def _full_duplex_request_iterator(requests_pb2): + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -227,22 +271,78 @@ class PythonPluginTest(unittest.TestCase): methods and does not exist for response-streaming methods. """ + def setUp(self): + self._directory = tempfile.mkdtemp(dir='.') + self._proto_path = path.join(self._directory, _RELATIVE_PROTO_PATH) + self._python_out = path.join(self._directory, _RELATIVE_PYTHON_OUT) + + os.makedirs(self._proto_path) + os.makedirs(self._python_out) + + directories_path_components = { + proto_file_path_components[:-1] + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS + } + _create_directory_tree(self._proto_path, directories_path_components) + self._proto_file_names = set() + for proto_file_path_components in _PROTO_FILES_PATH_COMPONENTS: + raw_proto_content = pkgutil.get_data( + 'tests.protoc_plugin.protos', + path.join(*proto_file_path_components[1:])) + massaged_proto_content = _massage_proto_content(raw_proto_content) + proto_file_name = path.join(self._proto_path, + *proto_file_path_components) + with open(proto_file_name, 'wb') as proto_file: + proto_file.write(massaged_proto_content) + self._proto_file_names.add(proto_file_name) + + def tearDown(self): + shutil.rmtree(self._directory) + + def _protoc(self): + args = [ + '', + '--proto_path={}'.format(self._proto_path), + '--python_out={}'.format(self._python_out), + '--grpc_python_out=grpc_1_0:{}'.format(self._python_out), + ] + list(self._proto_file_names) + protoc_exit_code = protoc.main(args) + self.assertEqual(0, protoc_exit_code) + + _packagify(self._python_out) + + with _system_path([ + self._python_out, + ]): + self._payload_pb2 = importlib.import_module(_PAYLOAD_PB2) + self._requests_pb2 = importlib.import_module(_REQUESTS_PB2) + self._responses_pb2 = importlib.import_module(_RESPONSES_PB2) + self._service_pb2 = importlib.import_module(_SERVICE_PB2) + def testImportAttributes(self): + self._protoc() + # check that we can access the generated module and its members. - self.assertIsNotNone(getattr(service_pb2, SERVICER_IDENTIFIER, None)) - self.assertIsNotNone(getattr(service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVICER_IDENTIFIER, None)) + self.assertIsNotNone(getattr(self._service_pb2, STUB_IDENTIFIER, None)) self.assertIsNotNone( - getattr(service_pb2, STUB_FACTORY_IDENTIFIER, None)) + getattr(self._service_pb2, SERVER_FACTORY_IDENTIFIER, None)) + self.assertIsNotNone( + getattr(self._service_pb2, STUB_FACTORY_IDENTIFIER, None)) def testUpDown(self): - with _CreateService(): - request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2): + self._requests_pb2.SimpleRequest(response_size=13) def testIncompleteServicer(self): - with _CreateIncompleteService() as (_, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateIncompleteService(self._service_pb2) as (_, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) try: stub.UnaryCall(request, test_constants.LONG_TIMEOUT) except face.AbortionError as error: @@ -250,15 +350,21 @@ class PythonPluginTest(unittest.TestCase): error.code) def testUnaryCall(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT) expected_response = methods.UnaryCall(request, 'not a real context!') self.assertEqual(expected_response, response) def testUnaryCallFuture(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) # Check that the call does not block waiting for the server to respond. with methods.pause(): response_future = stub.UnaryCall.future( @@ -268,8 +374,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testUnaryCallFutureExpired(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future( request, test_constants.SHORT_TIMEOUT) @@ -277,24 +386,33 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testUnaryCallFutureCancelled(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.pause(): response_future = stub.UnaryCall.future(request, 1) response_future.cancel() self.assertTrue(response_future.cancelled()) def testUnaryCallFutureFailed(self): - with _CreateService() as (methods, stub): - request = request_pb2.SimpleRequest(response_size=13) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = self._requests_pb2.SimpleRequest(response_size=13) with methods.fail(): response_future = stub.UnaryCall.future( request, test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testStreamingOutputCall(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) expected_responses = methods.StreamingOutputCall( @@ -304,8 +422,11 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testStreamingOutputCallExpired(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.pause(): responses = stub.StreamingOutputCall( request, test_constants.SHORT_TIMEOUT) @@ -313,8 +434,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testStreamingOutputCallCancelled(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) responses = stub.StreamingOutputCall(request, test_constants.LONG_TIMEOUT) next(responses) @@ -323,8 +447,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingOutputCallFailed(self): - with _CreateService() as (methods, stub): - request = _streaming_output_request() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request = _streaming_output_request(self._requests_pb2) with methods.fail(): responses = stub.StreamingOutputCall(request, 1) self.assertIsNotNone(responses) @@ -332,30 +459,46 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testStreamingInputCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): response = stub.StreamingInputCall( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFuture(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response = response_future.result() expected_response = methods.StreamingInputCall( - _streaming_input_request_iterator(), 'not a real RpcContext!') + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), + 'not a real RpcContext!') self.assertEqual(expected_response, response) def testStreamingInputCallFutureExpired(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.SHORT_TIMEOUT) with self.assertRaises(face.ExpirationError): response_future.result() @@ -363,10 +506,14 @@ class PythonPluginTest(unittest.TestCase): face.ExpirationError) def testStreamingInputCallFutureCancelled(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) response_future.cancel() self.assertTrue(response_future.cancelled()) @@ -374,26 +521,38 @@ class PythonPluginTest(unittest.TestCase): response_future.result() def testStreamingInputCallFutureFailed(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): response_future = stub.StreamingInputCall.future( - _streaming_input_request_iterator(), + _streaming_input_request_iterator(self._payload_pb2, + self._requests_pb2), test_constants.LONG_TIMEOUT) self.assertIsNotNone(response_future.exception()) def testFullDuplexCall(self): - with _CreateService() as (methods, stub): - responses = stub.FullDuplexCall(_full_duplex_request_iterator(), - test_constants.LONG_TIMEOUT) + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + responses = stub.FullDuplexCall( + _full_duplex_request_iterator(self._requests_pb2), + test_constants.LONG_TIMEOUT) expected_responses = methods.FullDuplexCall( - _full_duplex_request_iterator(), 'not a real RpcContext!') + _full_duplex_request_iterator(self._requests_pb2), + 'not a real RpcContext!') for expected_response, response in moves.zip_longest( expected_responses, responses): self.assertEqual(expected_response, response) def testFullDuplexCallExpired(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.pause(): responses = stub.FullDuplexCall(request_iterator, test_constants.SHORT_TIMEOUT) @@ -401,8 +560,11 @@ class PythonPluginTest(unittest.TestCase): list(responses) def testFullDuplexCallCancelled(self): - with _CreateService() as (methods, stub): - request_iterator = _full_duplex_request_iterator() + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): + request_iterator = _full_duplex_request_iterator(self._requests_pb2) responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) next(responses) @@ -411,8 +573,11 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testFullDuplexCallFailed(self): - request_iterator = _full_duplex_request_iterator() - with _CreateService() as (methods, stub): + self._protoc() + + request_iterator = _full_duplex_request_iterator(self._requests_pb2) + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with methods.fail(): responses = stub.FullDuplexCall(request_iterator, test_constants.LONG_TIMEOUT) @@ -421,13 +586,16 @@ class PythonPluginTest(unittest.TestCase): next(responses) def testHalfDuplexCall(self): - with _CreateService() as (methods, stub): + self._protoc() + + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=2, interval_us=0) request.response_parameters.add(size=3, interval_us=0) yield request @@ -441,6 +609,8 @@ class PythonPluginTest(unittest.TestCase): self.assertEqual(expected_response, response) def testHalfDuplexCallWedged(self): + self._protoc() + condition = threading.Condition() wait_cell = [False] @@ -455,14 +625,15 @@ class PythonPluginTest(unittest.TestCase): condition.notify_all() def half_duplex_request_iterator(): - request = request_pb2.StreamingOutputCallRequest() + request = self._requests_pb2.StreamingOutputCallRequest() request.response_parameters.add(size=1, interval_us=0) yield request with condition: while wait_cell[0]: condition.wait() - with _CreateService() as (methods, stub): + with _CreateService(self._payload_pb2, self._responses_pb2, + self._service_pb2) as (methods, stub): with wait(): responses = stub.HalfDuplexCall(half_duplex_request_iterator(), test_constants.SHORT_TIMEOUT) -- cgit v1.2.3 From 31ad68a7e0d7f2bb6c3e6684782a88887b5190cc Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 11:04:58 -0700 Subject: fix a bad typo --- src/core/lib/iomgr/closure.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index bb9e437b08..05401bc2bd 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -171,9 +171,9 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - closure, closure->file_created, closure->line_created, - closure->file_initiated, closure->line_initiated, - closure->run ? "true" : "false"); + c, c->file_created, c->line_created, + c->file_initiated, c->line_initiated, + c->run ? "true" : "false"); abort(); } c->scheduled = true; @@ -203,9 +203,9 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - closure, closure->file_created, closure->line_created, - closure->file_initiated, closure->line_initiated, - closure->run ? "true" : "false"); + c, c->file_created, c->line_created, + c->file_initiated, c->line_initiated, + c->run ? "true" : "false"); abort(); } c->scheduled = true; -- cgit v1.2.3 From 9667e8a57d0e4978f77a7fb35fab175dec2c9e10 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Mon, 18 Sep 2017 11:19:54 -0700 Subject: clang format --- src/core/lib/iomgr/closure.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c index 05401bc2bd..00edefc6ae 100644 --- a/src/core/lib/iomgr/closure.c +++ b/src/core/lib/iomgr/closure.c @@ -171,9 +171,8 @@ void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c, gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - c, c->file_created, c->line_created, - c->file_initiated, c->line_initiated, - c->run ? "true" : "false"); + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); abort(); } c->scheduled = true; @@ -203,9 +202,8 @@ void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { gpr_log(GPR_ERROR, "Closure already scheduled. (closure: %p, created: [%s:%d], " "previously scheduled at: [%s: %d] run?: %s", - c, c->file_created, c->line_created, - c->file_initiated, c->line_initiated, - c->run ? "true" : "false"); + c, c->file_created, c->line_created, c->file_initiated, + c->line_initiated, c->run ? "true" : "false"); abort(); } c->scheduled = true; -- cgit v1.2.3 From 4cd56837465234b7384bd3cb98abebced881f104 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 18 Sep 2017 11:22:03 -0700 Subject: Split C and C++ basic tests --- .../linux/pull_request/grpc_basictests_c_dbg.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_c_opt.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_cpp_dbg.cfg | 30 ++++++++++++++++++++++ .../linux/pull_request/grpc_basictests_cpp_opt.cfg | 30 ++++++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg new file mode 100644 index 0000000000..577cb28ae5 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg new file mode 100644 index 0000000000..9e0b724b2e --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg new file mode 100644 index 0000000000..0fda74cf44 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ dbg --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg new file mode 100644 index 0000000000..199a8905d9 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux c++ opt --inner_jobs 16 -j 1 --internal_ci --max_time=3600" +} -- cgit v1.2.3 From 998bd2c7f46dc4bdd0e3381fcd1f4430fe3c3e32 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 12:41:07 -0700 Subject: Fix leak in grpclb's lb_on_response_received_locked --- src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a776a07d99..4fcfa745f1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -1560,6 +1560,9 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops), &glb_policy->lb_on_response_received); /* loop */ GPR_ASSERT(GRPC_CALL_OK == call_error); + } else { + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "lb_on_response_received_locked_shutdown"); } } else { /* empty payload: call cancelled. */ /* dispose of the "lb_on_response_received_locked" weak ref taken in -- cgit v1.2.3 From 6a8c0ac2aaecacbdf28a7414109e175297cec003 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Sun, 14 May 2017 23:18:05 -0700 Subject: Fix ping policy --- .../ext/transport/chttp2/transport/chttp2_transport.c | 4 ++-- src/core/ext/transport/chttp2/transport/parsing.c | 5 +++++ src/core/ext/transport/chttp2/transport/writing.c | 16 +++++++++------- test/core/end2end/tests/bad_ping.c | 2 +- test/core/end2end/tests/ping.c | 5 ++++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 79a9ed827f..3a28762224 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -557,8 +557,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, } } - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; + /* No pings allowed before receiving a header or data frame. */ + t->ping_state.pings_before_data_required = 0; t->ping_state.is_delayed_ping_timer_set = false; t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 6c12c91365..e52e19f3d2 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -383,6 +383,8 @@ error_handler: /* t->parser = grpc_chttp2_data_parser_parse;*/ t->parser = grpc_chttp2_data_parser_parse; t->parser_data = &s->data_parser; + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; return GRPC_ERROR_NONE; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { /* handle stream errors by closing the stream */ @@ -559,6 +561,9 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, (t->incoming_frame_flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) != 0; } + t->ping_state.pings_before_data_required = + t->ping_policy.max_pings_without_data; + /* could be a new grpc_chttp2_stream or an existing grpc_chttp2_stream */ s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); if (s == NULL) { diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 3ded801985..d3699301e3 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -66,9 +66,17 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } return; } + if (t->keepalive_permit_without_calls == 0 && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { + gpr_log(GPR_DEBUG, "Ping delayed [%p]: no active stream", t->peer_string); + } + return; + } if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { - /* need to send something of substance before sending a ping again */ + /* need to receive something of substance before sending a ping again */ if (GRPC_TRACER_ON(grpc_http_trace) || GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: too many recent pings: %d/%d", @@ -297,8 +305,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_window_update_create(s->id, stream_announce, &s->stats.outgoing)); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); @@ -375,8 +381,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( send_bytes); s->sending_bytes += send_bytes; } - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); @@ -487,8 +491,6 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( grpc_slice_buffer_add( &t->outbuf, grpc_chttp2_window_update_create(0, transport_announce, &throwaway_stats)); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index 12aceda688..6713597fcc 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -71,7 +71,7 @@ static void test_bad_ping(grpc_end2end_test_config config) { .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}, + .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, .value.integer = 0}}; diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index 112ad9d7d2..b4d9b13a93 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -42,7 +42,10 @@ static void test_ping(grpc_end2end_test_config config, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}}; + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .value.integer = 1}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, -- cgit v1.2.3 From 27411495f6784a79890d676622e1c0d2f2a53584 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 May 2017 11:57:30 -0700 Subject: Fix keepalive_timeout --- test/core/end2end/tests/keepalive_timeout.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index e0ead4ab62..8d01f23c00 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -106,13 +106,13 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, - .value.integer = 1}}; + .value.integer = 0}}; - grpc_channel_args *client_args = NULL; - client_args = grpc_channel_args_copy_and_add(client_args, keepalive_args, 2); + grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(keepalive_args), + .args = keepalive_args}; grpc_end2end_test_fixture f = - begin_test(config, "keepalive_timeout", client_args, NULL); + begin_test(config, "keepalive_timeout", &client_args, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -216,12 +216,6 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { grpc_byte_buffer_destroy(response_payload); grpc_byte_buffer_destroy(response_payload_recv); - if (client_args != NULL) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_channel_args_destroy(&exec_ctx, client_args); - grpc_exec_ctx_finish(&exec_ctx); - } - end_test(&f); config.tear_down_data(&f); } -- cgit v1.2.3 From ad32075af84eba51466ef53ae68c2e6e0d899ee6 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 May 2017 15:38:06 -0700 Subject: Set last_ping_sent_time to inf_past after receiving data --- include/grpc/impl/codegen/grpc_types.h | 13 ++--- .../transport/chttp2/transport/chttp2_transport.c | 62 +++++++++++++--------- .../ext/transport/chttp2/transport/frame_ping.c | 2 +- src/core/ext/transport/chttp2/transport/internal.h | 4 +- src/core/ext/transport/chttp2/transport/parsing.c | 2 + src/core/ext/transport/chttp2/transport/writing.c | 41 +++++++------- test/core/client_channel/lb_policies_test.c | 2 +- test/core/end2end/tests/bad_ping.c | 21 ++++---- test/core/end2end/tests/ping.c | 21 ++++---- 9 files changed, 95 insertions(+), 73 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 748dc717a3..90f03f49a3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -188,9 +188,14 @@ typedef struct { #define GRPC_ARG_HTTP2_MAX_FRAME_SIZE "grpc.http2.max_frame_size" /** Should BDP probing be performed? */ #define GRPC_ARG_HTTP2_BDP_PROBE "grpc.http2.bdp_probe" -/** Minimum time (in milliseconds) between successive ping frames being sent */ -#define GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS \ +/** Minimum time between sending successive ping frames without receiving any + data frame, Int valued, milliseconds. */ +#define GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS \ "grpc.http2.min_time_between_pings_ms" +/** Minimum allowed time between receiving successive ping frames without + sending any data frame. Int valued, milliseconds */ +#define GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS \ + "grpc.http2.min_ping_interval_without_data_ms" /** Channel arg to override the http2 :scheme header */ #define GRPC_ARG_HTTP2_SCHEME "grpc.http2_scheme" /** How many pings can we send before needing to send a data frame or header @@ -202,10 +207,6 @@ typedef struct { closing the transport? (0 indicates that the server can bear an infinite number of misbehaving pings) */ #define GRPC_ARG_HTTP2_MAX_PING_STRIKES "grpc.http2.max_ping_strikes" -/** Minimum allowed time between two pings without sending any data frame. Int - valued, seconds */ -#define GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS \ - "grpc.http2.min_ping_interval_without_data_ms" /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 3a28762224..c742444902 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -152,10 +152,10 @@ static void send_ping_locked( static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); -#define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 +#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 2 #define DEFAULT_MAX_PING_STRIKES 2 -#define DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -364,11 +364,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy = (grpc_chttp2_repeated_ping_policy){ .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, - .min_time_between_pings = - gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), + .min_sent_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, - .min_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), + .min_recv_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), }; /* Keepalive setting */ @@ -434,23 +434,30 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){DEFAULT_MAX_PING_STRIKES, 0, INT_MAX}); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS)) { - t->ping_policy.min_time_between_pings = gpr_time_from_millis( - grpc_channel_arg_get_integer( - &channel_args->args[i], - (grpc_integer_options){DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, 0, - INT_MAX}), - GPR_TIMESPAN); } else if (0 == - strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS)) { - t->ping_policy.min_ping_interval_without_data = gpr_time_from_millis( - grpc_channel_arg_get_integer( - &channel_args->args[i], - (grpc_integer_options){ - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, 0, INT_MAX}), - GPR_TIMESPAN); + strcmp( + channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_sent_ping_interval_without_data = + gpr_time_from_millis( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){ + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 0, + INT_MAX}), + GPR_TIMESPAN); + } else if (0 == + strcmp( + channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_recv_ping_interval_without_data = + gpr_time_from_millis( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){ + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 0, + INT_MAX}), + GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = (uint32_t)grpc_channel_arg_get_integer( @@ -625,6 +632,9 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx, connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "close_transport"); grpc_endpoint_shutdown(exec_ctx, t->ep, GRPC_ERROR_REF(error)); + if (t->ping_state.is_delayed_ping_timer_set) { + grpc_timer_cancel(exec_ctx, &t->ping_state.delayed_ping_timer); + } switch (t->keepalive_state) { case GRPC_CHTTP2_KEEPALIVE_STATE_WAITING: grpc_timer_cancel(exec_ctx, &t->keepalive_ping_timer); @@ -1729,8 +1739,10 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)tp; t->ping_state.is_delayed_ping_timer_set = false; - grpc_chttp2_initiate_write(exec_ctx, t, - GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); + if (error == GRPC_ERROR_NONE) { + grpc_chttp2_initiate_write(exec_ctx, t, + GRPC_CHTTP2_INITIATE_WRITE_RETRY_SEND_PING); + } } void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 81bd02ae70..d431d6b2df 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -92,7 +92,7 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, - t->ping_policy.min_ping_interval_without_data); + t->ping_policy.min_recv_ping_interval_without_data); if (t->keepalive_permit_without_calls == 0 && grpc_chttp2_stream_map_size(&t->stream_map) == 0) { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index c2dfce7c9c..1682be28dd 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -112,10 +112,10 @@ typedef struct { } grpc_chttp2_ping_queue; typedef struct { - gpr_timespec min_time_between_pings; int max_pings_without_data; int max_ping_strikes; - gpr_timespec min_ping_interval_without_data; + gpr_timespec min_sent_ping_interval_without_data; + gpr_timespec min_recv_ping_interval_without_data; } grpc_chttp2_repeated_ping_policy; typedef struct { diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index e52e19f3d2..3db1ad4123 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -385,6 +385,7 @@ error_handler: t->parser_data = &s->data_parser; t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + t->ping_state.last_ping_sent_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); return GRPC_ERROR_NONE; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { /* handle stream errors by closing the stream */ @@ -563,6 +564,7 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + t->ping_state.last_ping_sent_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); /* could be a new grpc_chttp2_stream or an existing grpc_chttp2_stream */ s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index d3699301e3..d61063ed31 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -66,14 +66,6 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } return; } - if (t->keepalive_permit_without_calls == 0 && - grpc_chttp2_stream_map_size(&t->stream_map) == 0) { - if (GRPC_TRACER_ON(grpc_http_trace) || - GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { - gpr_log(GPR_DEBUG, "Ping delayed [%p]: no active stream", t->peer_string); - } - return; - } if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { /* need to receive something of substance before sending a ping again */ @@ -86,11 +78,18 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, return; } gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec elapsed = gpr_time_sub(now, t->ping_state.last_ping_sent_time); - /*gpr_log(GPR_DEBUG, "elapsed:%d.%09d min:%d.%09d", (int)elapsed.tv_sec, - elapsed.tv_nsec, (int)t->ping_policy.min_time_between_pings.tv_sec, - (int)t->ping_policy.min_time_between_pings.tv_nsec);*/ - if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < 0) { + gpr_timespec next_allowed_ping = + gpr_time_add(t->ping_state.last_ping_sent_time, + t->ping_policy.min_sent_ping_interval_without_data); + if (t->keepalive_permit_without_calls == 0 && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, + gpr_time_from_seconds(7200, GPR_TIMESPAN)); + } + /*gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", + (int)next_allowed_ping.tv_sec, (int)next_allowed_pingpsed.tv_nsec, + (int)now.tv_sec, (int)now.tv_nsec);*/ + if (gpr_time_cmp(next_allowed_ping, now) > 0) { /* not enough elapsed time between successive pings */ if (GRPC_TRACER_ON(grpc_http_trace) || GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { @@ -100,11 +99,11 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!t->ping_state.is_delayed_ping_timer_set) { t->ping_state.is_delayed_ping_timer_set = true; - grpc_timer_init(exec_ctx, &t->ping_state.delayed_ping_timer, - gpr_time_add(t->ping_state.last_ping_sent_time, - t->ping_policy.min_time_between_pings), - &t->retry_initiate_ping_locked, - gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_init( + exec_ctx, &t->ping_state.delayed_ping_timer, + gpr_time_add(t->ping_state.last_ping_sent_time, + t->ping_policy.min_sent_ping_interval_without_data), + &t->retry_initiate_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); } return; } @@ -127,6 +126,12 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_ping_create(false, pq->inflight_id)); GRPC_STATS_INC_HTTP2_PINGS_SENT(exec_ctx); t->ping_state.last_ping_sent_time = now; + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { + gpr_log(GPR_DEBUG, "Ping sent [%p]: %d/%d", t->peer_string, + t->ping_state.pings_before_data_required, + t->ping_policy.max_pings_without_data); + } t->ping_state.pings_before_data_required -= (t->ping_state.pings_before_data_required != 0); } diff --git a/test/core/client_channel/lb_policies_test.c b/test/core/client_channel/lb_policies_test.c index f70a9fc880..ba37cd673f 100644 --- a/test/core/client_channel/lb_policies_test.c +++ b/test/core/client_channel/lb_policies_test.c @@ -519,7 +519,7 @@ static grpc_channel *create_client(const servers_fixture *f) { arg_array[1].key = GRPC_ARG_LB_POLICY_NAME; arg_array[1].value.string = "ROUND_ROBIN"; arg_array[2].type = GRPC_ARG_INTEGER; - arg_array[2].key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS; + arg_array[2].key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS; arg_array[2].value.integer = 0; args.num_args = GPR_ARRAY_SIZE(arg_array); args.args = arg_array; diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index 6713597fcc..c97d11b306 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -66,18 +66,19 @@ static void end_test(grpc_end2end_test_fixture *f) { static void test_bad_ping(grpc_end2end_test_config config) { grpc_end2end_test_fixture f = config.create_fixture(NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); - grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_BDP_PROBE, - .value.integer = 0}}; + grpc_arg client_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, + .value.integer = 10}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_BDP_PROBE, + .value.integer = 0}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, + .key = GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, .value.integer = 300000 /* 5 minutes */}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PING_STRIKES, diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index b4d9b13a93..23c82569ba 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -37,18 +37,19 @@ static void test_ping(grpc_end2end_test_config config, grpc_connectivity_state state = GRPC_CHANNEL_IDLE; int i; - grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 0}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, - .value.integer = 1}}; + grpc_arg client_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, + .value.integer = 10}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .value.integer = 1}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, + .key = GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, -- cgit v1.2.3 From 1ecbebc08fe578236fbedefe770aa54b157221b4 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 18 May 2017 15:15:46 -0700 Subject: Set the ping behavior as described in the proposal --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- src/core/ext/transport/chttp2/transport/writing.c | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c742444902..94dff49509 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -154,7 +154,7 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, #define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ #define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 2 +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ #define DEFAULT_MAX_PING_STRIKES 2 /** keepalive-relevant functions */ diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index d61063ed31..8721803267 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -86,9 +86,9 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, gpr_time_from_seconds(7200, GPR_TIMESPAN)); } - /*gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", - (int)next_allowed_ping.tv_sec, (int)next_allowed_pingpsed.tv_nsec, - (int)now.tv_sec, (int)now.tv_nsec);*/ + /* gpr_log(GPR_DEBUG, "next_allowed_ping:%d.%09d now:%d.%09d", + (int)next_allowed_ping.tv_sec, (int)next_allowed_ping.tv_nsec, + (int)now.tv_sec, (int)now.tv_nsec); */ if (gpr_time_cmp(next_allowed_ping, now) > 0) { /* not enough elapsed time between successive pings */ if (GRPC_TRACER_ON(grpc_http_trace) || @@ -99,11 +99,9 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!t->ping_state.is_delayed_ping_timer_set) { t->ping_state.is_delayed_ping_timer_set = true; - grpc_timer_init( - exec_ctx, &t->ping_state.delayed_ping_timer, - gpr_time_add(t->ping_state.last_ping_sent_time, - t->ping_policy.min_sent_ping_interval_without_data), - &t->retry_initiate_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); + grpc_timer_init(exec_ctx, &t->ping_state.delayed_ping_timer, + next_allowed_ping, &t->retry_initiate_ping_locked, + gpr_now(GPR_CLOCK_MONOTONIC)); } return; } -- cgit v1.2.3 From 18ea66fe3787ed27cc444a68451de05e34c4d024 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 22 Jun 2017 11:32:36 -0700 Subject: Update writing.c with the trailers-only response change --- src/core/ext/transport/chttp2/transport/writing.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 8721803267..7d9d4867e1 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -268,8 +268,7 @@ grpc_chttp2_begin_write_result grpc_chttp2_begin_write( .stats = &s->stats.outgoing}; grpc_chttp2_encode_header(exec_ctx, &t->hpack_compressor, NULL, 0, s->send_initial_metadata, &hopt, &t->outbuf); - t->ping_state.pings_before_data_required = - t->ping_policy.max_pings_without_data; + now_writing = true; if (!t->is_client) { t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); -- cgit v1.2.3 From d7eda6127af92a311f78c57dabc24f679d493ae3 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 29 Jun 2017 18:55:45 -0700 Subject: Add more options in grpc_chttp2_config_default_keepalive_args --- .../transport/chttp2/transport/chttp2_transport.c | 64 +++++++++++++++++----- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 94dff49509..144780b67b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -64,6 +64,11 @@ #define DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS false #define KEEPALIVE_TIME_BACKOFF_MULTIPLIER 2 +#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ +#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ +#define DEFAULT_MAX_PING_STRIKES 2 + static int g_default_client_keepalive_time_ms = DEFAULT_CLIENT_KEEPALIVE_TIME_MS; static int g_default_client_keepalive_timeout_ms = @@ -75,6 +80,13 @@ static int g_default_server_keepalive_timeout_ms = static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; +static int g_default_min_sent_ping_interval_without_data_ms = + DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS; +static int g_default_min_recv_ping_interval_without_data_ms = + DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS; +static int g_default_max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA; +static int g_default_max_ping_strikes = DEFAULT_MAX_PING_STRIKES; + #define MAX_CLIENT_STREAM_ID 0x7fffffffu grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false, "http"); grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false, "flowctl"); @@ -152,11 +164,6 @@ static void send_ping_locked( static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error); -#define DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ -#define DEFAULT_MAX_PINGS_BETWEEN_DATA 0 /* unlimited */ -#define DEFAULT_MAX_PING_STRIKES 2 - /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); @@ -363,12 +370,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, 1); t->ping_policy = (grpc_chttp2_repeated_ping_policy){ - .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, + .max_pings_without_data = g_default_max_pings_without_data, .min_sent_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), - .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, + g_default_min_sent_ping_interval_without_data_ms, GPR_TIMESPAN), + .max_ping_strikes = g_default_max_ping_strikes, .min_recv_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), + g_default_min_recv_ping_interval_without_data_ms, GPR_TIMESPAN), }; /* Keepalive setting */ @@ -428,12 +435,13 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); + (grpc_integer_options){g_default_max_pings_without_data, 0, + INT_MAX}); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_PING_STRIKES, 0, INT_MAX}); + (grpc_integer_options){g_default_max_ping_strikes, 0, INT_MAX}); } else if (0 == strcmp( channel_args->args[i].key, @@ -443,7 +451,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){ - DEFAULT_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS, 0, + g_default_min_sent_ping_interval_without_data_ms, 0, INT_MAX}), GPR_TIMESPAN); } else if (0 == @@ -455,7 +463,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){ - DEFAULT_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS, 0, + g_default_min_recv_ping_interval_without_data_ms, 0, INT_MAX}), GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, @@ -2643,6 +2651,36 @@ void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args, &args->args[i], (grpc_integer_options){g_default_keepalive_permit_without_calls, 0, 1}); + } else if (0 == + strcmp(args->args[i].key, GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { + g_default_max_ping_strikes = grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){g_default_max_ping_strikes, 0, INT_MAX}); + } else if (0 == strcmp(args->args[i].key, + GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) { + g_default_max_pings_without_data = grpc_channel_arg_get_integer( + &args->args[i], (grpc_integer_options){ + g_default_max_pings_without_data, 0, INT_MAX}); + } else if (0 == + strcmp( + args->args[i].key, + GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS)) { + g_default_min_sent_ping_interval_without_data_ms = + grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){ + g_default_min_sent_ping_interval_without_data_ms, 0, + INT_MAX}); + } else if (0 == + strcmp( + args->args[i].key, + GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS)) { + g_default_min_recv_ping_interval_without_data_ms = + grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){ + g_default_min_recv_ping_interval_without_data_ms, 0, + INT_MAX}); } } } -- cgit v1.2.3 From a594ba0bd7be974cff1a8d1eeb3b21eb049e507e Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 7 Sep 2017 19:04:59 -0700 Subject: add a client side utility for extracting a google rpc status from a grpc::Status --- grpc.gemspec | 1 + src/ruby/lib/grpc/google_rpc_status_utils.rb | 28 ++++ src/ruby/spec/google_rpc_status_utils_spec.rb | 223 ++++++++++++++++++++++++++ templates/grpc.gemspec.template | 1 + 4 files changed, 253 insertions(+) create mode 100644 src/ruby/lib/grpc/google_rpc_status_utils.rb create mode 100644 src/ruby/spec/google_rpc_status_utils_spec.rb diff --git a/grpc.gemspec b/grpc.gemspec index 2d0f9fd450..96323aed10 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |s| s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' diff --git a/src/ruby/lib/grpc/google_rpc_status_utils.rb b/src/ruby/lib/grpc/google_rpc_status_utils.rb new file mode 100644 index 0000000000..fdadd6b76e --- /dev/null +++ b/src/ruby/lib/grpc/google_rpc_status_utils.rb @@ -0,0 +1,28 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative './grpc' +require 'google/rpc/status_pb' + +# GRPC contains the General RPC module. +module GRPC + # GoogleRpcStatusUtils provides utilities to convert between a + # GRPC::Core::Status and a deserialized Google::Rpc::Status proto + class GoogleRpcStatusUtils + def self.extract_google_rpc_status(status) + fail ArgumentError, 'bad type' unless status.is_a? Struct::Status + Google::Rpc::Status.decode(status.metadata['grpc-status-details-bin']) + end + end +end diff --git a/src/ruby/spec/google_rpc_status_utils_spec.rb b/src/ruby/spec/google_rpc_status_utils_spec.rb new file mode 100644 index 0000000000..fe221c30dd --- /dev/null +++ b/src/ruby/spec/google_rpc_status_utils_spec.rb @@ -0,0 +1,223 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'grpc' +require_relative '../lib/grpc/google_rpc_status_utils' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require_relative '../pb/src/proto/grpc/testing/messages_pb' +require 'google/protobuf/well_known_types' + +include GRPC::Core + +describe 'conversion from a status struct to a google protobuf status' do + it 'fails if the input is not a status struct' do + begin + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status('string') + rescue => e + exception = e + end + expect(exception.is_a?(ArgumentError)).to be true + expect(exception.message.include?('bad type')).to be true + end + + it 'fails with some error if the header key is missing' do + status = Struct::Status.new(1, 'details', key: 'val') + expect(status.metadata.nil?).to be false + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'fails with some error if the header key fails to deserialize' do + status = Struct::Status.new(1, 'details', + 'grpc-status-details-bin' => 'string_val') + expect do + GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + end.to raise_error(StandardError) + end + + it 'silently ignores erroneous mismatch between messages in '\ + 'status struct and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'proto message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'struct message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'silently ignores erroneous mismatch between codes in status struct '\ + 'and protobuf status' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(2, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(rpc_status).to eq(proto) + end + + it 'can succesfully convert a status struct into a google protobuf status '\ + 'when there are no rpcstatus details' do + proto = Google::Rpc::Status.new(code: 1, message: 'matching message') + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details).to eq([]) + end + + it 'can succesfully convert a status struct into a google protobuf '\ + 'status when there are multiple rpcstatus details' do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + proto = Google::Rpc::Status.new(code: 1, + message: 'matching message', + details: [ + simple_request_any, + simple_response_any, + payload_any + ]) + encoded_proto = Google::Rpc::Status.encode(proto) + status = Struct::Status.new(1, 'matching message', + 'grpc-status-details-bin' => encoded_proto) + out = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) + expect(out.code).to eq(1) + expect(out.message).to eq('matching message') + expect(out.details[0].unpack( + Grpc::Testing::SimpleRequest)).to eq(simple_request) + expect(out.details[1].unpack( + Grpc::Testing::SimpleResponse)).to eq(simple_response) + expect(out.details[2].unpack( + Grpc::Testing::Payload)).to eq(payload) + end +end + +# Test message +class EchoMsg + def self.marshal(_o) + '' + end + + def self.unmarshal(_o) + EchoMsg.new + end +end + +# A test service that fills in the "reserved" grpc-status-details-bin trailer, +# for client-side testing of GoogleRpcStatus protobuf extraction from trailers. +class GoogleRpcStatusTestService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + + def initialize(encoded_rpc_status) + @encoded_rpc_status = encoded_rpc_status + end + + def an_rpc(_, _) + # TODO: create a server-side utility API for sending a google rpc status. + # Applications are not expected to set the grpc-status-details-bin + # ("grpc"-fixed and reserved for library use) manually. + # Doing so here is only for testing of the client-side api for extracting + # a google rpc status, which is useful + # when the interacting with a server that does fill in this trailer. + fail GRPC::Unknown.new('test message', + 'grpc-status-details-bin' => @encoded_rpc_status) + end +end + +GoogleRpcStatusTestStub = GoogleRpcStatusTestService.rpc_stub_class + +describe 'receving a google rpc status from a remote endpoint' do + def start_server(encoded_rpc_status) + @srv = GRPC::RpcServer.new(pool_size: 1) + @server_port = @srv.add_http2_port('localhost:0', + :this_port_is_insecure) + @srv.handle(GoogleRpcStatusTestService.new(encoded_rpc_status)) + @server_thd = Thread.new { @srv.run } + @srv.wait_till_running + end + + def stop_server + expect(@srv.stopped?).to be(false) + @srv.stop + @server_thd.join + expect(@srv.stopped?).to be(true) + end + + before(:each) do + simple_request_any = Google::Protobuf::Any.new + simple_request = Grpc::Testing::SimpleRequest.new( + payload: Grpc::Testing::Payload.new(body: 'request')) + simple_request_any.pack(simple_request) + simple_response_any = Google::Protobuf::Any.new + simple_response = Grpc::Testing::SimpleResponse.new( + payload: Grpc::Testing::Payload.new(body: 'response')) + simple_response_any.pack(simple_response) + payload_any = Google::Protobuf::Any.new + payload = Grpc::Testing::Payload.new(body: 'payload') + payload_any.pack(payload) + @expected_proto = Google::Rpc::Status.new( + code: StatusCodes::UNKNOWN, + message: 'test message', + details: [simple_request_any, simple_response_any, payload_any]) + start_server(Google::Rpc::Status.encode(@expected_proto)) + end + + after(:each) do + stop_server + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from a BadStatus exception' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + begin + stub.an_rpc(EchoMsg.new) + rescue GRPC::BadStatus => e + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + e.to_status) + end + expect(rpc_status).to eq(@expected_proto) + end + + it 'should receive be able to extract a google rpc status from the '\ + 'status struct taken from the op view of a call' do + stub = GoogleRpcStatusTestStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + op = stub.an_rpc(EchoMsg.new, return_op: true) + begin + op.execute + rescue GRPC::BadStatus => e + status_from_exception = e.to_status + end + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + op.status) + expect(rpc_status).to eq(@expected_proto) + # "to_status" on the bad status should give the same result + # as "status" on the "op view". + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + status_from_exception)).to eq(rpc_status) + end +end diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index e62e5b2721..215d5f9df9 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -31,6 +31,7 @@ s.add_dependency 'google-protobuf', '~> 3.1' s.add_dependency 'googleauth', '~> 0.5.1' + s.add_dependency 'googleapis-common-protos-types', '~> 1.0.0' s.add_development_dependency 'bundler', '~> 1.9' s.add_development_dependency 'facter', '~> 2.4' -- cgit v1.2.3 From 6b56c4d953eb8f46c0d7d2aee9d9e6efa32e06e2 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 00:17:21 -0700 Subject: Redefine GRPC_SLICE_MALLOC to use grpc_slice_malloc function instead of designated initializers --- include/grpc/slice.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 9d2a9fbc81..3080d63e2b 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -65,11 +65,11 @@ GPRAPI grpc_slice grpc_slice_new_with_len(void *p, size_t len, GPRAPI grpc_slice grpc_slice_malloc(size_t length); GPRAPI grpc_slice grpc_slice_malloc_large(size_t length); -#define GRPC_SLICE_MALLOC(len) \ - ((len) <= GRPC_SLICE_INLINED_SIZE \ +#define GRPC_SLICE_MALLOC(len) grpc_slice_malloc(len) +/* ((len) <= GRPC_SLICE_INLINED_SIZE \ ? (grpc_slice){.refcount = NULL, \ .data.inlined = {.length = (uint8_t)(len)}} \ - : grpc_slice_malloc_large((len))) + : grpc_slice_malloc_large((len)))*/ /** Intern a slice: -- cgit v1.2.3 From 533d1184807869dfdd1cd239428be330a28a20d1 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 10:48:22 -0700 Subject: Removing designated initializers, implicit string literal to char * conversions and solving crosses initialization due to jump warnings --- src/core/ext/filters/client_channel/http_proxy.c | 4 +- .../resolver/dns/c_ares/grpc_ares_wrapper.c | 25 ++++---- .../ext/transport/chttp2/server/chttp2_server.c | 3 +- .../transport/chttp2/transport/chttp2_transport.c | 14 ++--- .../ext/transport/chttp2/transport/hpack_encoder.c | 34 +++++------ src/core/lib/http/httpcli.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 69 ++++++++++----------- src/core/lib/iomgr/ev_epollex_linux.c | 70 +++++++++++----------- src/core/lib/iomgr/ev_epollsig_linux.c | 59 +++++++++--------- src/core/lib/iomgr/ev_poll_posix.c | 56 ++++++++--------- src/core/lib/iomgr/iomgr.c | 2 +- src/core/lib/iomgr/is_epollexclusive_available.c | 12 ++-- src/core/lib/iomgr/socket_factory_posix.c | 4 +- src/core/lib/iomgr/socket_mutator.c | 4 +- src/core/lib/iomgr/tcp_server_posix.c | 4 +- src/core/lib/iomgr/timer_generic.c | 5 +- src/core/lib/surface/call.c | 31 ++++++---- src/core/lib/surface/completion_queue.c | 23 ++----- src/core/lib/transport/status_conversion.c | 2 +- src/core/lib/transport/transport.c | 8 ++- 20 files changed, 217 insertions(+), 214 deletions(-) diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index 0c3b009e44..c507a2750e 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -44,6 +44,8 @@ static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) { GPR_ASSERT(user_cred != NULL); char* proxy_name = NULL; char* uri_str = gpr_getenv("http_proxy"); + char** authority_strs = NULL; + size_t authority_nstrs; if (uri_str == NULL) return NULL; grpc_uri* uri = grpc_uri_parse(exec_ctx, uri_str, false /* suppress_errors */); @@ -56,8 +58,6 @@ static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) { goto done; } /* Split on '@' to separate user credentials from host */ - char** authority_strs = NULL; - size_t authority_nstrs; gpr_string_split(uri->authority, "@", &authority_strs, &authority_nstrs); GPR_ASSERT(authority_nstrs != 0); /* should have at least 1 string */ if (authority_nstrs == 1) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 0ffb38518a..9975b1ef79 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -275,14 +275,15 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, gpr_log(GPR_DEBUG, "on_txt_done_cb"); char *error_msg; grpc_ares_request *r = (grpc_ares_request *)arg; + const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; + struct ares_txt_ext *result; + struct ares_txt_ext *reply = NULL; + grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&r->mu); if (status != ARES_SUCCESS) goto fail; - struct ares_txt_ext *reply = NULL; status = ares_parse_txt_reply_ext(buf, len, &reply); if (status != ARES_SUCCESS) goto fail; // Find service config in TXT record. - const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; - struct ares_txt_ext *result; for (result = reply; result != NULL; result = result->next) { if (result->record_start && memcmp(result->txt, g_service_config_attribute_prefix, prefix_len) == @@ -313,7 +314,7 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, fail: gpr_asprintf(&error_msg, "C-ares TXT lookup status is not ARES_SUCCESS: %s", ares_strerror(status)); - grpc_error *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); gpr_free(error_msg); if (r->error == GRPC_ERROR_NONE) { r->error = error; @@ -331,6 +332,9 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb, char **service_config_json) { grpc_error *error = GRPC_ERROR_NONE; + grpc_ares_hostbyname_request *hr = NULL; + grpc_ares_request *r = NULL; + ares_channel *channel = NULL; /* TODO(zyc): Enable tracing after #9603 is checked in */ /* if (grpc_dns_trace) { gpr_log(GPR_DEBUG, "resolve_address (blocking): name=%s, default_port=%s", @@ -360,8 +364,7 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( error = grpc_ares_ev_driver_create(&ev_driver, interested_parties); if (error != GRPC_ERROR_NONE) goto error_cleanup; - grpc_ares_request *r = - (grpc_ares_request *)gpr_zalloc(sizeof(grpc_ares_request)); + r = (grpc_ares_request *)gpr_zalloc(sizeof(grpc_ares_request)); gpr_mu_init(&r->mu); r->ev_driver = ev_driver; r->on_done = on_done; @@ -369,7 +372,7 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( r->service_config_json_out = service_config_json; r->success = false; r->error = GRPC_ERROR_NONE; - ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver); + channel = grpc_ares_ev_driver_get_channel(r->ev_driver); // If dns_server is specified, use it. if (dns_server != NULL) { @@ -410,12 +413,12 @@ static grpc_ares_request *grpc_dns_lookup_ares_impl( } gpr_ref_init(&r->pending_queries, 1); if (grpc_ipv6_loopback_available()) { - grpc_ares_hostbyname_request *hr = create_hostbyname_request( - r, host, strhtons(port), false /* is_balancer */); + hr = create_hostbyname_request(r, host, strhtons(port), + false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_cb, hr); } - grpc_ares_hostbyname_request *hr = create_hostbyname_request( - r, host, strhtons(port), false /* is_balancer */); + hr = create_hostbyname_request(r, host, strhtons(port), + false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, hr); if (check_grpclb) { /* Query the SRV record */ diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index f5a409a403..60244e163b 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -201,6 +201,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, grpc_error *err = GRPC_ERROR_NONE; server_state *state = NULL; grpc_error **errors = NULL; + size_t naddrs = 0; *port_num = -1; @@ -225,7 +226,7 @@ grpc_error *grpc_chttp2_server_add_port(grpc_exec_ctx *exec_ctx, state->shutdown = true; gpr_mu_init(&state->mu); - const size_t naddrs = resolved->naddrs; + naddrs = resolved->naddrs; errors = (grpc_error **)gpr_malloc(sizeof(*errors) * naddrs); for (i = 0; i < naddrs; i++) { errors[i] = diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 79a9ed827f..d18df04851 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -362,14 +362,12 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, queue_setting_update(exec_ctx, t, GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA, 1); - t->ping_policy = (grpc_chttp2_repeated_ping_policy){ - .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, - .min_time_between_pings = - gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), - .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, - .min_ping_interval_without_data = gpr_time_from_millis( - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), - }; + t->ping_policy.max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA; + t->ping_policy.min_time_between_pings = + gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN); + t->ping_policy.max_ping_strikes = DEFAULT_MAX_PING_STRIKES; + t->ping_policy.min_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN); /* Keepalive setting */ if (t->is_client) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 3cd1a7ee5c..9537c2fa17 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -51,8 +51,9 @@ #define MAX_DECODER_SPACE_USAGE 512 static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; -static const grpc_slice terminal_slice = {&terminal_slice_refcount, - .data.refcounted = {0, 0}}; +static const grpc_slice terminal_slice = {&terminal_slice_refcount, /* refcount */ + {{0, 0}} /* refcounted */ +}; extern grpc_tracer_flag grpc_http_trace; @@ -283,29 +284,26 @@ typedef struct { } wire_value; static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { + wire_value wire_val; if (grpc_is_binary_header(GRPC_MDKEY(elem))) { if (true_binary_enabled) { - return (wire_value){ - .huffman_prefix = 0x00, - .insert_null_before_wire_value = true, - .data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x00; + wire_val.insert_null_before_wire_value = true; + wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); + } else { - return (wire_value){ - .huffman_prefix = 0x80, - .insert_null_before_wire_value = false, - .data = grpc_chttp2_base64_encode_and_huffman_compress( - GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x80; + wire_val.insert_null_before_wire_value = false; + wire_val.data = + grpc_chttp2_base64_encode_and_huffman_compress(GRPC_MDVALUE(elem)); } } else { /* TODO(ctiller): opportunistically compress non-binary headers */ - return (wire_value){ - .huffman_prefix = 0x00, - .insert_null_before_wire_value = false, - .data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)), - }; + wire_val.huffman_prefix = 0x00; + wire_val.insert_null_before_wire_value = false; + wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); } + return wire_val; } static size_t wire_value_length(wire_value v) { diff --git a/src/core/lib/http/httpcli.c b/src/core/lib/http/httpcli.c index 84cc39604c..db995943a9 100644 --- a/src/core/lib/http/httpcli.c +++ b/src/core/lib/http/httpcli.c @@ -217,7 +217,7 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req, GRPC_CLOSURE_INIT(&req->connected, on_connected, req, grpc_schedule_on_exec_ctx); grpc_arg arg = grpc_channel_arg_pointer_create( - GRPC_ARG_RESOURCE_QUOTA, req->resource_quota, + (char *)GRPC_ARG_RESOURCE_QUOTA, req->resource_quota, grpc_resource_quota_arg_vtable()); grpc_channel_args args = {1, &arg}; grpc_tcp_client_connect(exec_ctx, &req->connected, &req->ep, diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 0d57833d85..b5066e5f58 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -280,8 +280,9 @@ static grpc_fd *fd_create(int fd, const char *name) { #endif gpr_free(fd_name); - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), - .data.ptr = new_fd}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = new_fd; if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); } @@ -435,8 +436,9 @@ static grpc_error *pollset_global_init(void) { global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); if (err != GRPC_ERROR_NONE) return err; - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), - .data.ptr = &global_wakeup_fd}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = &global_wakeup_fd; if (epoll_ctl(g_epoll_set.epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); @@ -566,8 +568,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = { - .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -1172,34 +1173,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; /* It is possible that GLIBC has epoll but the underlying kernel doesn't. diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 51200fc064..671c4cd730 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -477,8 +477,9 @@ static grpc_error *pollable_materialize(pollable *p) { close(new_epfd); return err; } - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), - .data.ptr = (void *)(1 | (intptr_t)&p->wakeup)}; + struct epoll_event ev; + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = (void *)(1 | (intptr_t)&p->wakeup); if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { err = GRPC_OS_ERROR(errno, "epoll_ctl"); close(new_epfd); @@ -507,9 +508,9 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { gpr_mu_unlock(&fd->orphaned_mu); return GRPC_ERROR_NONE; } - struct epoll_event ev_fd = { - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE), - .data.ptr = fd}; + struct epoll_event ev_fd; + ev_fd.events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE); + ev_fd.data.ptr = fd; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { switch (errno) { case EEXIST: @@ -705,8 +706,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = { - .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -1390,34 +1390,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; const grpc_event_engine_vtable *grpc_init_epollex_linux( diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index b88c3ba111..87b6f363f8 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1132,7 +1132,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { } static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, - grpc_pollset *ps, char *reason) { + grpc_pollset *ps, + const char *reason) { if (ps->po.pi != NULL) { PI_UNREF(exec_ctx, ps->po.pi, reason); } @@ -1670,34 +1671,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; /* It is possible that GLIBC has epoll but the underlying kernel doesn't. diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 7f44eda138..53df2d7737 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1688,34 +1688,34 @@ static void shutdown_engine(void) { } static const grpc_event_engine_vtable vtable = { - .pollset_size = sizeof(grpc_pollset), - - .fd_create = fd_create, - .fd_wrapped_fd = fd_wrapped_fd, - .fd_orphan = fd_orphan, - .fd_shutdown = fd_shutdown, - .fd_is_shutdown = fd_is_shutdown, - .fd_notify_on_read = fd_notify_on_read, - .fd_notify_on_write = fd_notify_on_write, - .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, - - .pollset_init = pollset_init, - .pollset_shutdown = pollset_shutdown, - .pollset_destroy = pollset_destroy, - .pollset_work = pollset_work, - .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd, - - .pollset_set_create = pollset_set_create, - .pollset_set_destroy = pollset_set_destroy, - .pollset_set_add_pollset = pollset_set_add_pollset, - .pollset_set_del_pollset = pollset_set_del_pollset, - .pollset_set_add_pollset_set = pollset_set_add_pollset_set, - .pollset_set_del_pollset_set = pollset_set_del_pollset_set, - .pollset_set_add_fd = pollset_set_add_fd, - .pollset_set_del_fd = pollset_set_del_fd, - - .shutdown_engine = shutdown_engine, + sizeof(grpc_pollset), + + fd_create, + fd_wrapped_fd, + fd_orphan, + fd_shutdown, + fd_notify_on_read, + fd_notify_on_write, + fd_is_shutdown, + fd_get_read_notifier_pollset, + + pollset_init, + pollset_shutdown, + pollset_destroy, + pollset_work, + pollset_kick, + pollset_add_fd, + + pollset_set_create, + pollset_set_destroy, + pollset_set_add_pollset, + pollset_set_del_pollset, + pollset_set_add_pollset_set, + pollset_set_del_pollset_set, + pollset_set_add_fd, + pollset_set_del_fd, + + shutdown_engine, }; const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request) { diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 1feea6d628..f63f190155 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -50,7 +50,7 @@ void grpc_iomgr_init(grpc_exec_ctx *exec_ctx) { grpc_executor_init(exec_ctx); grpc_timer_list_init(gpr_now(GPR_CLOCK_MONOTONIC)); g_root_object.next = g_root_object.prev = &g_root_object; - g_root_object.name = "root"; + g_root_object.name = (char *)"root"; grpc_network_status_init(); grpc_iomgr_platform_init(); } diff --git a/src/core/lib/iomgr/is_epollexclusive_available.c b/src/core/lib/iomgr/is_epollexclusive_available.c index e8a7d4d52c..d08844c0df 100644 --- a/src/core/lib/iomgr/is_epollexclusive_available.c +++ b/src/core/lib/iomgr/is_epollexclusive_available.c @@ -57,12 +57,12 @@ bool grpc_is_epollexclusive_available(void) { close(fd); return false; } - struct epoll_event ev = { - /* choose events that should cause an error on - EPOLLEXCLUSIVE enabled kernels - specifically the combination of - EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), - .data.ptr = NULL}; + struct epoll_event ev; + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + ev.events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT); + ev.data.ptr = NULL; if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) { if (errno != EINVAL) { if (!logged_why_not) { diff --git a/src/core/lib/iomgr/socket_factory_posix.c b/src/core/lib/iomgr/socket_factory_posix.c index c81566575e..8e907703ae 100644 --- a/src/core/lib/iomgr/socket_factory_posix.c +++ b/src/core/lib/iomgr/socket_factory_posix.c @@ -85,8 +85,8 @@ static const grpc_arg_pointer_vtable socket_factory_arg_vtable = { socket_factory_arg_copy, socket_factory_arg_destroy, socket_factory_cmp}; grpc_arg grpc_socket_factory_to_arg(grpc_socket_factory *factory) { - return grpc_channel_arg_pointer_create(GRPC_ARG_SOCKET_FACTORY, factory, - &socket_factory_arg_vtable); + return grpc_channel_arg_pointer_create((char *)GRPC_ARG_SOCKET_FACTORY, + factory, &socket_factory_arg_vtable); } #endif diff --git a/src/core/lib/iomgr/socket_mutator.c b/src/core/lib/iomgr/socket_mutator.c index 300ac75b38..b0435d5a07 100644 --- a/src/core/lib/iomgr/socket_mutator.c +++ b/src/core/lib/iomgr/socket_mutator.c @@ -76,6 +76,6 @@ static const grpc_arg_pointer_vtable socket_mutator_arg_vtable = { socket_mutator_arg_copy, socket_mutator_arg_destroy, socket_mutator_cmp}; grpc_arg grpc_socket_mutator_to_arg(grpc_socket_mutator *mutator) { - return grpc_channel_arg_pointer_create(GRPC_ARG_SOCKET_MUTATOR, mutator, - &socket_mutator_arg_vtable); + return grpc_channel_arg_pointer_create((char *)GRPC_ARG_SOCKET_MUTATOR, + mutator, &socket_mutator_arg_vtable); } diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index c3ec3e447a..06612d639c 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -198,12 +198,12 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { /* event manager callback when reads are ready */ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { grpc_tcp_listener *sp = (grpc_tcp_listener *)arg; - + grpc_pollset *read_notifier_pollset; if (err != GRPC_ERROR_NONE) { goto error; } - grpc_pollset *read_notifier_pollset = + read_notifier_pollset = sp->server->pollsets[(size_t)gpr_atm_no_barrier_fetch_add( &sp->server->next_pollset_to_assign, 1) % sp->server->pollset_count]; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index c08bb525b7..c5955c869c 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -96,7 +96,10 @@ struct shared_mutables { } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); static struct shared_mutables g_shared_mutables = { - .checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER, .initialized = false, + 0, + GPR_SPINLOCK_STATIC_INITIALIZER, + false, + {{0}} }; static gpr_clock_type g_clock_type; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index a41eb336d4..09b107cf6b 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1673,6 +1673,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, batch_control *bctl; int num_completion_callbacks_needed = 1; grpc_call_error error = GRPC_CALL_OK; + grpc_transport_stream_op_batch *stream_op; + grpc_transport_stream_op_batch_payload *stream_op_payload; GPR_TIMER_BEGIN("grpc_call_start_batch", 0); GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag); @@ -1699,9 +1701,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, bctl->completion_data.notify_tag.is_closure = (uint8_t)(is_notify_tag_closure != 0); - grpc_transport_stream_op_batch *stream_op = &bctl->op; - grpc_transport_stream_op_batch_payload *stream_op_payload = - &call->stream_op_payload; + stream_op = &bctl->op; + stream_op_payload = &call->stream_op_payload; /* rewrite batch ops into a transport op */ for (i = 0; i < nops; i++) { @@ -1711,7 +1712,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, goto done_with_error; } switch (op->op) { - case GRPC_OP_SEND_INITIAL_METADATA: + case GRPC_OP_SEND_INITIAL_METADATA: { /* Flag validation: currently allow no flags */ if (!are_initial_metadata_flags_valid(op->flags, call->is_client)) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1805,7 +1806,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->peer_string; } break; - case GRPC_OP_SEND_MESSAGE: + } + case GRPC_OP_SEND_MESSAGE: { if (!are_write_flags_valid(op->flags)) { error = GRPC_CALL_ERROR_INVALID_FLAGS; goto done_with_error; @@ -1834,7 +1836,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_message.send_message = &call->sending_stream.base; break; - case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + } + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1853,7 +1856,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; - case GRPC_OP_SEND_STATUS_FROM_SERVER: + } + case GRPC_OP_SEND_STATUS_FROM_SERVER: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1915,7 +1919,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->send_trailing_metadata.send_trailing_metadata = &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]; break; - case GRPC_OP_RECV_INITIAL_METADATA: + } + case GRPC_OP_RECV_INITIAL_METADATA: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1942,7 +1947,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, } num_completion_callbacks_needed++; break; - case GRPC_OP_RECV_MESSAGE: + } + case GRPC_OP_RECV_MESSAGE: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1963,7 +1969,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, &call->receiving_stream_ready; num_completion_callbacks_needed++; break; - case GRPC_OP_RECV_STATUS_ON_CLIENT: + } + case GRPC_OP_RECV_STATUS_ON_CLIENT: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -1990,7 +1997,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; - case GRPC_OP_RECV_CLOSE_ON_SERVER: + } + case GRPC_OP_RECV_CLOSE_ON_SERVER: { /* Flag validation: currently allow no flags */ if (op->flags != 0) { error = GRPC_CALL_ERROR_INVALID_FLAGS; @@ -2014,6 +2022,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, stream_op_payload->collect_stats.collect_stats = &call->final_info.stats.transport_stream_stats; break; + } } } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 468360fbde..d60a0cd451 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -328,25 +328,12 @@ static void cq_destroy_pluck(void *data); /* Completion queue vtables based on the completion-type */ static const cq_vtable g_cq_vtable[] = { /* GRPC_CQ_NEXT */ - {.data_size = sizeof(cq_next_data), - .cq_completion_type = GRPC_CQ_NEXT, - .init = cq_init_next, - .shutdown = cq_shutdown_next, - .destroy = cq_destroy_next, - .begin_op = cq_begin_op_for_next, - .end_op = cq_end_op_for_next, - .next = cq_next, - .pluck = NULL}, + {GRPC_CQ_NEXT, sizeof(cq_next_data), cq_init_next, cq_shutdown_next, + cq_destroy_next, cq_begin_op_for_next, cq_end_op_for_next, cq_next, NULL}, /* GRPC_CQ_PLUCK */ - {.data_size = sizeof(cq_pluck_data), - .cq_completion_type = GRPC_CQ_PLUCK, - .init = cq_init_pluck, - .shutdown = cq_shutdown_pluck, - .destroy = cq_destroy_pluck, - .begin_op = cq_begin_op_for_pluck, - .end_op = cq_end_op_for_pluck, - .next = NULL, - .pluck = cq_pluck}, + {GRPC_CQ_PLUCK, sizeof(cq_pluck_data), cq_init_pluck, cq_shutdown_pluck, + cq_destroy_pluck, cq_begin_op_for_pluck, cq_end_op_for_pluck, NULL, + cq_pluck}, }; #define DATA_FROM_CQ(cq) ((void *)(cq + 1)) diff --git a/src/core/lib/transport/status_conversion.c b/src/core/lib/transport/status_conversion.c index 9a76977e4b..a40d333284 100644 --- a/src/core/lib/transport/status_conversion.c +++ b/src/core/lib/transport/status_conversion.c @@ -18,7 +18,7 @@ #include "src/core/lib/transport/status_conversion.h" -int grpc_status_to_http2_error(grpc_status_code status) { +grpc_http2_error_code grpc_status_to_http2_error(grpc_status_code status) { switch (status) { case GRPC_STATUS_OK: return GRPC_HTTP2_NO_ERROR; diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c index ae705195f3..682a820b48 100644 --- a/src/core/lib/transport/transport.c +++ b/src/core/lib/transport/transport.c @@ -102,9 +102,11 @@ static void slice_stream_unref(grpc_exec_ctx *exec_ctx, void *p) { grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount *refcount, void *buffer, size_t length) { slice_stream_ref(&refcount->slice_refcount); - return (grpc_slice){ - .refcount = &refcount->slice_refcount, - .data.refcounted = {.bytes = (uint8_t *)buffer, .length = length}}; + grpc_slice res; + res.refcount = &refcount->slice_refcount, + res.data.refcounted.bytes = (uint8_t *)buffer; + res.data.refcounted.length = length; + return res; } static const grpc_slice_refcount_vtable stream_ref_slice_vtable = { -- cgit v1.2.3 From 067611f19214af4a44435016234b65fc3918bbed Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 12:57:42 -0700 Subject: static metadata and clang format --- .../ext/transport/chttp2/transport/hpack_encoder.c | 5 +- src/core/lib/iomgr/timer_generic.c | 6 +- src/core/lib/transport/static_metadata.c | 816 +++++++-------------- tools/codegen/core/gen_static_metadata.py | 4 +- 4 files changed, 278 insertions(+), 553 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 9537c2fa17..52c9e217f1 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -51,8 +51,9 @@ #define MAX_DECODER_SPACE_USAGE 512 static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; -static const grpc_slice terminal_slice = {&terminal_slice_refcount, /* refcount */ - {{0, 0}} /* refcounted */ +static const grpc_slice terminal_slice = { + &terminal_slice_refcount, /* refcount */ + {{0, 0}} /* refcounted */ }; extern grpc_tracer_flag grpc_http_trace; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index c5955c869c..57b63cb229 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -96,11 +96,7 @@ struct shared_mutables { } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); static struct shared_mutables g_shared_mutables = { - 0, - GPR_SPINLOCK_STATIC_INITIALIZER, - false, - {{0}} -}; + 0, GPR_SPINLOCK_STATIC_INITIALIZER, false, {{0}}}; static gpr_clock_type g_clock_type; static gpr_timespec g_start_time; diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index b20d94aeac..472cf888ea 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -216,206 +216,106 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { - {.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[3], - .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[5], - .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[6], - .data.refcounted = {g_bytes + 38, 12}}, - {.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[8], - .data.refcounted = {g_bytes + 61, 16}}, - {.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[11], - .data.refcounted = {g_bytes + 110, 21}}, - {.refcount = &grpc_static_metadata_refcounts[12], - .data.refcounted = {g_bytes + 131, 13}}, - {.refcount = &grpc_static_metadata_refcounts[13], - .data.refcounted = {g_bytes + 144, 14}}, - {.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[17], - .data.refcounted = {g_bytes + 201, 30}}, - {.refcount = &grpc_static_metadata_refcounts[18], - .data.refcounted = {g_bytes + 231, 37}}, - {.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 268, 10}}, - {.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 278, 4}}, - {.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 282, 8}}, - {.refcount = &grpc_static_metadata_refcounts[22], - .data.refcounted = {g_bytes + 290, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}, - {.refcount = &grpc_static_metadata_refcounts[24], - .data.refcounted = {g_bytes + 302, 19}}, - {.refcount = &grpc_static_metadata_refcounts[25], - .data.refcounted = {g_bytes + 321, 12}}, - {.refcount = &grpc_static_metadata_refcounts[26], - .data.refcounted = {g_bytes + 333, 30}}, - {.refcount = &grpc_static_metadata_refcounts[27], - .data.refcounted = {g_bytes + 363, 31}}, - {.refcount = &grpc_static_metadata_refcounts[28], - .data.refcounted = {g_bytes + 394, 36}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 430, 1}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 431, 1}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 432, 1}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 452, 8}}, - {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 460, 16}}, - {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 476, 4}}, - {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 480, 3}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 483, 3}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 486, 4}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 490, 5}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 495, 4}}, - {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 499, 3}}, - {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 502, 3}}, - {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 505, 1}}, - {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 506, 11}}, - {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 517, 3}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 520, 3}}, - {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 523, 3}}, - {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 526, 3}}, - {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 529, 3}}, - {.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 532, 14}}, - {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 546, 13}}, - {.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 559, 15}}, - {.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 574, 13}}, - {.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 587, 6}}, - {.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 593, 27}}, - {.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 620, 3}}, - {.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 623, 5}}, - {.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 628, 13}}, - {.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 641, 13}}, - {.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 654, 19}}, - {.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 673, 16}}, - {.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 689, 14}}, - {.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 703, 16}}, - {.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 719, 13}}, - {.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 732, 6}}, - {.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 738, 4}}, - {.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 742, 4}}, - {.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 746, 6}}, - {.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 752, 7}}, - {.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 759, 4}}, - {.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 763, 8}}, - {.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 771, 17}}, - {.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 788, 13}}, - {.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 801, 8}}, - {.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 809, 19}}, - {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 828, 13}}, - {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 841, 11}}, - {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 852, 4}}, - {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 856, 8}}, - {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 864, 12}}, - {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 876, 18}}, - {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 894, 19}}, - {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 913, 5}}, - {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 918, 7}}, - {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 925, 7}}, - {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 932, 11}}, - {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 943, 6}}, - {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 949, 10}}, - {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 959, 25}}, - {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 984, 17}}, - {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 1001, 4}}, - {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 1005, 3}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 1008, 16}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1024, 16}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}, - {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1053, 12}}, - {.refcount = &grpc_static_metadata_refcounts[99], - .data.refcounted = {g_bytes + 1065, 21}}, + {&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[3], {{g_bytes + 19, 10}}}, + {&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[5], {{g_bytes + 36, 2}}}, + {&grpc_static_metadata_refcounts[6], {{g_bytes + 38, 12}}}, + {&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[8], {{g_bytes + 61, 16}}}, + {&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[11], {{g_bytes + 110, 21}}}, + {&grpc_static_metadata_refcounts[12], {{g_bytes + 131, 13}}}, + {&grpc_static_metadata_refcounts[13], {{g_bytes + 144, 14}}}, + {&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[17], {{g_bytes + 201, 30}}}, + {&grpc_static_metadata_refcounts[18], {{g_bytes + 231, 37}}}, + {&grpc_static_metadata_refcounts[19], {{g_bytes + 268, 10}}}, + {&grpc_static_metadata_refcounts[20], {{g_bytes + 278, 4}}}, + {&grpc_static_metadata_refcounts[21], {{g_bytes + 282, 8}}}, + {&grpc_static_metadata_refcounts[22], {{g_bytes + 290, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}, + {&grpc_static_metadata_refcounts[24], {{g_bytes + 302, 19}}}, + {&grpc_static_metadata_refcounts[25], {{g_bytes + 321, 12}}}, + {&grpc_static_metadata_refcounts[26], {{g_bytes + 333, 30}}}, + {&grpc_static_metadata_refcounts[27], {{g_bytes + 363, 31}}}, + {&grpc_static_metadata_refcounts[28], {{g_bytes + 394, 36}}}, + {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}, + {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}, + {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}, + {&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}, + {&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, + {&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, + {&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, + {&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, + {&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, + {&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, + {&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, + {&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, + {&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, + {&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, + {&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, + {&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, + {&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, + {&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, + {&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, + {&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, + {&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, + {&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, + {&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, + {&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, + {&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, + {&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, + {&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, + {&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, + {&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, + {&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, + {&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, + {&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, + {&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, + {&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, + {&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, + {&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, + {&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, + {&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, + {&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, + {&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, + {&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, + {&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, + {&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, + {&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, + {&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, + {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}, + {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { @@ -478,350 +378,178 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { } grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[29], - .data.refcounted = {g_bytes + 430, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[30], - .data.refcounted = {g_bytes + 431, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[7], - .data.refcounted = {g_bytes + 50, 11}}, - {.refcount = &grpc_static_metadata_refcounts[31], - .data.refcounted = {g_bytes + 432, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[9], - .data.refcounted = {g_bytes + 77, 13}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}}, - {{.refcount = &grpc_static_metadata_refcounts[5], - .data.refcounted = {g_bytes + 36, 2}}, - {.refcount = &grpc_static_metadata_refcounts[35], - .data.refcounted = {g_bytes + 452, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[36], - .data.refcounted = {g_bytes + 460, 16}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[37], - .data.refcounted = {g_bytes + 476, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[38], - .data.refcounted = {g_bytes + 480, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[39], - .data.refcounted = {g_bytes + 483, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[40], - .data.refcounted = {g_bytes + 486, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[41], - .data.refcounted = {g_bytes + 490, 5}}}, - {{.refcount = &grpc_static_metadata_refcounts[4], - .data.refcounted = {g_bytes + 29, 7}}, - {.refcount = &grpc_static_metadata_refcounts[42], - .data.refcounted = {g_bytes + 495, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[3], - .data.refcounted = {g_bytes + 19, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[43], - .data.refcounted = {g_bytes + 499, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[1], - .data.refcounted = {g_bytes + 5, 7}}, - {.refcount = &grpc_static_metadata_refcounts[44], - .data.refcounted = {g_bytes + 502, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[45], - .data.refcounted = {g_bytes + 505, 1}}}, - {{.refcount = &grpc_static_metadata_refcounts[0], - .data.refcounted = {g_bytes + 0, 5}}, - {.refcount = &grpc_static_metadata_refcounts[46], - .data.refcounted = {g_bytes + 506, 11}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[47], - .data.refcounted = {g_bytes + 517, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[48], - .data.refcounted = {g_bytes + 520, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[49], - .data.refcounted = {g_bytes + 523, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[50], - .data.refcounted = {g_bytes + 526, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[2], - .data.refcounted = {g_bytes + 12, 7}}, - {.refcount = &grpc_static_metadata_refcounts[51], - .data.refcounted = {g_bytes + 529, 3}}}, - {{.refcount = &grpc_static_metadata_refcounts[52], - .data.refcounted = {g_bytes + 532, 14}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[53], - .data.refcounted = {g_bytes + 546, 13}}}, - {{.refcount = &grpc_static_metadata_refcounts[54], - .data.refcounted = {g_bytes + 559, 15}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[55], - .data.refcounted = {g_bytes + 574, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[56], - .data.refcounted = {g_bytes + 587, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[57], - .data.refcounted = {g_bytes + 593, 27}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[58], - .data.refcounted = {g_bytes + 620, 3}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[59], - .data.refcounted = {g_bytes + 623, 5}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[60], - .data.refcounted = {g_bytes + 628, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[61], - .data.refcounted = {g_bytes + 641, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[62], - .data.refcounted = {g_bytes + 654, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[15], - .data.refcounted = {g_bytes + 170, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[63], - .data.refcounted = {g_bytes + 673, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[64], - .data.refcounted = {g_bytes + 689, 14}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[65], - .data.refcounted = {g_bytes + 703, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[66], - .data.refcounted = {g_bytes + 719, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[14], - .data.refcounted = {g_bytes + 158, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[67], - .data.refcounted = {g_bytes + 732, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[68], - .data.refcounted = {g_bytes + 738, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[69], - .data.refcounted = {g_bytes + 742, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[70], - .data.refcounted = {g_bytes + 746, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[71], - .data.refcounted = {g_bytes + 752, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[72], - .data.refcounted = {g_bytes + 759, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[20], - .data.refcounted = {g_bytes + 278, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[73], - .data.refcounted = {g_bytes + 763, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[74], - .data.refcounted = {g_bytes + 771, 17}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[75], - .data.refcounted = {g_bytes + 788, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[76], - .data.refcounted = {g_bytes + 801, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[77], - .data.refcounted = {g_bytes + 809, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 828, 13}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[21], - .data.refcounted = {g_bytes + 282, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 841, 11}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 852, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 856, 8}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 864, 12}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 876, 18}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 894, 19}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 913, 5}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 918, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 925, 7}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 932, 11}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 943, 6}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 949, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 959, 25}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 984, 17}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[19], - .data.refcounted = {g_bytes + 268, 10}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 1001, 4}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 1005, 3}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 1008, 16}}, - {.refcount = &grpc_static_metadata_refcounts[23], - .data.refcounted = {g_bytes + 302, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[34], - .data.refcounted = {g_bytes + 445, 7}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1024, 16}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[98], - .data.refcounted = {g_bytes + 1053, 12}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[99], - .data.refcounted = {g_bytes + 1065, 21}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[32], - .data.refcounted = {g_bytes + 433, 8}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[33], - .data.refcounted = {g_bytes + 441, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 186, 15}}, - {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1040, 13}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[29], {{g_bytes + 430, 1}}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[30], {{g_bytes + 431, 1}}}}, + {{&grpc_static_metadata_refcounts[7], {{g_bytes + 50, 11}}}, + {&grpc_static_metadata_refcounts[31], {{g_bytes + 432, 1}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[9], {{g_bytes + 77, 13}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {{&grpc_static_metadata_refcounts[5], {{g_bytes + 36, 2}}}, + {&grpc_static_metadata_refcounts[35], {{g_bytes + 452, 8}}}}, + {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[36], {{g_bytes + 460, 16}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[37], {{g_bytes + 476, 4}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[38], {{g_bytes + 480, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[39], {{g_bytes + 483, 3}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[40], {{g_bytes + 486, 4}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[41], {{g_bytes + 490, 5}}}}, + {{&grpc_static_metadata_refcounts[4], {{g_bytes + 29, 7}}}, + {&grpc_static_metadata_refcounts[42], {{g_bytes + 495, 4}}}}, + {{&grpc_static_metadata_refcounts[3], {{g_bytes + 19, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[43], {{g_bytes + 499, 3}}}}, + {{&grpc_static_metadata_refcounts[1], {{g_bytes + 5, 7}}}, + {&grpc_static_metadata_refcounts[44], {{g_bytes + 502, 3}}}}, + {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[45], {{g_bytes + 505, 1}}}}, + {{&grpc_static_metadata_refcounts[0], {{g_bytes + 0, 5}}}, + {&grpc_static_metadata_refcounts[46], {{g_bytes + 506, 11}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[47], {{g_bytes + 517, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[48], {{g_bytes + 520, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[49], {{g_bytes + 523, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[50], {{g_bytes + 526, 3}}}}, + {{&grpc_static_metadata_refcounts[2], {{g_bytes + 12, 7}}}, + {&grpc_static_metadata_refcounts[51], {{g_bytes + 529, 3}}}}, + {{&grpc_static_metadata_refcounts[52], {{g_bytes + 532, 14}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[53], {{g_bytes + 546, 13}}}}, + {{&grpc_static_metadata_refcounts[54], {{g_bytes + 559, 15}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[55], {{g_bytes + 574, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[56], {{g_bytes + 587, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[57], {{g_bytes + 593, 27}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[58], {{g_bytes + 620, 3}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[59], {{g_bytes + 623, 5}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[60], {{g_bytes + 628, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[61], {{g_bytes + 641, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[62], {{g_bytes + 654, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[15], {{g_bytes + 170, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[63], {{g_bytes + 673, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[64], {{g_bytes + 689, 14}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[65], {{g_bytes + 703, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[66], {{g_bytes + 719, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[14], {{g_bytes + 158, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[67], {{g_bytes + 732, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[68], {{g_bytes + 738, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[69], {{g_bytes + 742, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[70], {{g_bytes + 746, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[71], {{g_bytes + 752, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[72], {{g_bytes + 759, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[20], {{g_bytes + 278, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[73], {{g_bytes + 763, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[74], {{g_bytes + 771, 17}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[75], {{g_bytes + 788, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[76], {{g_bytes + 801, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[77], {{g_bytes + 809, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[78], {{g_bytes + 828, 13}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[21], {{g_bytes + 282, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[79], {{g_bytes + 841, 11}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[80], {{g_bytes + 852, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[81], {{g_bytes + 856, 8}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[82], {{g_bytes + 864, 12}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[83], {{g_bytes + 876, 18}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[84], {{g_bytes + 894, 19}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[85], {{g_bytes + 913, 5}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[86], {{g_bytes + 918, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[87], {{g_bytes + 925, 7}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[88], {{g_bytes + 932, 11}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[89], {{g_bytes + 943, 6}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[90], {{g_bytes + 949, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[91], {{g_bytes + 959, 25}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[92], {{g_bytes + 984, 17}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[19], {{g_bytes + 268, 10}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[93], {{g_bytes + 1001, 4}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[94], {{g_bytes + 1005, 3}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[95], {{g_bytes + 1008, 16}}}, + {&grpc_static_metadata_refcounts[23], {{g_bytes + 302, 0}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[34], {{g_bytes + 445, 7}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[96], {{g_bytes + 1024, 16}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[98], {{g_bytes + 1053, 12}}}}, + {{&grpc_static_metadata_refcounts[10], {{g_bytes + 90, 20}}}, + {&grpc_static_metadata_refcounts[99], {{g_bytes + 1065, 21}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[32], {{g_bytes + 433, 8}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[33], {{g_bytes + 441, 4}}}}, + {{&grpc_static_metadata_refcounts[16], {{g_bytes + 186, 15}}}, + {&grpc_static_metadata_refcounts[97], {{g_bytes + 1040, 13}}}}, }; bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = { true, // :path diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 6ee8a7cace..af7521da36 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -370,8 +370,8 @@ for i, elem in enumerate(all_strs): def slice_def(i): - return ('{.refcount = &grpc_static_metadata_refcounts[%d], .data.refcounted =' - ' {g_bytes+%d, %d}}') % ( + return ('{&grpc_static_metadata_refcounts[%d],' + ' {{g_bytes+%d, %d}}}') % ( i, id2strofs[i], len(all_strs[i])) -- cgit v1.2.3 From 341c7e7eca06e1c66ed2448fb3bcda6d278f5fd6 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 13:37:28 -0700 Subject: Timer generic correction --- src/core/lib/iomgr/timer_generic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index 57b63cb229..dc4df807b8 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -95,8 +95,7 @@ struct shared_mutables { gpr_mu mu; } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE); -static struct shared_mutables g_shared_mutables = { - 0, GPR_SPINLOCK_STATIC_INITIALIZER, false, {{0}}}; +static struct shared_mutables g_shared_mutables; static gpr_clock_type g_clock_type; static gpr_timespec g_start_time; @@ -154,6 +153,7 @@ void grpc_timer_list_init(gpr_timespec now) { uint32_t i; g_shared_mutables.initialized = true; + g_shared_mutables.checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER; gpr_mu_init(&g_shared_mutables.mu); g_clock_type = now.clock_type; g_start_time = now; -- cgit v1.2.3 From e203067bdcd9522446aa106ce77085c62a932c49 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 14:44:58 -0700 Subject: initializing checker_mu in timer_list to be safe --- src/core/lib/iomgr/timer_generic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index dc4df807b8..e9a7236c8c 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -153,7 +153,7 @@ void grpc_timer_list_init(gpr_timespec now) { uint32_t i; g_shared_mutables.initialized = true; - g_shared_mutables.checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER; + g_shared_mutables.checker_mu = GPR_SPINLOCK_INITIALIZER; gpr_mu_init(&g_shared_mutables.mu); g_clock_type = now.clock_type; g_start_time = now; -- cgit v1.2.3 From 06312bdb43c66150e59958479640913c3726ee10 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Mon, 18 Sep 2017 15:10:22 -0700 Subject: Minor formatting, commenting and clang format for C to C++ conversion --- include/grpc/slice.h | 4 ---- .../filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c | 2 +- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 6 +++++- src/core/lib/iomgr/ev_epollex_linux.c | 6 +++++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 3080d63e2b..3f3cff1408 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -66,10 +66,6 @@ GPRAPI grpc_slice grpc_slice_malloc(size_t length); GPRAPI grpc_slice grpc_slice_malloc_large(size_t length); #define GRPC_SLICE_MALLOC(len) grpc_slice_malloc(len) -/* ((len) <= GRPC_SLICE_INLINED_SIZE \ - ? (grpc_slice){.refcount = NULL, \ - .data.inlined = {.length = (uint8_t)(len)}} \ - : grpc_slice_malloc_large((len)))*/ /** Intern a slice: diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 9975b1ef79..04379975e1 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -276,7 +276,7 @@ static void on_txt_done_cb(void *arg, int status, int timeouts, char *error_msg; grpc_ares_request *r = (grpc_ares_request *)arg; const size_t prefix_len = sizeof(g_service_config_attribute_prefix) - 1; - struct ares_txt_ext *result; + struct ares_txt_ext *result = NULL; struct ares_txt_ext *reply = NULL; grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&r->mu); diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 52c9e217f1..9dfbd3c3a9 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -53,7 +53,7 @@ static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; static const grpc_slice terminal_slice = { &terminal_slice_refcount, /* refcount */ - {{0, 0}} /* refcounted */ + {{0, 0}} /* data.refcounted */ }; extern grpc_tracer_flag grpc_http_trace; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b5066e5f58..7fde45865e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -568,7 +568,11 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; + static const gpr_timespec round_up = { + 0, /* tv_sec */ + GPR_NS_PER_MS - 1, /* tv_nsec */ + GPR_TIMESPAN /* clock_type */ + }; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 671c4cd730..cff0c07a2b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -706,7 +706,11 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {0, GPR_NS_PER_MS - 1, GPR_TIMESPAN}; + static const gpr_timespec round_up = { + 0, /* tv_sec */ + GPR_NS_PER_MS - 1, /* tv_nsec */ + GPR_TIMESPAN /* clock_type */ + }; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; -- cgit v1.2.3 From 2c1a7e1375ade318d5925edb7bff63f4ce522a81 Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Tue, 19 Sep 2017 00:00:14 +0000 Subject: Flip gRPC Python Protoc Plug-In default behavior Now the grpc_2_0 flag is no longer required for the only-generate-code- elements-in-_pb2_grpc.py-files behavior and the grpc_1_0 flag is required for the add-generated-gRPC-using-code-elements-to-_pb2.py- files behavior. --- src/compiler/python_generator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index a60b528e3b..ef2d90de9e 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -767,9 +767,9 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file, ProtoBufFile pbfile(file); PrivateGenerator generator(config_, &pbfile); - if (parameter == "grpc_2_0") { + if (parameter == "" || parameter == "grpc_2_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true); - } else if (parameter == "grpc_1_0" || parameter == "") { + } else if (parameter == "grpc_1_0") { return GenerateGrpc(context, generator, pb2_grpc_file_name, true) && GenerateGrpc(context, generator, pb2_file_name, false); } else { -- cgit v1.2.3 From 75c27789026375122a33043833a25e5b935c906f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 18 Sep 2017 17:11:48 -0700 Subject: Update Dockerfiles from ubuntu:15.10 to debian:jessie --- templates/tools/dockerfile/clang_format.include | 6 ++-- .../grpc_clang_format/Dockerfile.template | 3 +- .../dockerfile/test/sanity/Dockerfile.template | 27 ++++++++++++------ tools/dockerfile/grpc_clang_format/Dockerfile | 8 +++--- tools/dockerfile/test/sanity/Dockerfile | 33 ++++++++++++++-------- 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang_format.include index 9a2b60ba8c..81bd2be797 100644 --- a/templates/tools/dockerfile/clang_format.include +++ b/templates/tools/dockerfile/clang_format.include @@ -1,5 +1,5 @@ RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template index 69cd4034b0..1ab667c95d 100644 --- a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template +++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM ubuntu:15.10 + FROM debian:jessie <%include file="../clang_format.include"/> ADD clang_format_all_the_things.sh / CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] + \ No newline at end of file diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 3f9ea32233..bf34c0ab0f 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM ubuntu:15.10 + FROM debian:jessie <%include file="../../apt_get_basic.include"/> <%include file="../../gcp_api_libraries.include"/> @@ -33,14 +33,23 @@ #====================================== # More sanity test dependencies (bazel) - RUN apt-get install -y openjdk-8-jdk - # Check out Bazel version 0.4.1 since this version allows running - # ./compile.sh without a local protoc dependency - # TODO(mattkwong): install dependencies to support latest Bazel version if newer - # version is needed - RUN git clone https://github.com/bazelbuild/bazel.git /bazel && ${"\\"} - cd /bazel && git checkout tags/0.4.1 && ./compile.sh - RUN ln -s /bazel/output/bazel /bin/ + RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list + RUN apt-get update + RUN apt-get install -y -t jessie-backports openjdk-8-jdk + + #======================== + # Bazel installation + RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list + RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - + RUN apt-get -y update + RUN apt-get -y install bazel + + # Pin Bazel to 0.4.4 + # Installing Bazel via apt-get first is required before installing 0.4.4 to + # allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 + RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh + RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh + RUN ./bazel-0.4.4-installer-linux-x86_64.sh <%include file="../../clang_format.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index 647cb52a7b..dff07feeb6 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 ADD clang_format_all_the_things.sh / diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 44732a5ae4..dff979dc37 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:15.10 +FROM debian:jessie # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -82,19 +82,28 @@ RUN pip install simplejson mako #====================================== # More sanity test dependencies (bazel) -RUN apt-get install -y openjdk-8-jdk -# Check out Bazel version 0.4.1 since this version allows running -# ./compile.sh without a local protoc dependency -# TODO(mattkwong): install dependencies to support latest Bazel version if newer -# version is needed -RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ - cd /bazel && git checkout tags/0.4.1 && ./compile.sh -RUN ln -s /bazel/output/bazel /bin/ +RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +RUN apt-get update +RUN apt-get install -y -t jessie-backports openjdk-8-jdk + +#======================== +# Bazel installation +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get -y update +RUN apt-get -y install bazel + +# Pin Bazel to 0.4.4 +# Installing Bazel via apt-get first is required before installing 0.4.4 to +# allow gRPC to build without errors. See https://github.com/grpc/grpc/issues/10553 +RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel-0.4.4-installer-linux-x86_64.sh +RUN chmod +x ./bazel-0.4.4-installer-linux-x86_64.sh +RUN ./bazel-0.4.4-installer-linux-x86_64.sh RUN apt-get update && apt-get -y install wget -RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list -RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN echo "deb http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN echo "deb-src http://llvm.org/apt/jessie/ llvm-toolchain-jessie-3.8 main" >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - RUN apt-get update && apt-get -y install clang-format-3.8 # Prepare ccache -- cgit v1.2.3 From 7d764673ef9a18503339747ab8ee002739f6e329 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 19:10:49 -0700 Subject: Backends must be up before some grpclb tests start. --- test/cpp/end2end/grpclb_end2end_test.cc | 87 ++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 570a3d1067..6896b54f1b 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -398,11 +398,40 @@ class GrpclbEnd2endTest : public ::testing::Test { return true; } - void WaitForAllBackends() { + void SendRpcAndCount(int* num_total, int* num_ok, int* num_failure, + int* num_drops) { + const Status status = SendRpc(); + if (status.ok()) { + ++*num_ok; + } else { + if (status.error_message() == "Call dropped by load balancing policy") { + ++*num_drops; + } else { + ++*num_failure; + } + } + ++*num_total; + } + + std::tuple WaitForAllBackends( + int num_requests_multiple_of = 1) { + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + int num_total = 0; while (!SeenAllBackends()) { - CheckRpcSendOk(); + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); + } + while (num_total % num_requests_multiple_of != 0) { + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); } ResetBackendCounters(); + gpr_log(GPR_INFO, + "Performed %d warm up requests (a multiple of %d) against the " + "backends. %d succeeded, %d failed, %d dropped.", + num_total, num_requests_multiple_of, num_ok, num_failure, + num_drops); + return std::make_tuple(num_ok, num_failure, num_drops); } void WaitForBackend(size_t backend_idx) { @@ -556,10 +585,8 @@ TEST_F(SingleBalancerTest, Vanilla) { 0); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); - // We need to wait for all backends to come online. WaitForAllBackends(); - // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); @@ -867,6 +894,8 @@ TEST_F(SingleBalancerTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), 0); + // Wait until all backends are ready. + WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { @@ -936,6 +965,11 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + // Wait until all backends are ready. + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); // Each backend should have gotten 100 requests. @@ -950,11 +984,12 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, + client_stats.num_calls_started); + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + (num_ok + num_drops), client_stats.num_calls_finished_known_received); EXPECT_THAT(client_stats.drop_token_counts, ::testing::ElementsAre()); } @@ -965,7 +1000,14 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), 0); - + // Wait until all backends are ready. + int num_warmup_ok = 0; + int num_warmup_failure = 0; + int num_warmup_drops = 0; + std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = + WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + const int num_total_warmup_requests = + num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { EchoResponse response; @@ -980,7 +1022,6 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { } } EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); - // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -993,17 +1034,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_finished); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_started); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, client_stats.num_calls_finished_known_received); - EXPECT_THAT(client_stats.drop_token_counts, - ::testing::ElementsAre( - ::testing::Pair("load_balancing", kNumRpcsPerAddress), - ::testing::Pair("rate_limiting", kNumRpcsPerAddress * 2))); + // The number of warmup request is a multiple of the number of addresses. + // Therefore, all addresses in the scheduled balancer response are hit the + // same number of times. + const int num_times_drop_addresses_hit = num_warmup_drops / 3; + EXPECT_THAT( + client_stats.drop_token_counts, + ::testing::ElementsAre( + ::testing::Pair("load_balancing", + (kNumRpcsPerAddress + num_times_drop_addresses_hit)), + ::testing::Pair( + "rate_limiting", + (kNumRpcsPerAddress + num_times_drop_addresses_hit) * 2))); } } // namespace -- cgit v1.2.3 From 588e36a3e3b43276f573c466d88c3dd60bd5be16 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 20:01:27 -0700 Subject: Get rid of magic numbers --- test/cpp/end2end/grpclb_end2end_test.cc | 45 ++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 6896b54f1b..77ed155292 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -890,15 +890,22 @@ TEST_F(UpdatesTest, UpdateBalancersDeadUpdate) { TEST_F(SingleBalancerTest, Drop) { const size_t kNumRpcsPerAddress = 100; + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 2; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -910,7 +917,7 @@ TEST_F(SingleBalancerTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { @@ -925,9 +932,12 @@ TEST_F(SingleBalancerTest, Drop) { TEST_F(SingleBalancerTest, DropAllFirst) { // All registered addresses are marked as "drop". + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); const Status status = SendRpc(); EXPECT_FALSE(status.ok()); @@ -938,9 +948,12 @@ TEST_F(SingleBalancerTest, DropAll) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 1000); // First call succeeds. @@ -996,20 +1009,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const size_t kNumRpcsPerAddress = 3; + const int num_of_drop_by_rate_limiting_addresses = 2; + const int num_of_drop_by_load_balancing_addresses = 1; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. int num_warmup_ok = 0; int num_warmup_failure = 0; int num_warmup_drops = 0; std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = - WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + WaitForAllBackends(num_total_addresses /* num_requests_multiple_of */); const int num_total_warmup_requests = num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -1021,7 +1041,7 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -1035,10 +1055,10 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const ClientStats client_stats = WaitForLoadReports(); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_started); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, @@ -1046,7 +1066,8 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { // The number of warmup request is a multiple of the number of addresses. // Therefore, all addresses in the scheduled balancer response are hit the // same number of times. - const int num_times_drop_addresses_hit = num_warmup_drops / 3; + const int num_times_drop_addresses_hit = + num_warmup_drops / num_of_drop_addresses; EXPECT_THAT( client_stats.drop_token_counts, ::testing::ElementsAre( -- cgit v1.2.3 From 350a4239fe3a56fb20b3eea99badaf307c652ad5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 18 Sep 2017 22:41:24 -0700 Subject: Fix compile error --- test/cpp/naming/resolver_component_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 0857fb6a32..396f4ceb9e 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -270,7 +270,8 @@ void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, CheckLBPolicyResultLocked(channel_args, args); gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); + GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, args->pollset, NULL)); gpr_mu_unlock(args->mu); } -- cgit v1.2.3 From 67cd21e6c7e1e9347760349c29f32d0d5b42c843 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 19 Sep 2017 16:21:20 +0200 Subject: adjust sanitizer timeouts --- .../internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg | 2 +- .../internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg | 2 +- .../internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_c_ubsan.cfg | 2 +- .../linux/sanitizer/pull_request/grpc_cpp_asan.cfg | 2 +- tools/run_tests/run_tests_matrix.py | 12 +++++++++++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index db9cb4f597..1daf7a514e 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index ba651d357c..a8503b7bcb 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index ccbd95ae63..12af4581eb 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index c240d3a573..0d3803bf23 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 38db2f4f60..557561810b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -17,7 +17,7 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 1440 +timeout_mins: 240 action { define_artifacts { regex: "**/*sponge_log.xml" diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 957e7b569e..2417c5ae55 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -35,6 +35,9 @@ _DEFAULT_RUNTESTS_TIMEOUT = 1*60*60 # clang docker. _CPP_RUNTESTS_TIMEOUT = 4*60*60 +# C++ TSAN takes longer than other sanitizers +_CPP_TSAN_RUNTESTS_TIMEOUT = 8*60*60 + # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 @@ -190,12 +193,19 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) test_jobs += _generate_jobs(languages=['c++'], - configs=['asan', 'tsan'], + configs=['asan'], platforms=['linux'], labels=['sanitizers', 'corelang'], extra_args=extra_args, inner_jobs=inner_jobs, timeout_seconds=_CPP_RUNTESTS_TIMEOUT) + test_jobs += _generate_jobs(languages=['c++'], + configs=['tsan'], + platforms=['linux'], + labels=['sanitizers', 'corelang'], + extra_args=extra_args, + inner_jobs=inner_jobs, + timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT) return test_jobs -- cgit v1.2.3 From 43b55221e4a109f9dc6845c2c081dd5ea17249ae Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 18 Sep 2017 15:03:53 -0700 Subject: Cancel outstanding streams when shutting down the client --- test/cpp/qps/client_async.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 1a4438047d..f5807da81e 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -56,6 +56,7 @@ class ClientRpcContext { } virtual void Start(CompletionQueue* cq, const ClientConfig& config) = 0; + virtual void TryCancel() = 0; }; template @@ -110,6 +111,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -142,8 +144,6 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { } }; -typedef std::forward_list context_list; - template class AsyncClient : public ClientImpl { // Specify which protected members we are using since there is no @@ -247,6 +247,7 @@ class AsyncClient : public ClientImpl { // this thread isn't supposed to shut down std::lock_guard l(shutdown_state_[thread_idx]->mutex); if (shutdown_state_[thread_idx]->shutdown) { + ctx->TryCancel(); delete ctx; return true; } @@ -388,6 +389,7 @@ class ClientRpcContextStreamingPingPongImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -527,6 +529,7 @@ class ClientRpcContextStreamingFromClientImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -644,6 +647,7 @@ class ClientRpcContextStreamingFromServerImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; @@ -786,6 +790,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { stub_, req_, next_issue_, prepare_req_, callback_); clone->StartInternal(cq, messages_per_stream_); } + void TryCancel() override { context_.TryCancel(); } private: grpc::ClientContext context_; -- cgit v1.2.3 From 0f7dc4f71d074157263462e49079d95277e83dfe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 19 Sep 2017 12:17:25 -0700 Subject: Use shortname for shortname for json_run_localhost tests --- tools/run_tests/run_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 176c1097ee..b38108d456 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -350,11 +350,12 @@ class CLanguage(object): environ=env)) else: cmdline = [binary] + target['args'] + shortname = target.get('shortname', ' '.join( + pipes.quote(arg) + for arg in cmdline)) + shortname += shortname_ext out.append(self.config.job_spec(cmdline, - shortname=' '.join( - pipes.quote(arg) - for arg in cmdline) + - shortname_ext, + shortname=shortname, cpu_cost=cpu_cost, flaky=target.get('flaky', False), timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS) * timeout_scaling, -- cgit v1.2.3 From 2746bd59982a5c186cf12d2007c73bd9fc4c3e33 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 19 Sep 2017 11:12:55 -0700 Subject: Fix cython definition to match c definition This is required to compile with Cython options that are more strict about const correctness. --- src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi | 2 +- src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 28cf114451..840af5c43a 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -523,7 +523,7 @@ cdef extern from "grpc/compression.h": int grpc_compression_algorithm_parse( grpc_slice value, grpc_compression_algorithm *algorithm) nogil int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, - char **name) nogil + const char **name) nogil grpc_compression_algorithm grpc_compression_algorithm_for_level( grpc_compression_level level, uint32_t accepted_encodings) nogil void grpc_compression_options_init(grpc_compression_options *opts) nogil diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi index 0a2a6eee05..d860173b5d 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi @@ -782,7 +782,7 @@ cdef class CompressionOptions: def compression_algorithm_name(grpc_compression_algorithm algorithm): - cdef char* name + cdef const char* name with nogil: grpc_compression_algorithm_name(algorithm, &name) # Let Cython do the right thing with string casting -- cgit v1.2.3 From 7fcdd7add07a4390c8d2e430d519602206433159 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 19 Sep 2017 14:11:21 -0700 Subject: Really spell out code generation --- src/core/lib/debug/stats_data.c | 14 ++++++++++++++ tools/codegen/core/gen_stats_data.py | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index e4723dac0a..8eaf308b71 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -278,6 +278,7 @@ const uint8_t grpc_stats_table_7[102] = { const int grpc_stats_table_8[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64}; const uint8_t grpc_stats_table_9[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5}; void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 262144); if (value < 6) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, @@ -303,6 +304,7 @@ void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_0, 64)); } void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 29) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -329,6 +331,7 @@ void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_2, 128)); } void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, @@ -354,6 +357,7 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -379,6 +383,7 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -404,6 +409,7 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, @@ -430,6 +436,7 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { } void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -457,6 +464,7 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM( @@ -484,6 +492,7 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_initial_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -513,6 +522,7 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write( } void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -540,6 +550,7 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -569,6 +580,7 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( } void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -596,6 +608,7 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -622,6 +635,7 @@ void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { + /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 64); if (value < 3) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 8359734c84..10ad0cc831 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -147,7 +147,8 @@ def gen_bucket_code(histogram): shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets) #print first_nontrivial, shift_data, bounds #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max + code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n' + code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) if first_nontrivial is None: code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n' @@ -407,4 +408,3 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: for counter in inst_map['Counter']: columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) - -- cgit v1.2.3 From 79c12b9dc02f5558af6868acd2cafdfaa600d89e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 18 Sep 2017 22:02:52 -0700 Subject: Drain readable fd --- .../dns/c_ares/grpc_ares_ev_driver_posix.c | 47 ++++++++++++++++++++-- test/cpp/naming/resolver_component_test.cc | 4 +- test/cpp/naming/resolver_component_tests_runner.sh | 8 ++++ test/cpp/naming/resolver_test_record_groups.yaml | 5 --- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index 7f1f57259a..d38fe66d06 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -20,6 +20,7 @@ #if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET) #include +#include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" @@ -54,6 +55,8 @@ typedef struct fd_node { bool readable_registered; /** if the writable closure has been registered */ bool writable_registered; + /** if the fd is being shut down */ + bool shutting_down; } fd_node; struct grpc_ares_ev_driver { @@ -100,7 +103,6 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { GPR_ASSERT(!fdn->readable_registered); GPR_ASSERT(!fdn->writable_registered); gpr_mu_destroy(&fdn->mu); - grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->fd); /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up immediately by another thread, and should not be closed by the following grpc_fd_orphan. */ @@ -109,6 +111,20 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { gpr_free(fdn); } +static void fd_node_shutdown(grpc_exec_ctx *exec_ctx, fd_node *fdn) { + gpr_mu_lock(&fdn->mu); + fdn->shutting_down = true; + if (!fdn->readable_registered && !fdn->writable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + } else { + grpc_fd_shutdown( + exec_ctx, fdn->fd, + GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown")); + gpr_mu_unlock(&fdn->mu); + } +} + grpc_error *grpc_ares_ev_driver_create(grpc_ares_ev_driver **ev_driver, grpc_pollset_set *pollset_set) { *ev_driver = (grpc_ares_ev_driver *)gpr_malloc(sizeof(grpc_ares_ev_driver)); @@ -175,18 +191,34 @@ static fd_node *pop_fd_node(fd_node **head, int fd) { return NULL; } +/* Check if \a fd is still readable */ +static bool grpc_ares_is_fd_still_readable(grpc_ares_ev_driver *ev_driver, + int fd) { + size_t bytes_available = 0; + return ioctl(fd, FIONREAD, &bytes_available) == 0 && bytes_available > 0; +} + static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); fdn->readable_registered = false; + if (fdn->shutting_down && !fdn->writable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + grpc_ares_ev_driver_unref(ev_driver); + return; + } gpr_mu_unlock(&fdn->mu); gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), - ARES_SOCKET_BAD); + do { + ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), + ARES_SOCKET_BAD); + } while ( + grpc_ares_is_fd_still_readable(ev_driver, grpc_fd_wrapped_fd(fdn->fd))); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -208,6 +240,12 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); fdn->writable_registered = false; + if (fdn->shutting_down && !fdn->readable_registered) { + gpr_mu_unlock(&fdn->mu); + fd_node_destroy(exec_ctx, fdn); + grpc_ares_ev_driver_unref(ev_driver); + return; + } gpr_mu_unlock(&fdn->mu); gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); @@ -256,6 +294,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, fdn->ev_driver = ev_driver; fdn->readable_registered = false; fdn->writable_registered = false; + fdn->shutting_down = false; gpr_mu_init(&fdn->mu); GRPC_CLOSURE_INIT(&fdn->read_closure, on_readable_cb, fdn, grpc_schedule_on_exec_ctx); @@ -296,7 +335,7 @@ static void grpc_ares_notify_on_event_locked(grpc_exec_ctx *exec_ctx, while (ev_driver->fds != NULL) { fd_node *cur = ev_driver->fds; ev_driver->fds = ev_driver->fds->next; - fd_node_destroy(exec_ctx, cur); + fd_node_shutdown(exec_ctx, cur); } ev_driver->fds = new_list; // If the ev driver has no working fd, all the tasks are done. diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 0857fb6a32..6a2ce37d00 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -267,7 +267,9 @@ void CheckResolverResultLocked(grpc_exec_ctx *exec_ctx, void *argsp, } EXPECT_THAT(args->expected_addrs, UnorderedElementsAreArray(found_lb_addrs)); CheckServiceConfigResultLocked(channel_args, args); - CheckLBPolicyResultLocked(channel_args, args); + if (args->expected_service_config_string == "") { + CheckLBPolicyResultLocked(channel_args, args); + } gpr_atm_rel_store(&args->done_atm, 1); gpr_mu_lock(args->mu); GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(args->pollset, NULL)); diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh index 83b03b67a3..cf71c9dcf9 100755 --- a/test/cpp/naming/resolver_component_tests_runner.sh +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -168,6 +168,14 @@ $FLAGS_test_bin_path \ --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & wait $! || EXIT_CODE=1 +$FLAGS_test_bin_path \ + --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='' \ + --local_dns_server_address=127.0.0.1:$FLAGS_test_dns_server_port & +wait $! || EXIT_CODE=1 + kill -SIGTERM $DNS_SERVER_PID || true wait exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 67c611d831..c2e8ddd0ed 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -137,11 +137,6 @@ resolver_component_tests: - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} srv-ipv6-target-has-backend-and-balancer: - {TTL: '2100', data: '2607:f8b0:400a:801::1002', type: AAAA} - -resolver_component_tests_TODO: -- 'TODO: enable this large-txt-record test once working. (it is much longer than 512 - bytes, likely to cause use of TCP even if max payload for UDP is changed somehow, - e.g. via notes in RFC 2671)' - expected_addrs: - {address: '1.2.3.4:443', is_balancer: false} expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' -- cgit v1.2.3 From c6f9cf3158d04359d74d14676c0b1d3b1e504919 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Tue, 19 Sep 2017 14:34:30 -0700 Subject: update composer installing protobuf v3 for php-qps --- src/php/tests/qps/composer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/php/tests/qps/composer.json b/src/php/tests/qps/composer.json index 8c1e7b6c74..f8512648c4 100644 --- a/src/php/tests/qps/composer.json +++ b/src/php/tests/qps/composer.json @@ -1,8 +1,7 @@ { - "minimum-stability": "dev", "require": { "grpc/grpc": "dev-master", - "google/protobuf": "^v3.3.0" + "google/protobuf": "v3.4.1" }, "autoload": { "psr-4": { -- cgit v1.2.3 From 85629d069638acb5f8834dcff053debde98c69a7 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 19 Sep 2017 16:47:12 -0700 Subject: Add missing exec_ctx parameter to pollset_uv.c --- src/core/lib/iomgr/pollset_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index a79fe89d3e..2651325e25 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -145,7 +145,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return GRPC_ERROR_NONE; } -grpc_error *grpc_pollset_kick(grpc_pollset *pollset, +grpc_error *grpc_pollset_kick(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { GRPC_UV_ASSERT_SAME_THREAD(); uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0); -- cgit v1.2.3 From b56cf2d3bf6b44be4291b7a9962a10e181a3a753 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 19 Sep 2017 15:01:39 -0700 Subject: Protect fdn->fd --- .../resolver/dns/c_ares/grpc_ares_ev_driver_posix.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index d38fe66d06..b5be69c428 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -38,8 +38,6 @@ typedef struct fd_node { /** the owner of this fd node */ grpc_ares_ev_driver *ev_driver; - /** the grpc_fd owned by this fd node */ - grpc_fd *fd; /** a closure wrapping on_readable_cb, which should be invoked when the grpc_fd in this node becomes readable. */ grpc_closure read_closure; @@ -51,6 +49,8 @@ typedef struct fd_node { /** mutex guarding the rest of the state */ gpr_mu mu; + /** the grpc_fd owned by this fd node */ + grpc_fd *fd; /** if the readable closure has been registered */ bool readable_registered; /** if the writable closure has been registered */ @@ -203,6 +203,7 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); + const int fd = grpc_fd_wrapped_fd(fdn->fd); fdn->readable_registered = false; if (fdn->shutting_down && !fdn->writable_registered) { gpr_mu_unlock(&fdn->mu); @@ -212,13 +213,11 @@ static void on_readable_cb(grpc_exec_ctx *exec_ctx, void *arg, } gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "readable on %d", grpc_fd_wrapped_fd(fdn->fd)); + gpr_log(GPR_DEBUG, "readable on %d", fd); if (error == GRPC_ERROR_NONE) { do { - ares_process_fd(ev_driver->channel, grpc_fd_wrapped_fd(fdn->fd), - ARES_SOCKET_BAD); - } while ( - grpc_ares_is_fd_still_readable(ev_driver, grpc_fd_wrapped_fd(fdn->fd))); + ares_process_fd(ev_driver->channel, fd, ARES_SOCKET_BAD); + } while (grpc_ares_is_fd_still_readable(ev_driver, fd)); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled @@ -239,6 +238,7 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, fd_node *fdn = (fd_node *)arg; grpc_ares_ev_driver *ev_driver = fdn->ev_driver; gpr_mu_lock(&fdn->mu); + const int fd = grpc_fd_wrapped_fd(fdn->fd); fdn->writable_registered = false; if (fdn->shutting_down && !fdn->readable_registered) { gpr_mu_unlock(&fdn->mu); @@ -248,10 +248,9 @@ static void on_writable_cb(grpc_exec_ctx *exec_ctx, void *arg, } gpr_mu_unlock(&fdn->mu); - gpr_log(GPR_DEBUG, "writable on %d", grpc_fd_wrapped_fd(fdn->fd)); + gpr_log(GPR_DEBUG, "writable on %d", fd); if (error == GRPC_ERROR_NONE) { - ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, - grpc_fd_wrapped_fd(fdn->fd)); + ares_process_fd(ev_driver->channel, ARES_SOCKET_BAD, fd); } else { // If error is not GRPC_ERROR_NONE, it means the fd has been shutdown or // timed out. The pending lookups made on this ev_driver will be cancelled -- cgit v1.2.3 From 4a6b08aa4706b870f51ec61d03fd786a2fc3066a Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 19 Sep 2017 18:49:52 -0700 Subject: Modifying uv tests to reflect exec_ctx parameter for grpc_pollset_kick --- test/core/iomgr/tcp_client_uv_test.c | 8 ++++---- test/core/iomgr/tcp_server_uv_test.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index 9927356613..b0ff11b39a 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -46,11 +46,11 @@ static gpr_timespec test_deadline(void) { return grpc_timeout_seconds_to_deadline(10); } -static void finish_connection() { +static void finish_connection(grpc_exec_ctx *exec_ctx) { gpr_mu_lock(g_mu); g_connections_complete++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } @@ -63,7 +63,7 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, GRPC_ERROR_CREATE_FROM_STATIC_STRING("must_succeed called")); grpc_endpoint_destroy(exec_ctx, g_connecting); g_connecting = NULL; - finish_connection(); + finish_connection(exec_ctx); } static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c index 8f4d553d1e..bd8ccb4f60 100644 --- a/test/core/iomgr/tcp_server_uv_test.c +++ b/test/core/iomgr/tcp_server_uv_test.c @@ -111,8 +111,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp, gpr_mu_lock(g_mu); g_result = temp_result; g_nconnects++; - GPR_ASSERT( - GRPC_LOG_IF_ERROR("pollset_kick", grpc_pollset_kick(g_pollset, NULL))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick", + grpc_pollset_kick(exec_ctx, g_pollset, NULL))); gpr_mu_unlock(g_mu); } -- cgit v1.2.3 From 5bb43935e11b2abc623bb278af497a5bd4e16b09 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 19 Sep 2017 18:54:36 -0700 Subject: Backport #12615 --- src/python/grpcio_testing/grpc_testing/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_testing/grpc_testing/__init__.py b/src/python/grpcio_testing/grpc_testing/__init__.py index 14e25f09e2..2cf6741fca 100644 --- a/src/python/grpcio_testing/grpc_testing/__init__.py +++ b/src/python/grpcio_testing/grpc_testing/__init__.py @@ -213,7 +213,7 @@ class StreamStreamChannelRpc(six.with_metaclass(abc.ABCMeta)): raise NotImplementedError() -class Channel(six.with_metaclass(abc.ABCMeta), grpc.Channel): +class Channel(six.with_metaclass(abc.ABCMeta, grpc.Channel)): """A grpc.Channel double with which to test a system that invokes RPCs.""" @abc.abstractmethod -- cgit v1.2.3 From 4b527a7d8c895abe537384b49465cfa6a53b6162 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 20 Sep 2017 13:35:05 -0700 Subject: Fix test flake of stream_compression_compressed_payload #12307 --- test/core/end2end/tests/compressed_payload.c | 9 +++++---- test/core/end2end/tests/stream_compression_compressed_payload.c | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 639af15454..ce86e97b83 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -151,6 +151,11 @@ static void request_for_disabled_algorithm( grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -187,10 +192,6 @@ static void request_for_disabled_algorithm( error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - error = - grpc_server_request_call(f.server, &s, &call_details, - &request_metadata_recv, f.cq, f.cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); cq_verify(cqv); diff --git a/test/core/end2end/tests/stream_compression_compressed_payload.c b/test/core/end2end/tests/stream_compression_compressed_payload.c index 5a69091a57..094c4de1c0 100644 --- a/test/core/end2end/tests/stream_compression_compressed_payload.c +++ b/test/core/end2end/tests/stream_compression_compressed_payload.c @@ -151,6 +151,11 @@ static void request_for_disabled_algorithm( grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -187,10 +192,6 @@ static void request_for_disabled_algorithm( error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - error = - grpc_server_request_call(f.server, &s, &call_details, - &request_metadata_recv, f.cq, f.cq, tag(101)); - GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(101), true); cq_verify(cqv); -- cgit v1.2.3 From a17ad0ce3cf83b5d252474e6d9d73e040a4e6f56 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 20 Sep 2017 14:59:49 -0700 Subject: Missing argument in finish_connection in tcp_client_uv_test --- test/core/iomgr/tcp_client_uv_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index b0ff11b39a..0f1db4705b 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -69,7 +69,7 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { GPR_ASSERT(g_connecting == NULL); GPR_ASSERT(error != GRPC_ERROR_NONE); - finish_connection(); + finish_connection(exec_ctx); } static void close_cb(uv_handle_t *handle) { gpr_free(handle); } -- cgit v1.2.3 From 8b255c7e4764d316adbdc90e49ec86f2336222f6 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 20 Sep 2017 15:47:29 -0700 Subject: Add line count job to Kokoro --- tools/internal_ci/linux/grpc_line_count.cfg | 19 +++++++++++++++++++ tools/internal_ci/linux/grpc_line_count.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tools/internal_ci/linux/grpc_line_count.cfg create mode 100755 tools/internal_ci/linux/grpc_line_count.sh diff --git a/tools/internal_ci/linux/grpc_line_count.cfg b/tools/internal_ci/linux/grpc_line_count.cfg new file mode 100644 index 0000000000..5854d9c9c2 --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.cfg @@ -0,0 +1,19 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_line_count.sh" +timeout_mins: 30 diff --git a/tools/internal_ci/linux/grpc_line_count.sh b/tools/internal_ci/linux/grpc_line_count.sh new file mode 100755 index 0000000000..22e484ba2f --- /dev/null +++ b/tools/internal_ci/linux/grpc_line_count.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script counts the numbers of line in gRPC's repo and uploads to BQ +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +git submodule update --init + +# Install cloc +git clone -b v1.72 https://github.com/AlDanial/cloc/ ~/cloc +PERL_MM_USE_DEFAULT=1 sudo perl -MCPAN -e 'install Regexp::Common; install Algorithm::Diff' +sudo make install -C ~/cloc/Unix + +./tools/line_count/collect-now.sh -- cgit v1.2.3 From 46c821fe811d0cd2ab18feaec368443d4fd39a75 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 20 Sep 2017 19:23:22 -0700 Subject: Remove extra record_to_resolve --- test/cpp/naming/resolver_test_record_groups.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index c2e8ddd0ed..33d774ca70 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -141,7 +141,6 @@ resolver_component_tests: - {address: '1.2.3.4:443', is_balancer: false} expected_chosen_service_config: '{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' expected_lb_policy: null - record_to_resolve: srv-ipv6-target-has-backend-and-balancer record_to_resolve: ipv4-config-causing-fallback-to-tcp records: ipv4-config-causing-fallback-to-tcp: -- cgit v1.2.3 From e232f1bd5a97a538bc880282416c39300cb1d62f Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Thu, 21 Sep 2017 05:52:25 +0000 Subject: Clean up even more uses of gRPC in _pb2.py files --- src/python/grpcio_tests/tests/http2/negative_http2_client.py | 4 ++-- src/python/grpcio_tests/tests/interop/client.py | 6 +++--- src/python/grpcio_tests/tests/qps/benchmark_client.py | 4 ++-- src/python/grpcio_tests/tests/stress/client.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/python/grpcio_tests/tests/http2/negative_http2_client.py b/src/python/grpcio_tests/tests/http2/negative_http2_client.py index 6d8a6bce77..8dab5b67f1 100644 --- a/src/python/grpcio_tests/tests/http2/negative_http2_client.py +++ b/src/python/grpcio_tests/tests/http2/negative_http2_client.py @@ -17,7 +17,7 @@ import argparse import grpc import time -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from src.proto.grpc.testing import messages_pb2 @@ -147,7 +147,7 @@ def _stub(server_host, server_port): target = '{}:{}'.format(server_host, server_port) channel = grpc.insecure_channel(target) grpc.channel_ready_future(channel).result() - return test_pb2.TestServiceStub(channel) + return test_pb2_grpc.TestServiceStub(channel) def main(): diff --git a/src/python/grpcio_tests/tests/interop/client.py b/src/python/grpcio_tests/tests/interop/client.py index 47ae96472d..e520c08290 100644 --- a/src/python/grpcio_tests/tests/interop/client.py +++ b/src/python/grpcio_tests/tests/interop/client.py @@ -19,7 +19,7 @@ import os from google import auth as google_auth from google.auth import jwt as google_auth_jwt import grpc -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from tests.interop import methods from tests.interop import resources @@ -106,9 +106,9 @@ def _stub(args): else: channel = grpc.insecure_channel(target) if args.test_case == "unimplemented_service": - return test_pb2.UnimplementedServiceStub(channel) + return test_pb2_grpc.UnimplementedServiceStub(channel) else: - return test_pb2.TestServiceStub(channel) + return test_pb2_grpc.TestServiceStub(channel) def _test_case_from_arg(test_case_arg): diff --git a/src/python/grpcio_tests/tests/qps/benchmark_client.py b/src/python/grpcio_tests/tests/qps/benchmark_client.py index 5f4df79c5b..17fa61ea36 100644 --- a/src/python/grpcio_tests/tests/qps/benchmark_client.py +++ b/src/python/grpcio_tests/tests/qps/benchmark_client.py @@ -22,7 +22,7 @@ from six.moves import queue import grpc from src.proto.grpc.testing import messages_pb2 -from src.proto.grpc.testing import services_pb2 +from src.proto.grpc.testing import services_pb2_grpc from tests.unit import resources from tests.unit import test_common @@ -58,7 +58,7 @@ class BenchmarkClient: if config.payload_config.WhichOneof('payload') == 'simple_params': self._generic = False - self._stub = services_pb2.BenchmarkServiceStub(channel) + self._stub = services_pb2_grpc.BenchmarkServiceStub(channel) payload = messages_pb2.Payload( body='\0' * config.payload_config.simple_params.req_size) self._request = messages_pb2.SimpleRequest( diff --git a/src/python/grpcio_tests/tests/stress/client.py b/src/python/grpcio_tests/tests/stress/client.py index d5ff0064fd..40caa3926a 100644 --- a/src/python/grpcio_tests/tests/stress/client.py +++ b/src/python/grpcio_tests/tests/stress/client.py @@ -20,7 +20,7 @@ import threading import grpc from six.moves import queue from src.proto.grpc.testing import metrics_pb2_grpc -from src.proto.grpc.testing import test_pb2 +from src.proto.grpc.testing import test_pb2_grpc from tests.interop import methods from tests.interop import resources @@ -133,7 +133,7 @@ def run_test(args): for _ in xrange(args.num_channels_per_server): channel = _get_channel(test_server_target, args) for _ in xrange(args.num_stubs_per_channel): - stub = test_pb2.TestServiceStub(channel) + stub = test_pb2_grpc.TestServiceStub(channel) runner = test_runner.TestRunner(stub, test_cases, hist, exception_queue, stop_event) runners.append(runner) -- cgit v1.2.3 From f1a25c7df1f4f18686fdb8de7bc13f103be34f57 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 21 Sep 2017 14:07:02 +0200 Subject: split windows PR basictests into dbg and opt --- .../windows/pull_request/grpc_basictests_dbg.cfg | 30 ++++++++++++++++++++++ .../windows/pull_request/grpc_basictests_opt.cfg | 30 ++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg create mode 100644 tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg new file mode 100644 index 0000000000..81a1e9d3c8 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows dbg -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg new file mode 100644 index 0000000000..3bb6510ca7 --- /dev/null +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests windows opt -j 1 --inner_jobs 8 --internal_ci --max_time=3600" +} -- cgit v1.2.3 From 7862097e5bb7eedd7e5d6af83973703054b3e5ef Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 21 Sep 2017 11:38:32 -0700 Subject: Add HPACK stats --- .../ext/transport/chttp2/transport/hpack_encoder.c | 61 ++++-- .../ext/transport/chttp2/transport/hpack_parser.c | 31 ++- src/core/lib/debug/stats_data.c | 48 +++++ src/core/lib/debug/stats_data.h | 80 ++++++++ src/core/lib/debug/stats_data.yaml | 44 +++++ src/core/lib/debug/stats_data_bq_schema.sql | 22 +++ tools/run_tests/performance/massage_qps_stats.py | 22 +++ .../performance/scenario_result_schema.json | 220 +++++++++++++++++++++ 8 files changed, 503 insertions(+), 25 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 9dfbd3c3a9..4e97595173 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -33,6 +33,7 @@ #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" #include "src/core/ext/transport/chttp2/transport/hpack_table.h" #include "src/core/ext/transport/chttp2/transport/varint.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/metadata.h" @@ -271,8 +272,10 @@ static void add_elem(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, } } -static void emit_indexed(grpc_chttp2_hpack_compressor *c, uint32_t elem_index, +static void emit_indexed(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t elem_index, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_INDEXED(exec_ctx); uint32_t len = GRPC_CHTTP2_VARINT_LENGTH(elem_index, 1); GRPC_CHTTP2_WRITE_VARINT(elem_index, 1, 0x80, add_tiny_header_data(st, len), len); @@ -284,15 +287,18 @@ typedef struct { bool insert_null_before_wire_value; } wire_value; -static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { +static wire_value get_wire_value(grpc_exec_ctx *exec_ctx, grpc_mdelem elem, + bool true_binary_enabled) { wire_value wire_val; if (grpc_is_binary_header(GRPC_MDKEY(elem))) { if (true_binary_enabled) { + GRPC_STATS_INC_HPACK_SEND_BINARY(exec_ctx); wire_val.huffman_prefix = 0x00; wire_val.insert_null_before_wire_value = true; wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); } else { + GRPC_STATS_INC_HPACK_SEND_BINARY_BASE64(exec_ctx); wire_val.huffman_prefix = 0x80; wire_val.insert_null_before_wire_value = false; wire_val.data = @@ -300,6 +306,7 @@ static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) { } } else { /* TODO(ctiller): opportunistically compress non-binary headers */ + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); wire_val.huffman_prefix = 0x00; wire_val.insert_null_before_wire_value = false; wire_val.data = grpc_slice_ref_internal(GRPC_MDVALUE(elem)); @@ -316,11 +323,14 @@ static void add_wire_value(framer_state *st, wire_value v) { add_header_data(st, v.data); } -static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_incidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t key_index, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX(exec_ctx); uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 2); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); size_t len_val = wire_value_length(value); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); @@ -332,11 +342,14 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_noidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, uint32_t key_index, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX(exec_ctx); uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 4); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); size_t len_val = wire_value_length(value); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); @@ -348,10 +361,13 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); uint32_t len_val = (uint32_t)wire_value_length(value); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); @@ -366,10 +382,13 @@ static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c, add_wire_value(st, value); } -static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c, +static void emit_lithdr_noidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { + GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); - wire_value value = get_wire_value(elem, st->use_true_binary_metadata); + wire_value value = + get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); uint32_t len_val = (uint32_t)wire_value_length(value); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); @@ -422,7 +441,7 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, gpr_free(v); } if (!GRPC_MDELEM_IS_INTERNED(elem)) { - emit_lithdr_noidx_v(c, elem, st); + emit_lithdr_noidx_v(exec_ctx, c, elem, st); return; } @@ -444,16 +463,16 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_2(elem_hash)], elem) && c->indices_elems[HASH_FRAGMENT_2(elem_hash)] > c->tail_remote_index) { /* HIT: complete element (first cuckoo hash) */ - emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]), - st); + emit_indexed(exec_ctx, c, + dynidx(c, c->indices_elems[HASH_FRAGMENT_2(elem_hash)]), st); return; } if (grpc_mdelem_eq(c->entries_elems[HASH_FRAGMENT_3(elem_hash)], elem) && c->indices_elems[HASH_FRAGMENT_3(elem_hash)] > c->tail_remote_index) { /* HIT: complete element (second cuckoo hash) */ - emit_indexed(c, dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]), - st); + emit_indexed(exec_ctx, c, + dynidx(c, c->indices_elems[HASH_FRAGMENT_3(elem_hash)]), st); return; } @@ -471,11 +490,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { - emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_incidx(exec_ctx, c, dynidx(c, indices_key), elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_noidx(exec_ctx, c, dynidx(c, indices_key), elem, st); return; } GPR_UNREACHABLE_CODE(return ); @@ -487,11 +506,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { - emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_incidx(exec_ctx, c, dynidx(c, indices_key), elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st); + emit_lithdr_noidx(exec_ctx, c, dynidx(c, indices_key), elem, st); return; } GPR_UNREACHABLE_CODE(return ); @@ -500,11 +519,11 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, /* no elem, key in the table... fall back to literal emission */ if (should_add_elem) { - emit_lithdr_incidx_v(c, elem, st); + emit_lithdr_incidx_v(exec_ctx, c, elem, st); add_elem(exec_ctx, c, elem); return; } else { - emit_lithdr_noidx_v(c, elem, st); + emit_lithdr_noidx_v(exec_ctx, c, elem, st); return; } GPR_UNREACHABLE_CODE(return ); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 901e7413c0..7e3ab565f5 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -30,6 +30,7 @@ #include #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" +#include "src/core/lib/debug/stats.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" @@ -777,8 +778,7 @@ static grpc_error *parse_stream_dep0(grpc_exec_ctx *exec_ctx, return parse_stream_dep1(exec_ctx, p, cur + 1, end); } -/* emit an indexed field; for now just logs it to console; jumps to - begin the next field on completion */ +/* emit an indexed field; jumps to begin the next field on completion */ static grpc_error *finish_indexed_field(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, @@ -792,6 +792,7 @@ static grpc_error *finish_indexed_field(grpc_exec_ctx *exec_ctx, GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents); } GRPC_MDELEM_REF(md); + GRPC_STATS_INC_HPACK_RECV_INDEXED(exec_ctx); grpc_error *err = on_hdr(exec_ctx, p, md, 0); if (err != GRPC_ERROR_NONE) return err; return parse_begin(exec_ctx, p, cur, end); @@ -820,14 +821,14 @@ static grpc_error *parse_indexed_field_x(grpc_exec_ctx *exec_ctx, return parse_value0(exec_ctx, p, cur + 1, end); } -/* finish a literal header with incremental indexing: just log, and jump to ' - begin */ +/* finish a literal header with incremental indexing */ static grpc_error *finish_lithdr_incidx(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -842,6 +843,7 @@ static grpc_error *finish_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -898,6 +900,7 @@ static grpc_error *finish_lithdr_notidx(grpc_exec_ctx *exec_ctx, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -912,6 +915,7 @@ static grpc_error *finish_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -968,6 +972,7 @@ static grpc_error *finish_lithdr_nvridx(grpc_exec_ctx *exec_ctx, const uint8_t *end) { grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), @@ -982,6 +987,7 @@ static grpc_error *finish_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(exec_ctx); grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), @@ -1310,9 +1316,11 @@ static grpc_error *append_string(grpc_exec_ctx *exec_ctx, /* 'true-binary' case */ ++cur; p->binary = NOT_BINARY; + GRPC_STATS_INC_HPACK_RECV_BINARY(exec_ctx); append_bytes(str, cur, (size_t)(end - cur)); return GRPC_ERROR_NONE; } + GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(exec_ctx); /* fallthrough */ b64_byte0: case B64_BYTE0: @@ -1510,6 +1518,7 @@ static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser_string *str) { if (!p->huff && binary == NOT_BINARY && (end - cur) >= (intptr_t)p->strlen && p->current_slice_refcount != NULL) { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx); str->copied = false; str->data.referenced.refcount = p->current_slice_refcount; str->data.referenced.data.refcounted.bytes = (uint8_t *)cur; @@ -1523,6 +1532,20 @@ static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, p->parsing.str = str; p->huff_state = 0; p->binary = binary; + switch (p->binary) { + case NOT_BINARY: + if (p->huff) { + GRPC_STATS_INC_HPACK_RECV_HUFFMAN(exec_ctx); + } else { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx); + } + break; + case BINARY_BEGIN: + /* stats incremented later: don't know true binary or not */ + break; + default: + abort(); + } return parse_string(exec_ctx, p, cur, end); } diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index 8eaf308b71..fb6055f795 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -77,6 +77,28 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "http2_initiate_write_due_to_transport_flow_control_unstalled", "http2_initiate_write_due_to_ping_response", "http2_initiate_write_due_to_force_rst_stream", + "hpack_recv_indexed", + "hpack_recv_lithdr_incidx", + "hpack_recv_lithdr_incidx_v", + "hpack_recv_lithdr_notidx", + "hpack_recv_lithdr_notidx_v", + "hpack_recv_lithdr_nvridx", + "hpack_recv_lithdr_nvridx_v", + "hpack_recv_uncompressed", + "hpack_recv_huffman", + "hpack_recv_binary", + "hpack_recv_binary_base64", + "hpack_send_indexed", + "hpack_send_lithdr_incidx", + "hpack_send_lithdr_incidx_v", + "hpack_send_lithdr_notidx", + "hpack_send_lithdr_notidx_v", + "hpack_send_lithdr_nvridx", + "hpack_send_lithdr_nvridx_v", + "hpack_send_uncompressed", + "hpack_send_huffman", + "hpack_send_binary", + "hpack_send_binary_base64", "combiner_locks_initiated", "combiner_locks_scheduled_items", "combiner_locks_scheduled_final_items", @@ -157,6 +179,32 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "'transport_flow_control_unstalled'", "Number of HTTP2 writes initiated due to 'ping_response'", "Number of HTTP2 writes initiated due to 'force_rst_stream'", + "Number of HPACK indexed fields received", + "Number of HPACK literal headers received with incremental indexing", + "Number of HPACK literal headers received with incremental indexing and " + "literal keys", + "Number of HPACK literal headers received with no indexing", + "Number of HPACK literal headers received with no indexing and literal " + "keys", + "Number of HPACK literal headers received with never-indexing", + "Number of HPACK literal headers received with never-indexing and literal " + "keys", + "Number of uncompressed strings received in metadata", + "Number of huffman encoded strings received in metadata", + "Number of binary strings received in metadata", + "Number of binary strings received encoded in base64 in metadata", + "Number of HPACK indexed fields sent", + "Number of HPACK literal headers sent with incremental indexing", + "Number of HPACK literal headers sent with incremental indexing and " + "literal keys", + "Number of HPACK literal headers sent with no indexing", + "Number of HPACK literal headers sent with no indexing and literal keys", + "Number of HPACK literal headers sent with never-indexing", + "Number of HPACK literal headers sent with never-indexing and literal keys", + "Number of uncompressed strings sent in metadata", + "Number of huffman encoded strings sent in metadata", + "Number of binary strings received in metadata", + "Number of binary strings received encoded in base64 in metadata", "Number of combiner lock entries by process (first items queued to a " "combiner)", "Number of items scheduled against combiner locks", diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 099302f29c..6c0ad30543 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -79,6 +79,28 @@ typedef enum { GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL_UNSTALLED, GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_PING_RESPONSE, GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM, + GRPC_STATS_COUNTER_HPACK_RECV_INDEXED, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX, + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX_V, + GRPC_STATS_COUNTER_HPACK_RECV_UNCOMPRESSED, + GRPC_STATS_COUNTER_HPACK_RECV_HUFFMAN, + GRPC_STATS_COUNTER_HPACK_RECV_BINARY, + GRPC_STATS_COUNTER_HPACK_RECV_BINARY_BASE64, + GRPC_STATS_COUNTER_HPACK_SEND_INDEXED, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX, + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX_V, + GRPC_STATS_COUNTER_HPACK_SEND_UNCOMPRESSED, + GRPC_STATS_COUNTER_HPACK_SEND_HUFFMAN, + GRPC_STATS_COUNTER_HPACK_SEND_BINARY, + GRPC_STATS_COUNTER_HPACK_SEND_BINARY_BASE64, GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_ITEMS, GRPC_STATS_COUNTER_COMBINER_LOCKS_SCHEDULED_FINAL_ITEMS, @@ -309,6 +331,64 @@ typedef enum { GRPC_STATS_INC_COUNTER( \ (exec_ctx), \ GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FORCE_RST_STREAM) +#define GRPC_STATS_INC_HPACK_RECV_INDEXED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_INDEXED) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_INCIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NOTIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX) +#define GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_LITHDR_NVRIDX_V) +#define GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_UNCOMPRESSED) +#define GRPC_STATS_INC_HPACK_RECV_HUFFMAN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_HUFFMAN) +#define GRPC_STATS_INC_HPACK_RECV_BINARY(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_RECV_BINARY) +#define GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_RECV_BINARY_BASE64) +#define GRPC_STATS_INC_HPACK_SEND_INDEXED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_INDEXED) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_INCIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NOTIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NVRIDX(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX) +#define GRPC_STATS_INC_HPACK_SEND_LITHDR_NVRIDX_V(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_LITHDR_NVRIDX_V) +#define GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_UNCOMPRESSED) +#define GRPC_STATS_INC_HPACK_SEND_HUFFMAN(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_HUFFMAN) +#define GRPC_STATS_INC_HPACK_SEND_BINARY(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_HPACK_SEND_BINARY) +#define GRPC_STATS_INC_HPACK_SEND_BINARY_BASE64(exec_ctx) \ + GRPC_STATS_INC_COUNTER((exec_ctx), \ + GRPC_STATS_COUNTER_HPACK_SEND_BINARY_BASE64) #define GRPC_STATS_INC_COMBINER_LOCKS_INITIATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), \ GRPC_STATS_COUNTER_COMBINER_LOCKS_INITIATED) diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index 2fd23ce675..de575f01c7 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -189,6 +189,50 @@ doc: Number of HTTP2 writes initiated due to 'ping_response' - counter: http2_initiate_write_due_to_force_rst_stream doc: Number of HTTP2 writes initiated due to 'force_rst_stream' +- counter: hpack_recv_indexed + doc: Number of HPACK indexed fields received +- counter: hpack_recv_lithdr_incidx + doc: Number of HPACK literal headers received with incremental indexing +- counter: hpack_recv_lithdr_incidx_v + doc: Number of HPACK literal headers received with incremental indexing and literal keys +- counter: hpack_recv_lithdr_notidx + doc: Number of HPACK literal headers received with no indexing +- counter: hpack_recv_lithdr_notidx_v + doc: Number of HPACK literal headers received with no indexing and literal keys +- counter: hpack_recv_lithdr_nvridx + doc: Number of HPACK literal headers received with never-indexing +- counter: hpack_recv_lithdr_nvridx_v + doc: Number of HPACK literal headers received with never-indexing and literal keys +- counter: hpack_recv_uncompressed + doc: Number of uncompressed strings received in metadata +- counter: hpack_recv_huffman + doc: Number of huffman encoded strings received in metadata +- counter: hpack_recv_binary + doc: Number of binary strings received in metadata +- counter: hpack_recv_binary_base64 + doc: Number of binary strings received encoded in base64 in metadata +- counter: hpack_send_indexed + doc: Number of HPACK indexed fields sent +- counter: hpack_send_lithdr_incidx + doc: Number of HPACK literal headers sent with incremental indexing +- counter: hpack_send_lithdr_incidx_v + doc: Number of HPACK literal headers sent with incremental indexing and literal keys +- counter: hpack_send_lithdr_notidx + doc: Number of HPACK literal headers sent with no indexing +- counter: hpack_send_lithdr_notidx_v + doc: Number of HPACK literal headers sent with no indexing and literal keys +- counter: hpack_send_lithdr_nvridx + doc: Number of HPACK literal headers sent with never-indexing +- counter: hpack_send_lithdr_nvridx_v + doc: Number of HPACK literal headers sent with never-indexing and literal keys +- counter: hpack_send_uncompressed + doc: Number of uncompressed strings sent in metadata +- counter: hpack_send_huffman + doc: Number of huffman encoded strings sent in metadata +- counter: hpack_send_binary + doc: Number of binary strings received in metadata +- counter: hpack_send_binary_base64 + doc: Number of binary strings received encoded in base64 in metadata # combiner locks - counter: combiner_locks_initiated doc: Number of combiner lock entries by process diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index b10985e449..0611ccaff0 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -52,6 +52,28 @@ http2_initiate_write_due_to_keepalive_ping_per_iteration:FLOAT, http2_initiate_write_due_to_transport_flow_control_unstalled_per_iteration:FLOAT, http2_initiate_write_due_to_ping_response_per_iteration:FLOAT, http2_initiate_write_due_to_force_rst_stream_per_iteration:FLOAT, +hpack_recv_indexed_per_iteration:FLOAT, +hpack_recv_lithdr_incidx_per_iteration:FLOAT, +hpack_recv_lithdr_incidx_v_per_iteration:FLOAT, +hpack_recv_lithdr_notidx_per_iteration:FLOAT, +hpack_recv_lithdr_notidx_v_per_iteration:FLOAT, +hpack_recv_lithdr_nvridx_per_iteration:FLOAT, +hpack_recv_lithdr_nvridx_v_per_iteration:FLOAT, +hpack_recv_uncompressed_per_iteration:FLOAT, +hpack_recv_huffman_per_iteration:FLOAT, +hpack_recv_binary_per_iteration:FLOAT, +hpack_recv_binary_base64_per_iteration:FLOAT, +hpack_send_indexed_per_iteration:FLOAT, +hpack_send_lithdr_incidx_per_iteration:FLOAT, +hpack_send_lithdr_incidx_v_per_iteration:FLOAT, +hpack_send_lithdr_notidx_per_iteration:FLOAT, +hpack_send_lithdr_notidx_v_per_iteration:FLOAT, +hpack_send_lithdr_nvridx_per_iteration:FLOAT, +hpack_send_lithdr_nvridx_v_per_iteration:FLOAT, +hpack_send_uncompressed_per_iteration:FLOAT, +hpack_send_huffman_per_iteration:FLOAT, +hpack_send_binary_per_iteration:FLOAT, +hpack_send_binary_base64_per_iteration:FLOAT, combiner_locks_initiated_per_iteration:FLOAT, combiner_locks_scheduled_items_per_iteration:FLOAT, combiner_locks_scheduled_final_items_per_iteration:FLOAT, diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index ef4f90cd93..e1620adad2 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -74,6 +74,28 @@ def massage_qps_stats(scenario_result): stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled") stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response") stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream") + stats["core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_indexed") + stats["core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx") + stats["core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx_v") + stats["core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx") + stats["core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx_v") + stats["core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx") + stats["core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx_v") + stats["core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_uncompressed") + stats["core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_huffman") + stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary") + stats["core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary_base64") + stats["core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_indexed") + stats["core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx") + stats["core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx_v") + stats["core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx") + stats["core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx_v") + stats["core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx") + stats["core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx_v") + stats["core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_uncompressed") + stats["core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_huffman") + stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary") + stats["core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary_base64") stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated") stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items") stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items") diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index e8fae6989e..c7b1904bd1 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -380,6 +380,116 @@ "name": "core_http2_initiate_write_due_to_force_rst_stream", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary_base64", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary_base64", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", @@ -1097,6 +1207,116 @@ "name": "core_http2_initiate_write_due_to_force_rst_stream", "type": "INTEGER" }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_recv_binary_base64", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_indexed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_incidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_notidx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_lithdr_nvridx_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_uncompressed", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_huffman", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "core_hpack_send_binary_base64", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "core_combiner_locks_initiated", -- cgit v1.2.3 From fb21821a08165628dd1e769cc59e5d38ba78e67d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 21 Sep 2017 11:43:09 -0700 Subject: Correct uncompressed string count --- src/core/ext/transport/chttp2/transport/hpack_encoder.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 4e97595173..a404b664e3 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -365,6 +365,7 @@ static void emit_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V(exec_ctx); + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); wire_value value = get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); @@ -386,6 +387,7 @@ static void emit_lithdr_noidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, grpc_mdelem elem, framer_state *st) { GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V(exec_ctx); + GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED(exec_ctx); uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)); wire_value value = get_wire_value(exec_ctx, elem, st->use_true_binary_metadata); -- cgit v1.2.3 From 3b6593679c28afd4455c7d38c417583ddc4341ef Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 21 Sep 2017 12:04:37 -0700 Subject: Fix format --- .../client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index b5be69c428..c30cc93b6f 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -118,9 +118,8 @@ static void fd_node_shutdown(grpc_exec_ctx *exec_ctx, fd_node *fdn) { gpr_mu_unlock(&fdn->mu); fd_node_destroy(exec_ctx, fdn); } else { - grpc_fd_shutdown( - exec_ctx, fdn->fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("c-ares fd shutdown")); + grpc_fd_shutdown(exec_ctx, fdn->fd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "c-ares fd shutdown")); gpr_mu_unlock(&fdn->mu); } } -- cgit v1.2.3 From d987d95423e7a410190157248f42537657f0b359 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 21 Sep 2017 12:21:17 -0700 Subject: Don't collect everything after running a build --- tools/internal_ci/linux/grpc_performance_profile_daily.cfg | 1 - tools/internal_ci/linux/grpc_performance_profile_master.cfg | 1 - 2 files changed, 2 deletions(-) diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg index 9831869edb..7320a19a76 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_daily.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_performance_profile_master.cfg b/tools/internal_ci/linux/grpc_performance_profile_master.cfg index e4cefbc3f4..ccb8aabb38 100644 --- a/tools/internal_ci/linux/grpc_performance_profile_master.cfg +++ b/tools/internal_ci/linux/grpc_performance_profile_master.cfg @@ -19,7 +19,6 @@ build_file: "grpc/tools/internal_ci/linux/grpc_performance_profile_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**" regex: "github/grpc/reports/**" } } -- cgit v1.2.3 From 3121da1f5fe3c1f93fba1cd4a83c7e537852a90d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 21 Sep 2017 12:24:33 -0700 Subject: Revert "Add automated PHP-to-C++ performance testing" --- tools/gce/linux_performance_worker_init.sh | 5 --- tools/run_tests/performance/run_worker_php.sh | 43 -------------------------- tools/run_tests/performance/scenario_config.py | 28 ----------------- 3 files changed, 76 deletions(-) delete mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 88d8de7402..8f0a0f65c3 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,11 +128,6 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh deleted file mode 100755 index 1da74cd0db..0000000000 --- a/tools/run_tests/performance/run_worker_php.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -source ~/.rvm/scripts/rvm -set -ex - -repo=$(dirname $0)/../../.. - -# First set up all dependences needed for PHP QPS test -cd $repo -cd src/php/tests/qps -composer install -# The proxy worker for PHP is implemented in Ruby -cd ../../../.. -ruby src/ruby/qps/proxy-worker.rb $@ - diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 5efc9f5648..a7560400dd 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -800,33 +800,6 @@ class RubyLanguage: return 'ruby' -class PhpLanguage: - - def __init__(self): - pass - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_php.sh'] - - def worker_port_offset(self): - return 800 - - def scenarios(self): - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - def __str__(self): - return 'php' - - class JavaLanguage: def __init__(self): @@ -1024,7 +997,6 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), - 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), -- cgit v1.2.3 From 4917b08a17cd74fbe3c4c4032b9455f00721969e Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Thu, 21 Sep 2017 12:52:50 -0700 Subject: Revert "Revert "Add automated PHP-to-C++ performance testing"" --- tools/gce/linux_performance_worker_init.sh | 5 +++ tools/run_tests/performance/run_worker_php.sh | 43 ++++++++++++++++++++++++++ tools/run_tests/performance/scenario_config.py | 28 +++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c3..88d8de7402 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,11 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 0000000000..1da74cd0db --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +composer install +# The proxy worker for PHP is implemented in Ruby +cd ../../../.. +ruby src/ruby/qps/proxy-worker.rb $@ + diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index a7560400dd..5efc9f5648 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -800,6 +800,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -997,6 +1024,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), -- cgit v1.2.3 From efce6e1e5017e8a6c6a81f348a4adcc4825b757d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 14 Sep 2017 09:07:50 -0700 Subject: Revert "Revert "Allow SerializationTraits to use grpc::ByteBuffer rather than only grpc_byte_buffer"" --- BUILD | 2 + CMakeLists.txt | 15 +++ Makefile | 15 +++ build.yaml | 2 + gRPC-Core.podspec | 1 + grpc.def | 20 +-- grpc.gemspec | 1 + include/grpc++/impl/codegen/byte_buffer.h | 141 +++++++++++++++++++++ include/grpc++/impl/codegen/call.h | 114 +++++++++++++---- include/grpc++/impl/codegen/method_handler_impl.h | 9 +- include/grpc++/impl/codegen/rpc_service_method.h | 14 +- include/grpc++/impl/codegen/serialization_traits.h | 25 ++-- include/grpc++/impl/codegen/slice.h | 78 ++++++++++++ include/grpc++/support/byte_buffer.h | 68 +--------- include/grpc++/support/slice.h | 80 +----------- include/grpc/byte_buffer.h | 64 +--------- include/grpc/impl/codegen/byte_buffer.h | 86 +++++++++++++ package.xml | 1 + src/core/lib/iomgr/resource_quota.c | 1 + .../server/health/default_health_check_service.cc | 1 + src/cpp/util/byte_buffer_cc.cc | 37 ++++-- src/cpp/util/slice_cc.cc | 2 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 40 +++--- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 62 ++++----- test/core/surface/public_headers_must_be_c89.c | 1 + test/cpp/util/byte_buffer_test.cc | 5 +- tools/doxygen/Doxyfile.c++ | 2 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core | 1 + tools/doxygen/Doxyfile.core.internal | 1 + tools/run_tests/generated/sources_and_headers.json | 4 + 31 files changed, 565 insertions(+), 330 deletions(-) create mode 100644 include/grpc++/impl/codegen/byte_buffer.h create mode 100644 include/grpc/impl/codegen/byte_buffer.h diff --git a/BUILD b/BUILD index 7b55e663cb..2135f59341 100644 --- a/BUILD +++ b/BUILD @@ -989,6 +989,7 @@ grpc_cc_library( name = "grpc_codegen", language = "c", public_hdrs = [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -1487,6 +1488,7 @@ grpc_cc_library( public_hdrs = [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index a5a7fad346..0a5f4024f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1250,6 +1250,7 @@ target_link_libraries(grpc ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1556,6 +1557,7 @@ target_link_libraries(grpc_cronet ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -1832,6 +1834,7 @@ target_link_libraries(grpc_test_util ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2092,6 +2095,7 @@ target_link_libraries(grpc_test_util_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2387,6 +2391,7 @@ target_link_libraries(grpc_unsecure ) foreach(_hdr + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2696,6 +2701,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -2706,6 +2712,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3188,6 +3195,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3198,6 +3206,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3558,6 +3567,7 @@ target_link_libraries(grpc++_test_util foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3585,6 +3595,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3696,6 +3707,7 @@ target_link_libraries(grpc++_test_util_unsecure foreach(_hdr include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h @@ -3723,6 +3735,7 @@ foreach(_hdr include/grpc++/impl/codegen/stub_options.h include/grpc++/impl/codegen/sync_stream.h include/grpc++/impl/codegen/time.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3925,6 +3938,7 @@ foreach(_hdr include/grpc/slice_buffer.h include/grpc/status.h include/grpc/support/workaround_list.h + include/grpc/impl/codegen/byte_buffer.h include/grpc/impl/codegen/byte_buffer_reader.h include/grpc/impl/codegen/compression_types.h include/grpc/impl/codegen/connectivity_state.h @@ -3935,6 +3949,7 @@ foreach(_hdr include/grpc/impl/codegen/status.h include/grpc++/impl/codegen/async_stream.h include/grpc++/impl/codegen/async_unary_call.h + include/grpc++/impl/codegen/byte_buffer.h include/grpc++/impl/codegen/call.h include/grpc++/impl/codegen/call_hook.h include/grpc++/impl/codegen/channel_interface.h diff --git a/Makefile b/Makefile index 61d6cae6ab..648689f93e 100644 --- a/Makefile +++ b/Makefile @@ -3206,6 +3206,7 @@ LIBGRPC_SRC = \ src/core/plugin_registry/grpc_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3512,6 +3513,7 @@ LIBGRPC_CRONET_SRC = \ src/core/plugin_registry/grpc_cronet_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -3789,6 +3791,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4040,6 +4043,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ src/core/ext/filters/http/server/http_server_filter.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4312,6 +4316,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/plugin_registry/grpc_unsecure_plugin_registry.c \ PUBLIC_HEADERS_C += \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4600,6 +4605,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -4610,6 +4616,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5093,6 +5100,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5103,6 +5111,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5456,6 +5465,7 @@ LIBGRPC++_TEST_UTIL_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5483,6 +5493,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5571,6 +5582,7 @@ LIBGRPC++_TEST_UTIL_UNSECURE_SRC = \ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -5598,6 +5610,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/stub_options.h \ include/grpc++/impl/codegen/sync_stream.h \ include/grpc++/impl/codegen/time.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5805,6 +5818,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/slice_buffer.h \ include/grpc/status.h \ include/grpc/support/workaround_list.h \ + include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ @@ -5815,6 +5829,7 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/status.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ + include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ diff --git a/build.yaml b/build.yaml index d15cf0339c..92ca2b6237 100644 --- a/build.yaml +++ b/build.yaml @@ -502,6 +502,7 @@ filegroups: - grpc_deadline_filter - name: grpc_codegen public_headers: + - include/grpc/impl/codegen/byte_buffer.h - include/grpc/impl/codegen/byte_buffer_reader.h - include/grpc/impl/codegen/compression_types.h - include/grpc/impl/codegen/connectivity_state.h @@ -969,6 +970,7 @@ filegroups: public_headers: - include/grpc++/impl/codegen/async_stream.h - include/grpc++/impl/codegen/async_unary_call.h + - include/grpc++/impl/codegen/byte_buffer.h - include/grpc++/impl/codegen/call.h - include/grpc++/impl/codegen/call_hook.h - include/grpc++/impl/codegen/channel_interface.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2f1f415866..6f84ef235d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -141,6 +141,7 @@ Pod::Spec.new do |s| 'include/grpc/impl/codegen/sync_generic.h', 'include/grpc/impl/codegen/sync_posix.h', 'include/grpc/impl/codegen/sync_windows.h', + 'include/grpc/impl/codegen/byte_buffer.h', 'include/grpc/impl/codegen/byte_buffer_reader.h', 'include/grpc/impl/codegen/compression_types.h', 'include/grpc/impl/codegen/connectivity_state.h', diff --git a/grpc.def b/grpc.def index a7a11601ed..558be60c3c 100644 --- a/grpc.def +++ b/grpc.def @@ -1,14 +1,4 @@ EXPORTS - grpc_raw_byte_buffer_create - grpc_raw_compressed_byte_buffer_create - grpc_byte_buffer_copy - grpc_byte_buffer_length - grpc_byte_buffer_destroy - grpc_byte_buffer_reader_init - grpc_byte_buffer_reader_destroy - grpc_byte_buffer_reader_next - grpc_byte_buffer_reader_readall - grpc_raw_byte_buffer_from_reader census_initialize census_shutdown census_supported @@ -145,6 +135,16 @@ EXPORTS grpc_server_add_secure_http2_port grpc_call_set_credentials grpc_server_credentials_set_auth_metadata_processor + grpc_raw_byte_buffer_create + grpc_raw_compressed_byte_buffer_create + grpc_byte_buffer_copy + grpc_byte_buffer_length + grpc_byte_buffer_destroy + grpc_byte_buffer_reader_init + grpc_byte_buffer_reader_destroy + grpc_byte_buffer_reader_next + grpc_byte_buffer_reader_readall + grpc_raw_byte_buffer_from_reader grpc_slice_ref grpc_slice_unref grpc_slice_copy diff --git a/grpc.gemspec b/grpc.gemspec index 96323aed10..c00a245693 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -146,6 +146,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/tmpfile_posix.c ) s.files += %w( src/core/lib/support/tmpfile_windows.c ) s.files += %w( src/core/lib/support/wrap_memcpy.c ) + s.files += %w( include/grpc/impl/codegen/byte_buffer.h ) s.files += %w( include/grpc/impl/codegen/byte_buffer_reader.h ) s.files += %w( include/grpc/impl/codegen/compression_types.h ) s.files += %w( include/grpc/impl/codegen/connectivity_state.h ) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h new file mode 100644 index 0000000000..87d390c688 --- /dev/null +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -0,0 +1,141 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#include +#include +#include +#include +#include + +#include + +namespace grpc { + +template +class CallOpRecvMessage; +class MethodHandler; +namespace internal { +template +class MessageDeserializer; +} + +/// A sequence of bytes. +class ByteBuffer final { + public: + /// Constuct an empty buffer. + ByteBuffer() : buffer_(nullptr) {} + + /// Construct buffer from \a slices, of which there are \a nslices. + ByteBuffer(const Slice* slices, size_t nslices); + + /// Constuct a byte buffer by referencing elements of existing buffer + /// \a buf. Wrapper of core function grpc_byte_buffer_copy + ByteBuffer(const ByteBuffer& buf); + + ~ByteBuffer() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + } + } + + ByteBuffer& operator=(const ByteBuffer&); + + /// Dump (read) the buffer contents into \a slices. + Status Dump(std::vector* slices) const; + + /// Remove all data. + void Clear() { + if (buffer_) { + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer_); + buffer_ = nullptr; + } + } + + /// Make a duplicate copy of the internals of this byte + /// buffer so that we have our own owned version of it. + /// bbuf.Duplicate(); is equivalent to bbuf=bbuf; but is actually readable + void Duplicate() { + buffer_ = g_core_codegen_interface->grpc_byte_buffer_copy(buffer_); + } + + /// Forget underlying byte buffer without destroying + /// Use this only for un-owned byte buffers + void Release() { buffer_ = nullptr; } + + /// Buffer size in bytes. + size_t Length() const; + + /// Swap the state of *this and *other. + void Swap(ByteBuffer* other); + + /// Is this ByteBuffer valid? + bool Valid() const { return (buffer_ != nullptr); } + + private: + friend class SerializationTraits; + friend class CallOpSendMessage; + template + friend class CallOpRecvMessage; + friend class CallOpGenericRecvMessage; + friend class MethodHandler; + template + friend class internal::MessageDeserializer; + + // takes ownership + void set_buffer(grpc_byte_buffer* buf) { + if (buffer_) { + Clear(); + } + buffer_ = buf; + } + + grpc_byte_buffer* c_buffer() { return buffer_; } + grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } + + // DEPRECATED: Implicit conversion to transparently + // support deprecated SerializationTraits API + // No need to inline since deprecated + operator grpc_byte_buffer*(); + operator const grpc_byte_buffer*() const; + + grpc_byte_buffer* buffer_; +}; + +template <> +class SerializationTraits { + public: + static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer.buffer_); + return Status::OK; + } + static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, + bool* own_buffer) { + *buffer = source; + *own_buffer = true; + return Status::OK; + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_BYTE_BUFFER_H diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 74ed5cbfb9..3c418c7ae2 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,6 @@ #include #include -struct grpc_byte_buffer; - namespace grpc { class ByteBuffer; @@ -281,7 +280,7 @@ class CallOpSendInitialMetadata { class CallOpSendMessage { public: - CallOpSendMessage() : send_buf_(nullptr) {} + CallOpSendMessage() : send_buf_() {} /// Send \a message using \a options for the write. The \a options are cleared /// after use. @@ -294,33 +293,67 @@ class CallOpSendMessage { protected: void AddOp(grpc_op* ops, size_t* nops) { - if (send_buf_ == nullptr) return; + if (!send_buf_.Valid()) return; grpc_op* op = &ops[(*nops)++]; op->op = GRPC_OP_SEND_MESSAGE; op->flags = write_options_.flags(); op->reserved = NULL; - op->data.send_message.send_message = send_buf_; + op->data.send_message.send_message = send_buf_.c_buffer(); // Flags are per-message: clear them after use. write_options_.Clear(); } - void FinishOp(bool* status) { - g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_); - send_buf_ = nullptr; - } + void FinishOp(bool* status) { send_buf_.Clear(); } private: - grpc_byte_buffer* send_buf_; + template + class MessageSerializer; + + ByteBuffer send_buf_; WriteOptions write_options_; }; +namespace internal { +template +T Example(); +} // namespace internal + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + internal::Example(), + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), + own_buf); + } +}; + +template +class CallOpSendMessage::MessageSerializer< + M, typename std::enable_if::Serialize( + internal::Example(), + internal::Example<::grpc::ByteBuffer*>(), + internal::Example()))>::value>::type> { + public: + static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, + bool* own_buf) { + return SerializationTraits::Serialize(message, bbuf, own_buf); + } +}; + template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; Status result = - SerializationTraits::Serialize(message, &send_buf_, &own_buf); + MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); if (!own_buf) { - send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_); + send_buf_.Duplicate(); } return result; } @@ -330,6 +363,36 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } +namespace internal { +template +class MessageDeserializer; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize(bbuf, message); + } +}; + +template +class MessageDeserializer< + M, typename std::enable_if::Deserialize( + internal::Example(), + internal::Example()))>::value>::type> { + public: + static Status Deserialize(const ByteBuffer& bbuf, M* message) { + return SerializationTraits::Deserialize( + const_cast(bbuf).c_buffer(), message); + } +}; +} // namespace internal + template class CallOpRecvMessage { public: @@ -352,18 +415,20 @@ class CallOpRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (message_ == nullptr) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = *status = - SerializationTraits::Deserialize(recv_buf_, message_).ok(); + internal::MessageDeserializer::Deserialize(recv_buf_, message_) + .ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -376,14 +441,14 @@ class CallOpRecvMessage { private: R* message_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(grpc_byte_buffer* buf) = 0; + virtual Status Deserialize(const ByteBuffer& buf) = 0; virtual ~DeserializeFunc() {} }; @@ -391,8 +456,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(grpc_byte_buffer* buf) override { - return SerializationTraits::Deserialize(buf, message_); + Status Deserialize(const ByteBuffer& buf) override { + return grpc::internal::MessageDeserializer::Deserialize(buf, message_); } ~DeserializeFuncType() override {} @@ -428,18 +493,19 @@ class CallOpGenericRecvMessage { op->op = GRPC_OP_RECV_MESSAGE; op->flags = 0; op->reserved = NULL; - op->data.recv_message.recv_message = &recv_buf_; + op->data.recv_message.recv_message = recv_buf_.c_buffer_ptr(); } void FinishOp(bool* status) { if (!deserialize_) return; - if (recv_buf_) { + if (recv_buf_.Valid()) { if (*status) { got_message = true; *status = deserialize_->Deserialize(recv_buf_).ok(); + recv_buf_.Release(); } else { got_message = false; - g_core_codegen_interface->grpc_byte_buffer_destroy(recv_buf_); + recv_buf_.Clear(); } } else { got_message = false; @@ -452,7 +518,7 @@ class CallOpGenericRecvMessage { private: std::unique_ptr deserialize_; - grpc_byte_buffer* recv_buf_; + ByteBuffer recv_buf_; bool allow_not_getting_message_; }; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 15e24bdcdc..8125e0a651 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H #define GRPCXX_IMPL_CODEGEN_METHOD_HANDLER_IMPL_H +#include #include #include #include @@ -37,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -123,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = - SerializationTraits::Deserialize(param.request, &req); + Status status = internal::MessageDeserializer::Deserialize( + param.request, &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/rpc_service_method.h b/include/grpc++/impl/codegen/rpc_service_method.h index 7165774172..d356012ad6 100644 --- a/include/grpc++/impl/codegen/rpc_service_method.h +++ b/include/grpc++/impl/codegen/rpc_service_method.h @@ -25,14 +25,11 @@ #include #include +#include #include #include #include -extern "C" { -struct grpc_byte_buffer; -} - namespace grpc { class ServerContext; class StreamContextInterface; @@ -43,11 +40,14 @@ class MethodHandler { virtual ~MethodHandler() {} struct HandlerParameter { HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req) - : call(c), server_context(context), request(req) {} + : call(c), server_context(context) { + request.set_buffer(req); + } + ~HandlerParameter() { request.Release(); } Call* call; ServerContext* server_context; - // Handler required to grpc_byte_buffer_destroy this - grpc_byte_buffer* request; + // Handler required to destroy these contents + ByteBuffer request; }; virtual void RunHandler(const HandlerParameter& param) = 0; }; diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index b72d474126..7fc4e24172 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -26,15 +26,24 @@ namespace grpc { /// Used for hooking different message serialization API's into GRPC. /// Each SerializationTraits implementation must provide the following /// functions: -/// static Status Serialize(const Message& msg, -/// grpc_byte_buffer** buffer, -/// bool* own_buffer); -/// static Status Deserialize(grpc_byte_buffer* buffer, -/// Message* msg, -/// int max_receive_message_size); +/// 1. static Status Serialize(const Message& msg, +/// ByteBuffer* buffer, +/// bool* own_buffer); +/// AND/OR +/// static Status Serialize(const Message& msg, +/// grpc_byte_buffer** buffer, +/// bool* own_buffer); +/// The former is preferred; the latter is deprecated /// -/// Serialize is required to convert message to a grpc_byte_buffer, and -/// to store a pointer to that byte buffer at *buffer. *own_buffer should +/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// Message* msg); +/// AND/OR +/// static Status Deserialize(grpc_byte_buffer* buffer, +/// Message* msg); +/// The former is preferred; the latter is deprecated +/// +/// Serialize is required to convert message to a ByteBuffer, and +/// return that byte buffer through *buffer. *own_buffer should /// be set to true if the caller owns said byte buffer, or false if /// ownership is retained elsewhere. /// diff --git a/include/grpc++/impl/codegen/slice.h b/include/grpc++/impl/codegen/slice.h index e682bdef6a..c185bf4fd0 100644 --- a/include/grpc++/impl/codegen/slice.h +++ b/include/grpc++/impl/codegen/slice.h @@ -19,11 +19,89 @@ #ifndef GRPCXX_IMPL_CODEGEN_SLICE_H #define GRPCXX_IMPL_CODEGEN_SLICE_H +#include #include #include +#include + namespace grpc { +/// A wrapper around \a grpc_slice. +/// +/// A slice represents a contiguous reference counted array of bytes. +/// It is cheap to take references to a slice, and it is cheap to create a +/// slice pointing to a subset of another slice. +class Slice final { + public: + /// Construct an empty slice. + Slice(); + /// Destructor - drops one reference. + ~Slice(); + + enum AddRef { ADD_REF }; + /// Construct a slice from \a slice, adding a reference. + Slice(grpc_slice slice, AddRef); + + enum StealRef { STEAL_REF }; + /// Construct a slice from \a slice, stealing a reference. + Slice(grpc_slice slice, StealRef); + + /// Allocate a slice of specified size + Slice(size_t len); + + /// Construct a slice from a copied buffer + Slice(const void* buf, size_t len); + + /// Construct a slice from a copied string + Slice(const grpc::string& str); + + enum StaticSlice { STATIC_SLICE }; + + /// Construct a slice from a static buffer + Slice(const void* buf, size_t len, StaticSlice); + + /// Copy constructor, adds a reference. + Slice(const Slice& other); + + /// Assignment, reference count is unchanged. + Slice& operator=(Slice other) { + std::swap(slice_, other.slice_); + return *this; + } + + /// Create a slice pointing at some data. Calls malloc to allocate a refcount + /// for the object, and arranges that destroy will be called with the + /// user data pointer passed in at destruction. Can be the same as buf or + /// different (e.g., if data is part of a larger structure that must be + /// destroyed when the data is no longer needed) + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + + /// Specialization of above for common case where buf == user_data + Slice(void* buf, size_t len, void (*destroy)(void*)) + : Slice(buf, len, destroy, buf) {} + + /// Similar to the above but has a destroy that also takes slice length + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + + /// Byte size. + size_t size() const { return GRPC_SLICE_LENGTH(slice_); } + + /// Raw pointer to the beginning (first element) of the slice. + const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } + + /// Raw pointer to the end (one byte \em past the last element) of the slice. + const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } + + /// Raw C slice. Caller needs to call grpc_slice_unref when done. + grpc_slice c_slice() const; + + private: + friend class ByteBuffer; + + grpc_slice slice_; +}; + inline grpc::string_ref StringRefFromSlice(const grpc_slice* slice) { return grpc::string_ref( reinterpret_cast(GRPC_SLICE_START_PTR(*slice)), diff --git a/include/grpc++/support/byte_buffer.h b/include/grpc++/support/byte_buffer.h index df16c6a75e..81fa3b0a18 100644 --- a/include/grpc++/support/byte_buffer.h +++ b/include/grpc++/support/byte_buffer.h @@ -19,6 +19,7 @@ #ifndef GRPCXX_SUPPORT_BYTE_BUFFER_H #define GRPCXX_SUPPORT_BYTE_BUFFER_H +#include #include #include #include @@ -27,71 +28,4 @@ #include #include -#include - -namespace grpc { - -/// A sequence of bytes. -class ByteBuffer final { - public: - /// Constuct an empty buffer. - ByteBuffer() : buffer_(nullptr) {} - - /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); - - /// Constuct a byte buffer by referencing elements of existing buffer - /// \a buf. Wrapper of core function grpc_byte_buffer_copy - ByteBuffer(const ByteBuffer& buf); - - ~ByteBuffer(); - - ByteBuffer& operator=(const ByteBuffer&); - - /// Dump (read) the buffer contents into \a slices. - Status Dump(std::vector* slices) const; - - /// Remove all data. - void Clear(); - - /// Buffer size in bytes. - size_t Length() const; - - /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); - - private: - friend class SerializationTraits; - - // takes ownership - void set_buffer(grpc_byte_buffer* buf) { - if (buffer_) { - Clear(); - } - buffer_ = buf; - } - - // For \a SerializationTraits's usage. - grpc_byte_buffer* buffer() const { return buffer_; } - - grpc_byte_buffer* buffer_; -}; - -template <> -class SerializationTraits { - public: - static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer); - return Status::OK; - } - static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, - bool* own_buffer) { - *buffer = grpc_byte_buffer_copy(source.buffer()); - *own_buffer = true; - return Status::OK; - } -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_BYTE_BUFFER_H diff --git a/include/grpc++/support/slice.h b/include/grpc++/support/slice.h index bbf97f280e..10db10d79c 100644 --- a/include/grpc++/support/slice.h +++ b/include/grpc++/support/slice.h @@ -19,86 +19,8 @@ #ifndef GRPCXX_SUPPORT_SLICE_H #define GRPCXX_SUPPORT_SLICE_H +#include #include #include -namespace grpc { - -/// A wrapper around \a grpc_slice. -/// -/// A slice represents a contiguous reference counted array of bytes. -/// It is cheap to take references to a slice, and it is cheap to create a -/// slice pointing to a subset of another slice. -class Slice final { - public: - /// Construct an empty slice. - Slice(); - /// Destructor - drops one reference. - ~Slice(); - - enum AddRef { ADD_REF }; - /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); - - enum StealRef { STEAL_REF }; - /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); - - /// Allocate a slice of specified size - Slice(size_t len); - - /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); - - /// Construct a slice from a copied string - Slice(const grpc::string& str); - - enum StaticSlice { STATIC_SLICE }; - - /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); - - /// Copy constructor, adds a reference. - Slice(const Slice& other); - - /// Assignment, reference count is unchanged. - Slice& operator=(Slice other) { - std::swap(slice_, other.slice_); - return *this; - } - - /// Create a slice pointing at some data. Calls malloc to allocate a refcount - /// for the object, and arranges that destroy will be called with the - /// user data pointer passed in at destruction. Can be the same as buf or - /// different (e.g., if data is part of a larger structure that must be - /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); - - /// Specialization of above for common case where buf == user_data - Slice(void* buf, size_t len, void (*destroy)(void*)) - : Slice(buf, len, destroy, buf) {} - - /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); - - /// Byte size. - size_t size() const { return GRPC_SLICE_LENGTH(slice_); } - - /// Raw pointer to the beginning (first element) of the slice. - const uint8_t* begin() const { return GRPC_SLICE_START_PTR(slice_); } - - /// Raw pointer to the end (one byte \em past the last element) of the slice. - const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } - - /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const { return grpc_slice_ref(slice_); } - - private: - friend class ByteBuffer; - - grpc_slice slice_; -}; - -} // namespace grpc - #endif // GRPCXX_SUPPORT_SLICE_H diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 55e191da31..7669582af2 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -19,69 +19,7 @@ #ifndef GRPC_BYTE_BUFFER_H #define GRPC_BYTE_BUFFER_H -#include +#include #include -#ifdef __cplusplus -extern "C" { -#endif - -/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, - size_t nslices); - -/** Returns a *compressed* RAW byte buffer instance over the given slices (up to - * \a nslices). The \a compression argument defines the compression algorithm - * used to generate the data in \a slices. - * - * Increases the reference count for all \a slices processed. The user is - * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ -GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( - grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); - -/** Copies input byte buffer \a bb. - * - * Increases the reference count of all the source slices. The user is - * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ -GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); - -/** Returns the size of the given byte buffer, in bytes. */ -GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); - -/** Destroys \a byte_buffer deallocating all its memory. */ -GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); - -/** Reader for byte buffers. Iterates over slices in the byte buffer */ -struct grpc_byte_buffer_reader; -typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; - -/** Initialize \a reader to read over \a buffer. - * Returns 1 upon success, 0 otherwise. */ -GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, - grpc_byte_buffer *buffer); - -/** Cleanup and destroy \a reader */ -GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); - -/** Updates \a slice with the next piece of data from from \a reader and returns - * 1. Returns 0 at the end of the stream. Caller is responsible for calling - * grpc_slice_unref on the result. */ -GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, - grpc_slice *slice); - -/** Merge all data from \a reader into single slice */ -GRPCAPI grpc_slice -grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); - -/** Returns a RAW byte buffer instance from the output of \a reader. */ -GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( - grpc_byte_buffer_reader *reader); - -#ifdef __cplusplus -} -#endif - #endif /* GRPC_BYTE_BUFFER_H */ diff --git a/include/grpc/impl/codegen/byte_buffer.h b/include/grpc/impl/codegen/byte_buffer.h new file mode 100644 index 0000000000..fc33305713 --- /dev/null +++ b/include/grpc/impl/codegen/byte_buffer.h @@ -0,0 +1,86 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_IMPL_CODEGEN_BYTE_BUFFER_H +#define GRPC_IMPL_CODEGEN_BYTE_BUFFER_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Returns a RAW byte buffer instance over the given slices (up to \a nslices). + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_create(grpc_slice *slices, + size_t nslices); + +/** Returns a *compressed* RAW byte buffer instance over the given slices (up to + * \a nslices). The \a compression argument defines the compression algorithm + * used to generate the data in \a slices. + * + * Increases the reference count for all \a slices processed. The user is + * responsible for invoking grpc_byte_buffer_destroy on the returned instance.*/ +GRPCAPI grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( + grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); + +/** Copies input byte buffer \a bb. + * + * Increases the reference count of all the source slices. The user is + * responsible for calling grpc_byte_buffer_destroy over the returned copy. */ +GRPCAPI grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb); + +/** Returns the size of the given byte buffer, in bytes. */ +GRPCAPI size_t grpc_byte_buffer_length(grpc_byte_buffer *bb); + +/** Destroys \a byte_buffer deallocating all its memory. */ +GRPCAPI void grpc_byte_buffer_destroy(grpc_byte_buffer *byte_buffer); + +/** Reader for byte buffers. Iterates over slices in the byte buffer */ +struct grpc_byte_buffer_reader; +typedef struct grpc_byte_buffer_reader grpc_byte_buffer_reader; + +/** Initialize \a reader to read over \a buffer. + * Returns 1 upon success, 0 otherwise. */ +GRPCAPI int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader *reader, + grpc_byte_buffer *buffer); + +/** Cleanup and destroy \a reader */ +GRPCAPI void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); + +/** Updates \a slice with the next piece of data from from \a reader and returns + * 1. Returns 0 at the end of the stream. Caller is responsible for calling + * grpc_slice_unref on the result. */ +GRPCAPI int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, + grpc_slice *slice); + +/** Merge all data from \a reader into single slice */ +GRPCAPI grpc_slice +grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader); + +/** Returns a RAW byte buffer instance from the output of \a reader. */ +GRPCAPI grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_IMPL_CODEGEN_BYTE_BUFFER_H */ diff --git a/package.xml b/package.xml index 0c201afdae..7e514bd541 100644 --- a/package.xml +++ b/package.xml @@ -158,6 +158,7 @@ + diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 4895e0d1c9..4d69986fbc 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 815b607032..d2cba6d662 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index b1ff25252a..1ebe22b6a4 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -16,11 +16,15 @@ * */ +#include #include +#include #include namespace grpc { +static internal::GrpcLibraryInitializer g_gli_initializer; + ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { // The following assertions check that the representation of a grpc::Slice is // identical to that of a grpc_slice: it has a grpc_slice field, and nothing @@ -29,6 +33,7 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the // slices it processes, and such an increase does not affect the semantics @@ -37,19 +42,6 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { reinterpret_cast(const_cast(slices)), nslices); } -ByteBuffer::~ByteBuffer() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - } -} - -void ByteBuffer::Clear() { - if (buffer_) { - grpc_byte_buffer_destroy(buffer_); - buffer_ = nullptr; - } -} - Status ByteBuffer::Dump(std::vector* slices) const { slices->clear(); if (!buffer_) { @@ -80,7 +72,9 @@ ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { - Clear(); // first remove existing data + if (this != &buf) { + Clear(); // first remove existing data + } if (buf.buffer_) { buffer_ = grpc_byte_buffer_copy(buf.buffer_); // then copy } @@ -93,4 +87,19 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } +ByteBuffer::operator grpc_byte_buffer*() { + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + return buffer_; +} + +ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } + } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc index 486d0cdf0e..3ae17e8052 100644 --- a/src/cpp/util/slice_cc.cc +++ b/src/cpp/util/slice_cc.cc @@ -50,4 +50,6 @@ Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} +grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } + } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 57b543967e..70831494fa 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -22,16 +22,6 @@ #include "rb_grpc_imports.generated.h" -grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; census_initialize_type census_initialize_import; census_shutdown_type census_shutdown_import; census_supported_type census_supported_import; @@ -168,6 +158,16 @@ grpc_ssl_server_credentials_create_ex_type grpc_ssl_server_credentials_create_ex grpc_server_add_secure_http2_port_type grpc_server_add_secure_http2_port_import; grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; +grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; grpc_slice_copy_type grpc_slice_copy_import; @@ -330,16 +330,6 @@ gpr_sleep_until_type gpr_sleep_until_import; gpr_timespec_to_micros_type gpr_timespec_to_micros_import; void grpc_rb_load_imports(HMODULE library) { - grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); - grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); - grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); - grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); - grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); - grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); - grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); - grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); - grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); - grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize"); census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown"); census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported"); @@ -476,6 +466,16 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_add_secure_http2_port_import = (grpc_server_add_secure_http2_port_type) GetProcAddress(library, "grpc_server_add_secure_http2_port"); grpc_call_set_credentials_import = (grpc_call_set_credentials_type) GetProcAddress(library, "grpc_call_set_credentials"); grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); + grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create"); + grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create"); + grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy"); + grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length"); + grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy"); + grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init"); + grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy"); + grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next"); + grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall"); + grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 55a7eed1a4..868772cfc8 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -25,12 +25,12 @@ #include -#include #include #include #include #include #include +#include #include #include #include @@ -47,36 +47,6 @@ #include #include -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); -extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; -#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); -extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; -#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import -typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; -#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import -typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); -extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; -#define grpc_byte_buffer_length grpc_byte_buffer_length_import -typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); -extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; -#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import -typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); -extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; -#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import -typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; -#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import -typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); -extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; -#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import -typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); -extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; -#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import -typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); -extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; -#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef int(*census_initialize_type)(int features); extern census_initialize_type census_initialize_import; #define census_initialize census_initialize_import @@ -485,6 +455,36 @@ extern grpc_call_set_credentials_type grpc_call_set_credentials_import; typedef void(*grpc_server_credentials_set_auth_metadata_processor_type)(grpc_server_credentials *creds, grpc_auth_metadata_processor processor); extern grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; #define grpc_server_credentials_set_auth_metadata_processor grpc_server_credentials_set_auth_metadata_processor_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(grpc_slice *slices, size_t nslices); +extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import; +#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(grpc_slice *slices, size_t nslices, grpc_compression_algorithm compression); +extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import; +#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import +typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import; +#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import +typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb); +extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import; +#define grpc_byte_buffer_length grpc_byte_buffer_length_import +typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer); +extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import; +#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import +typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer); +extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import; +#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import +typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import; +#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import +typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, grpc_slice *slice); +extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import; +#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import +typedef grpc_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader); +extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import; +#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import +typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader); +extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import; +#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import typedef grpc_slice(*grpc_slice_ref_type)(grpc_slice s); extern grpc_slice_ref_type grpc_slice_ref_import; #define grpc_slice_ref grpc_slice_ref_import diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 0d7f68c0ad..d36d116afb 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index cac01a7307..8fb51bc663 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -93,7 +93,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { std::vector slices; slices.push_back(Slice(hello, Slice::STEAL_REF)); slices.push_back(Slice(world, Slice::STEAL_REF)); - grpc_byte_buffer* send_buffer = nullptr; + ByteBuffer send_buffer; bool owned = false; ByteBuffer buffer(&slices[0], 2); slices.clear(); @@ -101,8 +101,7 @@ TEST_F(ByteBufferTest, SerializationMakesCopy) { buffer, &send_buffer, &owned); EXPECT_TRUE(status.ok()); EXPECT_TRUE(owned); - EXPECT_TRUE(send_buffer != nullptr); - grpc_byte_buffer_destroy(send_buffer); + EXPECT_TRUE(send_buffer.Valid()); } } // namespace diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 62f113907d..d81b7b4d11 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -881,6 +882,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 49919415bd..263c6c1afb 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -811,6 +811,7 @@ include/grpc++/impl/channel_argument_option.h \ include/grpc++/impl/client_unary_call.h \ include/grpc++/impl/codegen/async_stream.h \ include/grpc++/impl/codegen/async_unary_call.h \ +include/grpc++/impl/codegen/byte_buffer.h \ include/grpc++/impl/codegen/call.h \ include/grpc++/impl/codegen/call_hook.h \ include/grpc++/impl/codegen/channel_interface.h \ @@ -882,6 +883,7 @@ include/grpc/impl/codegen/atm.h \ include/grpc/impl/codegen/atm_gcc_atomic.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 632735342b..b8514fe311 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index e352cb78f1..77cd28e9df 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -811,6 +811,7 @@ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_gcc_sync.h \ include/grpc/impl/codegen/atm_windows.h \ include/grpc/impl/codegen/atm_windows.h \ +include/grpc/impl/codegen/byte_buffer.h \ include/grpc/impl/codegen/byte_buffer_reader.h \ include/grpc/impl/codegen/compression_types.h \ include/grpc/impl/codegen/connectivity_state.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 8d42c3e432..7cf4d441a0 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8475,6 +8475,7 @@ "gpr_codegen" ], "headers": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -8488,6 +8489,7 @@ "language": "c", "name": "grpc_codegen", "src": [ + "include/grpc/impl/codegen/byte_buffer.h", "include/grpc/impl/codegen/byte_buffer_reader.h", "include/grpc/impl/codegen/compression_types.h", "include/grpc/impl/codegen/connectivity_state.h", @@ -9366,6 +9368,7 @@ "headers": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", @@ -9400,6 +9403,7 @@ "src": [ "include/grpc++/impl/codegen/async_stream.h", "include/grpc++/impl/codegen/async_unary_call.h", + "include/grpc++/impl/codegen/byte_buffer.h", "include/grpc++/impl/codegen/call.h", "include/grpc++/impl/codegen/call_hook.h", "include/grpc++/impl/codegen/channel_interface.h", -- cgit v1.2.3 From a44cffa11ffd66b0f9ff6d2de0e90d782af88841 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 15 Sep 2017 12:32:20 -0700 Subject: Base the new SerializationTraits API on implicit type conversion rather than SFINAE --- include/grpc++/impl/codegen/byte_buffer.h | 45 ++++++++----- include/grpc++/impl/codegen/call.h | 76 ++-------------------- include/grpc++/impl/codegen/method_handler_impl.h | 8 +-- include/grpc++/impl/codegen/serialization_traits.h | 10 +-- src/cpp/util/byte_buffer_cc.cc | 24 +++---- 5 files changed, 55 insertions(+), 108 deletions(-) diff --git a/include/grpc++/impl/codegen/byte_buffer.h b/include/grpc++/impl/codegen/byte_buffer.h index 87d390c688..57d731be18 100644 --- a/include/grpc++/impl/codegen/byte_buffer.h +++ b/include/grpc++/impl/codegen/byte_buffer.h @@ -34,10 +34,14 @@ namespace grpc { template class CallOpRecvMessage; class MethodHandler; -namespace internal { -template -class MessageDeserializer; -} +template +class RpcMethodHandler; +template +class ServerStreamingHandler; +namespace CallOpGenericRecvMessageHelper { +template +class DeserializeFuncType; +} // namespace CallOpGenericRecvMessageHelper /// A sequence of bytes. class ByteBuffer final { @@ -98,8 +102,14 @@ class ByteBuffer final { friend class CallOpRecvMessage; friend class CallOpGenericRecvMessage; friend class MethodHandler; - template - friend class internal::MessageDeserializer; + template + friend class RpcMethodHandler; + template + friend class ServerStreamingHandler; + template + friend class CallOpGenericRecvMessageHelper::DeserializeFuncType; + + grpc_byte_buffer* buffer_; // takes ownership void set_buffer(grpc_byte_buffer* buf) { @@ -112,20 +122,25 @@ class ByteBuffer final { grpc_byte_buffer* c_buffer() { return buffer_; } grpc_byte_buffer** c_buffer_ptr() { return &buffer_; } - // DEPRECATED: Implicit conversion to transparently - // support deprecated SerializationTraits API - // No need to inline since deprecated - operator grpc_byte_buffer*(); - operator const grpc_byte_buffer*() const; - - grpc_byte_buffer* buffer_; + class ByteBufferPointer { + public: + ByteBufferPointer(const ByteBuffer* b) + : bbuf_(const_cast(b)) {} + operator ByteBuffer*() { return bbuf_; } + operator grpc_byte_buffer*() { return bbuf_->buffer_; } + operator grpc_byte_buffer**() { return &bbuf_->buffer_; } + + private: + ByteBuffer* bbuf_; + }; + ByteBufferPointer bbuf_ptr() const { return ByteBufferPointer(this); } }; template <> class SerializationTraits { public: - static Status Deserialize(const ByteBuffer& byte_buffer, ByteBuffer* dest) { - dest->set_buffer(byte_buffer.buffer_); + static Status Deserialize(ByteBuffer* byte_buffer, ByteBuffer* dest) { + dest->set_buffer(byte_buffer->buffer_); return Status::OK; } static Status Serialize(const ByteBuffer& source, ByteBuffer* buffer, diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index 3c418c7ae2..d9988e51fc 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -305,9 +305,6 @@ class CallOpSendMessage { void FinishOp(bool* status) { send_buf_.Clear(); } private: - template - class MessageSerializer; - ByteBuffer send_buf_; WriteOptions write_options_; }; @@ -317,41 +314,12 @@ template T Example(); } // namespace internal -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf->c_buffer_ptr(), - own_buf); - } -}; - -template -class CallOpSendMessage::MessageSerializer< - M, typename std::enable_if::Serialize( - internal::Example(), - internal::Example<::grpc::ByteBuffer*>(), - internal::Example()))>::value>::type> { - public: - static Status SendMessageInternal(const M& message, ByteBuffer* bbuf, - bool* own_buf) { - return SerializationTraits::Serialize(message, bbuf, own_buf); - } -}; - template Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) { write_options_ = options; bool own_buf; - Status result = - MessageSerializer::SendMessageInternal(message, &send_buf_, &own_buf); + Status result = SerializationTraits::Serialize( + message, send_buf_.bbuf_ptr(), &own_buf); if (!own_buf) { send_buf_.Duplicate(); } @@ -363,36 +331,6 @@ Status CallOpSendMessage::SendMessage(const M& message) { return SendMessage(message, WriteOptions()); } -namespace internal { -template -class MessageDeserializer; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize(bbuf, message); - } -}; - -template -class MessageDeserializer< - M, typename std::enable_if::Deserialize( - internal::Example(), - internal::Example()))>::value>::type> { - public: - static Status Deserialize(const ByteBuffer& bbuf, M* message) { - return SerializationTraits::Deserialize( - const_cast(bbuf).c_buffer(), message); - } -}; -} // namespace internal - template class CallOpRecvMessage { public: @@ -423,7 +361,7 @@ class CallOpRecvMessage { if (recv_buf_.Valid()) { if (*status) { got_message = *status = - internal::MessageDeserializer::Deserialize(recv_buf_, message_) + SerializationTraits::Deserialize(recv_buf_.bbuf_ptr(), message_) .ok(); recv_buf_.Release(); } else { @@ -448,7 +386,7 @@ class CallOpRecvMessage { namespace CallOpGenericRecvMessageHelper { class DeserializeFunc { public: - virtual Status Deserialize(const ByteBuffer& buf) = 0; + virtual Status Deserialize(ByteBuffer* buf) = 0; virtual ~DeserializeFunc() {} }; @@ -456,8 +394,8 @@ template class DeserializeFuncType final : public DeserializeFunc { public: DeserializeFuncType(R* message) : message_(message) {} - Status Deserialize(const ByteBuffer& buf) override { - return grpc::internal::MessageDeserializer::Deserialize(buf, message_); + Status Deserialize(ByteBuffer* buf) override { + return SerializationTraits::Deserialize(buf->bbuf_ptr(), message_); } ~DeserializeFuncType() override {} @@ -501,7 +439,7 @@ class CallOpGenericRecvMessage { if (recv_buf_.Valid()) { if (*status) { got_message = true; - *status = deserialize_->Deserialize(recv_buf_).ok(); + *status = deserialize_->Deserialize(&recv_buf_).ok(); recv_buf_.Release(); } else { got_message = false; diff --git a/include/grpc++/impl/codegen/method_handler_impl.h b/include/grpc++/impl/codegen/method_handler_impl.h index 8125e0a651..e14cb0e926 100644 --- a/include/grpc++/impl/codegen/method_handler_impl.h +++ b/include/grpc++/impl/codegen/method_handler_impl.h @@ -38,8 +38,8 @@ class RpcMethodHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = SerializationTraits::Deserialize( + param.request.bbuf_ptr(), &req); ResponseType rsp; if (status.ok()) { status = func_(service_, param.server_context, &req, &rsp); @@ -124,8 +124,8 @@ class ServerStreamingHandler : public MethodHandler { void RunHandler(const HandlerParameter& param) final { RequestType req; - Status status = internal::MessageDeserializer::Deserialize( - param.request, &req); + Status status = SerializationTraits::Deserialize( + param.request.bbuf_ptr(), &req); if (status.ok()) { ServerWriter writer(param.call, param.server_context); diff --git a/include/grpc++/impl/codegen/serialization_traits.h b/include/grpc++/impl/codegen/serialization_traits.h index 7fc4e24172..4d91739cc4 100644 --- a/include/grpc++/impl/codegen/serialization_traits.h +++ b/include/grpc++/impl/codegen/serialization_traits.h @@ -24,20 +24,20 @@ namespace grpc { /// Defines how to serialize and deserialize some type. /// /// Used for hooking different message serialization API's into GRPC. -/// Each SerializationTraits implementation must provide the following -/// functions: +/// Each SerializationTraits implementation must provide the +/// following functions: /// 1. static Status Serialize(const Message& msg, /// ByteBuffer* buffer, /// bool* own_buffer); -/// AND/OR +/// OR /// static Status Serialize(const Message& msg, /// grpc_byte_buffer** buffer, /// bool* own_buffer); /// The former is preferred; the latter is deprecated /// -/// 2. static Status Deserialize(const ByteBuffer& buffer, +/// 2. static Status Deserialize(ByteBuffer* buffer, /// Message* msg); -/// AND/OR +/// OR /// static Status Deserialize(grpc_byte_buffer* buffer, /// Message* msg); /// The former is preferred; the latter is deprecated diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index 1ebe22b6a4..180c813762 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -33,6 +33,15 @@ ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { "Slice must have same representation as grpc_slice"); static_assert(sizeof(Slice) == sizeof(grpc_slice), "Slice must have same representation as grpc_slice"); + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); g_gli_initializer.summon(); // Make sure that initializer linked in // The const_cast is legal if grpc_raw_byte_buffer_create() does no more // than its advertised side effect of increasing the reference count of the @@ -87,19 +96,4 @@ void ByteBuffer::Swap(ByteBuffer* other) { buffer_ = tmp; } -ByteBuffer::operator grpc_byte_buffer*() { - // The following assertions check that the representation of a ByteBuffer is - // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, - // and nothing else. - static_assert(std::is_same::value, - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - return buffer_; -} - -ByteBuffer::operator const grpc_byte_buffer*() const { return buffer_; } - } // namespace grpc -- cgit v1.2.3 From 7115862e98059b3fbd2069ae7ad1bf09232a416e Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Thu, 21 Sep 2017 13:23:35 -0700 Subject: update license --- tools/run_tests/performance/run_worker_php.sh | 35 ++++++++------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 1da74cd0db..5d0c4fa4fd 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -1,32 +1,17 @@ #!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. +# Copyright 2017 gRPC authors. # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. +# http://www.apache.org/licenses/LICENSE-2.0 # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. source ~/.rvm/scripts/rvm set -ex -- cgit v1.2.3 From 2f3e58809911fed67ba4ea79633d820ab6b719ef Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 21 Sep 2017 13:47:20 -0700 Subject: Dont assume that sigint wasnt masked when invoking ruby tests --- src/ruby/end2end/killed_client_thread_client.rb | 2 +- src/ruby/end2end/killed_client_thread_driver.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ruby/end2end/killed_client_thread_client.rb b/src/ruby/end2end/killed_client_thread_client.rb index 7d6ed8c8d7..493c0eb56a 100755 --- a/src/ruby/end2end/killed_client_thread_client.rb +++ b/src/ruby/end2end/killed_client_thread_client.rb @@ -35,7 +35,7 @@ def main :this_channel_is_insecure) stub.echo(Echo::EchoRequest.new(request: 'hello')) fail 'the clients rpc in this test shouldnt complete. ' \ - 'expecting SIGINT to happen in the middle of the call' + 'expecting SIGTERM to happen in the middle of the call' end thd.join end diff --git a/src/ruby/end2end/killed_client_thread_driver.rb b/src/ruby/end2end/killed_client_thread_driver.rb index 09f05a4487..fce5d13e82 100755 --- a/src/ruby/end2end/killed_client_thread_driver.rb +++ b/src/ruby/end2end/killed_client_thread_driver.rb @@ -69,9 +69,9 @@ def main call_started_cv.wait(call_started_mu) until call_started.val end - # SIGINT the child process now that it's + # SIGTERM the child process now that it's # in the middle of an RPC (happening on a non-main thread) - Process.kill('SIGINT', client_pid) + Process.kill('SIGTERM', client_pid) STDERR.puts 'sent shutdown' begin @@ -88,8 +88,8 @@ def main end client_exit_code = $CHILD_STATUS - if client_exit_code.termsig != 2 # SIGINT - fail 'expected client exit from SIGINT ' \ + if client_exit_code.termsig != 15 # SIGTERM + fail 'expected client exit from SIGTERM ' \ "but got child status: #{client_exit_code}" end -- cgit v1.2.3 From 9fa16599510ca4772ef0b2e3631f7eaa3b429695 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 21 Sep 2017 15:43:32 -0700 Subject: Fix a memory leak point --- test/core/end2end/fixtures/proxy.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/core/end2end/fixtures/proxy.c b/test/core/end2end/fixtures/proxy.c index 9ad862728f..6a2d75da09 100644 --- a/test/core/end2end/fixtures/proxy.c +++ b/test/core/end2end/fixtures/proxy.c @@ -227,6 +227,10 @@ static void on_c2p_recv_msg(void *arg, int success) { new_closure(on_p2s_sent_close, pc), NULL); GPR_ASSERT(err == GRPC_CALL_OK); } + } else { + if (pc->c2p_msg != NULL) { + grpc_byte_buffer_destroy(pc->c2p_msg); + } } unrefpc(pc, "on_c2p_recv_msg"); -- cgit v1.2.3 From 2046d0b3c50e555f339062b4fba298e8af8b3a1f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 19 Sep 2017 15:30:11 -0700 Subject: Add unary call mode to GenericStub to allow generic RPC with 1 CQ trip --- include/grpc++/generic/generic_stub.h | 10 ++++++ src/cpp/client/generic_stub.cc | 10 ++++++ test/cpp/end2end/generic_end2end_test.cc | 56 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/include/grpc++/generic/generic_stub.h b/include/grpc++/generic/generic_stub.h index 2b3ff59ea2..d5064318cf 100644 --- a/include/grpc++/generic/generic_stub.h +++ b/include/grpc++/generic/generic_stub.h @@ -20,6 +20,7 @@ #define GRPCXX_GENERIC_GENERIC_STUB_H #include +#include #include namespace grpc { @@ -27,6 +28,7 @@ namespace grpc { class CompletionQueue; typedef ClientAsyncReaderWriter GenericClientAsyncReaderWriter; +typedef ClientAsyncResponseReader GenericClientAsyncResponseReader; /// Generic stubs provide a type-unsafe interface to call gRPC methods /// by name. @@ -51,6 +53,14 @@ class GenericStub final { std::unique_ptr PrepareCall( ClientContext* context, const grpc::string& method, CompletionQueue* cq); + /// Setup a unary call to a named method \a method using \a context, and don't + /// start it. Let it be started explicitly with StartCall. + /// The return value only indicates whether or not registration of the call + /// succeeded (i.e. the call won't proceed if the return value is nullptr). + std::unique_ptr PrepareUnaryCall( + ClientContext* context, const grpc::string& method, + const ByteBuffer& request, CompletionQueue* cq); + private: std::shared_ptr channel_; }; diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index de2e449fe8..693b8bea56 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -47,4 +47,14 @@ std::unique_ptr GenericStub::PrepareCall( return CallInternal(channel_.get(), context, method, cq, false, nullptr); } +// setup a unary call to a named method +std::unique_ptr GenericStub::PrepareUnaryCall( + ClientContext* context, const grpc::string& method, + const ByteBuffer& request, CompletionQueue* cq) { + return std::unique_ptr( + GenericClientAsyncResponseReader::Create( + channel_.get(), cq, RpcMethod(method.c_str(), RpcMethod::NORMAL_RPC), + context, request, false)); +} + } // namespace grpc diff --git a/test/cpp/end2end/generic_end2end_test.cc b/test/cpp/end2end/generic_end2end_test.cc index b9e6e18ca7..33b35108d2 100644 --- a/test/cpp/end2end/generic_end2end_test.cc +++ b/test/cpp/end2end/generic_end2end_test.cc @@ -196,6 +196,62 @@ TEST_F(GenericEnd2endTest, SequentialRpcs) { SendRpc(10); } +TEST_F(GenericEnd2endTest, SequentialUnaryRpcs) { + ResetStub(); + const int num_rpcs = 10; + const grpc::string kMethodName("/grpc.cpp.test.util.EchoTestService/Echo"); + for (int i = 0; i < num_rpcs; i++) { + EchoRequest send_request; + EchoRequest recv_request; + EchoResponse send_response; + EchoResponse recv_response; + Status recv_status; + + ClientContext cli_ctx; + GenericServerContext srv_ctx; + GenericServerAsyncReaderWriter stream(&srv_ctx); + + // The string needs to be long enough to test heap-based slice. + send_request.set_message("Hello world. Hello world. Hello world."); + + std::unique_ptr cli_send_buffer = + SerializeToByteBuffer(&send_request); + std::unique_ptr call = + generic_stub_->PrepareUnaryCall(&cli_ctx, kMethodName, + *cli_send_buffer.get(), &cli_cq_); + call->StartCall(); + ByteBuffer cli_recv_buffer; + call->Finish(&cli_recv_buffer, &recv_status, tag(1)); + + generic_service_.RequestCall(&srv_ctx, &stream, srv_cq_.get(), + srv_cq_.get(), tag(4)); + + verify_ok(srv_cq_.get(), 4, true); + EXPECT_EQ(server_host_, srv_ctx.host().substr(0, server_host_.length())); + EXPECT_EQ(kMethodName, srv_ctx.method()); + + ByteBuffer srv_recv_buffer; + stream.Read(&srv_recv_buffer, tag(5)); + server_ok(5); + EXPECT_TRUE(ParseFromByteBuffer(&srv_recv_buffer, &recv_request)); + EXPECT_EQ(send_request.message(), recv_request.message()); + + send_response.set_message(recv_request.message()); + std::unique_ptr srv_send_buffer = + SerializeToByteBuffer(&send_response); + stream.Write(*srv_send_buffer, tag(6)); + server_ok(6); + + stream.Finish(Status::OK, tag(7)); + server_ok(7); + + client_ok(1); + EXPECT_TRUE(ParseFromByteBuffer(&cli_recv_buffer, &recv_response)); + EXPECT_EQ(send_response.message(), recv_response.message()); + EXPECT_TRUE(recv_status.ok()); + } +} + // One ping, one pong. TEST_F(GenericEnd2endTest, SimpleBidiStreaming) { ResetStub(); -- cgit v1.2.3 From b8e55626002932d0d4bb10f3050754c1b12f853a Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 21 Sep 2017 14:44:32 -0700 Subject: Revert "Temporarily take ownership of cpp generator to prevent changes" --- .github/CODEOWNERS | 1 - src/compiler/OWNERS | 1 - 2 files changed, 2 deletions(-) delete mode 100644 src/compiler/OWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a4eacd998..094e43e470 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,5 +3,4 @@ # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @ctiller /bazel/** @nicolasnoble @dgquintas @ctiller -/src/compiler/cpp_generator.cc @vjpai /src/core/ext/filters/client_channel/** @markdroth @dgquintas @ctiller diff --git a/src/compiler/OWNERS b/src/compiler/OWNERS deleted file mode 100644 index 96b89fc60f..0000000000 --- a/src/compiler/OWNERS +++ /dev/null @@ -1 +0,0 @@ -@vjpai cpp_generator.cc -- cgit v1.2.3 From af3b4eecbdb6a70b6a3d4b5050c8aaffc259b823 Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Thu, 21 Sep 2017 14:25:07 -0700 Subject: add php into jenkins performance tests --- tools/jenkins/run_full_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh index 0f101f6da9..a9661c7e26 100755 --- a/tools/jenkins/run_full_performance.sh +++ b/tools/jenkins/run_full_performance.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../.. # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp node ruby java python go node_express \ + -l c++ csharp node ruby java python go node_express php \ --netperf \ --category scalable \ --bq_result_table performance_test.performance_experiment \ -- cgit v1.2.3 From 604a7ca07b7aeb5ecff210ab4e854526c78ae863 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 22 Sep 2017 09:28:32 -0700 Subject: Reduce stack size of simple_request e2e test --- test/core/end2end/tests/simple_request.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/core/end2end/tests/simple_request.c b/test/core/end2end/tests/simple_request.c index 82ab0a1cfe..7ba5bd8f46 100644 --- a/test/core/end2end/tests/simple_request.c +++ b/test/core/end2end/tests/simple_request.c @@ -103,10 +103,10 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; char *peer; - grpc_stats_data before; - grpc_stats_data after; + grpc_stats_data *before = gpr_malloc(sizeof(grpc_stats_data)); + grpc_stats_data *after = gpr_malloc(sizeof(grpc_stats_data)); - grpc_stats_collect(&before); + grpc_stats_collect(before); gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( @@ -214,9 +214,9 @@ static void simple_request_body(grpc_end2end_test_config config, cq_verifier_destroy(cqv); - grpc_stats_collect(&after); + grpc_stats_collect(after); - char *stats = grpc_stats_data_as_json(&after); + char *stats = grpc_stats_data_as_json(after); gpr_log(GPR_DEBUG, "%s", stats); gpr_free(stats); @@ -224,11 +224,11 @@ static void simple_request_body(grpc_end2end_test_config config, if (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) { expected_calls *= 2; } - GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] - - before.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] == + GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] - + before->counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] == expected_calls); - GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - - before.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == + GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - + before->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == expected_calls); } -- cgit v1.2.3 From 3c5203886838d08f74f20a66d9d00af23f415644 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 22 Sep 2017 09:31:03 -0700 Subject: Fix memory leak --- test/core/end2end/tests/simple_request.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/core/end2end/tests/simple_request.c b/test/core/end2end/tests/simple_request.c index 7ba5bd8f46..7ce7e1f285 100644 --- a/test/core/end2end/tests/simple_request.c +++ b/test/core/end2end/tests/simple_request.c @@ -230,6 +230,8 @@ static void simple_request_body(grpc_end2end_test_config config, GPR_ASSERT(after->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] - before->counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] == expected_calls); + gpr_free(before); + gpr_free(after); } static void test_invoke_simple_request(grpc_end2end_test_config config) { -- cgit v1.2.3 From eec7a917baf37ed26449b106f99027c97aedaba6 Mon Sep 17 00:00:00 2001 From: Shaun McCormick Date: Fri, 11 Aug 2017 14:57:14 -0500 Subject: Add Ruby server interceptors --- src/ruby/lib/grpc.rb | 1 + src/ruby/lib/grpc/generic/active_call.rb | 43 +++- src/ruby/lib/grpc/generic/bidi_call.rb | 29 ++- src/ruby/lib/grpc/generic/client_stub.rb | 133 +++++++++---- src/ruby/lib/grpc/generic/interceptor_registry.rb | 53 +++++ src/ruby/lib/grpc/generic/interceptors.rb | 186 ++++++++++++++++++ src/ruby/lib/grpc/generic/rpc_desc.rb | 80 ++++++-- src/ruby/lib/grpc/generic/rpc_server.rb | 18 +- .../duplicate/echo_duplicate_services_pb.rb | 1 + src/ruby/spec/channel_connection_spec.rb | 35 +--- src/ruby/spec/generic/active_call_spec.rb | 19 +- src/ruby/spec/generic/client_interceptors_spec.rb | 153 +++++++++++++++ src/ruby/spec/generic/interceptor_registry_spec.rb | 65 ++++++ src/ruby/spec/generic/rpc_server_spec.rb | 35 +--- src/ruby/spec/generic/server_interceptors_spec.rb | 218 +++++++++++++++++++++ src/ruby/spec/spec_helper.rb | 4 + src/ruby/spec/support/helpers.rb | 73 +++++++ src/ruby/spec/support/services.rb | 147 ++++++++++++++ 18 files changed, 1143 insertions(+), 150 deletions(-) create mode 100644 src/ruby/lib/grpc/generic/interceptor_registry.rb create mode 100644 src/ruby/lib/grpc/generic/interceptors.rb create mode 100644 src/ruby/spec/generic/client_interceptors_spec.rb create mode 100644 src/ruby/spec/generic/interceptor_registry_spec.rb create mode 100644 src/ruby/spec/generic/server_interceptors_spec.rb create mode 100644 src/ruby/spec/support/helpers.rb create mode 100644 src/ruby/spec/support/services.rb diff --git a/src/ruby/lib/grpc.rb b/src/ruby/lib/grpc.rb index 98bfc0a0fa..37b0392072 100644 --- a/src/ruby/lib/grpc.rb +++ b/src/ruby/lib/grpc.rb @@ -24,6 +24,7 @@ require_relative 'grpc/generic/active_call' require_relative 'grpc/generic/client_stub' require_relative 'grpc/generic/service' require_relative 'grpc/generic/rpc_server' +require_relative 'grpc/generic/interceptors' begin file = File.open(ssl_roots_path) diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb index 10eb70b4a7..8c3aa284aa 100644 --- a/src/ruby/lib/grpc/generic/active_call.rb +++ b/src/ruby/lib/grpc/generic/active_call.rb @@ -154,6 +154,15 @@ module GRPC Operation.new(self) end + ## + # Returns a restricted view of this ActiveCall for use in interceptors + # + # @return [InterceptableView] + # + def interceptable + InterceptableView.new(self) + end + def receive_and_check_status batch_result = @call.run_batch(RECV_STATUS_ON_CLIENT => nil) set_input_stream_done @@ -515,15 +524,27 @@ module GRPC # This does not mean that must necessarily be one. E.g, the replies # produced by gen_each_reply could ignore the received_msgs # - # @param gen_each_reply [Proc] generates the BiDi stream replies - def run_server_bidi(gen_each_reply) - bd = BidiCall.new(@call, - @marshal, - @unmarshal, - metadata_received: @metadata_received, - req_view: MultiReqView.new(self)) - - bd.run_on_server(gen_each_reply, proc { set_input_stream_done }) + # @param mth [Proc] generates the BiDi stream replies + # @param interception_ctx [InterceptionContext] + # + def run_server_bidi(mth, interception_ctx) + view = multi_req_view + bidi_call = BidiCall.new( + @call, + @marshal, + @unmarshal, + metadata_received: @metadata_received, + req_view: view + ) + requests = bidi_call.read_next_loop(proc { set_input_stream_done }, false) + interception_ctx.intercept!( + :bidi_streamer, + call: view, + method: mth, + requests: requests + ) do + bidi_call.run_on_server(mth, requests) + end end # Waits till an operation completes @@ -645,5 +666,9 @@ module GRPC Operation = view_class(:cancel, :cancelled?, :deadline, :execute, :metadata, :status, :start_call, :wait, :write_flag, :write_flag=, :trailing_metadata) + + # InterceptableView further limits access to an ActiveCall's methods + # for use in interceptors on the client, exposing only the deadline + InterceptableView = view_class(:deadline) end end diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index c2239d0178..3bdcc0062e 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -87,23 +87,32 @@ module GRPC # This does not mean that must necessarily be one. E.g, the replies # produced by gen_each_reply could ignore the received_msgs # - # @param gen_each_reply [Proc] generates the BiDi stream replies. - # @param set_input_steam_done [Proc] call back to call when - # the reads have been completely read through. - def run_on_server(gen_each_reply, set_input_stream_done) + # @param [Proc] gen_each_reply generates the BiDi stream replies. + # @param [Enumerable] requests The enumerable of requests to run + def run_on_server(gen_each_reply, requests) + replies = nil + # Pass in the optional call object parameter if possible if gen_each_reply.arity == 1 - replys = gen_each_reply.call( - read_loop(set_input_stream_done, is_client: false)) + replies = gen_each_reply.call(requests) elsif gen_each_reply.arity == 2 - replys = gen_each_reply.call( - read_loop(set_input_stream_done, is_client: false), - @req_view) + replies = gen_each_reply.call(requests, @req_view) else fail 'Illegal arity of reply generator' end - write_loop(replys, is_client: false) + write_loop(replies, is_client: false) + end + + ## + # Read the next stream iteration + # + # @param [Proc] finalize_stream callback to call when the reads have been + # completely read through. + # @param [Boolean] is_client If this is a client or server request + # + def read_next_loop(finalize_stream, is_client = false) + read_loop(finalize_stream, is_client: is_client) end private diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb index 75a95a4e94..9a50f8a99d 100644 --- a/src/ruby/lib/grpc/generic/client_stub.rb +++ b/src/ruby/lib/grpc/generic/client_stub.rb @@ -89,17 +89,23 @@ module GRPC # used within a gRPC server. # @param channel_args [Hash] the channel arguments. Note: this argument is # ignored if the channel_override argument is provided. + # @param interceptors [Array] An array of + # GRPC::ClientInterceptor objects that will be used for + # intercepting calls before they are executed + # Interceptors are an EXPERIMENTAL API. def initialize(host, creds, channel_override: nil, timeout: nil, propagate_mask: nil, - channel_args: {}) + channel_args: {}, + interceptors: []) @ch = ClientStub.setup_channel(channel_override, host, creds, channel_args) alt_host = channel_args[Core::Channel::SSL_TARGET] @host = alt_host.nil? ? host : alt_host @propagate_mask = propagate_mask @timeout = timeout.nil? ? DEFAULT_TIMEOUT : timeout + @interceptors = InterceptorRegistry.new(interceptors) end # request_response sends a request to a GRPC server, and returns the @@ -149,16 +155,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.request_response(req, metadata: metadata) unless return_op - - # return the operation view of the active_call; define #execute as a - # new method for this instance that invokes #request_response. - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.request_response(req, metadata: metadata) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + request: req, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute as a + # new method for this instance that invokes #request_response. + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:request_response, intercept_args) do + c.request_response(req, metadata: metadata) + end + end + op + else + interception_context.intercept!(:request_response, intercept_args) do + c.request_response(req, metadata: metadata) + end end - op end # client_streamer sends a stream of requests to a GRPC server, and @@ -213,16 +232,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.client_streamer(requests, metadata: metadata) unless return_op - - # return the operation view of the active_call; define #execute as a - # new method for this instance that invokes #client_streamer. - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.client_streamer(requests) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + requests: requests, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute as a + # new method for this instance that invokes #client_streamer. + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:client_streamer, intercept_args) do + c.client_streamer(requests) + end + end + op + else + interception_context.intercept!(:client_streamer, intercept_args) do + c.client_streamer(requests, metadata: metadata) + end end - op end # server_streamer sends one request to the GRPC server, which yields a @@ -292,16 +324,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.server_streamer(req, metadata: metadata, &blk) unless return_op - - # return the operation view of the active_call; define #execute - # as a new method for this instance that invokes #server_streamer - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.server_streamer(req, &blk) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + request: req, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute + # as a new method for this instance that invokes #server_streamer + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:server_streamer, intercept_args) do + c.server_streamer(req, &blk) + end + end + op + else + interception_context.intercept!(:server_streamer, intercept_args) do + c.server_streamer(req, metadata: metadata, &blk) + end end - op end # bidi_streamer sends a stream of requests to the GRPC server, and yields @@ -405,17 +450,29 @@ module GRPC deadline: deadline, parent: parent, credentials: credentials) - return c.bidi_streamer(requests, metadata: metadata, - &blk) unless return_op - - # return the operation view of the active_call; define #execute - # as a new method for this instance that invokes #bidi_streamer - c.merge_metadata_to_send(metadata) - op = c.operation - op.define_singleton_method(:execute) do - c.bidi_streamer(requests, &blk) + interception_context = @interceptors.build_context + intercept_args = { + method: method, + requests: requests, + call: c.interceptable, + metadata: metadata + } + if return_op + # return the operation view of the active_call; define #execute + # as a new method for this instance that invokes #bidi_streamer + c.merge_metadata_to_send(metadata) + op = c.operation + op.define_singleton_method(:execute) do + interception_context.intercept!(:bidi_streamer, intercept_args) do + c.bidi_streamer(requests, &blk) + end + end + op + else + interception_context.intercept!(:bidi_streamer, intercept_args) do + c.bidi_streamer(requests, metadata: metadata, &blk) + end end - op end private diff --git a/src/ruby/lib/grpc/generic/interceptor_registry.rb b/src/ruby/lib/grpc/generic/interceptor_registry.rb new file mode 100644 index 0000000000..b241eb9a86 --- /dev/null +++ b/src/ruby/lib/grpc/generic/interceptor_registry.rb @@ -0,0 +1,53 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GRPC contains the General RPC module. +module GRPC + ## + # Represents a registry of added interceptors available for enumeration. + # The registry can be used for both server and client interceptors. + # This class is internal to gRPC and not meant for public usage. + # + class InterceptorRegistry + ## + # An error raised when an interceptor is attempted to be added + # that does not extend GRPC::Interceptor + # + class DescendantError < StandardError; end + + ## + # Initialize the registry with an empty interceptor list + # This is an EXPERIMENTAL API. + # + def initialize(interceptors = []) + @interceptors = [] + interceptors.each do |i| + base = GRPC::Interceptor + unless i.class.ancestors.include?(base) + fail DescendantError, "Interceptors must descend from #{base}" + end + @interceptors << i + end + end + + ## + # Builds an interception context from this registry + # + # @return [InterceptionContext] + # + def build_context + InterceptionContext.new(@interceptors) + end + end +end diff --git a/src/ruby/lib/grpc/generic/interceptors.rb b/src/ruby/lib/grpc/generic/interceptors.rb new file mode 100644 index 0000000000..73faec4b9c --- /dev/null +++ b/src/ruby/lib/grpc/generic/interceptors.rb @@ -0,0 +1,186 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require_relative 'interceptor_registry' + +# GRPC contains the General RPC module. +module GRPC + ## + # Base class for interception in GRPC + # + class Interceptor + ## + # @param [Hash] options A hash of options that will be used + # by the interceptor. This is an EXPERIMENTAL API. + # + def initialize(options = {}) + @options = options || {} + end + end + + ## + # ClientInterceptor allows for wrapping outbound gRPC client stub requests. + # This is an EXPERIMENTAL API. + # + class ClientInterceptor < Interceptor + ## + # Intercept a unary request response call + # + # @param [Object] request + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def request_response(request:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting request response method #{method}" \ + " for request #{request} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a client streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def client_streamer(requests:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting client streamer method #{method}" \ + " for requests #{requests} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a server streaming call + # + # @param [Object] request + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def server_streamer(request:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting server streamer method #{method}" \ + " for request #{request} with call #{call} and metadata: #{metadata}" + yield + end + + ## + # Intercept a BiDi streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall] call + # @param [Method] method + # @param [Hash] metadata + # + def bidi_streamer(requests:, call:, method:, metadata:) + GRPC.logger.debug "Intercepting bidi streamer method #{method}" \ + " for requests #{requests} with call #{call} and metadata: #{metadata}" + yield + end + end + + ## + # ServerInterceptor allows for wrapping gRPC server execution handling. + # This is an EXPERIMENTAL API. + # + class ServerInterceptor < Interceptor + ## + # Intercept a unary request response call. + # + # @param [Object] request + # @param [GRPC::ActiveCall::SingleReqView] call + # @param [Method] method + # + def request_response(request:, call:, method:) + GRPC.logger.debug "Intercepting request response method #{method}" \ + " for request #{request} with call #{call}" + yield + end + + ## + # Intercept a client streaming call + # + # @param [GRPC::ActiveCall::MultiReqView] call + # @param [Method] method + # + def client_streamer(call:, method:) + GRPC.logger.debug "Intercepting client streamer method #{method}" \ + " with call #{call}" + yield + end + + ## + # Intercept a server streaming call + # + # @param [Object] request + # @param [GRPC::ActiveCall::SingleReqView] call + # @param [Method] method + # + def server_streamer(request:, call:, method:) + GRPC.logger.debug "Intercepting server streamer method #{method}" \ + " for request #{request} with call #{call}" + yield + end + + ## + # Intercept a BiDi streaming call + # + # @param [Enumerable] requests + # @param [GRPC::ActiveCall::MultiReqView] call + # @param [Method] method + # + def bidi_streamer(requests:, call:, method:) + GRPC.logger.debug "Intercepting bidi streamer method #{method}" \ + " for requests #{requests} with call #{call}" + yield + end + end + + ## + # Represents the context in which an interceptor runs. Used to provide an + # injectable mechanism for handling interception. This is an EXPERIMENTAL API. + # + class InterceptionContext + ## + # @param [Array] + # + def initialize(interceptors = []) + @interceptors = interceptors.dup + end + + ## + # Intercept the call and fire out to interceptors in a FIFO execution. + # This is an EXPERIMENTAL API. + # + # @param [Symbol] type The request type + # @param [Hash] args The arguments for the call + # + def intercept!(type, args = {}) + return yield if @interceptors.none? + + i = @interceptors.pop + return yield unless i + + i.send(type, args) do + if @interceptors.any? + intercept!(type, args) do + yield + end + else + yield + end + end + end + end +end diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb index 6fb6c412fb..5fd1805aab 100644 --- a/src/ruby/lib/grpc/generic/rpc_desc.rb +++ b/src/ruby/lib/grpc/generic/rpc_desc.rb @@ -47,43 +47,85 @@ module GRPC proc { |o| unmarshal_class.method(unmarshal_method).call(o) } end - def handle_request_response(active_call, mth) + def handle_request_response(active_call, mth, inter_ctx) req = active_call.read_unary_request - resp = mth.call(req, active_call.single_req_view) - active_call.server_unary_response( - resp, trailing_metadata: active_call.output_metadata) + call = active_call.single_req_view + + inter_ctx.intercept!( + :request_response, + method: mth, + call: call, + request: req + ) do + resp = mth.call(req, call) + active_call.server_unary_response( + resp, + trailing_metadata: active_call.output_metadata + ) + end end - def handle_client_streamer(active_call, mth) - resp = mth.call(active_call.multi_req_view) - active_call.server_unary_response( - resp, trailing_metadata: active_call.output_metadata) + def handle_client_streamer(active_call, mth, inter_ctx) + call = active_call.multi_req_view + + inter_ctx.intercept!( + :client_streamer, + method: mth, + call: call + ) do + resp = mth.call(call) + active_call.server_unary_response( + resp, + trailing_metadata: active_call.output_metadata + ) + end end - def handle_server_streamer(active_call, mth) + def handle_server_streamer(active_call, mth, inter_ctx) req = active_call.read_unary_request - replys = mth.call(req, active_call.single_req_view) - replys.each { |r| active_call.remote_send(r) } - send_status(active_call, OK, 'OK', active_call.output_metadata) + call = active_call.single_req_view + + inter_ctx.intercept!( + :server_streamer, + method: mth, + call: call, + request: req + ) do + replies = mth.call(req, call) + replies.each { |r| active_call.remote_send(r) } + send_status(active_call, OK, 'OK', active_call.output_metadata) + end end - def handle_bidi_streamer(active_call, mth) - active_call.run_server_bidi(mth) + ## + # @param [GRPC::ActiveCall] active_call + # @param [Method] mth + # @param [Array] inter_ctx + # + def handle_bidi_streamer(active_call, mth, inter_ctx) + active_call.run_server_bidi(mth, inter_ctx) send_status(active_call, OK, 'OK', active_call.output_metadata) end - def run_server_method(active_call, mth) + ## + # @param [GRPC::ActiveCall] active_call The current active call object + # for the request + # @param [Method] mth The current RPC method being called + # @param [GRPC::InterceptionContext] inter_ctx The interception context + # being executed + # + def run_server_method(active_call, mth, inter_ctx = InterceptionContext.new) # While a server method is running, it might be cancelled, its deadline # might be reached, the handler could throw an unknown error, or a # well-behaved handler could throw a StatusError. if request_response? - handle_request_response(active_call, mth) + handle_request_response(active_call, mth, inter_ctx) elsif client_streamer? - handle_client_streamer(active_call, mth) + handle_client_streamer(active_call, mth, inter_ctx) elsif server_streamer? - handle_server_streamer(active_call, mth) + handle_server_streamer(active_call, mth, inter_ctx) else # is a bidi_stream - handle_bidi_streamer(active_call, mth) + handle_bidi_streamer(active_call, mth, inter_ctx) end rescue BadStatus => e # this is raised by handlers that want GRPC to send an application error diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 33b3cea1fc..d5fc11dc1c 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -196,11 +196,18 @@ module GRPC # # * server_args: # A server arguments hash to be passed down to the underlying core server + # + # * interceptors: + # Am array of GRPC::ServerInterceptor objects that will be used for + # intercepting server handlers to provide extra functionality. + # Interceptors are an EXPERIMENTAL API. + # def initialize(pool_size:DEFAULT_POOL_SIZE, max_waiting_requests:DEFAULT_MAX_WAITING_REQUESTS, poll_period:DEFAULT_POLL_PERIOD, connect_md_proc:nil, - server_args:{}) + server_args:{}, + interceptors:[]) @connect_md_proc = RpcServer.setup_connect_md_proc(connect_md_proc) @max_waiting_requests = max_waiting_requests @poll_period = poll_period @@ -212,6 +219,7 @@ module GRPC # :stopped. State transitions can only proceed in that order. @running_state = :not_started @server = Core::Server.new(server_args) + @interceptors = InterceptorRegistry.new(interceptors) end # stops a running server @@ -374,7 +382,11 @@ module GRPC @pool.schedule(active_call) do |ac| c, mth = ac begin - rpc_descs[mth].run_server_method(c, rpc_handlers[mth]) + rpc_descs[mth].run_server_method( + c, + rpc_handlers[mth], + @interceptors.build_context + ) rescue StandardError c.send_status(GRPC::Core::StatusCodes::INTERNAL, 'Server handler failed') @@ -382,7 +394,7 @@ module GRPC end end rescue Core::CallError, RuntimeError => e - # these might happen for various reasonse. The correct behaviour of + # these might happen for various reasons. The correct behavior of # the server is to log them and continue, if it's not shutting down. if running_state == :running GRPC.logger.warn("server call failed: #{e}") diff --git a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb index 683370121e..ab50d9b3a5 100644 --- a/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb +++ b/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb @@ -34,6 +34,7 @@ module Grpc self.service_name = 'grpc.testing.duplicate.EchoTestService' rpc :Echo, Grpc::Testing::EchoRequest, Grpc::Testing::EchoResponse + rpc :ResponseStream, Grpc::Testing::EchoRequest, stream(Grpc::Testing::EchoResponse) end Stub = Service.rpc_stub_class diff --git a/src/ruby/spec/channel_connection_spec.rb b/src/ruby/spec/channel_connection_spec.rb index c76056606b..ce3e3b1c93 100644 --- a/src/ruby/spec/channel_connection_spec.rb +++ b/src/ruby/spec/channel_connection_spec.rb @@ -11,45 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -require 'grpc' +require 'spec_helper' require 'timeout' include Timeout include GRPC::Core -# A test message -class EchoMsg - def self.marshal(_o) - '' - end - - def self.unmarshal(_o) - EchoMsg.new - end -end - -# A test service with an echo implementation. -class EchoService - include GRPC::GenericService - rpc :an_rpc, EchoMsg, EchoMsg - attr_reader :received_md - - def initialize(**kw) - @trailing_metadata = kw - @received_md = [] - end - - def an_rpc(req, call) - GRPC.logger.info('echo service received a request') - call.output_metadata.update(@trailing_metadata) - @received_md << call.metadata unless call.metadata.nil? - req - end -end - -EchoStub = EchoService.rpc_stub_class - def start_server(port = 0) @srv = GRPC::RpcServer.new(pool_size: 1) server_port = @srv.add_http2_port("localhost:#{port}", :this_port_is_insecure) diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb index a00df9236d..120acc35af 100644 --- a/src/ruby/spec/generic/active_call_spec.rb +++ b/src/ruby/spec/generic/active_call_spec.rb @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'grpc' +require 'spec_helper' include GRPC::Core::StatusCodes @@ -82,6 +82,16 @@ describe GRPC::ActiveCall do end end end + + describe '#interceptable' do + it 'exposes a fixed subset of the ActiveCall.methods' do + want = %w(deadline) + v = @client_call.interceptable + want.each do |w| + expect(v.methods.include?(w)) + end + end + end end describe '#remote_send' do @@ -609,9 +619,11 @@ describe GRPC::ActiveCall do msgs end + int_ctx = GRPC::InterceptionContext.new + @server_thread = Thread.new do @server_call.run_server_bidi( - fake_gen_each_reply_with_no_call_param) + fake_gen_each_reply_with_no_call_param, int_ctx) @server_call.send_status(@server_status) end end @@ -624,10 +636,11 @@ describe GRPC::ActiveCall do call_param.send_initial_metadata msgs end + int_ctx = GRPC::InterceptionContext.new @server_thread = Thread.new do @server_call.run_server_bidi( - fake_gen_each_reply_with_call_param) + fake_gen_each_reply_with_call_param, int_ctx) @server_call.send_status(@server_status) end end diff --git a/src/ruby/spec/generic/client_interceptors_spec.rb b/src/ruby/spec/generic/client_interceptors_spec.rb new file mode 100644 index 0000000000..f292715e4d --- /dev/null +++ b/src/ruby/spec/generic/client_interceptors_spec.rb @@ -0,0 +1,153 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe 'Client Interceptors' do + let(:interceptor) { TestClientInterceptor.new } + let(:interceptors_opts) { { interceptors: [interceptor] } } + let(:request) { EchoMsg.new } + let(:service) { EchoService } + + before(:each) do + build_rpc_server + end + + context 'when a client interceptor is added' do + context 'with a request/response call' do + it 'should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:request_response) + .once.and_call_original + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:request_response) + .with(request, metadata: { 'foo' => 'bar_from_request_response' }) + .once.and_call_original + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end + + context 'with a client streaming call' do + it 'should be called', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:client_streamer) + .once.and_call_original + requests = [EchoMsg.new, EchoMsg.new] + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + requests = [EchoMsg.new, EchoMsg.new] + expect_any_instance_of(GRPC::ActiveCall).to receive(:client_streamer) + .with(requests, metadata: { 'foo' => 'bar_from_client_streamer' }) + .once.and_call_original + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + end + + context 'with a server streaming call' do + it 'should be called', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + request = EchoMsg.new + expect_any_instance_of(GRPC::ActiveCall).to receive(:server_streamer) + .once.and_call_original + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + request = EchoMsg.new + expect_any_instance_of(GRPC::ActiveCall).to receive(:server_streamer) + .with(request, metadata: { 'foo' => 'bar_from_server_streamer' }) + .once.and_call_original + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + end + + context 'with a bidi call' do + it 'should be called', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + expect_any_instance_of(GRPC::ActiveCall).to receive(:bidi_streamer) + .once.and_call_original + requests = [EchoMsg.new, EchoMsg.new] + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify outgoing metadata', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub, opts: interceptors_opts) + requests = [EchoMsg.new, EchoMsg.new] + expect_any_instance_of(GRPC::ActiveCall).to receive(:bidi_streamer) + .with(requests, metadata: { 'foo' => 'bar_from_bidi_streamer' }) + .once.and_call_original + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + end + end +end diff --git a/src/ruby/spec/generic/interceptor_registry_spec.rb b/src/ruby/spec/generic/interceptor_registry_spec.rb new file mode 100644 index 0000000000..f93f5cec09 --- /dev/null +++ b/src/ruby/spec/generic/interceptor_registry_spec.rb @@ -0,0 +1,65 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe GRPC::InterceptorRegistry do + let(:server) { RpcServer.new } + let(:interceptor) { TestServerInterceptor.new } + let(:interceptors) { [interceptor] } + let(:registry) { described_class.new(interceptors) } + + describe 'initialization' do + subject { registry } + + context 'with an interceptor extending GRPC::ServerInterceptor' do + it 'should add the interceptor to the registry' do + subject + is = registry.instance_variable_get('@interceptors') + expect(is.count).to eq 1 + expect(is.first).to eq interceptor + end + end + + context 'with multiple interceptors' do + let(:interceptor2) { TestServerInterceptor.new } + let(:interceptor3) { TestServerInterceptor.new } + let(:interceptors) { [interceptor, interceptor2, interceptor3] } + + it 'should maintain order of insertion when iterated against' do + subject + is = registry.instance_variable_get('@interceptors') + expect(is.count).to eq 3 + is.each_with_index do |i, idx| + case idx + when 0 + expect(i).to eq interceptor + when 1 + expect(i).to eq interceptor2 + when 2 + expect(i).to eq interceptor3 + end + end + end + end + + context 'with an interceptor not extending GRPC::ServerInterceptor' do + let(:interceptor) { Class } + let(:err) { GRPC::InterceptorRegistry::DescendantError } + + it 'should raise an InvalidArgument exception' do + expect { subject }.to raise_error(err) + end + end + end +end diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index b887eaaf4e..05059fbecf 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -require 'grpc' +require 'spec_helper' def load_test_certs test_root = File.join(File.dirname(File.dirname(__FILE__)), 'testdata') @@ -28,17 +27,6 @@ def check_md(wanted_md, received_md) end end -# A test message -class EchoMsg - def self.marshal(_o) - '' - end - - def self.unmarshal(_o) - EchoMsg.new - end -end - # A test service with no methods. class EmptyService include GRPC::GenericService @@ -50,27 +38,6 @@ class NoRpcImplementation rpc :an_rpc, EchoMsg, EchoMsg end -# A test service with an echo implementation. -class EchoService - include GRPC::GenericService - rpc :an_rpc, EchoMsg, EchoMsg - attr_reader :received_md - - def initialize(**kw) - @trailing_metadata = kw - @received_md = [] - end - - def an_rpc(req, call) - GRPC.logger.info('echo service received a request') - call.output_metadata.update(@trailing_metadata) - @received_md << call.metadata unless call.metadata.nil? - req - end -end - -EchoStub = EchoService.rpc_stub_class - # A test service with an implementation that fails with BadStatus class FailingService include GRPC::GenericService diff --git a/src/ruby/spec/generic/server_interceptors_spec.rb b/src/ruby/spec/generic/server_interceptors_spec.rb new file mode 100644 index 0000000000..eb86686084 --- /dev/null +++ b/src/ruby/spec/generic/server_interceptors_spec.rb @@ -0,0 +1,218 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require 'spec_helper' + +describe 'Server Interceptors' do + let(:interceptor) { TestServerInterceptor.new } + let(:request) { EchoMsg.new } + let(:trailing_metadata) { {} } + let(:service) { EchoService.new(trailing_metadata) } + let(:interceptors) { [] } + + before(:each) do + build_rpc_server(server_opts: { interceptors: interceptors }) + end + + context 'when a server interceptor is added' do + let(:interceptors) { [interceptor] } + let(:client_metadata) { { client_md: 'test' } } + let(:client_call_opts) { { metadata: client_metadata, return_op: true } } + + context 'with a request/response call' do + let(:trailing_metadata) { { server_om: 'from_request_response' } } + + it 'should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:request_response).with(request, metadata: client_metadata) + .once.and_call_original + ) + op = stub.an_rpc(request, client_call_opts) + msg = op.execute + expect(op.trailing_metadata).to eq( + 'interc' => 'from_request_response', + 'server_om' => 'from_request_response' + ) + expect(msg).to be_a(EchoMsg) + end + end + end + + context 'with a client streaming call' do + let(:trailing_metadata) { { server_om: 'from_client_streamer' } } + let(:requests) { [EchoMsg.new, EchoMsg.new] } + + it 'should be called', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.a_client_streaming_rpc(requests)).to be_a(EchoMsg) + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:client_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:client_streamer).with(requests) + .once.and_call_original + ) + op = stub.a_client_streaming_rpc(requests, client_call_opts) + msg = op.execute + expect(op.trailing_metadata).to eq( + 'interc' => 'from_client_streamer', + 'server_om' => 'from_client_streamer' + ) + expect(msg).to be_a(EchoMsg) + end + end + end + + context 'with a server streaming call' do + let(:trailing_metadata) { { server_om: 'from_server_streamer' } } + let(:request) { EchoMsg.new } + + it 'should be called', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + responses = stub.a_server_streaming_rpc(request) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:server_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:server_streamer).with(request) + .once.and_call_original + ) + op = stub.a_server_streaming_rpc(request, client_call_opts) + responses = op.execute + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + expect(op.trailing_metadata).to eq( + 'interc' => 'from_server_streamer', + 'server_om' => 'from_server_streamer' + ) + end + end + end + + context 'with a bidi call' do + let(:trailing_metadata) { { server_om: 'from_bidi_streamer' } } + let(:requests) { [EchoMsg.new, EchoMsg.new] } + + it 'should be called', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + responses = stub.a_bidi_rpc(requests) + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + end + end + + it 'can modify trailing metadata', server: true do + expect(interceptor).to receive(:bidi_streamer) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect_any_instance_of(GRPC::ActiveCall).to( + receive(:bidi_streamer).with(requests) + .once.and_call_original + ) + op = stub.a_bidi_rpc(requests, client_call_opts) + responses = op.execute + responses.each do |r| + expect(r).to be_a(EchoMsg) + end + expect(op.trailing_metadata).to eq( + 'interc' => 'from_bidi_streamer', + 'server_om' => 'from_bidi_streamer' + ) + end + end + end + end + + context 'when multiple interceptors are added' do + let(:interceptor2) { TestServerInterceptor.new } + let(:interceptor3) { TestServerInterceptor.new } + let(:interceptors) do + [ + interceptor, + interceptor2, + interceptor3 + ] + end + + it 'each should be called', server: true do + expect(interceptor).to receive(:request_response) + .once.and_call_original + expect(interceptor2).to receive(:request_response) + .once.and_call_original + expect(interceptor3).to receive(:request_response) + .once.and_call_original + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end + + context 'when an interceptor is not added' do + it 'should not be called', server: true do + expect(interceptor).to_not receive(:call) + + run_services_on_server(@server, services: [service]) do + stub = build_insecure_stub(EchoStub) + expect(stub.an_rpc(request)).to be_a(EchoMsg) + end + end + end +end diff --git a/src/ruby/spec/spec_helper.rb b/src/ruby/spec/spec_helper.rb index 6e1eba1945..8fe9e6e808 100644 --- a/src/ruby/spec/spec_helper.rb +++ b/src/ruby/spec/spec_helper.rb @@ -32,6 +32,9 @@ require 'rspec' require 'logging' require 'rspec/logging_helper' +require_relative 'support/services' +require_relative 'support/helpers' + # GRPC is the general RPC module # # Configure its logging for fine-grained log control during test runs @@ -49,6 +52,7 @@ Logging.logger['GRPC::BidiCall'].level = :info RSpec.configure do |config| include RSpec::LoggingHelper config.capture_log_messages # comment this out to see logs during test runs + include GRPC::Spec::Helpers end RSpec::Expectations.configuration.warn_about_potential_false_positives = false diff --git a/src/ruby/spec/support/helpers.rb b/src/ruby/spec/support/helpers.rb new file mode 100644 index 0000000000..65fffff9e7 --- /dev/null +++ b/src/ruby/spec/support/helpers.rb @@ -0,0 +1,73 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GRPC contains the General RPC module. +module GRPC + ## + # GRPC RSpec base module + # + module Spec + ## + # A module that is used for providing generic helpers across the + # GRPC test suite + # + module Helpers + # Shortcut syntax for a GRPC RPC Server + RpcServer = GRPC::RpcServer + + ## + # Build an RPC server used for testing + # + def build_rpc_server(server_opts: {}, + client_opts: {}) + @server = RpcServer.new({ poll_period: 1 }.merge(server_opts)) + @port = @server.add_http2_port('0.0.0.0:0', :this_port_is_insecure) + @host = "0.0.0.0:#{@port}" + @client_opts = client_opts + @server + end + + ## + # Run services on an RPC server, yielding to allow testing within + # + # @param [RpcServer] server + # @param [Array] services + # + def run_services_on_server(server, services: []) + services.each do |s| + server.handle(s) + end + t = Thread.new { server.run } + server.wait_till_running + + yield + + server.stop + t.join + end + + ## + # Build an insecure stub from a given stub class + # + # @param [Class] klass + # @param [String] host + # + def build_insecure_stub(klass, host: nil, opts: nil) + host ||= @host + opts ||= @client_opts + klass.new(host, :this_channel_is_insecure, **opts) + end + end + end +end diff --git a/src/ruby/spec/support/services.rb b/src/ruby/spec/support/services.rb new file mode 100644 index 0000000000..27cc8e61ac --- /dev/null +++ b/src/ruby/spec/support/services.rb @@ -0,0 +1,147 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test stubs for various scenarios +require 'grpc' + +# A test message +class EchoMsg + def self.marshal(_o) + '' + end + + def self.unmarshal(_o) + EchoMsg.new + end +end + +# A test service with an echo implementation. +class EchoService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + rpc :a_client_streaming_rpc, stream(EchoMsg), EchoMsg + rpc :a_server_streaming_rpc, EchoMsg, stream(EchoMsg) + rpc :a_bidi_rpc, stream(EchoMsg), stream(EchoMsg) + attr_reader :received_md + + def initialize(**kw) + @trailing_metadata = kw + @received_md = [] + end + + def an_rpc(req, call) + GRPC.logger.info('echo service received a request') + call.output_metadata.update(@trailing_metadata) + @received_md << call.metadata unless call.metadata.nil? + req + end + + def a_client_streaming_rpc(call) + # iterate through requests so call can complete + call.output_metadata.update(@trailing_metadata) + call.each_remote_read.each { |r| p r } + EchoMsg.new + end + + def a_server_streaming_rpc(_req, call) + call.output_metadata.update(@trailing_metadata) + [EchoMsg.new, EchoMsg.new] + end + + def a_bidi_rpc(requests, call) + call.output_metadata.update(@trailing_metadata) + requests.each { |r| p r } + [EchoMsg.new, EchoMsg.new] + end +end + +EchoStub = EchoService.rpc_stub_class + +# For testing server interceptors +class TestServerInterceptor < GRPC::ServerInterceptor + def request_response(request:, call:, method:) + p "Received request/response call at method #{method}" \ + " with request #{request} for call #{call}" + call.output_metadata[:interc] = 'from_request_response' + p "[GRPC::Ok] (#{method.owner.name}.#{method.name})" + yield + end + + def client_streamer(call:, method:) + call.output_metadata[:interc] = 'from_client_streamer' + call.each_remote_read.each do |r| + p "In interceptor: #{r}" + end + p "Received client streamer call at method #{method} for call #{call}" + yield + end + + def server_streamer(request:, call:, method:) + p "Received server streamer call at method #{method} with request" \ + " #{request} for call #{call}" + call.output_metadata[:interc] = 'from_server_streamer' + yield + end + + def bidi_streamer(requests:, call:, method:) + requests.each do |r| + p "Bidi request: #{r}" + end + p "Received bidi streamer call at method #{method} with requests" \ + " #{requests} for call #{call}" + call.output_metadata[:interc] = 'from_bidi_streamer' + yield + end +end + +# For testing client interceptors +class TestClientInterceptor < GRPC::ClientInterceptor + def request_response(request:, call:, method:, metadata: {}) + p "Intercepted request/response call at method #{method}" \ + " with request #{request} for call #{call}" \ + " and metadata: #{metadata}" + metadata['foo'] = 'bar_from_request_response' + yield + end + + def client_streamer(requests:, call:, method:, metadata: {}) + p "Received client streamer call at method #{method}" \ + " with requests #{requests} for call #{call}" \ + " and metadata: #{metadata}" + requests.each do |r| + p "In client interceptor: #{r}" + end + metadata['foo'] = 'bar_from_client_streamer' + yield + end + + def server_streamer(request:, call:, method:, metadata: {}) + p "Received server streamer call at method #{method}" \ + " with request #{request} for call #{call}" \ + " and metadata: #{metadata}" + metadata['foo'] = 'bar_from_server_streamer' + yield + end + + def bidi_streamer(requests:, call:, method:, metadata: {}) + p "Received bidi streamer call at method #{method}" \ + "with requests #{requests} for call #{call}" \ + " and metadata: #{metadata}" + requests.each do |r| + p "In client interceptor: #{r}" + end + metadata['foo'] = 'bar_from_bidi_streamer' + yield + end +end -- cgit v1.2.3