diff options
67 files changed, 517 insertions, 979 deletions
@@ -1,3 +1,14 @@ +[VARIABLES] +# TODO(https://github.com/PyCQA/pylint/issues/1345): How does the inspection +# not include "unused_" and "ignored_" by default? +dummy-variables-rgx=^ignored_|^unused_ + +[MISCELLANEOUS] +# NOTE(nathaniel): We are big fans of "TODO(<issue link>): " and +# "NOTE(<username or issue link>): ". We do not allow "TODO:", +# "TODO(<username>):", "FIXME:", or anything else. +notes=FIXME,XXX + [MESSAGES CONTROL] #TODO: Enable missing-docstring @@ -11,15 +22,11 @@ #TODO: Enable protected-access #TODO: Enable no-name-in-module #TODO: Enable unused-argument -#TODO: Enable fixme #TODO: Enable wrong-import-order -#TODO: Enable no-value-for-parameter -#TODO: Enable cyclic-import -#TODO: Enable unused-variable -#TODO: Enable redefined-outer-name -#TODO: Enable unused-import +# TODO(https://github.com/PyCQA/pylint/issues/59#issuecomment-283774279): +# enable cyclic-import after a 1.7-or-later pylint release that recognizes our +# disable=cyclic-import suppressions. #TODO: Enable too-many-instance-attributes -#TODO: Enable broad-except #TODO: Enable too-many-locals #TODO: Enable too-many-lines #TODO: Enable redefined-variable-type @@ -29,6 +36,5 @@ #TODO: Enable too-many-return-statements #TODO: Enable too-many-nested-blocks #TODO: Enable super-init-not-called -#TODO: Enable no-self-use -disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,fixme,wrong-import-order,no-value-for-parameter,cyclic-import,unused-variable,redefined-outer-name,unused-import,too-many-instance-attributes,broad-except,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called,no-self-use +disable=missing-docstring,too-few-public-methods,too-many-arguments,no-init,duplicate-code,invalid-name,suppressed-message,locally-disabled,protected-access,no-name-in-module,unused-argument,wrong-import-order,cyclic-import,too-many-instance-attributes,too-many-locals,too-many-lines,redefined-variable-type,next-method-called,import-error,useless-else-on-loop,too-many-return-statements,too-many-nested-blocks,super-init-not-called diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d62c48380..53e1a7454b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1448,10 +1448,8 @@ add_library(grpc_test_util test/core/util/mock_endpoint.c test/core/util/parse_hexstring.c test/core/util/passthru_endpoint.c - test/core/util/port_posix.c + test/core/util/port.c test/core/util/port_server_client.c - test/core/util/port_uv.c - test/core/util/port_windows.c test/core/util/slice_splitter.c test/core/util/trickle_endpoint.c src/core/lib/channel/channel_args.c @@ -1657,10 +1655,8 @@ add_library(grpc_test_util_unsecure test/core/util/mock_endpoint.c test/core/util/parse_hexstring.c test/core/util/passthru_endpoint.c - test/core/util/port_posix.c + test/core/util/port.c test/core/util/port_server_client.c - test/core/util/port_uv.c - test/core/util/port_windows.c test/core/util/slice_splitter.c test/core/util/trickle_endpoint.c ) @@ -3323,10 +3323,8 @@ LIBGRPC_TEST_UTIL_SRC = \ test/core/util/mock_endpoint.c \ test/core/util/parse_hexstring.c \ test/core/util/passthru_endpoint.c \ - test/core/util/port_posix.c \ + test/core/util/port.c \ test/core/util/port_server_client.c \ - test/core/util/port_uv.c \ - test/core/util/port_windows.c \ test/core/util/slice_splitter.c \ test/core/util/trickle_endpoint.c \ src/core/lib/channel/channel_args.c \ @@ -3525,10 +3523,8 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \ test/core/util/mock_endpoint.c \ test/core/util/parse_hexstring.c \ test/core/util/passthru_endpoint.c \ - test/core/util/port_posix.c \ + test/core/util/port.c \ test/core/util/port_server_client.c \ - test/core/util/port_uv.c \ - test/core/util/port_windows.c \ test/core/util/slice_splitter.c \ test/core/util/trickle_endpoint.c \ diff --git a/build.yaml b/build.yaml index caa7dec83c..89895d83c4 100644 --- a/build.yaml +++ b/build.yaml @@ -611,10 +611,8 @@ filegroups: - test/core/util/mock_endpoint.c - test/core/util/parse_hexstring.c - test/core/util/passthru_endpoint.c - - test/core/util/port_posix.c + - test/core/util/port.c - test/core/util/port_server_client.c - - test/core/util/port_uv.c - - test/core/util/port_windows.c - test/core/util/slice_splitter.c - test/core/util/trickle_endpoint.c deps: diff --git a/composer.json b/composer.json index 53ae6388fb..0cafb94808 100644 --- a/composer.json +++ b/composer.json @@ -7,7 +7,7 @@ "license": "BSD-3-Clause", "require": { "php": ">=5.5.0", - "google/protobuf": "v3.1.0" + "google/protobuf": "^v3.1.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/doc/server_side_auth.md b/doc/server_side_auth.md new file mode 100644 index 0000000000..288c6e9cb3 --- /dev/null +++ b/doc/server_side_auth.md @@ -0,0 +1,60 @@ +Server-side API for Authenticating Clients +========================================== + +NOTE: This document describes how server-side authentication works in C-core based gRPC implementations only. In gRPC Java and Go, server side authentication is handled differently. + +## AuthContext + +To perform server-side authentication, gRPC exposes the *authentication context* for each call. The context exposes important authentication-related information about the RPC such as the type of security/authentication type being used and the peer identity. + +The authentication context is structured as a multi-map of key-value pairs - the *auth properties*. In addition to that, for authenticated RPCs, the set of properties corresponding to a selected key will represent the verified identity of the caller - the *peer identity*. + +The contents of the *auth properties* are populated by an *auth interceptor*. The interceptor also chooses which property key will act as the peer identity (e.g. for client certificate authentication this property will be `"x509_common_name"` or `"x509_subject_alternative_name"`). + +WARNING: AuthContext is the only reliable source of truth when it comes to authenticating RPCs. Using any other call/context properties for authentication purposes is wrong and inherently unsafe. + +####Example AuthContext contents + +For secure channel using mutual TLS authentication with both client and server certificates (test certificates from this repository are used). + +Populated auth properties: +``` +"transport_security_type": "ssl" # connection is secured using TLS/SSL +"x509_common_name": "*.test.google.com" # from client's certificate +"x509_pem_cert": "-----BEGIN CERTIFICATE-----\n..." # client's PEM encoded certificate +"x509_subject_alternative_name": "*.test.google.fr" +"x509_subject_alternative_name": "waterzooi.test.google.be" +"x509_subject_alternative_name": "*.test.youtube.com" +"x509_subject_alternative_name": "192.168.1.3" +``` + +The peer identity is set of all properties named `"x509_subject_alternative_name"`: +``` +peer_identity_property_name = "x509_subject_alternative_name" +``` + +## AuthProperty + +Auth properties are elements of the AuthContext. They have a name (a key of type string) and a value which can be a string or binary data. + +## Auth Interceptors + +Auth interceptors are gRPC components that populate contents of the auth context based on gRPC's internal state and/or call metadata. +gRPC comes with some basic "interceptors" already built-in. + +WARNING: While there is a public API that allows anyone to write their own custom interceptor, please think twice before using it. +There are legitimate uses for custom interceptors but you should keep in mind that as auth interceptors essentially decide which RPCs are authenticated and which are not, their code is very sensitive from the security perspective and getting things wrong might have serious consequences. If unsure, we strongly recommend to rely on official & proven interceptors that come with gRPC. + +####Available auth interceptors +- TLS/SSL certificate authentication (built into gRPC's security layer, automatically used whenever you use a secure connection) +- (coming soon) JWT auth token authentication +- more will be added over time + +## Status (by language) +C-core exposes low level API to access auth context contents and to implement an auth interceptor. +In C++, the auth interceptor API is exposed as `AuthMetadataProcessor`. + +A high level API to access AuthContext contents is available in these languages: +- C++ +- C# (implementation in-progress) +- other languages coming soon diff --git a/examples/php/composer.json b/examples/php/composer.json index f0b9977252..f4b177c271 100644 --- a/examples/php/composer.json +++ b/examples/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-demo", "description": "gRPC example for PHP", "require": { - "grpc/grpc": "v1.1.0" + "grpc/grpc": "^v1.1.0" }, "autoload": { "psr-4": { diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index beb72276d2..909ea5af96 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -917,7 +917,7 @@ Pod::Spec.new do |s| 'test/core/util/debugger_macros.c', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', - 'test/core/util/port_posix.c', + 'test/core/util/port.c', 'test/core/util/port_server_client.{c,h}' end end diff --git a/package.xml b/package.xml index d82f2e49a8..e16161d7a2 100644 --- a/package.xml +++ b/package.xml @@ -10,7 +10,7 @@ <email>grpc-packages@google.com</email> <active>yes</active> </lead> - <date>2017-01-13</date> + <date>2017-03-01</date> <time>16:06:07</time> <version> <release>1.2.0dev</release> @@ -22,8 +22,8 @@ </stability> <license>BSD</license> <notes> -- PHP Proto3 adoption #8179 -- Various bug fixes +- Added arg info macros #9751 +- Updated codegen to be consistent with protobuf #9492 </notes> <contents> <dir baseinstalldir="/" name="/"> @@ -1276,8 +1276,8 @@ Update to wrap gRPC C Core version 0.10.0 </release> <release> <version> - <release>1.1.0dev</release> - <api>1.1.0dev</api> + <release>1.1.0RC1</release> + <api>1.1.0RC1</api> </version> <stability> <release>beta</release> @@ -1287,7 +1287,39 @@ Update to wrap gRPC C Core version 0.10.0 <license>BSD</license> <notes> - PHP Proto3 adoption #8179 -- Various bug fixes +- Various bug fixes + </notes> + </release> + <release> + <version> + <release>1.1.0</release> + <api>1.1.0</api> + </version> + <stability> + <release>stable</release> + <api>stable</api> + </stability> + <date>2017-01-31</date> + <license>BSD</license> + <notes> +- PHP Proto3 adoption #8179 +- Various bug fixes + </notes> + </release> + <release> + <version> + <release>1.2.0RC1</release> + <api>1.2.0RC1</api> + </version> + <stability> + <release>beta</release> + <api>beta</api> + </stability> + <date>2017-03-01</date> + <license>BSD</license> + <notes> +- Added arg info macros #9751 +- Updated codegen to be consistent with protobuf #9492 </notes> </release> </changelog> diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index 2f84adc187..b9c56c103c 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -284,9 +284,9 @@ static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte, case GRPC_HTTP_HEADERS: if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) { if (grpc_http1_trace) - gpr_log(GPR_ERROR, "HTTP client max line length (%d) exceeded", + gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded", GRPC_HTTP_PARSER_MAX_HEADER_LENGTH); - return GRPC_ERROR_NONE; + return GRPC_ERROR_CREATE("HTTP header max line length exceeded"); } parser->cur_line[parser->cur_line_length] = byte; parser->cur_line_length++; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index cc57654ea4..c2547c5147 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -161,6 +161,7 @@ struct grpc_call { bool receiving_message; bool requested_final_op; bool received_final_op; + bool sent_any_op; /* have we received initial metadata */ bool has_initial_md_been_received; @@ -488,7 +489,7 @@ void grpc_call_destroy(grpc_call *c) { gpr_mu_lock(&c->mu); GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = !c->received_final_op; + cancel = c->sent_any_op && !c->received_final_op; gpr_mu_unlock(&c->mu); if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, @@ -1678,6 +1679,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx, grpc_closure_init(&bctl->finish_batch, finish_batch, bctl, grpc_schedule_on_exec_ctx); stream_op->on_complete = &bctl->finish_batch; + call->sent_any_op = true; gpr_mu_unlock(&call->mu); execute_op(exec_ctx, call, stream_op); diff --git a/src/core/lib/tsi/test_creds/BUILD b/src/core/lib/tsi/test_creds/BUILD index dcd6d930a8..5cf04caf17 100644 --- a/src/core/lib/tsi/test_creds/BUILD +++ b/src/core/lib/tsi/test_creds/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + exports_files([ "ca.pem", "server1.key", diff --git a/src/cpp/server/health/default_health_check_service.cc b/src/cpp/server/health/default_health_check_service.cc index 46def70e8a..bc98ce79a7 100644 --- a/src/cpp/server/health/default_health_check_service.cc +++ b/src/cpp/server/health/default_health_check_service.cc @@ -63,7 +63,9 @@ Status DefaultHealthCheckService::HealthCheckServiceImpl::Check( ServerContext* context, const ByteBuffer* request, ByteBuffer* response) { // Decode request. std::vector<Slice> slices; - request->Dump(&slices); + if (!request->Dump(&slices).ok()) { + return Status(StatusCode::INVALID_ARGUMENT, ""); + } uint8_t* request_bytes = nullptr; bool request_bytes_owned = false; size_t request_size = 0; diff --git a/src/php/README.md b/src/php/README.md index 821ea16aab..f08541f16c 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -158,7 +158,7 @@ $ composer install ### Protobuf compiler Again if you don't have it already, you need to install the protobuf compiler -`protoc`, version 3.2.0+. +`protoc`, version 3.1.0+ (the newer the better). If `protoc` hasn't been installed, you can download the `protoc` binaries from [the protocol buffers Github repository](https://github.com/google/protobuf/releases). diff --git a/src/php/composer.json b/src/php/composer.json index 8528304c81..491e34795a 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -5,7 +5,7 @@ "version": "1.2.0", "require": { "php": ">=5.5.0", - "google/protobuf": "v3.1.0" + "google/protobuf": "^v3.1.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index b64a708cc7..a4481b2ac3 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1004,7 +1004,7 @@ def unary_unary_rpc_method_handler(behavior, An RpcMethodHandler for a unary-unary RPC method constructed from the given parameters. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.RpcMethodHandler(False, False, request_deserializer, response_serializer, behavior, None, None, None) @@ -1025,7 +1025,7 @@ def unary_stream_rpc_method_handler(behavior, An RpcMethodHandler for a unary-stream RPC method constructed from the given parameters. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.RpcMethodHandler(False, True, request_deserializer, response_serializer, None, behavior, None, None) @@ -1046,7 +1046,7 @@ def stream_unary_rpc_method_handler(behavior, An RpcMethodHandler for a stream-unary RPC method constructed from the given parameters. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.RpcMethodHandler(True, False, request_deserializer, response_serializer, None, None, behavior, None) @@ -1068,7 +1068,7 @@ def stream_stream_rpc_method_handler(behavior, An RpcMethodHandler for a stream-stream RPC method constructed from the given parameters. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.RpcMethodHandler(True, True, request_deserializer, response_serializer, None, None, None, behavior) @@ -1085,7 +1085,7 @@ def method_handlers_generic_handler(service, method_handlers): Returns: A GenericRpcHandler constructed from the given parameters. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.DictionaryGenericHandler(service, method_handlers) @@ -1124,7 +1124,7 @@ def metadata_call_credentials(metadata_plugin, name=None): Returns: A CallCredentials. """ - from grpc import _plugin_wrapping + from grpc import _plugin_wrapping # pylint: disable=cyclic-import if name is None: try: effective_name = metadata_plugin.__name__ @@ -1147,7 +1147,7 @@ def access_token_call_credentials(access_token): Returns: A CallCredentials. """ - from grpc import _auth + from grpc import _auth # pylint: disable=cyclic-import return metadata_call_credentials( _auth.AccessTokenCallCredentials(access_token)) @@ -1161,7 +1161,7 @@ def composite_call_credentials(*call_credentials): Returns: A CallCredentials object composed of the given CallCredentials objects. """ - from grpc import _credential_composition + from grpc import _credential_composition # pylint: disable=cyclic-import cygrpc_call_credentials = tuple( single_call_credentials._credentials for single_call_credentials in call_credentials) @@ -1180,7 +1180,7 @@ def composite_channel_credentials(channel_credentials, *call_credentials): A ChannelCredentials composed of the given ChannelCredentials and CallCredentials objects. """ - from grpc import _credential_composition + from grpc import _credential_composition # pylint: disable=cyclic-import cygrpc_call_credentials = tuple( single_call_credentials._credentials for single_call_credentials in call_credentials) @@ -1237,7 +1237,7 @@ def channel_ready_future(channel): A Future that matures when the given Channel has connectivity ChannelConnectivity.READY. """ - from grpc import _utilities + from grpc import _utilities # pylint: disable=cyclic-import return _utilities.channel_ready_future(channel) @@ -1252,7 +1252,7 @@ def insecure_channel(target, options=None): Returns: A Channel to the target through which RPCs may be conducted. """ - from grpc import _channel + from grpc import _channel # pylint: disable=cyclic-import return _channel.Channel(target, () if options is None else options, None) @@ -1268,7 +1268,7 @@ def secure_channel(target, credentials, options=None): Returns: A Channel to the target through which RPCs may be conducted. """ - from grpc import _channel + from grpc import _channel # pylint: disable=cyclic-import return _channel.Channel(target, () if options is None else options, credentials._credentials) @@ -1290,7 +1290,7 @@ def server(thread_pool, handlers=None, options=None): Returns: A Server with which RPCs can be serviced. """ - from grpc import _server + from grpc import _server # pylint: disable=cyclic-import return _server.Server(thread_pool, () if handlers is None else handlers, () if options is None else options) diff --git a/src/python/grpcio/grpc/_auth.py b/src/python/grpcio/grpc/_auth.py index 21131f85f1..cb7c6fe4fd 100644 --- a/src/python/grpcio/grpc/_auth.py +++ b/src/python/grpcio/grpc/_auth.py @@ -39,6 +39,19 @@ def _sign_request(callback, token, error): callback(metadata, error) +def _create_get_token_callback(callback): + + def get_token_callback(future): + try: + access_token = future.result().access_token + except Exception as exception: # pylint: disable=broad-except + _sign_request(callback, None, exception) + else: + _sign_request(callback, access_token, None) + + return get_token_callback + + class GoogleCallCredentials(grpc.AuthMetadataPlugin): """Metadata wrapper for GoogleCredentials from the oauth2client library.""" @@ -59,16 +72,7 @@ class GoogleCallCredentials(grpc.AuthMetadataPlugin): additional_claims={'aud': context.service_url}) else: future = self._pool.submit(self._credentials.get_access_token) - future.add_done_callback( - lambda x: self._get_token_callback(callback, x)) - - def _get_token_callback(self, callback, future): - try: - access_token = future.result().access_token - except Exception as e: - _sign_request(callback, None, e) - else: - _sign_request(callback, access_token, None) + future.add_done_callback(_create_get_token_callback(callback)) def __del__(self): self._pool.shutdown(wait=False) diff --git a/src/python/grpcio/grpc/_channel.py b/src/python/grpcio/grpc/_channel.py index af86f5eabe..c86cb25682 100644 --- a/src/python/grpcio/grpc/_channel.py +++ b/src/python/grpcio/grpc/_channel.py @@ -200,7 +200,7 @@ def _consume_request_iterator(request_iterator, state, call, request = next(request_iterator) except StopIteration: break - except Exception as e: + except Exception: # pylint: disable=broad-except logging.exception("Exception iterating requests!") call.cancel() _abort(state, grpc.StatusCode.UNKNOWN, @@ -387,13 +387,14 @@ class _Rendezvous(grpc.RpcError, grpc.Future, grpc.Call): with self._state.condition: while self._state.initial_metadata is None: self._state.condition.wait() - return _common.application_metadata(self._state.initial_metadata) + return _common.to_application_metadata(self._state.initial_metadata) def trailing_metadata(self): with self._state.condition: while self._state.trailing_metadata is None: self._state.condition.wait() - return _common.application_metadata(self._state.trailing_metadata) + return _common.to_application_metadata( + self._state.trailing_metadata) def code(self): with self._state.condition: @@ -473,7 +474,7 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), _EMPTY_FLAGS), + _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), cygrpc.operation_send_message(serialized_request, _EMPTY_FLAGS), cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), @@ -563,7 +564,7 @@ class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): )), event_handler) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), + _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), cygrpc.operation_send_message( serialized_request, _EMPTY_FLAGS), cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), @@ -603,7 +604,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): None) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), _EMPTY_FLAGS), + _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), cygrpc.operation_receive_message(_EMPTY_FLAGS), cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) call_error = call.start_client_batch( @@ -657,7 +658,7 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): event_handler) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), _EMPTY_FLAGS), + _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), cygrpc.operation_receive_message(_EMPTY_FLAGS), cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) call_error = call.start_client_batch( @@ -700,7 +701,7 @@ class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): event_handler) operations = ( cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(metadata), _EMPTY_FLAGS), + _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) call_error = call.start_client_batch( cygrpc.Operations(operations), event_handler) diff --git a/src/python/grpcio/grpc/_common.py b/src/python/grpcio/grpc/_common.py index 6879e1780b..2e369013f5 100644 --- a/src/python/grpcio/grpc/_common.py +++ b/src/python/grpcio/grpc/_common.py @@ -97,22 +97,22 @@ def decode(b): def channel_args(options): - channel_args = [] + cygrpc_args = [] for key, value in options: if isinstance(value, six.string_types): - channel_args.append(cygrpc.ChannelArg(encode(key), encode(value))) + cygrpc_args.append(cygrpc.ChannelArg(encode(key), encode(value))) else: - channel_args.append(cygrpc.ChannelArg(encode(key), value)) - return cygrpc.ChannelArgs(channel_args) + cygrpc_args.append(cygrpc.ChannelArg(encode(key), value)) + return cygrpc.ChannelArgs(cygrpc_args) -def cygrpc_metadata(application_metadata): +def to_cygrpc_metadata(application_metadata): return EMPTY_METADATA if application_metadata is None else cygrpc.Metadata( cygrpc.Metadatum(encode(key), encode(value)) for key, value in application_metadata) -def application_metadata(cygrpc_metadata): +def to_application_metadata(cygrpc_metadata): if cygrpc_metadata is None: return () else: diff --git a/src/python/grpcio/grpc/_plugin_wrapping.py b/src/python/grpcio/grpc/_plugin_wrapping.py index 69c46aa546..1e44561c97 100644 --- a/src/python/grpcio/grpc/_plugin_wrapping.py +++ b/src/python/grpcio/grpc/_plugin_wrapping.py @@ -66,9 +66,9 @@ class _WrappedCygrpcCallback(object): def _invoke_success(self, metadata): try: - cygrpc_metadata = _common.cygrpc_metadata(metadata) - except Exception as error: - self._invoke_failure(error) + cygrpc_metadata = _common.to_cygrpc_metadata(metadata) + except Exception as exception: # pylint: disable=broad-except + self._invoke_failure(exception) return self.cygrpc_callback(cygrpc_metadata, cygrpc.StatusCode.ok, b'') diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index b8e7ea17f7..84e096d4c0 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -142,14 +142,14 @@ def _abort(state, call, code, details): effective_details = details if state.details is None else state.details if state.initial_metadata_allowed: operations = (cygrpc.operation_send_initial_metadata( - _common.EMPTY_METADATA, _EMPTY_FLAGS), - cygrpc.operation_send_status_from_server( - _common.cygrpc_metadata(state.trailing_metadata), - effective_code, effective_details, _EMPTY_FLAGS),) + _common.EMPTY_METADATA, + _EMPTY_FLAGS), cygrpc.operation_send_status_from_server( + _common.to_cygrpc_metadata(state.trailing_metadata), + effective_code, effective_details, _EMPTY_FLAGS),) token = _SEND_INITIAL_METADATA_AND_SEND_STATUS_FROM_SERVER_TOKEN else: operations = (cygrpc.operation_send_status_from_server( - _common.cygrpc_metadata(state.trailing_metadata), + _common.to_cygrpc_metadata(state.trailing_metadata), effective_code, effective_details, _EMPTY_FLAGS),) token = _SEND_STATUS_FROM_SERVER_TOKEN call.start_server_batch( @@ -250,7 +250,7 @@ class _Context(grpc.ServicerContext): self._state.disable_next_compression = True def invocation_metadata(self): - return _common.application_metadata(self._rpc_event.request_metadata) + return _common.to_application_metadata(self._rpc_event.request_metadata) def peer(self): return _common.decode(self._rpc_event.operation_call.peer()) @@ -262,7 +262,8 @@ class _Context(grpc.ServicerContext): else: if self._state.initial_metadata_allowed: operation = cygrpc.operation_send_initial_metadata( - _common.cygrpc_metadata(initial_metadata), _EMPTY_FLAGS) + _common.to_cygrpc_metadata(initial_metadata), + _EMPTY_FLAGS) self._rpc_event.operation_call.start_server_batch( cygrpc.Operations((operation,)), _send_initial_metadata(self._state)) @@ -273,7 +274,7 @@ class _Context(grpc.ServicerContext): def set_trailing_metadata(self, trailing_metadata): with self._state.condition: - self._state.trailing_metadata = _common.cygrpc_metadata( + self._state.trailing_metadata = _common.to_cygrpc_metadata( trailing_metadata) def set_code(self, code): @@ -342,7 +343,7 @@ def _unary_request(rpc_event, state, request_deserializer): if state.client is _CANCELLED or state.statused: return None else: - start_server_batch_result = rpc_event.operation_call.start_server_batch( + rpc_event.operation_call.start_server_batch( cygrpc.Operations( (cygrpc.operation_receive_message(_EMPTY_FLAGS),)), _receive_message(state, rpc_event.operation_call, @@ -436,7 +437,8 @@ def _send_response(rpc_event, state, serialized_response): def _status(rpc_event, state, serialized_response): with state.condition: if state.client is not _CANCELLED: - trailing_metadata = _common.cygrpc_metadata(state.trailing_metadata) + trailing_metadata = _common.to_cygrpc_metadata( + state.trailing_metadata) code = _completion_code(state) details = _details(state) operations = [ diff --git a/src/python/grpcio/grpc/beta/_client_adaptations.py b/src/python/grpcio/grpc/beta/_client_adaptations.py index b53395e2a2..901cb85612 100644 --- a/src/python/grpcio/grpc/beta/_client_adaptations.py +++ b/src/python/grpcio/grpc/beta/_client_adaptations.py @@ -30,7 +30,6 @@ import grpc from grpc import _common -from grpc._cython import cygrpc from grpc.beta import interfaces from grpc.framework.common import cardinality from grpc.framework.foundation import future @@ -621,8 +620,8 @@ class _GenericStub(face.GenericStub): class _DynamicStub(face.DynamicStub): - def __init__(self, generic_stub, group, cardinalities): - self._generic_stub = generic_stub + def __init__(self, backing_generic_stub, group, cardinalities): + self._generic_stub = backing_generic_stub self._group = group self._cardinalities = cardinalities diff --git a/src/python/grpcio/grpc/beta/_connectivity_channel.py b/src/python/grpcio/grpc/beta/_connectivity_channel.py deleted file mode 100644 index bfb847f80a..0000000000 --- a/src/python/grpcio/grpc/beta/_connectivity_channel.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Affords a connectivity-state-listenable channel.""" - -import threading -import time - -from grpc._adapter import _low -from grpc._adapter import _types -from grpc.beta import interfaces -from grpc.framework.foundation import callable_util - -_CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE = ( - 'Exception calling channel subscription callback!') - -_LOW_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY = { - state: connectivity - for state, connectivity in zip(_types.ConnectivityState, - interfaces.ChannelConnectivity) -} - - -class ConnectivityChannel(object): - - def __init__(self, low_channel): - self._lock = threading.Lock() - self._low_channel = low_channel - - self._polling = False - self._connectivity = None - self._try_to_connect = False - self._callbacks_and_connectivities = [] - self._delivering = False - - def _deliveries(self, connectivity): - callbacks_needing_update = [] - for callback_and_connectivity in self._callbacks_and_connectivities: - callback, callback_connectivity = callback_and_connectivity - if callback_connectivity is not connectivity: - callbacks_needing_update.append(callback) - callback_and_connectivity[1] = connectivity - return callbacks_needing_update - - def _deliver(self, initial_connectivity, initial_callbacks): - connectivity = initial_connectivity - callbacks = initial_callbacks - while True: - for callback in callbacks: - callable_util.call_logging_exceptions( - callback, _CHANNEL_SUBSCRIPTION_CALLBACK_ERROR_LOG_MESSAGE, - connectivity) - with self._lock: - callbacks = self._deliveries(self._connectivity) - if callbacks: - connectivity = self._connectivity - else: - self._delivering = False - return - - def _spawn_delivery(self, connectivity, callbacks): - delivering_thread = threading.Thread( - target=self._deliver, args=(connectivity, callbacks,)) - delivering_thread.start() - self._delivering = True - - # TODO(issue 3064): Don't poll. - def _poll_connectivity(self, low_channel, initial_try_to_connect): - try_to_connect = initial_try_to_connect - low_connectivity = low_channel.check_connectivity_state(try_to_connect) - with self._lock: - self._connectivity = _LOW_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[ - low_connectivity] - callbacks = tuple( - callback - for callback, unused_but_known_to_be_none_connectivity in - self._callbacks_and_connectivities) - for callback_and_connectivity in self._callbacks_and_connectivities: - callback_and_connectivity[1] = self._connectivity - if callbacks: - self._spawn_delivery(self._connectivity, callbacks) - completion_queue = _low.CompletionQueue() - while True: - low_channel.watch_connectivity_state(low_connectivity, - time.time() + 0.2, - completion_queue, None) - event = completion_queue.next() - with self._lock: - if not self._callbacks_and_connectivities and not self._try_to_connect: - self._polling = False - self._connectivity = None - completion_queue.shutdown() - break - try_to_connect = self._try_to_connect - self._try_to_connect = False - if event.success or try_to_connect: - low_connectivity = low_channel.check_connectivity_state( - try_to_connect) - with self._lock: - self._connectivity = _LOW_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY[ - low_connectivity] - if not self._delivering: - callbacks = self._deliveries(self._connectivity) - if callbacks: - self._spawn_delivery(self._connectivity, callbacks) - - def subscribe(self, callback, try_to_connect): - with self._lock: - if not self._callbacks_and_connectivities and not self._polling: - polling_thread = threading.Thread( - target=self._poll_connectivity, - args=(self._low_channel, bool(try_to_connect))) - polling_thread.start() - self._polling = True - self._callbacks_and_connectivities.append([callback, None]) - elif not self._delivering and self._connectivity is not None: - self._spawn_delivery(self._connectivity, (callback,)) - self._try_to_connect |= bool(try_to_connect) - self._callbacks_and_connectivities.append( - [callback, self._connectivity]) - else: - self._try_to_connect |= bool(try_to_connect) - self._callbacks_and_connectivities.append([callback, None]) - - def unsubscribe(self, callback): - with self._lock: - for index, (subscribed_callback, unused_connectivity - ) in enumerate(self._callbacks_and_connectivities): - if callback == subscribed_callback: - self._callbacks_and_connectivities.pop(index) - break - - def low_channel(self): - return self._low_channel diff --git a/src/python/grpcio/grpc/beta/_server_adaptations.py b/src/python/grpcio/grpc/beta/_server_adaptations.py index 174af2d642..81348d5d87 100644 --- a/src/python/grpcio/grpc/beta/_server_adaptations.py +++ b/src/python/grpcio/grpc/beta/_server_adaptations.py @@ -78,7 +78,7 @@ class _FaceServicerContext(face.ServicerContext): return _ServerProtocolContext(self._servicer_context) def invocation_metadata(self): - return _common.cygrpc_metadata( + return _common.to_cygrpc_metadata( self._servicer_context.invocation_metadata()) def initial_metadata(self, initial_metadata): @@ -351,27 +351,27 @@ class _GenericRpcHandler(grpc.GenericRpcHandler): class _Server(interfaces.Server): - def __init__(self, server): - self._server = server + def __init__(self, grpc_server): + self._grpc_server = grpc_server def add_insecure_port(self, address): - return self._server.add_insecure_port(address) + return self._grpc_server.add_insecure_port(address) def add_secure_port(self, address, server_credentials): - return self._server.add_secure_port(address, server_credentials) + return self._grpc_server.add_secure_port(address, server_credentials) def start(self): - self._server.start() + self._grpc_server.start() def stop(self, grace): - return self._server.stop(grace) + return self._grpc_server.stop(grace) def __enter__(self): - self._server.start() + self._grpc_server.start() return self def __exit__(self, exc_type, exc_val, exc_tb): - self._server.stop(None) + self._grpc_server.stop(None) return False diff --git a/src/python/grpcio/grpc/beta/implementations.py b/src/python/grpcio/grpc/beta/implementations.py index 7093852278..0b79577689 100644 --- a/src/python/grpcio/grpc/beta/implementations.py +++ b/src/python/grpcio/grpc/beta/implementations.py @@ -29,16 +29,15 @@ """Entry points into the Beta API of gRPC Python.""" # threading is referenced from specification in this module. -import abc -import enum import threading # pylint: disable=unused-import -# cardinality and face are referenced from specification in this module. +# interfaces, cardinality, and face are referenced from specification in this +# module. import grpc from grpc import _auth from grpc.beta import _client_adaptations from grpc.beta import _server_adaptations -from grpc.beta import interfaces +from grpc.beta import interfaces # pylint: disable=unused-import from grpc.framework.common import cardinality # pylint: disable=unused-import from grpc.framework.interfaces.face import face # pylint: disable=unused-import @@ -218,7 +217,7 @@ def dynamic_stub(channel, service, cardinalities, options=None): Returns: A face.DynamicStub with which RPCs can be invoked. """ - effective_options = StubOptions() if options is None else options + effective_options = _EMPTY_STUB_OPTIONS if options is None else options return _client_adaptations.dynamic_stub( channel._channel, # pylint: disable=protected-access service, diff --git a/src/python/grpcio/grpc/framework/foundation/logging_pool.py b/src/python/grpcio/grpc/framework/foundation/logging_pool.py index 7ee37373fa..0912fba139 100644 --- a/src/python/grpcio/grpc/framework/foundation/logging_pool.py +++ b/src/python/grpcio/grpc/framework/foundation/logging_pool.py @@ -39,7 +39,7 @@ def _wrap(behavior): def _wrapping(*args, **kwargs): try: return behavior(*args, **kwargs) - except Exception as e: + except Exception: logging.exception( 'Unexpected exception from %s executed in logging pool!', behavior) diff --git a/src/python/grpcio_tests/tests/http2/negative_http2_client.py b/src/python/grpcio_tests/tests/http2/negative_http2_client.py index b8adf093a5..b184e62cfd 100644 --- a/src/python/grpcio_tests/tests/http2/negative_http2_client.py +++ b/src/python/grpcio_tests/tests/http2/negative_http2_client.py @@ -31,6 +31,7 @@ import argparse import grpc +import time from src.proto.grpc.testing import test_pb2 from src.proto.grpc.testing import messages_pb2 @@ -75,6 +76,7 @@ def _goaway(stub): first_response = stub.UnaryCall(_SIMPLE_REQUEST) _validate_payload_type_and_length(first_response, messages_pb2.COMPRESSABLE, _RESPONSE_SIZE) + time.sleep(1) second_response = stub.UnaryCall(_SIMPLE_REQUEST) _validate_payload_type_and_length(second_response, messages_pb2.COMPRESSABLE, _RESPONSE_SIZE) diff --git a/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py b/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py index be3522f46f..eb5f459848 100644 --- a/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py +++ b/src/python/grpcio_tests/tests/unit/_thread_cleanup_test.py @@ -35,8 +35,8 @@ import unittest from grpc import _common _SHORT_TIME = 0.5 -_LONG_TIME = 2.0 -_EPSILON = 0.1 +_LONG_TIME = 5.0 +_EPSILON = 0.5 def cleanup(timeout): diff --git a/templates/composer.json.template b/templates/composer.json.template index f56ecb69d6..94f0c236a9 100644 --- a/templates/composer.json.template +++ b/templates/composer.json.template @@ -9,7 +9,7 @@ "license": "BSD-3-Clause", "require": { "php": ">=5.5.0", - "google/protobuf": "v3.1.0" + "google/protobuf": "^v3.1.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index f05ee4c05f..9ed32e31cf 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -191,7 +191,7 @@ 'test/core/util/debugger_macros.c', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', - 'test/core/util/port_posix.c', + 'test/core/util/port.c', 'test/core/util/port_server_client.{c,h}' end end diff --git a/templates/package.xml.template b/templates/package.xml.template index 80f1a1fe97..8655cfa1d9 100644 --- a/templates/package.xml.template +++ b/templates/package.xml.template @@ -12,7 +12,7 @@ <email>grpc-packages@google.com</email> <active>yes</active> </lead> - <date>2017-01-13</date> + <date>2017-03-01</date> <time>16:06:07</time> <version> <release>${settings.php_version.php()}</release> @@ -24,8 +24,8 @@ </stability> <license>BSD</license> <notes> - - PHP Proto3 adoption #8179 - - Various bug fixes + - Added arg info macros #9751 + - Updated codegen to be consistent with protobuf #9492 </notes> <contents> <dir baseinstalldir="/" name="/"> @@ -312,8 +312,8 @@ </release> <release> <version> - <release>1.1.0dev</release> - <api>1.1.0dev</api> + <release>1.1.0RC1</release> + <api>1.1.0RC1</api> </version> <stability> <release>beta</release> @@ -323,7 +323,39 @@ <license>BSD</license> <notes> - PHP Proto3 adoption #8179 - - Various bug fixes + - Various bug fixes + </notes> + </release> + <release> + <version> + <release>1.1.0</release> + <api>1.1.0</api> + </version> + <stability> + <release>stable</release> + <api>stable</api> + </stability> + <date>2017-01-31</date> + <license>BSD</license> + <notes> + - PHP Proto3 adoption #8179 + - Various bug fixes + </notes> + </release> + <release> + <version> + <release>1.2.0RC1</release> + <api>1.2.0RC1</api> + </version> + <stability> + <release>beta</release> + <api>beta</api> + </stability> + <date>2017-03-01</date> + <license>BSD</license> + <notes> + - Added arg info macros #9751 + - Updated codegen to be consistent with protobuf #9492 </notes> </release> </changelog> diff --git a/templates/src/php/composer.json.template b/templates/src/php/composer.json.template index ecc516d1c4..1887ee3c82 100644 --- a/templates/src/php/composer.json.template +++ b/templates/src/php/composer.json.template @@ -7,7 +7,7 @@ "version": "${settings.php_version.php_composer()}", "require": { "php": ">=5.5.0", - "google/protobuf": "v3.1.0" + "google/protobuf": "^v3.1.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/test/core/census/BUILD b/test/core/census/BUILD index 9ec48bdfe2..49680ab91f 100644 --- a/test/core/census/BUILD +++ b/test/core/census/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "context_test", srcs = ["context_test.c"], diff --git a/test/core/channel/BUILD b/test/core/channel/BUILD index 42cb468485..c6590465f1 100644 --- a/test/core/channel/BUILD +++ b/test/core/channel/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "channel_args_test", srcs = ["channel_args_test.c"], diff --git a/test/core/compression/BUILD b/test/core/compression/BUILD index a243a72029..9ddb4c52b4 100644 --- a/test/core/compression/BUILD +++ b/test/core/compression/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "algorithm_test", srcs = ["algorithm_test.c"], diff --git a/test/core/handshake/BUILD b/test/core/handshake/BUILD index 864e0db00b..eb8f3a9beb 100644 --- a/test/core/handshake/BUILD +++ b/test/core/handshake/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "client_ssl", srcs = ["client_ssl.c"], diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index efadddc011..6e514324a5 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -454,7 +454,8 @@ int main(int argc, char **argv) { const grpc_channel_args channel_args = {1, chan_args}; struct ifaddrs *ifa = NULL; struct ifaddrs *ifa_it; - test_addrs dst_addrs; + // Zalloc dst_addrs to avoid oversized frames. + test_addrs *dst_addrs = gpr_zalloc(sizeof(*dst_addrs)); grpc_test_init(argc, argv); grpc_init(); g_pollset = gpr_zalloc(grpc_pollset_size()); @@ -469,24 +470,25 @@ int main(int argc, char **argv) { gpr_log(GPR_ERROR, "getifaddrs: %s", strerror(errno)); return EXIT_FAILURE; } - dst_addrs.naddrs = 0; - for (ifa_it = ifa; ifa_it != NULL && dst_addrs.naddrs < MAX_ADDRS; + dst_addrs->naddrs = 0; + for (ifa_it = ifa; ifa_it != NULL && dst_addrs->naddrs < MAX_ADDRS; ifa_it = ifa_it->ifa_next) { if (ifa_it->ifa_addr == NULL) { continue; } else if (ifa_it->ifa_addr->sa_family == AF_INET) { - dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in); + dst_addrs->addrs[dst_addrs->naddrs].addr.len = sizeof(struct sockaddr_in); } else if (ifa_it->ifa_addr->sa_family == AF_INET6) { - dst_addrs.addrs[dst_addrs.naddrs].addr.len = sizeof(struct sockaddr_in6); + dst_addrs->addrs[dst_addrs->naddrs].addr.len = + sizeof(struct sockaddr_in6); } else { continue; } - memcpy(dst_addrs.addrs[dst_addrs.naddrs].addr.addr, ifa_it->ifa_addr, - dst_addrs.addrs[dst_addrs.naddrs].addr.len); + memcpy(dst_addrs->addrs[dst_addrs->naddrs].addr.addr, ifa_it->ifa_addr, + dst_addrs->addrs[dst_addrs->naddrs].addr.len); GPR_ASSERT( - grpc_sockaddr_set_port(&dst_addrs.addrs[dst_addrs.naddrs].addr, 0)); - test_addr_init_str(&dst_addrs.addrs[dst_addrs.naddrs]); - ++dst_addrs.naddrs; + grpc_sockaddr_set_port(&dst_addrs->addrs[dst_addrs->naddrs].addr, 0)); + test_addr_init_str(&dst_addrs->addrs[dst_addrs->naddrs]); + ++dst_addrs->naddrs; } freeifaddrs(ifa); ifa = NULL; @@ -495,20 +497,21 @@ int main(int argc, char **argv) { test_connect(1, NULL, NULL, false); test_connect(10, NULL, NULL, false); - /* Set dst_addrs.addrs[i].len=0 for dst_addrs that are unreachable with a "::" - listener. */ - test_connect(1, NULL, &dst_addrs, true); + /* Set dst_addrs->addrs[i].len=0 for dst_addrs that are unreachable with a + "::" listener. */ + test_connect(1, NULL, dst_addrs, true); /* Test connect(2) with dst_addrs. */ - test_connect(1, &channel_args, &dst_addrs, false); + test_connect(1, &channel_args, dst_addrs, false); /* Test connect(2) with dst_addrs. */ - test_connect(10, &channel_args, &dst_addrs, false); + test_connect(10, &channel_args, dst_addrs, false); grpc_closure_init(&destroyed, destroy_pollset, g_pollset, grpc_schedule_on_exec_ctx); grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed); grpc_exec_ctx_finish(&exec_ctx); grpc_shutdown(); + gpr_free(dst_addrs); gpr_free(g_pollset); return EXIT_SUCCESS; } diff --git a/test/core/support/BUILD b/test/core/support/BUILD index 08cee1441b..3183510db9 100644 --- a/test/core/support/BUILD +++ b/test/core/support/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "alloc_test", srcs = ["alloc_test.c"], diff --git a/test/core/surface/BUILD b/test/core/surface/BUILD index c158413122..3d5e26ced3 100644 --- a/test/core/surface/BUILD +++ b/test/core/surface/BUILD @@ -27,6 +27,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +licenses(["notice"]) # 3-clause BSD + cc_test( name = "alarm_test", srcs = ["alarm_test.c"], diff --git a/test/core/util/BUILD b/test/core/util/BUILD index 5c90f1db44..e6d0d247db 100644 --- a/test/core/util/BUILD +++ b/test/core/util/BUILD @@ -51,9 +51,8 @@ cc_library( "mock_endpoint.c", "parse_hexstring.c", "passthru_endpoint.c", - "port_posix.c", + "port.c", "port_server_client.c", - "port_windows.c", "reconnect_server.c", "slice_splitter.c", "test_tcp_server.c", diff --git a/test/core/util/port_uv.c b/test/core/util/port.c index 0c9c0d87d6..da1ed4e052 100644 --- a/test/core/util/port_uv.c +++ b/test/core/util/port.c @@ -33,17 +33,24 @@ #include "src/core/lib/iomgr/port.h" #include "test/core/util/test_config.h" -#if defined(GRPC_UV) && defined(GRPC_TEST_PICK_PORT) +#if defined(GRPC_TEST_PICK_PORT) +#include "test/core/util/port.h" + +#include <stdbool.h> +#include <stdio.h> +#include <string.h> + +#include <grpc/grpc.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> -#include "src/core/lib/support/env.h" -#include "test/core/util/port.h" +#include "src/core/lib/http/httpcli.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "test/core/util/port_server_client.h" -// Almost everything in this file has been copied from port_posix.c - static int *chosen_ports = NULL; static size_t num_chosen_ports = 0; @@ -51,7 +58,6 @@ static int free_chosen_port(int port) { size_t i; int found = 0; size_t found_at = 0; - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); /* Find the port and erase it from the list, then tell the server it can be freed. */ for (i = 0; i < num_chosen_ports; i++) { @@ -64,24 +70,16 @@ static int free_chosen_port(int port) { if (found) { chosen_ports[found_at] = chosen_ports[num_chosen_ports - 1]; num_chosen_ports--; - if (env) { - grpc_free_port_using_server(env, port); - } + grpc_free_port_using_server(port); } - gpr_free(env); return found; } static void free_chosen_ports(void) { - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env != NULL) { - size_t i; - for (i = 0; i < num_chosen_ports; i++) { - grpc_free_port_using_server(env, chosen_ports[i]); - } - gpr_free(env); + size_t i; + for (i = 0; i < num_chosen_ports; i++) { + grpc_free_port_using_server(chosen_ports[i]); } - gpr_free(chosen_ports); } @@ -95,23 +93,27 @@ static void chose_port(int port) { } int grpc_pick_unused_port(void) { - // Currently only works with the port server - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - GPR_ASSERT(env); - int port = grpc_pick_port_using_server(env); - gpr_free(env); + int port = grpc_pick_port_using_server(); if (port != 0) { chose_port(port); } + return port; } int grpc_pick_unused_port_or_die(void) { int port = grpc_pick_unused_port(); - GPR_ASSERT(port > 0); + if (port == 0) { + fprintf(stderr, + "gRPC tests require a helper port server to allocate ports used \n" + "during the test.\n\n" + "This server is not currently running.\n\n" + "To start it, run tools/run_tests/start_port_server.py\n\n"); + exit(1); + } return port; } void grpc_recycle_unused_port(int port) { GPR_ASSERT(free_chosen_port(port)); } -#endif /* GRPC_UV && GRPC_TEST_PICK_PORT */ +#endif /* GRPC_TEST_PICK_PORT */ diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c deleted file mode 100644 index 4a42e4c702..0000000000 --- a/test/core/util/port_posix.c +++ /dev/null @@ -1,256 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/iomgr/port.h" -#include "test/core/util/test_config.h" -#if defined(GRPC_POSIX_SOCKET) && defined(GRPC_TEST_PICK_PORT) - -#include "test/core/util/port.h" - -#include <errno.h> -#include <netinet/in.h> -#include <stdbool.h> -#include <stdio.h> -#include <string.h> -#include <sys/socket.h> -#include <unistd.h> - -#include <grpc/grpc.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> -#include <grpc/support/string_util.h> - -#include "src/core/lib/http/httpcli.h" -#include "src/core/lib/iomgr/resolve_address.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" -#include "src/core/lib/support/env.h" -#include "test/core/util/port_server_client.h" - -#define NUM_RANDOM_PORTS_TO_PICK 100 - -static int *chosen_ports = NULL; -static size_t num_chosen_ports = 0; - -static int has_port_been_chosen(int port) { - size_t i; - for (i = 0; i < num_chosen_ports; i++) { - if (chosen_ports[i] == port) { - return 1; - } - } - return 0; -} - -static int free_chosen_port(int port) { - size_t i; - int found = 0; - size_t found_at = 0; - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - /* Find the port and erase it from the list, then tell the server it can be - freed. */ - for (i = 0; i < num_chosen_ports; i++) { - if (chosen_ports[i] == port) { - GPR_ASSERT(found == 0); - found = 1; - found_at = i; - } - } - if (found) { - chosen_ports[found_at] = chosen_ports[num_chosen_ports - 1]; - num_chosen_ports--; - if (env) { - grpc_free_port_using_server(env, port); - } - } - gpr_free(env); - return found; -} - -static void free_chosen_ports(void) { - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env != NULL) { - size_t i; - for (i = 0; i < num_chosen_ports; i++) { - grpc_free_port_using_server(env, chosen_ports[i]); - } - gpr_free(env); - } - - gpr_free(chosen_ports); -} - -static void chose_port(int port) { - if (chosen_ports == NULL) { - atexit(free_chosen_ports); - } - num_chosen_ports++; - chosen_ports = gpr_realloc(chosen_ports, sizeof(int) * num_chosen_ports); - chosen_ports[num_chosen_ports - 1] = port; -} - -static bool is_port_available(int *port, bool is_tcp) { - GPR_ASSERT(*port >= 0); - GPR_ASSERT(*port <= 65535); - - /* For a port to be considered available, the kernel must support - at least one of (IPv6, IPv4), and the port must be available - on each supported family. */ - bool got_socket = false; - for (int is_ipv6 = 1; is_ipv6 >= 0; is_ipv6--) { - const int fd = - socket(is_ipv6 ? AF_INET6 : AF_INET, is_tcp ? SOCK_STREAM : SOCK_DGRAM, - is_tcp ? IPPROTO_TCP : 0); - if (fd >= 0) { - got_socket = true; - } else { - continue; - } - - /* Reuseaddr lets us start up a server immediately after it exits */ - const int one = 1; - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) < 0) { - gpr_log(GPR_ERROR, "setsockopt() failed: %s", strerror(errno)); - close(fd); - return false; - } - - /* Try binding to port */ - grpc_resolved_address addr; - if (is_ipv6) { - grpc_sockaddr_make_wildcard6(*port, &addr); /* [::]:port */ - } else { - grpc_sockaddr_make_wildcard4(*port, &addr); /* 0.0.0.0:port */ - } - if (bind(fd, (struct sockaddr *)addr.addr, (socklen_t)addr.len) < 0) { - gpr_log(GPR_DEBUG, "bind(port=%d) failed: %s", *port, strerror(errno)); - close(fd); - return false; - } - - /* Get the bound port number */ - if (getsockname(fd, (struct sockaddr *)addr.addr, (socklen_t *)&addr.len) < - 0) { - gpr_log(GPR_ERROR, "getsockname() failed: %s", strerror(errno)); - close(fd); - return false; - } - GPR_ASSERT(addr.len <= sizeof(addr.addr)); - const int actual_port = grpc_sockaddr_get_port(&addr); - GPR_ASSERT(actual_port > 0); - if (*port == 0) { - *port = actual_port; - } else { - GPR_ASSERT(*port == actual_port); - } - - close(fd); - } - if (!got_socket) { - gpr_log(GPR_ERROR, "socket() failed: %s", strerror(errno)); - return false; - } - return true; -} - -int grpc_pick_unused_port(void) { - /* We repeatedly pick a port and then see whether or not it is - available for use both as a TCP socket and a UDP socket. First, we - pick a random large port number. For subsequent - iterations, we bind to an anonymous port and let the OS pick the - port number. The random port picking reduces the probability of - races with other processes on kernels that want to reuse the same - port numbers over and over. */ - - /* In alternating iterations we trial UDP ports before TCP ports UDP - ports -- it could be the case that this machine has been using up - UDP ports and they are scarcer. */ - - /* Type of port to first pick in next iteration */ - bool is_tcp = true; - int trial = 0; - - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env) { - int port = grpc_pick_port_using_server(env); - gpr_free(env); - if (port != 0) { - chose_port(port); - } - return port; - } - - for (;;) { - int port; - trial++; - if (trial == 1) { - port = getpid() % (65536 - 30000) + 30000; - } else if (trial <= NUM_RANDOM_PORTS_TO_PICK) { - port = rand() % (65536 - 30000) + 30000; - } else { - port = 0; - } - - if (has_port_been_chosen(port)) { - continue; - } - - if (!is_port_available(&port, is_tcp)) { - continue; - } - - GPR_ASSERT(port > 0); - /* Check that the port # is free for the other type of socket also */ - if (!is_port_available(&port, !is_tcp)) { - /* In the next iteration trial to bind to the other type first - because perhaps it is more rare. */ - is_tcp = !is_tcp; - continue; - } - - chose_port(port); - return port; - } - - /* The port iterator reached the end without finding a suitable port. */ - return 0; -} - -int grpc_pick_unused_port_or_die(void) { - int port = grpc_pick_unused_port(); - GPR_ASSERT(port > 0); - return port; -} - -void grpc_recycle_unused_port(int port) { GPR_ASSERT(free_chosen_port(port)); } - -#endif /* GRPC_POSIX_SOCKET && GRPC_TEST_PICK_PORT */ diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index f7d723a7d2..a851d01635 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -74,7 +74,7 @@ static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, gpr_mu_unlock(pr->mu); } -void grpc_free_port_using_server(char *server, int port) { +void grpc_free_port_using_server(int port) { grpc_httpcli_context context; grpc_httpcli_request req; grpc_httpcli_response rsp; @@ -95,7 +95,7 @@ void grpc_free_port_using_server(char *server, int port) { shutdown_closure = grpc_closure_create(destroy_pops_and_shutdown, &pr.pops, grpc_schedule_on_exec_ctx); - req.host = server; + req.host = GRPC_PORT_SERVER_ADDRESS; gpr_asprintf(&path, "/drop/%d", port); req.http.path = path; @@ -198,7 +198,7 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, gpr_mu_unlock(pr->mu); } -int grpc_pick_port_using_server(char *server) { +int grpc_pick_port_using_server(void) { grpc_httpcli_context context; grpc_httpcli_request req; portreq pr; @@ -215,10 +215,10 @@ int grpc_pick_port_using_server(char *server) { shutdown_closure = grpc_closure_create(destroy_pops_and_shutdown, &pr.pops, grpc_schedule_on_exec_ctx); pr.port = -1; - pr.server = server; + pr.server = GRPC_PORT_SERVER_ADDRESS; pr.ctx = &context; - req.host = server; + req.host = GRPC_PORT_SERVER_ADDRESS; req.http.path = "/get"; grpc_httpcli_context_init(&context); diff --git a/test/core/util/port_server_client.h b/test/core/util/port_server_client.h index 437006495c..70471ecb8a 100644 --- a/test/core/util/port_server_client.h +++ b/test/core/util/port_server_client.h @@ -36,7 +36,10 @@ // C interface to port_server.py -int grpc_pick_port_using_server(char *server); -void grpc_free_port_using_server(char *server, int port); +// must be synchronized with tools/run_tests/python_utils/start_port_server.py +#define GRPC_PORT_SERVER_ADDRESS "localhost:32766" + +int grpc_pick_port_using_server(void); +void grpc_free_port_using_server(int port); #endif // GRPC_TEST_CORE_UTIL_PORT_SERVER_CLIENT_H diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c deleted file mode 100644 index 0c50a46644..0000000000 --- a/test/core/util/port_windows.c +++ /dev/null @@ -1,247 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "src/core/lib/iomgr/port.h" -#include "test/core/util/test_config.h" -#if defined(GRPC_WINSOCK_SOCKET) && defined(GRPC_TEST_PICK_PORT) - -#include "src/core/lib/iomgr/sockaddr.h" - -#include "test/core/util/port.h" - -#include <errno.h> -#include <process.h> -#include <stdio.h> -#include <string.h> - -#include <grpc/grpc.h> -#include <grpc/support/alloc.h> -#include <grpc/support/log.h> - -#include "src/core/lib/http/httpcli.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" -#include "src/core/lib/support/env.h" -#include "test/core/util/port_server_client.h" - -#if GPR_GETPID_IN_UNISTD_H -#include <sys/unistd.h> -static int _getpid() { return getpid(); } -#endif - -#define NUM_RANDOM_PORTS_TO_PICK 100 - -static int *chosen_ports = NULL; -static size_t num_chosen_ports = 0; - -static int has_port_been_chosen(int port) { - size_t i; - for (i = 0; i < num_chosen_ports; i++) { - if (chosen_ports[i] == port) { - return 1; - } - } - return 0; -} - -static int free_chosen_port(int port) { - size_t i; - int found = 0; - size_t found_at = 0; - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env != NULL) { - /* Find the port and erase it from the list, then tell the server it can be - freed. */ - for (i = 0; i < num_chosen_ports; i++) { - if (chosen_ports[i] == port) { - GPR_ASSERT(found == 0); - found = 1; - found_at = i; - } - } - if (found) { - chosen_ports[found_at] = chosen_ports[num_chosen_ports - 1]; - grpc_free_port_using_server(env, port); - num_chosen_ports--; - } - } - return found; -} - -static void free_chosen_ports(void) { - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env != NULL) { - size_t i; - for (i = 0; i < num_chosen_ports; i++) { - grpc_free_port_using_server(env, chosen_ports[i]); - } - gpr_free(env); - } - - gpr_free(chosen_ports); -} - -static void chose_port(int port) { - if (chosen_ports == NULL) { - atexit(free_chosen_ports); - } - num_chosen_ports++; - chosen_ports = gpr_realloc(chosen_ports, sizeof(int) * num_chosen_ports); - chosen_ports[num_chosen_ports - 1] = port; -} - -static int is_port_available(int *port, int is_tcp) { - const int proto = is_tcp ? IPPROTO_TCP : 0; - const SOCKET fd = socket(AF_INET, is_tcp ? SOCK_STREAM : SOCK_DGRAM, proto); - int one = 1; - struct sockaddr_in addr; - socklen_t alen = sizeof(addr); - int actual_port; - - GPR_ASSERT(*port >= 0); - GPR_ASSERT(*port <= 65535); - if (INVALID_SOCKET == fd) { - gpr_log(GPR_ERROR, "socket() failed: %s", strerror(errno)); - return 0; - } - - /* Reuseaddr lets us start up a server immediately after it exits */ - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char *)&one, - sizeof(one)) < 0) { - gpr_log(GPR_ERROR, "setsockopt() failed: %s", strerror(errno)); - closesocket(fd); - return 0; - } - - /* Try binding to port */ - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = INADDR_ANY; - addr.sin_port = htons((u_short)*port); - if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { - gpr_log(GPR_DEBUG, "bind(port=%d) failed: %s", *port, strerror(errno)); - closesocket(fd); - return 0; - } - - /* Get the bound port number */ - if (getsockname(fd, (struct sockaddr *)&addr, &alen) < 0) { - gpr_log(GPR_ERROR, "getsockname() failed: %s", strerror(errno)); - closesocket(fd); - return 0; - } - GPR_ASSERT(alen <= (socklen_t)sizeof(addr)); - actual_port = ntohs(addr.sin_port); - GPR_ASSERT(actual_port > 0); - if (*port == 0) { - *port = actual_port; - } else { - GPR_ASSERT(*port == actual_port); - } - - closesocket(fd); - return 1; -} - -int grpc_pick_unused_port(void) { - /* We repeatedly pick a port and then see whether or not it is - available for use both as a TCP socket and a UDP socket. First, we - pick a random large port number. For subsequent - iterations, we bind to an anonymous port and let the OS pick the - port number. The random port picking reduces the probability of - races with other processes on kernels that want to reuse the same - port numbers over and over. */ - - /* In alternating iterations we trial UDP ports before TCP ports UDP - ports -- it could be the case that this machine has been using up - UDP ports and they are scarcer. */ - - /* Type of port to first pick in next iteration */ - int is_tcp = 1; - int trial = 0; - - char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); - if (env) { - int port = grpc_pick_port_using_server(env); - gpr_free(env); - if (port != 0) { - return port; - } - } - - for (;;) { - int port; - trial++; - if (trial == 1) { - port = _getpid() % (65536 - 30000) + 30000; - } else if (trial <= NUM_RANDOM_PORTS_TO_PICK) { - port = rand() % (65536 - 30000) + 30000; - } else { - port = 0; - } - - if (has_port_been_chosen(port)) { - continue; - } - - if (!is_port_available(&port, is_tcp)) { - continue; - } - - GPR_ASSERT(port > 0); - /* Check that the port # is free for the other type of socket also */ - if (!is_port_available(&port, !is_tcp)) { - /* In the next iteration trial to bind to the other type first - because perhaps it is more rare. */ - is_tcp = !is_tcp; - continue; - } - - /* TODO(ctiller): consider caching this port in some structure, to avoid - handing it out again */ - - chose_port(port); - return port; - } - - /* The port iterator reached the end without finding a suitable port. */ - return 0; -} - -int grpc_pick_unused_port_or_die(void) { - int port = grpc_pick_unused_port(); - GPR_ASSERT(port > 0); - return port; -} - -void grpc_recycle_unused_port(int port) { GPR_ASSERT(free_chosen_port(port)); } - -#endif /* GRPC_WINSOCK_SOCKET && GRPC_TEST_PICK_PORT */ diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index b96e9fac36..01c07823cf 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -65,7 +65,9 @@ Http2Client::ServiceStub::ServiceStub(std::shared_ptr<Channel> channel) TestService::Stub* Http2Client::ServiceStub::Get() { return stub_.get(); } Http2Client::Http2Client(std::shared_ptr<Channel> channel) - : serviceStub_(channel), channel_(channel) {} + : serviceStub_(channel), + channel_(channel), + defaultRequest_(BuildDefaultRequest()) {} bool Http2Client::AssertStatusCode(const Status& s, StatusCode expected_code) { if (s.error_code() == expected_code) { @@ -77,18 +79,24 @@ bool Http2Client::AssertStatusCode(const Status& s, StatusCode expected_code) { abort(); } -bool Http2Client::DoRstAfterHeader() { - gpr_log(GPR_DEBUG, "Sending RPC and expecting reset stream after header"); - +Status Http2Client::SendUnaryCall(SimpleResponse* response) { ClientContext context; + return serviceStub_.Get()->UnaryCall(&context, defaultRequest_, response); +} + +SimpleRequest Http2Client::BuildDefaultRequest() { SimpleRequest request; - SimpleResponse response; request.set_response_size(kLargeResponseSize); grpc::string payload(kLargeRequestSize, '\0'); request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); + return request; +} + +bool Http2Client::DoRstAfterHeader() { + gpr_log(GPR_DEBUG, "Sending RPC and expecting reset stream after header"); - Status s = serviceStub_.Get()->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::UNKNOWN); + SimpleResponse response; + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::INTERNAL); GPR_ASSERT(!response.has_payload()); // no data should be received gpr_log(GPR_DEBUG, "Done testing reset stream after header"); @@ -98,15 +106,8 @@ bool Http2Client::DoRstAfterHeader() { bool Http2Client::DoRstAfterData() { gpr_log(GPR_DEBUG, "Sending RPC and expecting reset stream after data"); - ClientContext context; - SimpleRequest request; SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - - Status s = serviceStub_.Get()->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::UNKNOWN); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::INTERNAL); GPR_ASSERT(response.has_payload()); // data should be received gpr_log(GPR_DEBUG, "Done testing reset stream after data"); @@ -116,15 +117,8 @@ bool Http2Client::DoRstAfterData() { bool Http2Client::DoRstDuringData() { gpr_log(GPR_DEBUG, "Sending RPC and expecting reset stream during data"); - ClientContext context; - SimpleRequest request; SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - - Status s = serviceStub_.Get()->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::UNKNOWN); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::INTERNAL); GPR_ASSERT(!response.has_payload()); // no data should be received gpr_log(GPR_DEBUG, "Done testing reset stream during data"); @@ -133,56 +127,37 @@ bool Http2Client::DoRstDuringData() { bool Http2Client::DoGoaway() { gpr_log(GPR_DEBUG, "Sending two RPCs and expecting goaway"); + SimpleResponse response; + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::OK); + GPR_ASSERT(response.payload().body() == + grpc::string(kLargeResponseSize, '\0')); - int numCalls = 2; - for (int i = 0; i < numCalls; i++) { - ClientContext context; - SimpleRequest request; - SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - - Status s = serviceStub_.Get()->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::OK); - GPR_ASSERT(response.payload().body() == - grpc::string(kLargeResponseSize, '\0')); - } + // Sleep for one second to give time for client to receive goaway frame. + gpr_timespec sleep_time = gpr_time_add( + gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(1, GPR_TIMESPAN)); + gpr_sleep_until(sleep_time); + response.Clear(); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::OK); + GPR_ASSERT(response.payload().body() == + grpc::string(kLargeResponseSize, '\0')); gpr_log(GPR_DEBUG, "Done testing goaway"); return true; } bool Http2Client::DoPing() { gpr_log(GPR_DEBUG, "Sending RPC and expecting ping"); - - ClientContext context; - SimpleRequest request; SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - - Status s = serviceStub_.Get()->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::OK); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::OK); GPR_ASSERT(response.payload().body() == grpc::string(kLargeResponseSize, '\0')); - gpr_log(GPR_DEBUG, "Done testing ping"); return true; } void Http2Client::MaxStreamsWorker(std::shared_ptr<grpc::Channel> channel) { - ClientContext context; - SimpleRequest request; SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - - Status s = - TestService::NewStub(channel)->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::OK); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::OK); GPR_ASSERT(response.payload().body() == grpc::string(kLargeResponseSize, '\0')); } @@ -192,15 +167,8 @@ bool Http2Client::DoMaxStreams() { // Make an initial call on the channel to ensure the server's max streams // setting is received - ClientContext context; - SimpleRequest request; SimpleResponse response; - request.set_response_size(kLargeResponseSize); - grpc::string payload(kLargeRequestSize, '\0'); - request.mutable_payload()->set_body(payload.c_str(), kLargeRequestSize); - Status s = - TestService::NewStub(channel_)->UnaryCall(&context, request, &response); - AssertStatusCode(s, grpc::StatusCode::OK); + AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::OK); GPR_ASSERT(response.payload().body() == grpc::string(kLargeResponseSize, '\0')); @@ -240,7 +208,11 @@ int main(int argc, char** argv) { char host_port[host_port_buf_size]; snprintf(host_port, host_port_buf_size, "%s:%d", FLAGS_server_host.c_str(), FLAGS_server_port); - grpc::testing::Http2Client client(grpc::CreateTestChannel(host_port, false)); + std::shared_ptr<grpc::Channel> channel = + grpc::CreateTestChannel(host_port, false); + GPR_ASSERT(channel->WaitForConnected(gpr_time_add( + gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(300, GPR_TIMESPAN)))); + grpc::testing::Http2Client client(channel); gpr_log(GPR_INFO, "Testing case: %s", FLAGS_test_case.c_str()); int ret = 0; if (FLAGS_test_case == "rst_after_header") { diff --git a/test/cpp/interop/http2_client.h b/test/cpp/interop/http2_client.h index 6a315f5abb..12df5d26bc 100644 --- a/test/cpp/interop/http2_client.h +++ b/test/cpp/interop/http2_client.h @@ -70,8 +70,11 @@ class Http2Client { void MaxStreamsWorker(std::shared_ptr<grpc::Channel> channel); bool AssertStatusCode(const Status& s, StatusCode expected_code); + Status SendUnaryCall(SimpleResponse* response); + SimpleRequest BuildDefaultRequest(); ServiceStub serviceStub_; std::shared_ptr<Channel> channel_; + SimpleRequest defaultRequest_; }; } // namespace testing diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 76d5030276..e2e5bbbe00 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -62,21 +62,48 @@ static struct Init { ~Init() { grpc_shutdown(); } } g_init; -static void BM_InsecureChannelWithDefaults(benchmark::State &state) { - grpc_channel *channel = - grpc_insecure_channel_create("localhost:12345", NULL, NULL); +class BaseChannelFixture { + public: + BaseChannelFixture(grpc_channel *channel) : channel_(channel) {} + ~BaseChannelFixture() { grpc_channel_destroy(channel_); } + + grpc_channel *channel() const { return channel_; } + + private: + grpc_channel *const channel_; +}; + +class InsecureChannel : public BaseChannelFixture { + public: + InsecureChannel() + : BaseChannelFixture( + grpc_insecure_channel_create("localhost:1234", NULL, NULL)) {} +}; + +class LameChannel : public BaseChannelFixture { + public: + LameChannel() + : BaseChannelFixture(grpc_lame_client_channel_create( + "localhost:1234", GRPC_STATUS_UNAUTHENTICATED, "blah")) {} +}; + +template <class Fixture> +static void BM_CallCreateDestroy(benchmark::State &state) { + Fixture fixture; grpc_completion_queue *cq = grpc_completion_queue_create(NULL); - grpc_slice method = grpc_slice_from_static_string("/foo/bar"); gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + void *method_hdl = + grpc_channel_register_call(fixture.channel(), "/foo/bar", NULL, NULL); while (state.KeepRunning()) { - grpc_call_destroy(grpc_channel_create_call(channel, NULL, - GRPC_PROPAGATE_DEFAULTS, cq, - method, NULL, deadline, NULL)); + grpc_call_destroy(grpc_channel_create_registered_call( + fixture.channel(), NULL, GRPC_PROPAGATE_DEFAULTS, cq, method_hdl, + deadline, NULL)); } - grpc_channel_destroy(channel); grpc_completion_queue_destroy(cq); } -BENCHMARK(BM_InsecureChannelWithDefaults); + +BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel); +BENCHMARK_TEMPLATE(BM_CallCreateDestroy, LameChannel); static void FilterDestroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh index 7a7ca0d3d1..0a61334103 100755 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh @@ -47,3 +47,6 @@ make install-certs # build C++ interop client & server make interop_client interop_server + +# build C++ http2 client +make http2_client diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 4d2c429957..965259746c 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -786,6 +786,7 @@ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 498f572dc6..d98475fb00 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -786,6 +786,7 @@ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index fa41e56b43..478bdb2d04 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -785,6 +785,7 @@ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index bffa050041..a7d6aaeb9e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -785,6 +785,7 @@ doc/naming.md \ doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ +doc/server_side_auth.md \ doc/service_config.md \ doc/statuscodes.md \ doc/stress_test_framework.md \ diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index ab29e015e0..3380f3de3e 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -142,7 +142,7 @@ gem install bundler # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.7.1 +GO_VERSION=1.8 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz diff --git a/tools/internal_ci/windows/grpc_master.bat b/tools/internal_ci/windows/grpc_master.bat index 4041c50313..8943390a8d 100644 --- a/tools/internal_ci/windows/grpc_master.bat +++ b/tools/internal_ci/windows/grpc_master.bat @@ -27,18 +27,17 @@ @rem (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -setlocal +@rem make sure msys binaries are preferred over cygwin binaries +@rem set path to python 2.7 +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem enter repo root cd /d %~dp0\..\..\.. git submodule update --init -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_dbg_sponge_log.xml --report_suite_name c_windows_dbg -l c -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x c_windows_opt_sponge_log.xml --report_suite_name c_windows_opt -l c -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_dbg_sponge_log.xml --report_suite_name csharp_windows_dbg -l csharp -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x csharp_windows_opt_sponge_log.xml --report_suite_name csharp_windows_opt -l csharp -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_dbg_sponge_log.xml --report_suite_name node_windows_dbg -l node -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x node_windows_opt_sponge_log.xml --report_suite_name node_windows_opt -l node -c opt -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_dbg_sponge_log.xml --report_suite_name python_windows_dbg -l python -c dbg -sh tools\run_tests\helper_scripts\run_tests_in_workspace.sh -t -j 4 -x python_windows_opt_sponge_log.xml --report_suite_name python_windows_opt -l python -c opt +python tools/run_tests/run_tests_matrix.py -f basictests windows -j 1 --inner_jobs 8 || goto :error +goto :EOF + +:error +exit /b %errorlevel% diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 76ed0fef0d..a83b3be89c 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -149,6 +149,10 @@ bm_specs = { 'tpl': ['fixture'], 'dyn': [], }, + 'BM_CallCreateDestroy' : { + 'tpl': ['fixture'], + 'dyn': [], + }, } def numericalize(s): diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fed1b8ceb7..018a84bd21 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7969,12 +7969,10 @@ "test/core/util/parse_hexstring.h", "test/core/util/passthru_endpoint.c", "test/core/util/passthru_endpoint.h", + "test/core/util/port.c", "test/core/util/port.h", - "test/core/util/port_posix.c", "test/core/util/port_server_client.c", "test/core/util/port_server_client.h", - "test/core/util/port_uv.c", - "test/core/util/port_windows.c", "test/core/util/slice_splitter.c", "test/core/util/slice_splitter.h", "test/core/util/trickle_endpoint.c", diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 4c9f6aac63..deb7354438 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -40,7 +40,10 @@ import tempfile import time -def start_port_server(port_server_port): +# must be synchronized with test/core/utils/port_server_client.h +_PORT_SERVER_PORT = 32766 + +def start_port_server(): # check if a compatible port server is running # if incompatible (version mismatch) ==> start a new one # if not running ==> start a new one @@ -48,7 +51,7 @@ def start_port_server(port_server_port): try: version = int( urllib.request.urlopen( - 'http://localhost:%d/version_number' % port_server_port, + 'http://localhost:%d/version_number' % _PORT_SERVER_PORT, timeout=10).read()) logging.info('detected port server running version %d', version) running = True @@ -67,7 +70,7 @@ def start_port_server(port_server_port): if not running: logging.info('port_server version mismatch: killing the old one') urllib.request.urlopen('http://localhost:%d/quitquitquit' % - port_server_port).read() + _PORT_SERVER_PORT).read() time.sleep(1) if not running: fd, logfile = tempfile.mkstemp() @@ -76,7 +79,7 @@ def start_port_server(port_server_port): args = [ sys.executable, os.path.abspath('tools/run_tests/python_utils/port_server.py'), - '-p', '%d' % port_server_port, '-l', logfile + '-p', '%d' % _PORT_SERVER_PORT, '-l', logfile ] env = dict(os.environ) env['BUILD_ID'] = 'pleaseDontKillMeJenkins' @@ -107,7 +110,7 @@ def start_port_server(port_server_port): time.sleep(1) try: urllib.request.urlopen( - 'http://localhost:%d/get' % port_server_port, + 'http://localhost:%d/get' % _PORT_SERVER_PORT, timeout=1).read() logging.info( 'last ditch attempt to contact port server succeeded') @@ -119,7 +122,7 @@ def start_port_server(port_server_port): print(port_log) sys.exit(1) try: - port_server_url = 'http://localhost:%d/get' % port_server_port + port_server_url = 'http://localhost:%d/get' % _PORT_SERVER_PORT urllib.request.urlopen(port_server_url, timeout=1).read() logging.info('port server is up and ready') break diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index d29bfa9b0c..0d5bec1d67 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -77,11 +77,15 @@ class CXXLanguage: def __init__(self): self.client_cwd = None self.server_cwd = None + self.http2_cwd = None self.safename = 'cxx' def client_cmd(self, args): return ['bins/opt/interop_client'] + args + def client_cmd_http2interop(self, args): + return ['bins/opt/http2_client'] + args + def cloud_to_prod_env(self): return {} @@ -474,7 +478,7 @@ _HTTP2_TEST_CASES = ['tls', 'framing'] _HTTP2_BADSERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', 'goaway', 'ping', 'max_streams'] -_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java', 'go', 'python'] +_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java', 'go', 'python', 'c++'] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index c6cc60715e..3a9461ecd3 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -44,8 +44,7 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) if not os.path.exists('reports'): os.makedirs('reports') -port_server_port = 32766 -start_port_server.start_port_server(port_server_port) +start_port_server.start_port_server() def fnize(s): out = '' @@ -110,8 +109,7 @@ def collect_latency(bm_name, args): if len(benchmarks) >= min(16, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] @@ -119,8 +117,7 @@ def collect_latency(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2), - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) @@ -156,8 +153,7 @@ def collect_perf(bm_name, args): if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) - jobset.run(benchmarks, maxjobs=1, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] @@ -165,17 +161,16 @@ def collect_perf(bm_name, args): cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): - jobset.run(benchmarks, maxjobs=1, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) + jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) -def run_summary(cfg): +def run_summary(bm_name, cfg, base_json_name): subprocess.check_call( ['make', bm_name, 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) cmd = ['bins/%s/%s' % (cfg, bm_name), - '--benchmark_out=out.%s.json' % cfg, + '--benchmark_out=%s.%s.json' % (base_json_name, cfg), '--benchmark_out_format=json'] if args.summary_time is not None: cmd += ['--benchmark_min_time=%d' % args.summary_time] @@ -183,9 +178,9 @@ def run_summary(cfg): def collect_summary(bm_name, args): heading('Summary: %s [no counters]' % bm_name) - text(run_summary('opt')) + text(run_summary(bm_name, 'opt', 'out')) heading('Summary: %s [with counters]' % bm_name) - text(run_summary('counters')) + text(run_summary(bm_name, 'counters', 'out')) if args.bigquery_upload: with open('out.csv', 'w') as f: f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json'])) @@ -200,7 +195,7 @@ collectors = { argp = argparse.ArgumentParser(description='Collect data from microbenchmarks') argp.add_argument('-c', '--collect', choices=sorted(collectors.keys()), - nargs='+', + nargs='*', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', @@ -214,6 +209,10 @@ argp.add_argument('-b', '--benchmarks', nargs='+', type=str, help='Which microbenchmarks should be run') +argp.add_argument('--diff_perf', + default=None, + type=str, + help='Diff microbenchmarks against this git revision') argp.add_argument('--bigquery_upload', default=False, action='store_const', @@ -228,6 +227,26 @@ args = argp.parse_args() for bm_name in args.benchmarks: for collect in args.collect: collectors[collect](bm_name, args) +if args.diff_perf: + for bm_name in args.benchmarks: + run_summary(bm_name, 'opt', '%s.new' % bm_name) + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_perf]) + comparables = [] + subprocess.check_call(['make', 'clean']) + try: + for bm_name in args.benchmarks: + try: + run_summary(bm_name, 'opt', '%s.old' % bm_name) + comparables.append(bm_name) + except subprocess.CalledProcessError, e: + pass + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + for bm_name in comparables: + subprocess.check_call(['third_party/benchmark/tools/compare_bench.py', + '%s.new.opt.json' % bm_name, + '%s.old.opt.json' % bm_name]) index_html += "</body>\n</html>\n" with open('reports/index.html', 'w') as f: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9741624c4f..d4983403cc 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -692,18 +692,11 @@ class RubyLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - #TODO(apolcyn) turn mac ruby tests back on once ruby 2.4 issues done - if platform_string() == 'mac': - print('skipping ruby test_specs on mac until running on 2.4') - return [] return [self.config.job_spec(['tools/run_tests/helper_scripts/run_ruby.sh'], timeout_seconds=10*60, environ=_FORCE_ENVIRON_FOR_WRAPPERS)] def pre_build_steps(self): - if platform_string() == 'mac': - print('skipping ruby pre_build_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/pre_build_ruby.sh']] def make_targets(self): @@ -713,15 +706,9 @@ class RubyLanguage(object): return [] def build_steps(self): - if platform_string() == 'mac': - print('skipping ruby build_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/build_ruby.sh']] def post_tests_steps(self): - if platform_string() == 'mac': - print('skipping ruby post_test_steps on mac until running on 2.4') - return [] return [['tools/run_tests/helper_scripts/post_tests_ruby.sh']] def makefile_name(self): @@ -1451,8 +1438,7 @@ def _build_and_run( # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] - port_server_port = 32766 - start_port_server.start_port_server(port_server_port) + start_port_server.start_port_server() resultset = None num_test_failures = 0 try: @@ -1495,7 +1481,6 @@ def _build_and_run( all_runs, check_cancelled, newline_on_success=newline_on_success, travis=args.travis, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, - add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}, quiet_success=args.quiet_success) if resultset: for k, v in sorted(resultset.items()): diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index bc4fdaba71..5f5df70d1d 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -70,7 +70,8 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_ workspace_name = 'workspace_%s' % name env = {'WORKSPACE_NAME': workspace_name} test_job = jobset.JobSpec( - cmdline=['tools/run_tests/helper_scripts/run_tests_in_workspace.sh', + cmdline=['bash', + 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', '-t', '-j', str(inner_jobs), '-x', '../report_%s.xml' % name, diff --git a/tools/run_tests/start_port_server.py b/tools/run_tests/start_port_server.py new file mode 100755 index 0000000000..e33ac12bd3 --- /dev/null +++ b/tools/run_tests/start_port_server.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Wrapper around port server starting code. + +Used by developers who wish to run individual C/C++ tests outside of the +run_tests.py infrastructure. + +The path to this file is called out in test/core/util/port.c, and printed as +an error message to users. +""" + +import python_utils.start_port_server as start_port_server + +start_port_server.start_port_server() + +print "Port server started successfully" diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 6d4b4f70f1..d08ceb6828 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -335,14 +335,10 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.c"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_posix.c"> + <ClCompile Include="$(SolutionDir)\..\test\core\util\port.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\port_server_client.c"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_uv.c"> - </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_windows.c"> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c"> diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 5444f6f8c7..3beaa80994 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -49,18 +49,12 @@ <ClCompile Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.c"> <Filter>test\core\util</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_posix.c"> + <ClCompile Include="$(SolutionDir)\..\test\core\util\port.c"> <Filter>test\core\util</Filter> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\port_server_client.c"> <Filter>test\core\util</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_uv.c"> - <Filter>test\core\util</Filter> - </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_windows.c"> - <Filter>test\core\util</Filter> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c"> <Filter>test\core\util</Filter> </ClCompile> diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj index 1ea64654e5..5a58cae83f 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj @@ -186,14 +186,10 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.c"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_posix.c"> + <ClCompile Include="$(SolutionDir)\..\test\core\util\port.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\port_server_client.c"> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_uv.c"> - </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_windows.c"> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\trickle_endpoint.c"> diff --git a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters index e2ad88c96e..88c875cb01 100644 --- a/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util_unsecure/grpc_test_util_unsecure.vcxproj.filters @@ -34,18 +34,12 @@ <ClCompile Include="$(SolutionDir)\..\test\core\util\passthru_endpoint.c"> <Filter>test\core\util</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_posix.c"> + <ClCompile Include="$(SolutionDir)\..\test\core\util\port.c"> <Filter>test\core\util</Filter> </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\port_server_client.c"> <Filter>test\core\util</Filter> </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_uv.c"> - <Filter>test\core\util</Filter> - </ClCompile> - <ClCompile Include="$(SolutionDir)\..\test\core\util\port_windows.c"> - <Filter>test\core\util</Filter> - </ClCompile> <ClCompile Include="$(SolutionDir)\..\test\core\util\slice_splitter.c"> <Filter>test\core\util</Filter> </ClCompile> |