diff options
30 files changed, 799 insertions, 182 deletions
diff --git a/.gitignore b/.gitignore index f5ca501db2..45f8fdaa46 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ cache.mk # Temporary test reports report.xml + +# port server log +portlog.txt @@ -132,10 +132,6 @@ cc_library( cc_library( name = "grpc", srcs = [ - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/httpcli_security_connector.h", - "src/core/httpcli/parser.h", "src/core/security/auth_filters.h", "src/core/security/base64.h", "src/core/security/credentials.h", @@ -175,6 +171,9 @@ cc_library( "src/core/client_config/uri_parser.h", "src/core/compression/message_compress.h", "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", "src/core/iomgr/alarm_internal.h", @@ -247,10 +246,7 @@ cc_library( "src/core/transport/transport_impl.h", "src/core/census/context.h", "src/core/census/rpc_stat_id.h", - "src/core/httpcli/format_request.c", - "src/core/httpcli/httpcli.c", "src/core/httpcli/httpcli_security_connector.c", - "src/core/httpcli/parser.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", "src/core/security/credentials.c", @@ -297,6 +293,9 @@ cc_library( "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", "src/core/debug/trace.c", + "src/core/httpcli/format_request.c", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/parser.c", "src/core/iomgr/alarm.c", "src/core/iomgr/alarm_heap.c", "src/core/iomgr/endpoint.c", @@ -435,6 +434,9 @@ cc_library( "src/core/client_config/uri_parser.h", "src/core/compression/message_compress.h", "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", "src/core/iomgr/alarm_internal.h", @@ -534,6 +536,9 @@ cc_library( "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", "src/core/debug/trace.c", + "src/core/httpcli/format_request.c", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/parser.c", "src/core/iomgr/alarm.c", "src/core/iomgr/alarm_heap.c", "src/core/iomgr/endpoint.c", @@ -971,10 +976,7 @@ objc_library( objc_library( name = "grpc_objc", srcs = [ - "src/core/httpcli/format_request.c", - "src/core/httpcli/httpcli.c", "src/core/httpcli/httpcli_security_connector.c", - "src/core/httpcli/parser.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", "src/core/security/credentials.c", @@ -1021,6 +1023,9 @@ objc_library( "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", "src/core/debug/trace.c", + "src/core/httpcli/format_request.c", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/parser.c", "src/core/iomgr/alarm.c", "src/core/iomgr/alarm_heap.c", "src/core/iomgr/endpoint.c", @@ -1118,10 +1123,6 @@ objc_library( "include/grpc/grpc.h", "include/grpc/status.h", "include/grpc/census.h", - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/httpcli_security_connector.h", - "src/core/httpcli/parser.h", "src/core/security/auth_filters.h", "src/core/security/base64.h", "src/core/security/credentials.h", @@ -1161,6 +1162,9 @@ objc_library( "src/core/client_config/uri_parser.h", "src/core/compression/message_compress.h", "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", "src/core/iomgr/alarm_internal.h", @@ -3955,10 +3955,7 @@ endif LIBGRPC_SRC = \ - src/core/httpcli/format_request.c \ - src/core/httpcli/httpcli.c \ src/core/httpcli/httpcli_security_connector.c \ - src/core/httpcli/parser.c \ src/core/security/base64.c \ src/core/security/client_auth_filter.c \ src/core/security/credentials.c \ @@ -4005,6 +4002,9 @@ LIBGRPC_SRC = \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ src/core/debug/trace.c \ + src/core/httpcli/format_request.c \ + src/core/httpcli/httpcli.c \ + src/core/httpcli/parser.c \ src/core/iomgr/alarm.c \ src/core/iomgr/alarm_heap.c \ src/core/iomgr/endpoint.c \ @@ -4274,6 +4274,9 @@ LIBGRPC_UNSECURE_SRC = \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ src/core/debug/trace.c \ + src/core/httpcli/format_request.c \ + src/core/httpcli/httpcli.c \ + src/core/httpcli/parser.c \ src/core/iomgr/alarm.c \ src/core/iomgr/alarm_heap.c \ src/core/iomgr/endpoint.c \ @@ -20369,10 +20372,7 @@ ifneq ($(OPENSSL_DEP),) # otherwise parallel compilation will fail if a source is compiled first. examples/pubsub/publisher.cc: $(OPENSSL_DEP) examples/pubsub/subscriber.cc: $(OPENSSL_DEP) -src/core/httpcli/format_request.c: $(OPENSSL_DEP) -src/core/httpcli/httpcli.c: $(OPENSSL_DEP) src/core/httpcli/httpcli_security_connector.c: $(OPENSSL_DEP) -src/core/httpcli/parser.c: $(OPENSSL_DEP) src/core/security/base64.c: $(OPENSSL_DEP) src/core/security/client_auth_filter.c: $(OPENSSL_DEP) src/core/security/credentials.c: $(OPENSSL_DEP) diff --git a/build.json b/build.json index e0f768882b..515cecdc5a 100644 --- a/build.json +++ b/build.json @@ -141,6 +141,9 @@ "src/core/client_config/uri_parser.h", "src/core/compression/message_compress.h", "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", "src/core/iomgr/alarm_internal.h", @@ -239,6 +242,9 @@ "src/core/compression/algorithm.c", "src/core/compression/message_compress.c", "src/core/debug/trace.c", + "src/core/httpcli/format_request.c", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/parser.c", "src/core/iomgr/alarm.c", "src/core/iomgr/alarm_heap.c", "src/core/iomgr/endpoint.c", @@ -462,10 +468,6 @@ "include/grpc/grpc_security.h" ], "headers": [ - "src/core/httpcli/format_request.h", - "src/core/httpcli/httpcli.h", - "src/core/httpcli/httpcli_security_connector.h", - "src/core/httpcli/parser.h", "src/core/security/auth_filters.h", "src/core/security/base64.h", "src/core/security/credentials.h", @@ -481,10 +483,7 @@ "src/core/tsi/transport_security_interface.h" ], "src": [ - "src/core/httpcli/format_request.c", - "src/core/httpcli/httpcli.c", "src/core/httpcli/httpcli_security_connector.c", - "src/core/httpcli/parser.c", "src/core/security/base64.c", "src/core/security/client_auth_filter.c", "src/core/security/credentials.c", diff --git a/doc/naming.md b/doc/naming.md new file mode 100644 index 0000000000..5ad7e6622e --- /dev/null +++ b/doc/naming.md @@ -0,0 +1,52 @@ +#gRPC Naming and Discovery Support + +## Overview + +gRPC supports DNS as the default name-system. A number of alternative name-systems are used in various deployments. We propose an API that is general enough to support a range of name-systems and the corresponding syntax for names. The gRPC client library in various languages will provide a plugin mechanism so resolvers for different name-systems can be plugged in. + +## Detailed Proposal + + A fully qualified, self contained name used for gRPC channel construction uses the syntax: + +``` +scheme://authority/endpoint_name +``` + +Here, scheme indicates the name-system to be used. Example schemes to be supported include: + +* `dns` + +* `zookeeper` + +* `etcd` + +Authority indicates some scheme-specific bootstrap information, e.g., for DNS, the authority may include the IP[:port] of the DNS server to use. Often, a DNS name may used as the authority, since the ability to resolve DNS names is already built into all gRPC client libraries. + +Finally, the endpoint_name indicates a concrete name to be looked up in a given name-system identified by the scheme and the authority. The syntax of endpoint name is dictated by the scheme in use. + +### Plugins + +The gRPC client library will switch on the scheme to pick the right resolver plugin and pass it the fully qualified name string. + +Resolvers should be able to contact the authority and get a resolution that they return back to the gRPC client library. The returned contents include a list of IP:port, an optional config and optional auth config data to be used for channel authentication. The plugin API allows the resolvers to continuously watch an endpoint_name and return updated resolutions as needed. + +## Zookeeper + +Apache [ZooKeeper](https://zookeeper.apache.org/) is a popular solution for building name-systems. Curator is a service discovery system built on to of ZooKeeper. We propose to organize names hierarchically as `/path/service/instance` similar to Apache Curator. + +A fully-qualified ZooKeeper name used to construct a gRPC channel will look as follows: + +``` +zookeeper://host:port/path/service/instance +``` +Here `zookeeper` is the scheme identifying the name-system. `host:port` identifies an authoritative name-server for this scheme (i.e., a Zookeeper server). The host can be an IP address or a DNS name. +Finally `/path/service/instance` is the Zookeeper name to be resolved. + +## Service Registration + + +Service providers can register their services in Zookeeper by using a Zookeeper client. + +Each service is a zookeeper node, and each instance is a child node of the corresponding service. For example, a MySQL service may have multiple instances, `/mysql/1`, `/mysql/2`, `/mysql/3`. The name of the service or instance, as well as an optional path is specified by the service provider. + +The data in service nodes is empty. Each instance node stores its address in the format of `host:port`, where host can be either hostname or IP address. diff --git a/doc/server-reflection.md b/doc/server-reflection.md new file mode 100644 index 0000000000..cceee1647f --- /dev/null +++ b/doc/server-reflection.md @@ -0,0 +1,183 @@ +GRPC Server Reflection Protocol +=============================== + +This document describes server reflection as an optional extension for servers +to assist clients in runtime construction of requests without having stub +information precompiled into the client. + +The primary usecase for server reflection is to write (typically) command line +debugging tools for talking to a grpc server. In particular, such a tool will +take in a method and a payload (in human readable text format) send it to the +server (typically in binary proto wire format), and then take the response and +decode it to text to present to the user. + +This broadly involves two problems: determining what formats (which protobuf +messages) a server’s method uses, and determining how to convert messages +between human readable format and the (likely binary) wire format. + +## Method reflection + +We want to be able to answer the following queries: + 1. What methods does a server export? + 2. For a particular method, how do we call it? +Specifically, what are the names of the methods, are those methods unary or +streaming, and what are the types of the argument and result? + +``` +#TODO(dklempner): link to an actual .proto later. +package grpc.reflection.v1alpha; + +message ListApisRequest { +} + +message ListApisResponse { + repeated google.protobuf.Api apis = 1; +} + +message GetMethodRequest { + string method = 1; +} +message GetMethodResponse { + google.protobuf.Method method = 1; +} + +service ServerReflection { + rpc ListApis (ListApisRequest) returns (ListApisResponse); + rpc GetMethod (GetMethodRequest) returns (GetMethodResponse); +} +``` + +Note that a server is under no obligation to return a complete list of all +methods it supports. For example, a reverse proxy may support server reflection +for methods implemented directly on the proxy but not enumerate all methods +supported by its backends. + + +### Open questions on method reflection + * Consider how to extend this protocol to support non-protobuf methods. + +## Argument reflection +The second half of the problem is converting between the human readable +input/output of a debugging tool and the binary format understood by the +method. + +This is obviously dependent on protocol type. At one extreme, if both the +server and the debugging tool accept JSON, there may be no need for such a +conversion in the first place. At the opposite extreme, a server using a custom +binary format has no hope of being supported by a generic system. The +intermediate interesting common case is a server which speaks binary-proto and +a debugging client which speaks either ascii-proto or json-proto. + +One approach would be to require servers directly support human readable input. +In the future method reflection may be extended to document such support, +should it become widespread or standardized. + +## Protobuf descriptors + +A second would be for the server to export its +google::protobuf::DescriptorDatabase over the wire. This is very easy to +implement in C++, and Google implementations of a similar protocol already +exist in C++, Go, and Java. + +This protocol mostly returns FileDescriptorProtos, which are a proto encoding +of a parsed .proto file. It supports four queries: + 1. The FileDescriptorProto for a given file name + 2. The FileDescriptorProto for the file with a given symbol + 3. The FileDescriptorProto for the file with a given extension + 4. The list of known extension tag numbers of a given type + +These directly correspond to the methods of +google::protobuf::DescriptorDatabase. Note that this protocol includes support +for extensions, which have been removed from proto3 but are still in widespread +use in Google’s codebase. + +Because most usecases will require also requesting the transitive dependencies +of requested files, the queries will also return all transitive dependencies of +the returned file. Should interesting usecases for non-transitive queries turn +up later, we can easily extend the protocol to support them. + +### Reverse proxy traversal + +One potential issue with naive reverse proxies is that, while any individual +server will have a consistent and valid picture of the proto DB which is +sufficient to handle incoming requests, incompatibilities will arise if the +backend servers have a mix of builds. For example, if a given message is moved +from foo.proto to bar.proto, and the client requests foo.proto from an old +server and bar.proto from a new server, the resulting database will have a +double definition. + +To solve this problem, the protocol is structured as a bidirectional stream, +ensuring all related requests go to a single server. This has the additional +benefit that overlapping recursive requests don’t require sending a lot of +redundant information, because there is a single stream to maintain context +between queries. + +``` +package grpc.reflection.v1alpha; +message DescriptorDatabaseRequest { + string host = 1; + oneof message_request { + string files_for_file_name = 3; + string files_for_symbol_name = 4; + FileContainingExtensionRequest file_containing_extension = 5; + string list_all_extensions_of_type = 6; + } +} + +message FileContainingExtensionRequest { + string base_message = 1; + int64 extension_id = 2; +} + +message DescriptorDatabaseResponse { + string valid_host = 1; + DescriptorDatabaseRequest original_request = 2; + oneof message_response { + // These are proto2 type google.protobuf.FileDescriptorProto, but + // we avoid taking a dependency on descriptor.proto, which uses + // proto2 only features, by making them opaque + // bytes instead + repeated bytes fd_proto = 4; + ListAllExtensionsResponse extensions_response = 5; + // Notably includes error code 5, NOT FOUND + int32 error_code = 6; + } +} + +message ListAllExtensionsResponse { + string base_type_name; + repeated int64 extension_number; +} + +service ProtoDescriptorDatabase { + rpc DescriptorDatabaseInfo(stream DescriptorDatabaseRequest) returns (stream DescriptorDatabaseResponse); +} +``` + +Any given request must either result in an error code or an answer, usually in +the form of a series of FileDescriptorProtos with the requested file itself +and all previously unsent transitive imports of that file. Servers may track +which FileDescriptorProtos have been sent on a given stream, for a given value +of valid_host, and avoid sending them repeatedly for overlapping requests. + +| message_request message | Result | +| files_for_file_name | transitive closure of file name | +| files_for_symbol_name | transitive closure file containing symbol | +| file_containing_extension | transitive closure of file containing a given extension number of a given symbol | +| list_all_extensions_of_type | ListAllExtensionsResponse containing all known extension numbers of a given type | + +At some point it would make sense to additionally also support any.proto’s +format. Note that known any.proto messages can be queried by symbol using this +protocol even without any such support, by parsing the url and extracting the +symbol name from it. + +## Language specific implementation thoughts +All of the information needed to implement Proto reflection is available to the +code generator, but I’m not certain we actually generate this in every +language. If the proto implementation in the language doesn’t have something +like google::protobuf::DescriptorPool the grpc implementation for that language +will need to index those FileDescriptorProtos by file and symbol and imports. + +One issue is that some grpc implementations are very loosely coupled with +protobufs; in such implementations it probably makes sense to split apart these +reflection APIs so as not to take an additional proto dependency. diff --git a/gRPC.podspec b/gRPC.podspec index e04767ce1f..12ce7c1e7b 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -134,10 +134,6 @@ Pod::Spec.new do |s| 'src/core/support/time_posix.c', 'src/core/support/time_win32.c', 'src/core/support/tls_pthread.c', - 'src/core/httpcli/format_request.h', - 'src/core/httpcli/httpcli.h', - 'src/core/httpcli/httpcli_security_connector.h', - 'src/core/httpcli/parser.h', 'src/core/security/auth_filters.h', 'src/core/security/base64.h', 'src/core/security/credentials.h', @@ -177,6 +173,9 @@ Pod::Spec.new do |s| 'src/core/client_config/uri_parser.h', 'src/core/compression/message_compress.h', 'src/core/debug/trace.h', + 'src/core/httpcli/format_request.h', + 'src/core/httpcli/httpcli.h', + 'src/core/httpcli/parser.h', 'src/core/iomgr/alarm.h', 'src/core/iomgr/alarm_heap.h', 'src/core/iomgr/alarm_internal.h', @@ -256,10 +255,7 @@ Pod::Spec.new do |s| 'grpc/grpc.h', 'grpc/status.h', 'grpc/census.h', - 'src/core/httpcli/format_request.c', - 'src/core/httpcli/httpcli.c', 'src/core/httpcli/httpcli_security_connector.c', - 'src/core/httpcli/parser.c', 'src/core/security/base64.c', 'src/core/security/client_auth_filter.c', 'src/core/security/credentials.c', @@ -306,6 +302,9 @@ Pod::Spec.new do |s| 'src/core/compression/algorithm.c', 'src/core/compression/message_compress.c', 'src/core/debug/trace.c', + 'src/core/httpcli/format_request.c', + 'src/core/httpcli/httpcli.c', + 'src/core/httpcli/parser.c', 'src/core/iomgr/alarm.c', 'src/core/iomgr/alarm_heap.c', 'src/core/iomgr/endpoint.c', @@ -402,10 +401,6 @@ Pod::Spec.new do |s| 'src/core/support/string.h', 'src/core/support/string_win32.h', 'src/core/support/thd_internal.h', - 'src/core/httpcli/format_request.h', - 'src/core/httpcli/httpcli.h', - 'src/core/httpcli/httpcli_security_connector.h', - 'src/core/httpcli/parser.h', 'src/core/security/auth_filters.h', 'src/core/security/base64.h', 'src/core/security/credentials.h', @@ -445,6 +440,9 @@ Pod::Spec.new do |s| 'src/core/client_config/uri_parser.h', 'src/core/compression/message_compress.h', 'src/core/debug/trace.h', + 'src/core/httpcli/format_request.h', + 'src/core/httpcli/httpcli.h', + 'src/core/httpcli/parser.h', 'src/core/iomgr/alarm.h', 'src/core/iomgr/alarm_heap.h', 'src/core/iomgr/alarm_internal.h', diff --git a/src/core/httpcli/httpcli.c b/src/core/httpcli/httpcli.c index 65997d5f44..9012070e8e 100644 --- a/src/core/httpcli/httpcli.c +++ b/src/core/httpcli/httpcli.c @@ -40,9 +40,7 @@ #include "src/core/iomgr/resolve_address.h" #include "src/core/iomgr/tcp_client.h" #include "src/core/httpcli/format_request.h" -#include "src/core/httpcli/httpcli_security_connector.h" #include "src/core/httpcli/parser.h" -#include "src/core/security/secure_transport_setup.h" #include "src/core/support/string.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> @@ -57,7 +55,7 @@ typedef struct { char *host; gpr_timespec deadline; int have_read_byte; - int use_ssl; + const grpc_httpcli_handshaker *handshaker; grpc_httpcli_response_cb on_response; void *user_data; grpc_httpcli_context *context; @@ -68,6 +66,16 @@ typedef struct { static grpc_httpcli_get_override g_get_override = NULL; static grpc_httpcli_post_override g_post_override = NULL; +static void plaintext_handshake(void *arg, grpc_endpoint *endpoint, + const char *host, + void (*on_done)(void *arg, + grpc_endpoint *endpoint)) { + on_done(arg, endpoint); +} + +const grpc_httpcli_handshaker grpc_httpcli_plaintext = {"http", + plaintext_handshake}; + void grpc_httpcli_context_init(grpc_httpcli_context *context) { grpc_pollset_set_init(&context->pollset_set); } @@ -163,18 +171,16 @@ static void start_write(internal_request *req) { } } -static void on_secure_transport_setup_done(void *rp, - grpc_security_status status, - grpc_endpoint *wrapped_endpoint, - grpc_endpoint *secure_endpoint) { - internal_request *req = rp; - if (status != GRPC_SECURITY_OK) { - gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status); - finish(req, 0); - } else { - req->ep = secure_endpoint; - start_write(req); +static void on_handshake_done(void *arg, grpc_endpoint *ep) { + internal_request *req = arg; + + if (!ep) { + next_address(req); + return; } + + req->ep = ep; + start_write(req); } static void on_connected(void *arg, grpc_endpoint *tcp) { @@ -184,25 +190,7 @@ static void on_connected(void *arg, grpc_endpoint *tcp) { next_address(req); return; } - req->ep = tcp; - if (req->use_ssl) { - grpc_channel_security_connector *sc = NULL; - const unsigned char *pem_root_certs = NULL; - size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); - if (pem_root_certs == NULL || pem_root_certs_size == 0) { - gpr_log(GPR_ERROR, "Could not get default pem root certs."); - finish(req, 0); - return; - } - GPR_ASSERT(grpc_httpcli_ssl_channel_security_connector_create( - pem_root_certs, pem_root_certs_size, req->host, &sc) == - GRPC_SECURITY_OK); - grpc_setup_secure_transport(&sc->base, tcp, on_secure_transport_setup_done, - req); - GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli"); - } else { - start_write(req); - } + req->handshaker->handshake(req, tcp, req->host, on_handshake_done); } static void next_address(internal_request *req) { @@ -245,18 +233,17 @@ void grpc_httpcli_get(grpc_httpcli_context *context, grpc_pollset *pollset, req->on_response = on_response; req->user_data = user_data; req->deadline = deadline; - req->use_ssl = request->use_ssl; + req->handshaker = + request->handshaker ? request->handshaker : &grpc_httpcli_plaintext; req->context = context; req->pollset = pollset; gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->path); grpc_iomgr_register_object(&req->iomgr_obj, name); gpr_free(name); - if (req->use_ssl) { - req->host = gpr_strdup(request->host); - } + req->host = gpr_strdup(request->host); grpc_pollset_set_add_pollset(&req->context->pollset_set, req->pollset); - grpc_resolve_address(request->host, req->use_ssl ? "https" : "http", + grpc_resolve_address(request->host, req->handshaker->default_port, on_resolved, req); } @@ -279,18 +266,17 @@ void grpc_httpcli_post(grpc_httpcli_context *context, grpc_pollset *pollset, req->on_response = on_response; req->user_data = user_data; req->deadline = deadline; - req->use_ssl = request->use_ssl; + req->handshaker = + request->handshaker ? request->handshaker : &grpc_httpcli_plaintext; req->context = context; req->pollset = pollset; gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->path); grpc_iomgr_register_object(&req->iomgr_obj, name); gpr_free(name); - if (req->use_ssl) { - req->host = gpr_strdup(request->host); - } + req->host = gpr_strdup(request->host); grpc_pollset_set_add_pollset(&req->context->pollset_set, req->pollset); - grpc_resolve_address(request->host, req->use_ssl ? "https" : "http", + grpc_resolve_address(request->host, req->handshaker->default_port, on_resolved, req); } diff --git a/src/core/httpcli/httpcli.h b/src/core/httpcli/httpcli.h index ab98178f8a..c45966714c 100644 --- a/src/core/httpcli/httpcli.h +++ b/src/core/httpcli/httpcli.h @@ -38,6 +38,7 @@ #include <grpc/support/time.h> +#include "src/core/iomgr/endpoint.h" #include "src/core/iomgr/pollset_set.h" /* User agent this library reports */ @@ -58,6 +59,15 @@ typedef struct grpc_httpcli_context { grpc_pollset_set pollset_set; } grpc_httpcli_context; +typedef struct { + const char *default_port; + void (*handshake)(void *arg, grpc_endpoint *endpoint, const char *host, + void (*on_done)(void *arg, grpc_endpoint *endpoint)); +} grpc_httpcli_handshaker; + +extern const grpc_httpcli_handshaker grpc_httpcli_plaintext; +extern const grpc_httpcli_handshaker grpc_httpcli_ssl; + /* A request */ typedef struct grpc_httpcli_request { /* The host name to connect to */ @@ -69,8 +79,8 @@ typedef struct grpc_httpcli_request { Host, Connection, User-Agent */ size_t hdr_count; grpc_httpcli_header *hdrs; - /* whether to use ssl for the request */ - int use_ssl; + /* handshaker to use ssl for the request */ + const grpc_httpcli_handshaker *handshaker; } grpc_httpcli_request; /* A response */ diff --git a/src/core/httpcli/httpcli_security_connector.c b/src/core/httpcli/httpcli_security_connector.c index ce0d3d5a70..7887f9d530 100644 --- a/src/core/httpcli/httpcli_security_connector.c +++ b/src/core/httpcli/httpcli_security_connector.c @@ -31,7 +31,7 @@ * */ -#include "src/core/httpcli/httpcli_security_connector.h" +#include "src/core/httpcli/httpcli.h" #include <string.h> @@ -96,7 +96,7 @@ static grpc_security_status httpcli_ssl_check_peer(grpc_security_connector *sc, static grpc_security_connector_vtable httpcli_ssl_vtable = { httpcli_ssl_destroy, httpcli_ssl_create_handshaker, httpcli_ssl_check_peer}; -grpc_security_status grpc_httpcli_ssl_channel_security_connector_create( +static grpc_security_status httpcli_ssl_channel_security_connector_create( const unsigned char *pem_root_certs, size_t pem_root_certs_size, const char *secure_peer_name, grpc_channel_security_connector **sc) { tsi_result result = TSI_OK; @@ -130,3 +130,48 @@ grpc_security_status grpc_httpcli_ssl_channel_security_connector_create( *sc = &c->base; return GRPC_SECURITY_OK; } + +/* handshaker */ + +typedef struct { + void (*func)(void *arg, grpc_endpoint *endpoint); + void *arg; +} on_done_closure; + +static void on_secure_transport_setup_done(void *rp, + grpc_security_status status, + grpc_endpoint *wrapped_endpoint, + grpc_endpoint *secure_endpoint) { + on_done_closure *c = rp; + if (status != GRPC_SECURITY_OK) { + gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status); + c->func(c->arg, NULL); + } else { + c->func(c->arg, secure_endpoint); + } + gpr_free(c); +} + +static void ssl_handshake(void *arg, grpc_endpoint *tcp, const char *host, + void (*on_done)(void *arg, grpc_endpoint *endpoint)) { + grpc_channel_security_connector *sc = NULL; + const unsigned char *pem_root_certs = NULL; + on_done_closure *c = gpr_malloc(sizeof(*c)); + size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); + if (pem_root_certs == NULL || pem_root_certs_size == 0) { + gpr_log(GPR_ERROR, "Could not get default pem root certs."); + on_done(arg, NULL); + gpr_free(c); + return; + } + c->func = on_done; + c->arg = arg; + GPR_ASSERT(httpcli_ssl_channel_security_connector_create( + pem_root_certs, pem_root_certs_size, host, &sc) == + GRPC_SECURITY_OK); + grpc_setup_secure_transport(&sc->base, tcp, on_secure_transport_setup_done, + c); + GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli"); +} + +const grpc_httpcli_handshaker grpc_httpcli_ssl = {"https", ssl_handshake}; diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c index 2239f57378..6421ce673d 100644 --- a/src/core/security/credentials.c +++ b/src/core/security/credentials.c @@ -685,7 +685,7 @@ static void service_account_fetch_oauth2( request.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH; request.hdr_count = 1; request.hdrs = &header; - request.use_ssl = 1; + request.handshaker = &grpc_httpcli_ssl; grpc_httpcli_post(httpcli_context, pollset, &request, body, strlen(body), deadline, response_cb, metadata_req); gpr_free(body); @@ -744,7 +744,7 @@ static void refresh_token_fetch_oauth2( request.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH; request.hdr_count = 1; request.hdrs = &header; - request.use_ssl = 1; + request.handshaker = &grpc_httpcli_ssl; grpc_httpcli_post(httpcli_context, pollset, &request, body, strlen(body), deadline, response_cb, metadata_req); gpr_free(body); diff --git a/src/core/security/jwt_verifier.c b/src/core/security/jwt_verifier.c index 1276693da7..38ad134a6a 100644 --- a/src/core/security/jwt_verifier.c +++ b/src/core/security/jwt_verifier.c @@ -628,7 +628,7 @@ static void on_openid_config_retrieved(void *user_data, goto error; } jwks_uri += 8; - req.use_ssl = 1; + req.handshaker = &grpc_httpcli_ssl; req.host = gpr_strdup(jwks_uri); req.path = strchr(jwks_uri, '/'); if (req.path == NULL) { @@ -685,7 +685,7 @@ static void retrieve_key_and_verify(verifier_cb_ctx *ctx) { const char *iss; grpc_httpcli_request req; memset(&req, 0, sizeof(grpc_httpcli_request)); - req.use_ssl = 1; + req.handshaker = &grpc_httpcli_ssl; GPR_ASSERT(ctx != NULL && ctx->header != NULL && ctx->claims != NULL); iss = ctx->claims->iss; diff --git a/src/core/surface/call.c b/src/core/surface/call.c index d3e66e9c4c..6e566e6a8f 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -456,20 +456,6 @@ void grpc_call_internal_ref(grpc_call *c) { static void destroy_call(void *call, int ignored_success) { size_t i; grpc_call *c = call; - grpc_call *parent = c->parent; - if (parent) { - gpr_mu_lock(&parent->mu); - if (call == parent->first_child) { - parent->first_child = c->sibling_next; - if (c == parent->first_child) { - parent->first_child = NULL; - } - c->sibling_prev->sibling_next = c->sibling_next; - c->sibling_next->sibling_prev = c->sibling_prev; - } - gpr_mu_unlock(&parent->mu); - GRPC_CALL_INTERNAL_UNREF(parent, "child", 1); - } grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c)); GRPC_CHANNEL_INTERNAL_UNREF(c->channel, "call"); gpr_mu_destroy(&c->mu); @@ -1257,6 +1243,22 @@ grpc_call_error grpc_call_start_ioreq_and_call_back( void grpc_call_destroy(grpc_call *c) { int cancel; + grpc_call *parent = c->parent; + + if (parent) { + gpr_mu_lock(&parent->mu); + if (c == parent->first_child) { + parent->first_child = c->sibling_next; + if (c == parent->first_child) { + parent->first_child = NULL; + } + c->sibling_prev->sibling_next = c->sibling_next; + c->sibling_next->sibling_prev = c->sibling_prev; + } + gpr_mu_unlock(&parent->mu); + GRPC_CALL_INTERNAL_UNREF(parent, "child", 1); + } + lock(c); GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; diff --git a/src/core/httpcli/httpcli_security_connector.h b/src/objective-c/GRPCClient/GRPCCall+OAuth2.h index c50f25905e..2e379a7157 100644 --- a/src/core/httpcli/httpcli_security_connector.h +++ b/src/objective-c/GRPCClient/GRPCCall+OAuth2.h @@ -31,13 +31,19 @@ * */ -#ifndef GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H -#define GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H +#import "GRPCCall.h" -#include "src/core/security/security_connector.h" +// Helpers for setting and reading headers compatible with OAuth2. +@interface GRPCCall (OAuth2) -grpc_security_status grpc_httpcli_ssl_channel_security_connector_create( - const unsigned char *pem_root_certs, size_t pem_root_certs_size, - const char *secure_peer_name, grpc_channel_security_connector **sc); +// Setting this property is equivalent to setting "Bearer <passed token>" as the value of the +// request header with key "authorization" (the authorization header). Setting it to nil removes the +// authorization header from the request. +// The value obtained by getting the property is the OAuth2 bearer token if the authorization header +// of the request has the form "Bearer <token>", or nil otherwise. +@property(atomic, copy) NSString *oauth2AccessToken; -#endif /* GRPC_INTERNAL_CORE_HTTPCLI_HTTPCLI_SECURITY_CONNECTOR_H */ +// Returns the value (if any) of the "www-authenticate" response header (the challenge header). +@property(atomic, readonly) NSString *oauth2ChallengeHeader; + +@end diff --git a/src/objective-c/GRPCClient/GRPCCall+OAuth2.m b/src/objective-c/GRPCClient/GRPCCall+OAuth2.m new file mode 100644 index 0000000000..ed39d4b0f7 --- /dev/null +++ b/src/objective-c/GRPCClient/GRPCCall+OAuth2.m @@ -0,0 +1,63 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#import "GRPCCall+OAuth2.h" + +static NSString * const kAuthorizationHeader = @"authorization"; +static NSString * const kBearerPrefix = @"Bearer "; +static NSString * const kChallengeHeader = @"www-authenticate"; + +@implementation GRPCCall (OAuth2) + +- (NSString *)oauth2AccessToken { + NSString *headerValue = self.requestMetadata[kAuthorizationHeader]; + if ([headerValue hasPrefix:kBearerPrefix]) { + return [headerValue substringFromIndex:kBearerPrefix.length]; + } else { + return nil; + } +} + +- (void)setOauth2AccessToken:(NSString *)token { + if (token) { + self.requestMetadata[kAuthorizationHeader] = [kBearerPrefix stringByAppendingString:token]; + } else { + [self.requestMetadata removeObjectForKey:kAuthorizationHeader]; + } +} + +- (NSString *)oauth2ChallengeHeader { + return self.responseMetadata[kChallengeHeader]; +} + +@end diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 6636c48620..d902f95b51 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -79,10 +79,11 @@ return cachedHost; } - if ((self = [super init])) { - _address = address; - _secure = YES; - hostCache[address] = self; + if ((self = [super init])) { + _address = address; + _secure = YES; + hostCache[address] = self; + } } return self; } diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index e5d7e43ed9..e85dd6e65c 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -35,6 +35,7 @@ #import <XCTest/XCTest.h> #import <GRPCClient/GRPCCall.h> +#import <GRPCClient/GRPCCall+OAuth2.h> #import <GRPCClient/GRPCCall+Tests.h> #import <ProtoRPC/ProtoMethod.h> #import <RemoteTest/Messages.pbobjc.h> @@ -160,7 +161,7 @@ static ProtoMethod *kUnaryCallMethod; path:kUnaryCallMethod.HTTPPath requestsWriter:requestsWriter]; - call.requestMetadata[@"Authorization"] = @"Bearer bogusToken"; + call.oauth2AccessToken = @"bogusToken"; id<GRXWriteable> responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { XCTFail(@"Received unexpected response: %@", value); @@ -169,7 +170,7 @@ static ProtoMethod *kUnaryCallMethod; XCTAssertEqual(errorOrNil.code, 16, @"Finished with unexpected error: %@", errorOrNil); XCTAssertEqualObjects(call.responseMetadata, errorOrNil.userInfo[kGRPCStatusMetadataKey], @"Metadata in the NSError object and call object differ."); - NSString *challengeHeader = call.responseMetadata[@"www-authenticate"]; + NSString *challengeHeader = call.oauth2ChallengeHeader; XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", call.responseMetadata); [expectation fulfill]; diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c index 390afcdf63..8dddfbee98 100644 --- a/test/core/httpcli/httpcli_test.c +++ b/test/core/httpcli/httpcli_test.c @@ -81,7 +81,7 @@ static void test_get(int use_ssl, int port) { memset(&req, 0, sizeof(req)); req.host = host; req.path = "/get"; - req.use_ssl = use_ssl; + req.handshaker = use_ssl ? &grpc_httpcli_ssl : &grpc_httpcli_plaintext; grpc_httpcli_get(&g_context, &g_pollset, &req, n_seconds_time(15), on_finish, (void *)42); @@ -107,7 +107,7 @@ static void test_post(int use_ssl, int port) { memset(&req, 0, sizeof(req)); req.host = host; req.path = "/post"; - req.use_ssl = use_ssl; + req.handshaker = use_ssl ? &grpc_httpcli_ssl : &grpc_httpcli_plaintext; grpc_httpcli_post(&g_context, &g_pollset, &req, "hello", 5, n_seconds_time(15), on_finish, (void *)42); diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c index ecbff75de7..e4a8144eaf 100644 --- a/test/core/security/credentials_test.c +++ b/test/core/security/credentials_test.c @@ -477,7 +477,7 @@ static void on_oauth2_creds_get_metadata_failure( static void validate_compute_engine_http_request( const grpc_httpcli_request *request) { - GPR_ASSERT(!request->use_ssl); + GPR_ASSERT(request->handshaker != &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, "metadata") == 0); GPR_ASSERT( strcmp(request->path, @@ -573,7 +573,7 @@ static void validate_refresh_token_http_request( GPR_ASSERT(strlen(expected_body) == body_size); GPR_ASSERT(memcmp(expected_body, body, body_size) == 0); gpr_free(expected_body); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, GRPC_GOOGLE_OAUTH2_SERVICE_HOST) == 0); GPR_ASSERT(strcmp(request->path, GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH) == 0); GPR_ASSERT(request->hdr_count == 1); @@ -697,7 +697,7 @@ static void validate_service_account_http_request( GPR_ASSERT(strlen(expected_body) == body_size); GPR_ASSERT(memcmp(expected_body, body, body_size) == 0); gpr_free(expected_body); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, GRPC_GOOGLE_OAUTH2_SERVICE_HOST) == 0); GPR_ASSERT(strcmp(request->path, GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH) == 0); GPR_ASSERT(request->hdr_count == 1); diff --git a/test/core/security/jwt_verifier_test.c b/test/core/security/jwt_verifier_test.c index 98db56c0ef..440286ea1a 100644 --- a/test/core/security/jwt_verifier_test.c +++ b/test/core/security/jwt_verifier_test.c @@ -286,7 +286,7 @@ static int httpcli_get_google_keys_for_email( const grpc_httpcli_request *request, gpr_timespec deadline, grpc_httpcli_response_cb on_response, void *user_data) { grpc_httpcli_response response = http_response(200, good_google_email_keys()); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, "www.googleapis.com") == 0); GPR_ASSERT(strcmp(request->path, "/robot/v1/metadata/x509/" @@ -331,7 +331,7 @@ static int httpcli_get_custom_keys_for_email( const grpc_httpcli_request *request, gpr_timespec deadline, grpc_httpcli_response_cb on_response, void *user_data) { grpc_httpcli_response response = http_response(200, gpr_strdup(good_jwk_set)); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, "keys.bar.com") == 0); GPR_ASSERT(strcmp(request->path, "/jwk/foo@bar.com") == 0); on_response(user_data, &response); @@ -363,7 +363,7 @@ static int httpcli_get_jwk_set( const grpc_httpcli_request *request, gpr_timespec deadline, grpc_httpcli_response_cb on_response, void *user_data) { grpc_httpcli_response response = http_response(200, gpr_strdup(good_jwk_set)); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, "www.googleapis.com") == 0); GPR_ASSERT(strcmp(request->path, "/oauth2/v3/certs") == 0); on_response(user_data, &response); @@ -377,7 +377,7 @@ static int httpcli_get_openid_config(const grpc_httpcli_request *request, void *user_data) { grpc_httpcli_response response = http_response(200, gpr_strdup(good_openid_config)); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); GPR_ASSERT(strcmp(request->host, "accounts.google.com") == 0); GPR_ASSERT(strcmp(request->path, GRPC_OPENID_CONFIG_URL_SUFFIX) == 0); grpc_httpcli_set_override(httpcli_get_jwk_set, @@ -421,7 +421,7 @@ static int httpcli_get_bad_json(const grpc_httpcli_request *request, void *user_data) { grpc_httpcli_response response = http_response(200, gpr_strdup("{\"bad\": \"stuff\"}")); - GPR_ASSERT(request->use_ssl); + GPR_ASSERT(request->handshaker == &grpc_httpcli_ssl); on_response(user_data, &response); gpr_free(response.body); return 1; diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c index b07df391f9..9bff18d311 100644 --- a/test/core/util/port_posix.c +++ b/test/core/util/port_posix.c @@ -44,9 +44,13 @@ #include <string.h> #include <unistd.h> +#include <grpc/grpc.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/httpcli/httpcli.h" +#include "src/core/support/env.h" + #define NUM_RANDOM_PORTS_TO_PICK 100 static int *chosen_ports = NULL; @@ -126,6 +130,67 @@ static int is_port_available(int *port, int is_tcp) { return 1; } +typedef struct portreq { + grpc_pollset pollset; + int port; +} portreq; + +static void got_port_from_server(void *arg, + const grpc_httpcli_response *response) { + size_t i; + int port = 0; + portreq *pr = arg; + GPR_ASSERT(response); + GPR_ASSERT(response->status == 200); + for (i = 0; i < response->body_length; i++) { + GPR_ASSERT(response->body[i] >= '0' && response->body[i] <= '9'); + port = port * 10 + response->body[i] - '0'; + } + GPR_ASSERT(port > 1024); + gpr_mu_lock(GRPC_POLLSET_MU(&pr->pollset)); + pr->port = port; + grpc_pollset_kick(&pr->pollset, NULL); + gpr_mu_unlock(GRPC_POLLSET_MU(&pr->pollset)); +} + +static void destroy_pollset_and_shutdown(void *p) { + grpc_pollset_destroy(p); + grpc_shutdown(); +} + +static int pick_port_using_server(char *server) { + grpc_httpcli_context context; + grpc_httpcli_request req; + portreq pr; + + grpc_init(); + + memset(&pr, 0, sizeof(pr)); + memset(&req, 0, sizeof(req)); + grpc_pollset_init(&pr.pollset); + pr.port = -1; + + req.host = server; + req.path = "/get"; + + grpc_httpcli_context_init(&context); + grpc_httpcli_get(&context, &pr.pollset, &req, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server, + &pr); + gpr_mu_lock(GRPC_POLLSET_MU(&pr.pollset)); + while (pr.port == -1) { + grpc_pollset_worker worker; + grpc_pollset_work(&pr.pollset, &worker, + GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1)); + } + gpr_mu_unlock(GRPC_POLLSET_MU(&pr.pollset)); + + grpc_httpcli_context_destroy(&context); + grpc_pollset_shutdown(&pr.pollset, destroy_pollset_and_shutdown, &pr.pollset); + + return pr.port; +} + int grpc_pick_unused_port(void) { /* We repeatedly pick a port and then see whether or not it is available for use both as a TCP socket and a UDP socket. First, we @@ -143,6 +208,15 @@ int grpc_pick_unused_port(void) { int is_tcp = 1; int try = 0; + char *env = gpr_getenv("GRPC_TEST_PORT_SERVER"); + if (env) { + int port = pick_port_using_server(env); + gpr_free(env); + if (port != 0) { + return port; + } + } + for (;;) { int port; try++; diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a4d7764dfb..cbf5c50a65 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -767,10 +767,6 @@ include/grpc/compression.h \ include/grpc/grpc.h \ include/grpc/status.h \ include/grpc/census.h \ -src/core/httpcli/format_request.h \ -src/core/httpcli/httpcli.h \ -src/core/httpcli/httpcli_security_connector.h \ -src/core/httpcli/parser.h \ src/core/security/auth_filters.h \ src/core/security/base64.h \ src/core/security/credentials.h \ @@ -810,6 +806,9 @@ src/core/client_config/subchannel_factory_decorators/merge_channel_args.h \ src/core/client_config/uri_parser.h \ src/core/compression/message_compress.h \ src/core/debug/trace.h \ +src/core/httpcli/format_request.h \ +src/core/httpcli/httpcli.h \ +src/core/httpcli/parser.h \ src/core/iomgr/alarm.h \ src/core/iomgr/alarm_heap.h \ src/core/iomgr/alarm_internal.h \ @@ -882,10 +881,7 @@ src/core/transport/transport.h \ src/core/transport/transport_impl.h \ src/core/census/context.h \ src/core/census/rpc_stat_id.h \ -src/core/httpcli/format_request.c \ -src/core/httpcli/httpcli.c \ src/core/httpcli/httpcli_security_connector.c \ -src/core/httpcli/parser.c \ src/core/security/base64.c \ src/core/security/client_auth_filter.c \ src/core/security/credentials.c \ @@ -932,6 +928,9 @@ src/core/client_config/uri_parser.c \ src/core/compression/algorithm.c \ src/core/compression/message_compress.c \ src/core/debug/trace.c \ +src/core/httpcli/format_request.c \ +src/core/httpcli/httpcli.c \ +src/core/httpcli/parser.c \ src/core/iomgr/alarm.c \ src/core/iomgr/alarm_heap.c \ src/core/iomgr/endpoint.c \ diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index e5e778a3f1..b7e0089269 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -164,13 +164,15 @@ class JobSpec(object): class Job(object): """Manages one job.""" - def __init__(self, spec, bin_hash, newline_on_success, travis, xml_report): + def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report): self._spec = spec self._bin_hash = bin_hash self._tempfile = tempfile.TemporaryFile() env = os.environ.copy() for k, v in spec.environ.iteritems(): env[k] = v + for k, v in add_env.iteritems(): + env[k] = v self._start = time.time() self._process = subprocess.Popen(args=spec.cmdline, stderr=subprocess.STDOUT, @@ -229,7 +231,7 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, cache, xml_report): + stop_on_failure, add_env, cache, xml_report): self._running = set() self._check_cancelled = check_cancelled self._cancelled = False @@ -242,6 +244,7 @@ class Jobset(object): self._stop_on_failure = stop_on_failure self._hashes = {} self._xml_report = xml_report + self._add_env = add_env def start(self, spec): """Start a job. Return True on success, False on failure.""" @@ -264,16 +267,12 @@ class Jobset(object): bin_hash = None should_run = True if should_run: - try: - self._running.add(Job(spec, - bin_hash, - self._newline_on_success, - self._travis, - self._xml_report)) - except: - message('FAILED', spec.shortname) - self._cancelled = True - return False + self._running.add(Job(spec, + bin_hash, + self._newline_on_success, + self._travis, + self._add_env, + self._xml_report)) return True def reap(self): @@ -344,10 +343,11 @@ def run(cmdlines, infinite_runs=False, stop_on_failure=False, cache=None, - xml_report=None): + xml_report=None, + add_env={}): js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, - newline_on_success, travis, stop_on_failure, + newline_on_success, travis, stop_on_failure, add_env, cache if cache is not None else NoCache(), xml_report) for cmdline in cmdlines: diff --git a/tools/run_tests/port_server.py b/tools/run_tests/port_server.py new file mode 100755 index 0000000000..0f81470d28 --- /dev/null +++ b/tools/run_tests/port_server.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Manage TCP ports for unit tests; started by run_tests.py""" + +import argparse +import BaseHTTPServer +import hashlib +import os +import socket +import sys +import time + +argp = argparse.ArgumentParser(description='Server for httpcli_test') +argp.add_argument('-p', '--port', default=12345, type=int) +args = argp.parse_args() + +print 'port server running on port %d' % args.port + +pool = [] +in_use = {} + +with open(__file__) as f: + _MY_VERSION = hashlib.sha1(f.read()).hexdigest() + + +def refill_pool(): + """Scan for ports not marked for being in use""" + for i in range(10000, 65000): + if len(pool) > 100: break + if i in in_use: + age = time.time() - in_use[i] + if age < 600: + continue + del in_use[i] + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind(('localhost', i)) + pool.append(i) + except: + pass # we really don't care about failures + finally: + s.close() + + +def allocate_port(): + global pool + global in_use + if not pool: + refill_pool() + port = pool[0] + pool = pool[1:] + in_use[port] = time.time() + return port + + +keep_running = True + + +class Handler(BaseHTTPServer.BaseHTTPRequestHandler): + + def do_GET(self): + global keep_running + if self.path == '/get': + # allocate a new port, it will stay bound for ten minutes and until + # it's unused + self.send_response(200) + self.send_header('Content-Type', 'text/plain') + self.end_headers() + p = allocate_port() + self.log_message('allocated port %d' % p) + self.wfile.write('%d' % p) + elif self.path == '/version': + # fetch a version string and the current process pid + self.send_response(200) + self.send_header('Content-Type', 'text/plain') + self.end_headers() + self.wfile.write(_MY_VERSION) + elif self.path == '/quit': + self.send_response(200) + self.end_headers() + keep_running = False + + +httpd = BaseHTTPServer.HTTPServer(('', args.port), Handler) +while keep_running: + httpd.handle_request() + +print 'done' + diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0d4caa66aa..47d2b6183e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -32,6 +32,7 @@ import argparse import glob +import hashlib import itertools import json import multiprocessing @@ -43,6 +44,7 @@ import subprocess import sys import time import xml.etree.cElementTree as ET +import urllib2 import jobset import watch_dirs @@ -530,7 +532,43 @@ class TestCache(object): self.parse(json.loads(f.read())) -def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report=None): +def _start_port_server(port_server_port): + # check if a compatible port server is running + # if incompatible (version mismatch) ==> start a new one + # if not running ==> start a new one + # otherwise, leave it up + try: + version = urllib2.urlopen('http://localhost:%d/version' % port_server_port).read() + running = True + except Exception: + running = False + if running: + with open('tools/run_tests/port_server.py') as f: + current_version = hashlib.sha1(f.read()).hexdigest() + running = (version == current_version) + if not running: + urllib2.urlopen('http://localhost:%d/quit' % port_server_port).read() + time.sleep(1) + if not running: + port_log = open('portlog.txt', 'w') + port_server = subprocess.Popen( + ['python', 'tools/run_tests/port_server.py', '-p', '%d' % port_server_port], + stderr=subprocess.STDOUT, + stdout=port_log) + # ensure port server is up + while True: + try: + urllib2.urlopen('http://localhost:%d/get' % port_server_port).read() + break + except urllib2.URLError: + time.sleep(0.5) + except: + port_server.kill() + raise + + +def _build_and_run( + check_cancelled, newline_on_success, travis, cache, xml_report=None): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, @@ -540,6 +578,8 @@ def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_repor # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] + port_server_port = 9999 + _start_port_server(port_server_port) try: infinite_runs = runs_per_test == 0 # When running on travis, we want out test runs to be as similar as possible @@ -566,7 +606,8 @@ def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_repor maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, - xml_report=testsuite): + xml_report=testsuite, + add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}): return 2 finally: for antagonist in antagonists: diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e3490c93af..5d23bf9e88 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -12252,7 +12252,6 @@ "src/core/debug/trace.h", "src/core/httpcli/format_request.h", "src/core/httpcli/httpcli.h", - "src/core/httpcli/httpcli_security_connector.h", "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", @@ -12410,7 +12409,6 @@ "src/core/httpcli/httpcli.c", "src/core/httpcli/httpcli.h", "src/core/httpcli/httpcli_security_connector.c", - "src/core/httpcli/httpcli_security_connector.h", "src/core/httpcli/parser.c", "src/core/httpcli/parser.h", "src/core/iomgr/alarm.c", @@ -12723,6 +12721,9 @@ "src/core/client_config/uri_parser.h", "src/core/compression/message_compress.h", "src/core/debug/trace.h", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.h", "src/core/iomgr/alarm_internal.h", @@ -12860,6 +12861,12 @@ "src/core/compression/message_compress.h", "src/core/debug/trace.c", "src/core/debug/trace.h", + "src/core/httpcli/format_request.c", + "src/core/httpcli/format_request.h", + "src/core/httpcli/httpcli.c", + "src/core/httpcli/httpcli.h", + "src/core/httpcli/parser.c", + "src/core/httpcli/parser.h", "src/core/iomgr/alarm.c", "src/core/iomgr/alarm.h", "src/core/iomgr/alarm_heap.c", diff --git a/vsprojects/grpc/grpc.vcxproj b/vsprojects/grpc/grpc.vcxproj index 005a1b411c..067f341b95 100644 --- a/vsprojects/grpc/grpc.vcxproj +++ b/vsprojects/grpc/grpc.vcxproj @@ -229,10 +229,6 @@ <ClInclude Include="..\..\include\grpc\census.h" /> </ItemGroup> <ItemGroup> - <ClInclude Include="..\..\src\core\httpcli\format_request.h" /> - <ClInclude Include="..\..\src\core\httpcli\httpcli.h" /> - <ClInclude Include="..\..\src\core\httpcli\httpcli_security_connector.h" /> - <ClInclude Include="..\..\src\core\httpcli\parser.h" /> <ClInclude Include="..\..\src\core\security\auth_filters.h" /> <ClInclude Include="..\..\src\core\security\base64.h" /> <ClInclude Include="..\..\src\core\security\credentials.h" /> @@ -272,6 +268,9 @@ <ClInclude Include="..\..\src\core\client_config\uri_parser.h" /> <ClInclude Include="..\..\src\core\compression\message_compress.h" /> <ClInclude Include="..\..\src\core\debug\trace.h" /> + <ClInclude Include="..\..\src\core\httpcli\format_request.h" /> + <ClInclude Include="..\..\src\core\httpcli\httpcli.h" /> + <ClInclude Include="..\..\src\core\httpcli\parser.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm_heap.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm_internal.h" /> @@ -346,14 +345,8 @@ <ClInclude Include="..\..\src\core\census\rpc_stat_id.h" /> </ItemGroup> <ItemGroup> - <ClCompile Include="..\..\src\core\httpcli\format_request.c"> - </ClCompile> - <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> - </ClCompile> <ClCompile Include="..\..\src\core\httpcli\httpcli_security_connector.c"> </ClCompile> - <ClCompile Include="..\..\src\core\httpcli\parser.c"> - </ClCompile> <ClCompile Include="..\..\src\core\security\base64.c"> </ClCompile> <ClCompile Include="..\..\src\core\security\client_auth_filter.c"> @@ -446,6 +439,12 @@ </ClCompile> <ClCompile Include="..\..\src\core\debug\trace.c"> </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\format_request.c"> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\parser.c"> + </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm.c"> </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm_heap.c"> diff --git a/vsprojects/grpc/grpc.vcxproj.filters b/vsprojects/grpc/grpc.vcxproj.filters index ab03d96a1b..fcc40c3a4b 100644 --- a/vsprojects/grpc/grpc.vcxproj.filters +++ b/vsprojects/grpc/grpc.vcxproj.filters @@ -1,18 +1,9 @@ <?xml version="1.0" encoding="utf-8"?> <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <ItemGroup> - <ClCompile Include="..\..\src\core\httpcli\format_request.c"> - <Filter>src\core\httpcli</Filter> - </ClCompile> - <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> - <Filter>src\core\httpcli</Filter> - </ClCompile> <ClCompile Include="..\..\src\core\httpcli\httpcli_security_connector.c"> <Filter>src\core\httpcli</Filter> </ClCompile> - <ClCompile Include="..\..\src\core\httpcli\parser.c"> - <Filter>src\core\httpcli</Filter> - </ClCompile> <ClCompile Include="..\..\src\core\security\base64.c"> <Filter>src\core\security</Filter> </ClCompile> @@ -151,6 +142,15 @@ <ClCompile Include="..\..\src\core\debug\trace.c"> <Filter>src\core\debug</Filter> </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\format_request.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\parser.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm.c"> <Filter>src\core\iomgr</Filter> </ClCompile> @@ -440,18 +440,6 @@ </ClInclude> </ItemGroup> <ItemGroup> - <ClInclude Include="..\..\src\core\httpcli\format_request.h"> - <Filter>src\core\httpcli</Filter> - </ClInclude> - <ClInclude Include="..\..\src\core\httpcli\httpcli.h"> - <Filter>src\core\httpcli</Filter> - </ClInclude> - <ClInclude Include="..\..\src\core\httpcli\httpcli_security_connector.h"> - <Filter>src\core\httpcli</Filter> - </ClInclude> - <ClInclude Include="..\..\src\core\httpcli\parser.h"> - <Filter>src\core\httpcli</Filter> - </ClInclude> <ClInclude Include="..\..\src\core\security\auth_filters.h"> <Filter>src\core\security</Filter> </ClInclude> @@ -569,6 +557,15 @@ <ClInclude Include="..\..\src\core\debug\trace.h"> <Filter>src\core\debug</Filter> </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\format_request.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\httpcli.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\parser.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> <ClInclude Include="..\..\src\core\iomgr\alarm.h"> <Filter>src\core\iomgr</Filter> </ClInclude> diff --git a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj index a6113fd971..b95658b70c 100644 --- a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj @@ -251,6 +251,9 @@ <ClInclude Include="..\..\src\core\client_config\uri_parser.h" /> <ClInclude Include="..\..\src\core\compression\message_compress.h" /> <ClInclude Include="..\..\src\core\debug\trace.h" /> + <ClInclude Include="..\..\src\core\httpcli\format_request.h" /> + <ClInclude Include="..\..\src\core\httpcli\httpcli.h" /> + <ClInclude Include="..\..\src\core\httpcli\parser.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm_heap.h" /> <ClInclude Include="..\..\src\core\iomgr\alarm_internal.h" /> @@ -379,6 +382,12 @@ </ClCompile> <ClCompile Include="..\..\src\core\debug\trace.c"> </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\format_request.c"> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\parser.c"> + </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm.c"> </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm_heap.c"> diff --git a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters index 0b86f87412..05e9d139e0 100644 --- a/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -82,6 +82,15 @@ <ClCompile Include="..\..\src\core\debug\trace.c"> <Filter>src\core\debug</Filter> </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\format_request.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\httpcli.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> + <ClCompile Include="..\..\src\core\httpcli\parser.c"> + <Filter>src\core\httpcli</Filter> + </ClCompile> <ClCompile Include="..\..\src\core\iomgr\alarm.c"> <Filter>src\core\iomgr</Filter> </ClCompile> @@ -446,6 +455,15 @@ <ClInclude Include="..\..\src\core\debug\trace.h"> <Filter>src\core\debug</Filter> </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\format_request.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\httpcli.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> + <ClInclude Include="..\..\src\core\httpcli\parser.h"> + <Filter>src\core\httpcli</Filter> + </ClInclude> <ClInclude Include="..\..\src\core\iomgr\alarm.h"> <Filter>src\core\iomgr</Filter> </ClInclude> @@ -701,6 +719,9 @@ <Filter Include="src\core\debug"> <UniqueIdentifier>{6d8d5774-7291-554d-fafa-583463cd3fd9}</UniqueIdentifier> </Filter> + <Filter Include="src\core\httpcli"> + <UniqueIdentifier>{1ba3a245-47e7-89b5-b0c9-aca758bd0277}</UniqueIdentifier> + </Filter> <Filter Include="src\core\iomgr"> <UniqueIdentifier>{a9df8b24-ecea-ff6d-8999-d8fa54cd70bf}</UniqueIdentifier> </Filter> |