diff options
74 files changed, 1163 insertions, 532 deletions
diff --git a/INSTALL.md b/INSTALL.md index 6cfa1b6cba..5406fec84d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -51,6 +51,15 @@ If you plan to build from source and run tests, install the following as well: $ brew install gflags ``` +*Tip*: when building, +you *may* want to explicitly set the `LIBTOOL` and `LIBTOOLIZE` +environment variables when running `make` to ensure the version +installed by `brew` is being used: + +```sh + $ LIBTOOL=glibtool LIBTOOLIZE=glibtoolize make +``` + ## Protoc By default gRPC uses [protocol buffers](https://github.com/google/protobuf), diff --git a/composer.json b/composer.json index 0cafb94808..284b57a809 100644 --- a/composer.json +++ b/composer.json @@ -7,7 +7,7 @@ "license": "BSD-3-Clause", "require": { "php": ">=5.5.0", - "google/protobuf": "^v3.1.0" + "google/protobuf": "^v3.3.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/include/grpc++/impl/codegen/config_protobuf.h b/include/grpc++/impl/codegen/config_protobuf.h index 8620d4b218..e66189f8d1 100644 --- a/include/grpc++/impl/codegen/config_protobuf.h +++ b/include/grpc++/impl/codegen/config_protobuf.h @@ -34,6 +34,8 @@ #ifndef GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H #define GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H +#define GRPC_OPEN_SOURCE_PROTO + #ifndef GRPC_CUSTOM_PROTOBUF_INT64 #include <google/protobuf/stubs/common.h> #define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64 diff --git a/include/grpc++/impl/codegen/core_codegen.h b/include/grpc++/impl/codegen/core_codegen.h index a1593729eb..7a1236a716 100644 --- a/include/grpc++/impl/codegen/core_codegen.h +++ b/include/grpc++/impl/codegen/core_codegen.h @@ -90,11 +90,15 @@ class CoreCodegen final : public CoreCodegenInterface { grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) override; - + grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) override; grpc_slice grpc_empty_slice() override; grpc_slice grpc_slice_malloc(size_t length) override; void grpc_slice_unref(grpc_slice slice) override; + grpc_slice grpc_slice_ref(grpc_slice slice) override; grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) override; + grpc_slice grpc_slice_split_head(grpc_slice* s, size_t split) override; void grpc_slice_buffer_add(grpc_slice_buffer* sb, grpc_slice slice) override; void grpc_slice_buffer_pop(grpc_slice_buffer* sb) override; grpc_slice grpc_slice_from_static_buffer(const void* buffer, diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h index 7cc3a82476..4ac37f5255 100644 --- a/include/grpc++/impl/codegen/core_codegen_interface.h +++ b/include/grpc++/impl/codegen/core_codegen_interface.h @@ -101,15 +101,18 @@ class CoreCodegenInterface { virtual grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) = 0; - + virtual grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) = 0; virtual void grpc_call_ref(grpc_call* call) = 0; virtual void grpc_call_unref(grpc_call* call) = 0; virtual void* grpc_call_arena_alloc(grpc_call* call, size_t length) = 0; - virtual grpc_slice grpc_empty_slice() = 0; virtual grpc_slice grpc_slice_malloc(size_t length) = 0; virtual void grpc_slice_unref(grpc_slice slice) = 0; + virtual grpc_slice grpc_slice_ref(grpc_slice slice) = 0; virtual grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) = 0; + virtual grpc_slice grpc_slice_split_head(grpc_slice* s, size_t split) = 0; virtual void grpc_slice_buffer_add(grpc_slice_buffer* sb, grpc_slice slice) = 0; virtual void grpc_slice_buffer_pop(grpc_slice_buffer* sb) = 0; diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h index 8c0c32bfc5..5cdb2a7a15 100644 --- a/include/grpc++/impl/codegen/proto_utils.h +++ b/include/grpc++/impl/codegen/proto_utils.h @@ -54,8 +54,7 @@ class GrpcBufferWriterPeer; const int kGrpcBufferWriterMaxBufferLength = 1024 * 1024; -class GrpcBufferWriter final - : public ::grpc::protobuf::io::ZeroCopyOutputStream { +class GrpcBufferWriter : public ::grpc::protobuf::io::ZeroCopyOutputStream { public: explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size) : block_size_(block_size), byte_count_(0), have_backup_(false) { @@ -103,6 +102,8 @@ class GrpcBufferWriter final grpc::protobuf::int64 ByteCount() const override { return byte_count_; } + grpc_slice_buffer* SliceBuffer() { return slice_buffer_; } + private: friend class GrpcBufferWriterPeer; const int block_size_; @@ -113,8 +114,7 @@ class GrpcBufferWriter final grpc_slice slice_; }; -class GrpcBufferReader final - : public ::grpc::protobuf::io::ZeroCopyInputStream { +class GrpcBufferReader : public ::grpc::protobuf::io::ZeroCopyInputStream { public: explicit GrpcBufferReader(grpc_byte_buffer* buffer) : byte_count_(0), backup_count_(0), status_() { @@ -175,64 +175,91 @@ class GrpcBufferReader final return byte_count_ - backup_count_; } - private: + protected: int64_t byte_count_; int64_t backup_count_; grpc_byte_buffer_reader reader_; grpc_slice slice_; Status status_; }; + +template <class BufferWriter, class T> +Status GenericSerialize(const grpc::protobuf::Message& msg, + grpc_byte_buffer** bp, bool* own_buffer) { + static_assert( + std::is_base_of<protobuf::io::ZeroCopyOutputStream, BufferWriter>::value, + "BufferWriter must be a subclass of io::ZeroCopyOutputStream"); + *own_buffer = true; + int byte_size = msg.ByteSize(); + if (byte_size <= internal::kGrpcBufferWriterMaxBufferLength) { + grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size); + GPR_CODEGEN_ASSERT( + GRPC_SLICE_END_PTR(slice) == + msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); + *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); + g_core_codegen_interface->grpc_slice_unref(slice); + return g_core_codegen_interface->ok(); + } else { + BufferWriter writer(bp, internal::kGrpcBufferWriterMaxBufferLength); + return msg.SerializeToZeroCopyStream(&writer) + ? g_core_codegen_interface->ok() + : Status(StatusCode::INTERNAL, "Failed to serialize message"); + } +} + +template <class BufferReader, class T> +Status GenericDeserialize(grpc_byte_buffer* buffer, + grpc::protobuf::Message* msg) { + static_assert( + std::is_base_of<protobuf::io::ZeroCopyInputStream, BufferReader>::value, + "BufferReader must be a subclass of io::ZeroCopyInputStream"); + if (buffer == nullptr) { + return Status(StatusCode::INTERNAL, "No payload"); + } + Status result = g_core_codegen_interface->ok(); + { + BufferReader reader(buffer); + if (!reader.status().ok()) { + return reader.status(); + } + ::grpc::protobuf::io::CodedInputStream decoder(&reader); + decoder.SetTotalBytesLimit(INT_MAX, INT_MAX); + if (!msg->ParseFromCodedStream(&decoder)) { + result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); + } + if (!decoder.ConsumedEntireMessage()) { + result = Status(StatusCode::INTERNAL, "Did not read entire message"); + } + } + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); + return result; +} + } // namespace internal +// this is needed so the following class does not conflict with protobuf +// serializers that utilize internal-only tools. +#ifdef GRPC_OPEN_SOURCE_PROTO +// This class provides a protobuf serializer. It translates between protobuf +// objects and grpc_byte_buffers. More information about SerializationTraits can +// be found in include/grpc++/impl/codegen/serialization_traits.h. template <class T> class SerializationTraits<T, typename std::enable_if<std::is_base_of< grpc::protobuf::Message, T>::value>::type> { public: static Status Serialize(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp, bool* own_buffer) { - *own_buffer = true; - int byte_size = msg.ByteSize(); - if (byte_size <= internal::kGrpcBufferWriterMaxBufferLength) { - grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size); - GPR_CODEGEN_ASSERT( - GRPC_SLICE_END_PTR(slice) == - msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); - *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); - g_core_codegen_interface->grpc_slice_unref(slice); - return g_core_codegen_interface->ok(); - } else { - internal::GrpcBufferWriter writer( - bp, internal::kGrpcBufferWriterMaxBufferLength); - return msg.SerializeToZeroCopyStream(&writer) - ? g_core_codegen_interface->ok() - : Status(StatusCode::INTERNAL, "Failed to serialize message"); - } + return internal::GenericSerialize<internal::GrpcBufferWriter, T>( + msg, bp, own_buffer); } static Status Deserialize(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg) { - if (buffer == nullptr) { - return Status(StatusCode::INTERNAL, "No payload"); - } - Status result = g_core_codegen_interface->ok(); - { - internal::GrpcBufferReader reader(buffer); - if (!reader.status().ok()) { - return reader.status(); - } - ::grpc::protobuf::io::CodedInputStream decoder(&reader); - decoder.SetTotalBytesLimit(INT_MAX, INT_MAX); - if (!msg->ParseFromCodedStream(&decoder)) { - result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); - } - if (!decoder.ConsumedEntireMessage()) { - result = Status(StatusCode::INTERNAL, "Did not read entire message"); - } - } - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); - return result; + return internal::GenericDeserialize<internal::GrpcBufferReader, T>(buffer, + msg); } }; +#endif } // namespace grpc @@ -116,7 +116,7 @@ if EXTRA_ENV_COMPILE_ARGS is None: elif 'win32' in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC' elif "linux" in sys.platform: - EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -fvisibility=hidden -fno-wrapv' + EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -std=gnu99 -fvisibility=hidden -fno-wrapv' elif "darwin" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -fvisibility=hidden -fno-wrapv' @@ -237,7 +237,7 @@ INSTALL_REQUIRES = ( 'six>=1.5.2', # TODO(atash): eventually split the grpcio package into a metapackage # depending on protobuf and the runtime component (independent of protobuf) - 'protobuf>=3.2.0', + 'protobuf>=3.3.0', ) if not PY3: diff --git a/src/boringssl/gen_build_yaml.py b/src/boringssl/gen_build_yaml.py index c53beb0da5..d01c2b4ec5 100755 --- a/src/boringssl/gen_build_yaml.py +++ b/src/boringssl/gen_build_yaml.py @@ -138,7 +138,7 @@ class Grpc(object): { 'name': 'boringssl_%s' % os.path.basename(test[0]), 'args': [map_testarg(arg) for arg in test[1:]], - 'exclude_configs': ['asan'], + 'exclude_configs': ['asan', 'ubsan'], 'ci_platforms': ['linux', 'mac', 'posix', 'windows'], 'platforms': ['linux', 'mac', 'posix', 'windows'], 'flaky': False, diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 30738080ee..3c5216e104 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2554,7 +2554,7 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, add_max_recv_bytes); if ((int64_t)s->incoming_window_delta + (int64_t)initial_window_size - (int64_t)s->announce_window > - 2 * (int64_t)initial_window_size) { + (int64_t)initial_window_size / 2) { write_type = GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK; } grpc_chttp2_become_writable(exec_ctx, t, s, write_type, diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index d0b94bd6ce..b0dd685116 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -418,7 +418,10 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s, incoming_frame_size); - if ((int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= 0) { + if ((int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= + -(int64_t)t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] / + 2) { grpc_chttp2_become_writable(exec_ctx, t, s, GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED, "window-update-required"); diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index f679a33945..2b484c0a08 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -134,6 +134,12 @@ grpc_byte_buffer* CoreCodegen::grpc_raw_byte_buffer_create(grpc_slice* slice, return ::grpc_raw_byte_buffer_create(slice, nslices); } +grpc_slice CoreCodegen::grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) { + return ::grpc_slice_new_with_user_data(p, len, destroy, user_data); +} + grpc_slice CoreCodegen::grpc_empty_slice() { return ::grpc_empty_slice(); } grpc_slice CoreCodegen::grpc_slice_malloc(size_t length) { @@ -144,10 +150,18 @@ void CoreCodegen::grpc_slice_unref(grpc_slice slice) { ::grpc_slice_unref(slice); } +grpc_slice CoreCodegen::grpc_slice_ref(grpc_slice slice) { + return ::grpc_slice_ref(slice); +} + grpc_slice CoreCodegen::grpc_slice_split_tail(grpc_slice* s, size_t split) { return ::grpc_slice_split_tail(s, split); } +grpc_slice CoreCodegen::grpc_slice_split_head(grpc_slice* s, size_t split) { + return ::grpc_slice_split_head(s, split); +} + grpc_slice CoreCodegen::grpc_slice_from_static_buffer(const void* buffer, size_t length) { return ::grpc_slice_from_static_buffer(buffer, length); diff --git a/src/node/index.js b/src/node/index.js index 071bfd7927..76ab1744b0 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -31,6 +31,10 @@ * */ +/** + * @module + */ + 'use strict'; var path = require('path'); @@ -256,5 +260,10 @@ exports.getClientChannel = client.getClientChannel; exports.waitForClientReady = client.waitForClientReady; exports.closeClient = function closeClient(client_obj) { - client.getClientChannel(client_obj).close(); + client.Client.prototype.close.apply(client_obj); }; + +/** + * @see module:src/client.Client + */ +exports.Client = client.Client; diff --git a/src/node/src/client.js b/src/node/src/client.js index 1aaf35c16c..43502da6af 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -37,7 +37,7 @@ * This module contains the factory method for creating Client classes, and the * method calling code for all types of methods. * - * For example, to create a client and call a method on it: + * @example <caption>Create a client and call a method on it</caption> * * var proto_obj = grpc.load(proto_file_path); * var Client = proto_obj.package.subpackage.ServiceName; @@ -68,14 +68,33 @@ var Duplex = stream.Duplex; var util = require('util'); var version = require('../../../package.json').version; +util.inherits(ClientUnaryCall, EventEmitter); + +/** + * An EventEmitter. Used for unary calls + * @constructor + * @extends external:EventEmitter + * @param {grpc.Call} call The call object associated with the request + */ +function ClientUnaryCall(call) { + EventEmitter.call(this); + this.call = call; +} + util.inherits(ClientWritableStream, Writable); /** * A stream that the client can write to. Used for calls that are streaming from * the client side. * @constructor + * @extends external:Writable + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientWritableStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientWritableStream#getPeer * @param {grpc.Call} call The call object to send data with - * @param {function(*):Buffer=} serialize Serialization function for writes. + * @param {module:src/common~serialize=} [serialize=identity] Serialization + * function for writes. */ function ClientWritableStream(call, serialize) { Writable.call(this, {objectMode: true}); @@ -134,8 +153,14 @@ util.inherits(ClientReadableStream, Readable); * A stream that the client can read from. Used for calls that are streaming * from the server side. * @constructor + * @extends external:Readable + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientReadableStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientReadableStream#getPeer * @param {grpc.Call} call The call object to read data with - * @param {function(Buffer):*=} deserialize Deserialization function for reads + * @param {module:src/common~deserialize=} [deserialize=identity] + * Deserialization function for reads */ function ClientReadableStream(call, deserialize) { Readable.call(this, {objectMode: true}); @@ -155,6 +180,7 @@ function ClientReadableStream(call, deserialize) { * parameter indicates that the call should end with that status. status * defaults to OK if not provided. * @param {Object!} status The status that the call should end with + * @access private */ function _readsDone(status) { /* jshint validthis: true */ @@ -173,6 +199,7 @@ ClientReadableStream.prototype._readsDone = _readsDone; /** * Called to indicate that we have received a status from the server. + * @access private */ function _receiveStatus(status) { /* jshint validthis: true */ @@ -185,6 +212,7 @@ ClientReadableStream.prototype._receiveStatus = _receiveStatus; /** * If we have both processed all incoming messages and received the status from * the server, emit the status. Otherwise, do nothing. + * @access private */ function _emitStatusIfDone() { /* jshint validthis: true */ @@ -270,10 +298,16 @@ util.inherits(ClientDuplexStream, Duplex); * A stream that the client can read from or write to. Used for calls with * duplex streaming. * @constructor + * @extends external:Duplex + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientDuplexStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientDuplexStream#getPeer * @param {grpc.Call} call Call object to proxy - * @param {function(*):Buffer=} serialize Serialization function for requests - * @param {function(Buffer):*=} deserialize Deserialization function for - * responses + * @param {module:src/common~serialize=} [serialize=identity] Serialization + * function for requests + * @param {module:src/common~deserialize=} [deserialize=identity] + * Deserialization function for responses */ function ClientDuplexStream(call, serialize, deserialize) { Duplex.call(this, {objectMode: true}); @@ -300,12 +334,14 @@ ClientDuplexStream.prototype._write = _write; /** * Cancel the ongoing call + * @alias module:src/client~ClientUnaryCall#cancel */ function cancel() { /* jshint validthis: true */ this.call.cancel(); } +ClientUnaryCall.prototype.cancel = cancel; ClientReadableStream.prototype.cancel = cancel; ClientWritableStream.prototype.cancel = cancel; ClientDuplexStream.prototype.cancel = cancel; @@ -313,21 +349,49 @@ ClientDuplexStream.prototype.cancel = cancel; /** * Get the endpoint this call/stream is connected to. * @return {string} The URI of the endpoint + * @alias module:src/client~ClientUnaryCall#getPeer */ function getPeer() { /* jshint validthis: true */ return this.call.getPeer(); } +ClientUnaryCall.prototype.getPeer = getPeer; ClientReadableStream.prototype.getPeer = getPeer; ClientWritableStream.prototype.getPeer = getPeer; ClientDuplexStream.prototype.getPeer = getPeer; /** + * Any client call type + * @typedef {(ClientUnaryCall|ClientReadableStream| + * ClientWritableStream|ClientDuplexStream)} + * module:src/client~Call + */ + +/** + * Options that can be set on a call. + * @typedef {Object} module:src/client~CallOptions + * @property {(date|number)} deadline The deadline for the entire call to + * complete. A value of Infinity indicates that no deadline should be set. + * @property {(string)} host Server hostname to set on the call. Only meaningful + * if different from the server address used to construct the client. + * @property {module:src/client~Call} parent Parent call. Used in servers when + * making a call as part of the process of handling a call. Used to + * propagate some information automatically, as specified by + * propagate_flags. + * @property {number} propagate_flags Indicates which properties of a parent + * call should propagate to this call. Bitwise combination of flags in + * [grpc.propagate]{@link module:index.propagate}. + * @property {module:src/credentials~CallCredentials} credentials The + * credentials that should be used to make this particular call. + */ + +/** * Get a call object built with the provided options. Keys for options are * 'deadline', which takes a date or number, and 'host', which takes a string * and overrides the hostname to connect to. - * @param {Object} options Options map. + * @access private + * @param {module:src/client~CallOptions=} options Options object. */ function getCall(channel, method, options) { var deadline; @@ -354,315 +418,380 @@ function getCall(channel, method, options) { } /** - * Get a function that can make unary requests to the specified method. - * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeUnaryRequest + * A generic gRPC client. Primarily useful as a base class for generated clients + * @alias module:src/client.Client + * @constructor + * @param {string} address Server address to connect to + * @param {module:src/credentials~ChannelCredentials} credentials Credentials to + * use to connect to the server + * @param {Object} options Options to apply to channel creation */ -function makeUnaryRequestFunction(method, serialize, deserialize) { - /** - * Make a unary request with this method on the given channel with the given - * argument, callback, etc. - * @this {Client} Client object. Must have a channel member. - * @param {*} argument The argument to the call. Should be serializable with - * serialize - * @param {Metadata=} metadata Metadata to add to the call - * @param {Object=} options Options map - * @param {function(?Error, value=)} callback The callback to for when the - * response is received - * @return {EventEmitter} An event emitter for stream related events +var Client = exports.Client = function Client(address, credentials, options) { + if (!options) { + options = {}; + } + /* Append the grpc-node user agent string after the application user agent + * string, and put the combination at the beginning of the user agent string */ - function makeUnaryRequest(argument, metadata, options, callback) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. This allows for simple handling of optional arguments in the - * middle of the argument list, and also provides type checking. */ - var args = arguejs({argument: null, metadata: [Metadata, new Metadata()], - options: [Object], callback: Function}, arguments); - var emitter = new EventEmitter(); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - emitter.cancel = function cancel() { - call.cancel(); - }; - emitter.getPeer = function getPeer() { - return call.getPeer(); - }; - var client_batch = {}; - var message = serialize(args.argument); - if (args.options) { - message.grpcWriteFlags = args.options.flags; - } - - client_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - client_batch[grpc.opType.SEND_MESSAGE] = message; - client_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; - client_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - client_batch[grpc.opType.RECV_MESSAGE] = true; - client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(client_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - var status = response.status; - var error; - var deserialized; - emitter.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - if (status.code === grpc.status.OK) { - if (err) { - // Got a batch error, but OK status. Something went wrong - args.callback(err); - return; - } else { - try { - deserialized = deserialize(response.read); - } catch (e) { - /* Change status to indicate bad server response. This will result - * in passing an error to the callback */ - status = { - code: grpc.status.INTERNAL, - details: 'Failed to parse server response' - }; - } - } - } - if (status.code !== grpc.status.OK) { - error = new Error(status.details); - error.code = status.code; - error.metadata = status.metadata; - args.callback(error); - } else { - args.callback(null, deserialized); - } - emitter.emit('status', status); - }); - return emitter; + if (options['grpc.primary_user_agent']) { + options['grpc.primary_user_agent'] += ' '; + } else { + options['grpc.primary_user_agent'] = ''; } - return makeUnaryRequest; -} + options['grpc.primary_user_agent'] += 'grpc-node/' + version; + /* Private fields use $ as a prefix instead of _ because it is an invalid + * prefix of a method name */ + this.$channel = new grpc.Channel(address, credentials, options); +}; /** - * Get a function that can make client stream requests to the specified method. + * @typedef {Error} module:src/client.Client~ServiceError + * @property {number} code The error code, a key of + * [grpc.status]{@link module:src/client.status} + * @property {module:metadata.Metadata} metadata Metadata sent with the status + * by the server, if any + */ + +/** + * @callback module:src/client.Client~requestCallback + * @param {?module:src/client.Client~ServiceError} error The error, if the call + * failed + * @param {*} value The response value, if the call succeeded + */ + +/** + * Make a unary request to the given method, using the given serialize + * and deserialize functions, with the given argument. * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeClientStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {*} argument The argument to the call. Should be serializable with + * serialize + * @param {module:src/metadata.Metadata=} metadata Metadata to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @param {module:src/client.Client~requestCallback} callback The callback to + * for when the response is received + * @return {EventEmitter} An event emitter for stream related events */ -function makeClientStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a client stream request with this method on the given channel with the - * given callback, etc. - * @this {Client} Client object. Must have a channel member. - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Object=} options Options map - * @param {function(?Error, value=)} callback The callback to for when the - * response is received - * @return {EventEmitter} An event emitter for stream related events - */ - function makeClientStreamRequest(metadata, options, callback) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. This allows for simple handling of optional arguments in the - * middle of the argument list, and also provides type checking. */ - var args = arguejs({metadata: [Metadata, new Metadata()], - options: [Object], callback: Function}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientWritableStream(call, serialize); - var metadata_batch = {}; - metadata_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - metadata_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - call.startBatch(metadata_batch, function(err, response) { +Client.prototype.makeUnaryRequest = function(method, serialize, deserialize, + argument, metadata, options, + callback) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. This allows for simple handling of optional arguments in the + * middle of the argument list, and also provides type checking. */ + var args = arguejs({method: String, serialize: Function, + deserialize: Function, + argument: null, metadata: [Metadata, new Metadata()], + options: [Object], callback: Function}, arguments); + var call = getCall(this.$channel, method, args.options); + var emitter = new ClientUnaryCall(call); + metadata = args.metadata.clone(); + var client_batch = {}; + var message = serialize(args.argument); + if (args.options) { + message.grpcWriteFlags = args.options.flags; + } + + client_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + client_batch[grpc.opType.SEND_MESSAGE] = message; + client_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; + client_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + client_batch[grpc.opType.RECV_MESSAGE] = true; + client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(client_batch, function(err, response) { + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + var status = response.status; + var error; + var deserialized; + emitter.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + if (status.code === grpc.status.OK) { if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. + // Got a batch error, but OK status. Something went wrong + args.callback(err); return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var client_batch = {}; - client_batch[grpc.opType.RECV_MESSAGE] = true; - client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(client_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - var status = response.status; - var error; - var deserialized; - if (status.code === grpc.status.OK) { - if (err) { - // Got a batch error, but OK status. Something went wrong - args.callback(err); - return; - } else { - try { - deserialized = deserialize(response.read); - } catch (e) { - /* Change status to indicate bad server response. This will result - * in passing an error to the callback */ - status = { - code: grpc.status.INTERNAL, - details: 'Failed to parse server response' - }; - } - } - } - if (status.code !== grpc.status.OK) { - error = new Error(response.status.details); - error.code = status.code; - error.metadata = status.metadata; - args.callback(error); } else { - args.callback(null, deserialized); + try { + deserialized = deserialize(response.read); + } catch (e) { + /* Change status to indicate bad server response. This will result + * in passing an error to the callback */ + status = { + code: grpc.status.INTERNAL, + details: 'Failed to parse server response' + }; + } } - stream.emit('status', status); - }); - return stream; - } - return makeClientStreamRequest; -} + } + if (status.code !== grpc.status.OK) { + error = new Error(status.details); + error.code = status.code; + error.metadata = status.metadata; + args.callback(error); + } else { + args.callback(null, deserialized); + } + emitter.emit('status', status); + }); + return emitter; +}; /** - * Get a function that can make server stream requests to the specified method. + * Make a client stream request to the given method, using the given serialize + * and deserialize functions, with the given argument. * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeServerStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @param {Client~requestCallback} callback The callback to for when the + * response is received + * @return {module:src/client~ClientWritableStream} An event emitter for stream + * related events */ -function makeServerStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a server stream request with this method on the given channel with the - * given argument, etc. - * @this {SurfaceClient} Client object. Must have a channel member. - * @param {*} argument The argument to the call. Should be serializable with - * serialize - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Object} options Options map - * @return {EventEmitter} An event emitter for stream related events - */ - function makeServerStreamRequest(argument, metadata, options) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. */ - var args = arguejs({argument: null, metadata: [Metadata, new Metadata()], - options: [Object]}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientReadableStream(call, deserialize); - var start_batch = {}; - var message = serialize(args.argument); - if (args.options) { - message.grpcWriteFlags = args.options.flags; +Client.prototype.makeClientStreamRequest = function(method, serialize, + deserialize, metadata, + options, callback) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. This allows for simple handling of optional arguments in the + * middle of the argument list, and also provides type checking. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + metadata: [Metadata, new Metadata()], + options: [Object], callback: Function}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientWritableStream(call, serialize); + var metadata_batch = {}; + metadata_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + metadata_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + call.startBatch(metadata_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; } - start_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - start_batch[grpc.opType.SEND_MESSAGE] = message; - start_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; - call.startBatch(start_batch, function(err, response) { - if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. - return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var status_batch = {}; - status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(status_batch, function(err, response) { + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var client_batch = {}; + client_batch[grpc.opType.RECV_MESSAGE] = true; + client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(client_batch, function(err, response) { + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + var status = response.status; + var error; + var deserialized; + if (status.code === grpc.status.OK) { if (err) { - stream.emit('error', err); + // Got a batch error, but OK status. Something went wrong + args.callback(err); return; + } else { + try { + deserialized = deserialize(response.read); + } catch (e) { + /* Change status to indicate bad server response. This will result + * in passing an error to the callback */ + status = { + code: grpc.status.INTERNAL, + details: 'Failed to parse server response' + }; + } } - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream._receiveStatus(response.status); - }); - return stream; - } - return makeServerStreamRequest; -} + } + if (status.code !== grpc.status.OK) { + error = new Error(response.status.details); + error.code = status.code; + error.metadata = status.metadata; + args.callback(error); + } else { + args.callback(null, deserialized); + } + stream.emit('status', status); + }); + return stream; +}; /** - * Get a function that can make bidirectional stream requests to the specified - * method. + * Make a server stream request to the given method, with the given serialize + * and deserialize function, using the given argument * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeBidiStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {*} argument The argument to the call. Should be serializable with + * serialize + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @return {module:src/client~ClientReadableStream} An event emitter for stream + * related events */ -function makeBidiStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a bidirectional stream request with this method on the given channel. - * @this {SurfaceClient} Client object. Must have a channel member. - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Options} options Options map - * @return {EventEmitter} An event emitter for stream related events - */ - function makeBidiStreamRequest(metadata, options) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. */ - var args = arguejs({metadata: [Metadata, new Metadata()], - options: [Object]}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientDuplexStream(call, serialize, deserialize); - var start_batch = {}; - start_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - call.startBatch(start_batch, function(err, response) { - if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. - return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var status_batch = {}; - status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(status_batch, function(err, response) { - if (err) { - stream.emit('error', err); - return; - } - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream._receiveStatus(response.status); - }); - return stream; +Client.prototype.makeServerStreamRequest = function(method, serialize, + deserialize, argument, + metadata, options) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + argument: null, metadata: [Metadata, new Metadata()], + options: [Object]}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientReadableStream(call, deserialize); + var start_batch = {}; + var message = serialize(args.argument); + if (args.options) { + message.grpcWriteFlags = args.options.flags; } - return makeBidiStreamRequest; -} + start_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + start_batch[grpc.opType.SEND_MESSAGE] = message; + start_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; + call.startBatch(start_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; + } + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var status_batch = {}; + status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(status_batch, function(err, response) { + if (err) { + stream.emit('error', err); + return; + } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); + }); + return stream; +}; + + +/** + * Make a bidirectional stream request with this method on the given channel. + * @param {string} method The name of the method to request + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @return {module:src/client~ClientDuplexStream} An event emitter for stream + * related events + */ +Client.prototype.makeBidiStreamRequest = function(method, serialize, + deserialize, metadata, + options) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + metadata: [Metadata, new Metadata()], + options: [Object]}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientDuplexStream(call, serialize, deserialize); + var start_batch = {}; + start_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + call.startBatch(start_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; + } + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var status_batch = {}; + status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(status_batch, function(err, response) { + if (err) { + stream.emit('error', err); + return; + } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); + }); + return stream; +}; +Client.prototype.close = function() { + this.$channel.close(); +}; + +/** + * Return the underlying channel object for the specified client + * @return {Channel} The channel + */ +Client.prototype.getChannel = function() { + return this.$channel; +}; + +/** + * Wait for the client to be ready. The callback will be called when the + * client has successfully connected to the server, and it will be called + * with an error if the attempt to connect to the server has unrecoverablly + * failed or if the deadline expires. This function will make the channel + * start connecting if it has not already done so. + * @param {(Date|Number)} deadline When to stop waiting for a connection. Pass + * Infinity to wait forever. + * @param {function(Error)} callback The callback to call when done attempting + * to connect. + */ +Client.prototype.waitForReady = function(deadline, callback) { + var self = this; + var checkState = function(err) { + if (err) { + callback(new Error('Failed to connect before the deadline')); + return; + } + var new_state = self.$channel.getConnectivityState(true); + if (new_state === grpc.connectivityState.READY) { + callback(); + } else if (new_state === grpc.connectivityState.FATAL_FAILURE) { + callback(new Error('Failed to connect to server')); + } else { + self.$channel.watchConnectivityState(new_state, deadline, checkState); + } + }; + checkState(); +}; /** * Map with short names for each of the requester maker functions. Used in * makeClientConstructor + * @access private */ -var requester_makers = { - unary: makeUnaryRequestFunction, - server_stream: makeServerStreamRequestFunction, - client_stream: makeClientStreamRequestFunction, - bidi: makeBidiStreamRequestFunction +var requester_funcs = { + unary: Client.prototype.makeUnaryRequest, + server_stream: Client.prototype.makeServerStreamRequest, + client_stream: Client.prototype.makeClientStreamRequest, + bidi: Client.prototype.makeBidiStreamRequest }; function getDefaultValues(metadata, options) { @@ -675,6 +804,7 @@ function getDefaultValues(metadata, options) { /** * Map with wrappers for each type of requester function to make it use the old * argument order with optional arguments after the callback. + * @access private */ var deprecated_request_wrap = { unary: function(makeUnaryRequest) { @@ -700,55 +830,33 @@ var deprecated_request_wrap = { }; /** - * Creates a constructor for a client with the given methods. The methods object - * maps method name to an object with the following keys: - * path: The path on the server for accessing the method. For example, for - * protocol buffers, we use "/service_name/method_name" - * requestStream: bool indicating whether the client sends a stream - * resonseStream: bool indicating whether the server sends a stream - * requestSerialize: function to serialize request objects - * responseDeserialize: function to deserialize response objects - * @param {Object} methods An object mapping method names to method attributes + * Creates a constructor for a client with the given methods, as specified in + * the methods argument. + * @param {module:src/common~ServiceDefinition} methods An object mapping + * method names to method attributes * @param {string} serviceName The fully qualified name of the service - * @param {Object} class_options An options object. Currently only uses the key - * deprecatedArgumentOrder, a boolean that Indicates that the old argument - * order should be used for methods, with optional arguments at the end - * instead of the callback at the end. Defaults to false. This option is - * only a temporary stopgap measure to smooth an API breakage. + * @param {Object} class_options An options object. + * @param {boolean=} [class_options.deprecatedArgumentOrder=false] Indicates + * that the old argument order should be used for methods, with optional + * arguments at the end instead of the callback at the end. This option + * is only a temporary stopgap measure to smooth an API breakage. * It is deprecated, and new code should not use it. - * @return {function(string, Object)} New client constructor + * @return {function(string, Object)} New client constructor, which is a + * subclass of [grpc.Client]{@link module:src/client.Client}, and has the + * same arguments as that constructor. */ exports.makeClientConstructor = function(methods, serviceName, class_options) { if (!class_options) { class_options = {}; } - /** - * Create a client with the given methods - * @constructor - * @param {string} address The address of the server to connect to - * @param {grpc.Credentials} credentials Credentials to use to connect - * to the server - * @param {Object} options Options to pass to the underlying channel - */ - function Client(address, credentials, options) { - if (!options) { - options = {}; - } - /* Append the grpc-node user agent string after the application user agent - * string, and put the combination at the beginning of the user agent string - */ - if (options['grpc.primary_user_agent']) { - options['grpc.primary_user_agent'] += ' '; - } else { - options['grpc.primary_user_agent'] = ''; - } - options['grpc.primary_user_agent'] += 'grpc-node/' + version; - /* Private fields use $ as a prefix instead of _ because it is an invalid - * prefix of a method name */ - this.$channel = new grpc.Channel(address, credentials, options); + + function ServiceClient(address, credentials, options) { + Client.call(this, address, credentials, options); } + util.inherits(ServiceClient, Client); + _.each(methods, function(attrs, name) { var method_type; if (_.startsWith(name, '$')) { @@ -769,20 +877,20 @@ exports.makeClientConstructor = function(methods, serviceName, } var serialize = attrs.requestSerialize; var deserialize = attrs.responseDeserialize; - var method_func = requester_makers[method_type]( - attrs.path, serialize, deserialize); + var method_func = _.partial(requester_funcs[method_type], attrs.path, + serialize, deserialize); if (class_options.deprecatedArgumentOrder) { - Client.prototype[name] = deprecated_request_wrap(method_func); + ServiceClient.prototype[name] = deprecated_request_wrap(method_func); } else { - Client.prototype[name] = method_func; + ServiceClient.prototype[name] = method_func; } // Associate all provided attributes with the method - _.assign(Client.prototype[name], attrs); + _.assign(ServiceClient.prototype[name], attrs); }); - Client.service = methods; + ServiceClient.service = methods; - return Client; + return ServiceClient; }; /** @@ -791,7 +899,7 @@ exports.makeClientConstructor = function(methods, serviceName, * @return {Channel} The channel */ exports.getClientChannel = function(client) { - return client.$channel; + return Client.prototype.getChannel.call(client); }; /** @@ -807,21 +915,7 @@ exports.getClientChannel = function(client) { * to connect. */ exports.waitForClientReady = function(client, deadline, callback) { - var checkState = function(err) { - if (err) { - callback(new Error('Failed to connect before the deadline')); - return; - } - var new_state = client.$channel.getConnectivityState(true); - if (new_state === grpc.connectivityState.READY) { - callback(); - } else if (new_state === grpc.connectivityState.FATAL_FAILURE) { - callback(new Error('Failed to connect to server')); - } else { - client.$channel.watchConnectivityState(new_state, deadline, checkState); - } - }; - checkState(); + Client.prototype.waitForReady.call(client, deadline, callback); }; /** diff --git a/src/node/src/common.js b/src/node/src/common.js index 757969dbdd..4dad60e630 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -43,7 +43,8 @@ var _ = require('lodash'); /** * Wrap a function to pass null-like values through without calling it. If no - * function is given, just uses the identity; + * function is given, just uses the identity. + * @private * @param {?function} func The function to wrap * @return {function} The wrapped function */ @@ -90,3 +91,67 @@ exports.defaultGrpcOptions = { enumsAsStrings: true, deprecatedArgumentOrder: false }; + +// JSDoc definitions that are used in multiple other modules + +/** + * The EventEmitter class in the event standard module + * @external EventEmitter + * @see https://nodejs.org/api/events.html#events_class_eventemitter + */ + +/** + * The Readable class in the stream standard module + * @external Readable + * @see https://nodejs.org/api/stream.html#stream_readable_streams + */ + +/** + * The Writable class in the stream standard module + * @external Writable + * @see https://nodejs.org/api/stream.html#stream_writable_streams + */ + +/** + * The Duplex class in the stream standard module + * @external Duplex + * @see https://nodejs.org/api/stream.html#stream_class_stream_duplex + */ + +/** + * A serialization function + * @callback module:src/common~serialize + * @param {*} value The value to serialize + * @return {Buffer} The value serialized as a byte sequence + */ + +/** + * A deserialization function + * @callback module:src/common~deserialize + * @param {Buffer} data The byte sequence to deserialize + * @return {*} The data deserialized as a value + */ + +/** + * An object that completely defines a service method signature. + * @typedef {Object} module:src/common~MethodDefinition + * @property {string} path The method's URL path + * @property {boolean} requestStream Indicates whether the method accepts + * a stream of requests + * @property {boolean} responseStream Indicates whether the method returns + * a stream of responses + * @property {module:src/common~serialize} requestSerialize Serialization + * function for request values + * @property {module:src/common~serialize} responseSerialize Serialization + * function for response values + * @property {module:src/common~deserialize} requestDeserialize Deserialization + * function for request data + * @property {module:src/common~deserialize} responseDeserialize Deserialization + * function for repsonse data + */ + +/** + * An object that completely defines a service. + * @typedef {Object.<string, module:src/common~MethodDefinition>} + * module:src/common~ServiceDefinition + */ diff --git a/src/node/src/metadata.js b/src/node/src/metadata.js index 612361b0ea..92cf23998b 100644 --- a/src/node/src/metadata.js +++ b/src/node/src/metadata.js @@ -35,12 +35,7 @@ * Metadata module * * This module defines the Metadata class, which represents header and trailer - * metadata for gRPC calls. Here is an example of how to use it: - * - * var metadata = new metadata_module.Metadata(); - * metadata.set('key1', 'value1'); - * metadata.add('key1', 'value2'); - * metadata.get('key1') // returns ['value1', 'value2'] + * metadata for gRPC calls. * * @module */ @@ -54,6 +49,12 @@ var grpc = require('./grpc_extension'); /** * Class for storing metadata. Keys are normalized to lowercase ASCII. * @constructor + * @alias module:src/metadata.Metadata + * @example + * var metadata = new metadata_module.Metadata(); + * metadata.set('key1', 'value1'); + * metadata.add('key1', 'value2'); + * metadata.get('key1') // returns ['value1', 'value2'] */ function Metadata() { this._internal_repr = {}; diff --git a/src/php/composer.json b/src/php/composer.json index 24c17c3b57..a4fba7e4f6 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -5,7 +5,7 @@ "version": "1.4.0", "require": { "php": ">=5.5.0", - "google/protobuf": "^v3.1.0" + "google/protobuf": "^v3.3.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 4f072809c4..f4ccb1ab94 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -284,9 +284,9 @@ class BuildExt(build_ext.build_ext): stderr=subprocess.PIPE) make_out, make_err = make_process.communicate() if make_out and make_process.returncode != 0: - sys.stdout.write(make_out + '\n') + sys.stdout.write(str(make_out) + '\n') if make_err: - sys.stderr.write(make_err + '\n') + sys.stderr.write(str(make_err) + '\n') if make_process.returncode != 0: raise Exception("make command failed!") diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 1806d311b4..f1c09de2eb 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -46,7 +46,7 @@ PACKAGE_DIRECTORIES = { SETUP_REQUIRES = ( 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) -INSTALL_REQUIRES = ('protobuf>=3.2.0', +INSTALL_REQUIRES = ('protobuf>=3.3.0', 'grpcio>={version}'.format(version=grpc_version.VERSION),) COMMAND_CLASS = { diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index e6cf54745e..cb49c3d57e 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -47,7 +47,7 @@ PACKAGE_DIRECTORIES = { SETUP_REQUIRES = ( 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) -INSTALL_REQUIRES = ('protobuf>=3.2.0', +INSTALL_REQUIRES = ('protobuf>=3.3.0', 'grpcio>={version}'.format(version=grpc_version.VERSION),) COMMAND_CLASS = { diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index b9f0264dae..7ee5336a7d 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -56,7 +56,7 @@ INSTALL_REQUIRES = ( 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.2.0', 'six>=1.10',) + 'oauth2client>=1.4.7', 'protobuf>=3.3.0', 'six>=1.10',) COMMAND_CLASS = { # Run `preprocess` *before* doing any packaging! diff --git a/templates/composer.json.template b/templates/composer.json.template index 94f0c236a9..2d4cb11919 100644 --- a/templates/composer.json.template +++ b/templates/composer.json.template @@ -9,7 +9,7 @@ "license": "BSD-3-Clause", "require": { "php": ">=5.5.0", - "google/protobuf": "^v3.1.0" + "google/protobuf": "^v3.3.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/templates/src/php/composer.json.template b/templates/src/php/composer.json.template index 1887ee3c82..36932cf8f6 100644 --- a/templates/src/php/composer.json.template +++ b/templates/src/php/composer.json.template @@ -7,7 +7,7 @@ "version": "${settings.php_version.php_composer()}", "require": { "php": ">=5.5.0", - "google/protobuf": "^v3.1.0" + "google/protobuf": "^v3.3.0" }, "require-dev": { "google/auth": "v0.9" diff --git a/templates/tools/dockerfile/gcp_api_libraries.include b/templates/tools/dockerfile/gcp_api_libraries.include index 669b0f887c..adecb92c15 100644 --- a/templates/tools/dockerfile/gcp_api_libraries.include +++ b/templates/tools/dockerfile/gcp_api_libraries.include @@ -1,4 +1,3 @@ # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean RUN pip install --upgrade google-api-python-client - diff --git a/templates/tools/dockerfile/test/csharp_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/csharp_jessie_x64/Dockerfile.template index 092f04dac6..f869cf9106 100644 --- a/templates/tools/dockerfile/test/csharp_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/csharp_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../csharp_deps.include"/> <%include file="../../csharp_dotnetcli_deps.include"/> diff --git a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template index 35b0e177fb..a42215d257 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../clang_update.include"/> diff --git a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template index 643b5cb65b..9df8158382 100644 --- a/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_jessie_x86/Dockerfile.template @@ -32,6 +32,7 @@ FROM 32bit/debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template index 8a95cad649..98854ff1c4 100644 --- a/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM ubuntu:14.04 <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../run_tests_addons_nocache.include"/> diff --git a/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template index 42ad6c130d..3ad65e9eee 100644 --- a/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM ubuntu:16.04 <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template index 5c38e9a0e9..6d74f4c8d1 100644 --- a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:wheezy <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> diff --git a/templates/tools/dockerfile/test/fuzzer/Dockerfile.template b/templates/tools/dockerfile/test/fuzzer/Dockerfile.template index 6d7cb72f27..32a843a482 100644 --- a/templates/tools/dockerfile/test/fuzzer/Dockerfile.template +++ b/templates/tools/dockerfile/test/fuzzer/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../clang_update.include"/> diff --git a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template index 93d26b5594..bf64ba210a 100644 --- a/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/multilang_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../csharp_deps.include"/> <%include file="../../cxx_deps.include"/> <%include file="../../node_deps.include"/> diff --git a/templates/tools/dockerfile/test/node_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/node_jessie_x64/Dockerfile.template index ceaa9aa5ab..103b1ef848 100644 --- a/templates/tools/dockerfile/test/node_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/node_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> # Install Electron apt dependencies RUN apt-get update && apt-get install -y ${'\\'} diff --git a/templates/tools/dockerfile/test/php7_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/php7_jessie_x64/Dockerfile.template index e6a213d90d..1581002830 100644 --- a/templates/tools/dockerfile/test/php7_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/php7_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../php7_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../run_tests_addons.include"/> # Define the default command. diff --git a/templates/tools/dockerfile/test/php_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/php_jessie_x64/Dockerfile.template index 0cfa373c90..ced93d0406 100644 --- a/templates/tools/dockerfile/test/php_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/php_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../php_deps.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template index 46fb84ba83..26ff987ac2 100644 --- a/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_jessie_x64/Dockerfile.template @@ -32,8 +32,8 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] -
\ No newline at end of file diff --git a/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template index f9a4dcb7b6..62d9b92400 100644 --- a/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_pyenv_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../apt_get_pyenv.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/ruby_jessie_x64/Dockerfile.template b/templates/tools/dockerfile/test/ruby_jessie_x64/Dockerfile.template index 35838bc11e..d1899853e6 100644 --- a/templates/tools/dockerfile/test/ruby_jessie_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/ruby_jessie_x64/Dockerfile.template @@ -32,6 +32,7 @@ FROM debian:jessie <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> <%include file="../../ruby_deps.include"/> <%include file="../../run_tests_addons.include"/> diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 8617666b21..6943134b6a 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -32,6 +32,7 @@ FROM ubuntu:15.10 <%include file="../../apt_get_basic.include"/> + <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> #======================== # Sanity test dependencies diff --git a/test/core/end2end/cq_verifier.c b/test/core/end2end/cq_verifier.c index 5eea5d43fe..0fafb0c8c9 100644 --- a/test/core/end2end/cq_verifier.c +++ b/test/core/end2end/cq_verifier.c @@ -77,7 +77,7 @@ struct cq_verifier { }; cq_verifier *cq_verifier_create(grpc_completion_queue *cq) { - cq_verifier *v = gpr_malloc(sizeof(cq_verifier)); + cq_verifier *v = (cq_verifier *)gpr_malloc(sizeof(cq_verifier)); v->cq = cq; v->first_expectation = NULL; return v; @@ -314,7 +314,7 @@ void cq_verify_empty(cq_verifier *v) { cq_verify_empty_timeout(v, 1); } static void add(cq_verifier *v, const char *file, int line, grpc_completion_type type, void *tag, bool success) { - expectation *e = gpr_malloc(sizeof(expectation)); + expectation *e = (expectation *)gpr_malloc(sizeof(expectation)); e->type = type; e->file = file; e->line = line; diff --git a/test/core/end2end/fake_resolver.c b/test/core/end2end/fake_resolver.c index df902a24bf..736b224fd6 100644 --- a/test/core/end2end/fake_resolver.c +++ b/test/core/end2end/fake_resolver.c @@ -136,7 +136,7 @@ struct grpc_fake_resolver_response_generator { grpc_fake_resolver_response_generator* grpc_fake_resolver_response_generator_create() { grpc_fake_resolver_response_generator* generator = - gpr_zalloc(sizeof(*generator)); + (grpc_fake_resolver_response_generator*)gpr_zalloc(sizeof(*generator)); gpr_ref_init(&generator->refcount, 1); return generator; } @@ -157,7 +157,8 @@ void grpc_fake_resolver_response_generator_unref( static void set_response_cb(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - grpc_fake_resolver_response_generator* generator = arg; + grpc_fake_resolver_response_generator* generator = + (grpc_fake_resolver_response_generator*)arg; fake_resolver* r = generator->resolver; if (r->next_results != NULL) { grpc_channel_args_destroy(exec_ctx, r->next_results); @@ -180,11 +181,13 @@ void grpc_fake_resolver_response_generator_set_response( } static void* response_generator_arg_copy(void* p) { - return grpc_fake_resolver_response_generator_ref(p); + return grpc_fake_resolver_response_generator_ref( + (grpc_fake_resolver_response_generator*)p); } static void response_generator_arg_destroy(grpc_exec_ctx* exec_ctx, void* p) { - grpc_fake_resolver_response_generator_unref(p); + grpc_fake_resolver_response_generator_unref( + (grpc_fake_resolver_response_generator*)p); } static int response_generator_cmp(void* a, void* b) { return GPR_ICMP(a, b); } @@ -208,7 +211,7 @@ grpc_fake_resolver_get_response_generator(const grpc_channel_args* args) { const grpc_arg* arg = grpc_channel_args_find(args, GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR); if (arg == NULL || arg->type != GRPC_ARG_POINTER) return NULL; - return arg->value.pointer.p; + return (grpc_fake_resolver_response_generator*)arg->value.pointer.p; } // @@ -222,7 +225,7 @@ static void fake_resolver_factory_unref(grpc_resolver_factory* factory) {} static grpc_resolver* fake_resolver_create(grpc_exec_ctx* exec_ctx, grpc_resolver_factory* factory, grpc_resolver_args* args) { - fake_resolver* r = gpr_zalloc(sizeof(*r)); + fake_resolver* r = (fake_resolver*)gpr_zalloc(sizeof(*r)); r->channel_args = grpc_channel_args_copy(args->args); grpc_resolver_init(&r->base, &fake_resolver_vtable, args->combiner); grpc_fake_resolver_response_generator* response_generator = diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index f0d09487c6..5df43b9384 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -156,7 +156,7 @@ static void proxy_connection_failed(grpc_exec_ctx* exec_ctx, // Callback for writing proxy data to the client. static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy client write", error); @@ -181,7 +181,7 @@ static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg, // Callback for writing proxy data to the backend server. static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, false /* is_client */, "HTTP proxy server write", error); @@ -207,7 +207,7 @@ static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg, // the backend server. static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy client read", error); @@ -239,7 +239,7 @@ static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg, // proxied to the client. static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, false /* is_client */, "HTTP proxy server read", error); @@ -270,7 +270,7 @@ static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg, // Callback to write the HTTP response for the CONNECT request. static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy write response", error); @@ -294,7 +294,7 @@ static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg, // CONNECT request. static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { // TODO(roth): Technically, in this case, we should handle the error // by returning an HTTP response to the client indicating that the @@ -324,7 +324,7 @@ static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg, // which will cause the client connection to be dropped. static void on_read_request_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; gpr_log(GPR_DEBUG, "on_read_request_done: %p %s", conn, grpc_error_string(error)); if (error != GRPC_ERROR_NONE) { @@ -389,9 +389,9 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, grpc_endpoint* endpoint, grpc_pollset* accepting_pollset, grpc_tcp_server_acceptor* acceptor) { gpr_free(acceptor); - grpc_end2end_http_proxy* proxy = arg; + grpc_end2end_http_proxy* proxy = (grpc_end2end_http_proxy*)arg; // Instantiate proxy_connection. - proxy_connection* conn = gpr_zalloc(sizeof(*conn)); + proxy_connection* conn = (proxy_connection*)gpr_zalloc(sizeof(*conn)); gpr_ref(&proxy->users); conn->client_endpoint = endpoint; conn->proxy = proxy; @@ -430,7 +430,7 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, // static void thread_main(void* arg) { - grpc_end2end_http_proxy* proxy = arg; + grpc_end2end_http_proxy* proxy = (grpc_end2end_http_proxy*)arg; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; do { gpr_ref(&proxy->users); @@ -450,7 +450,8 @@ static void thread_main(void* arg) { grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_end2end_http_proxy* proxy = gpr_malloc(sizeof(*proxy)); + grpc_end2end_http_proxy* proxy = + (grpc_end2end_http_proxy*)gpr_malloc(sizeof(*proxy)); memset(proxy, 0, sizeof(*proxy)); gpr_ref_init(&proxy->users, 1); // Construct proxy address. @@ -473,7 +474,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { GPR_ASSERT(error == GRPC_ERROR_NONE); GPR_ASSERT(port == proxy_port); // Start server. - proxy->pollset = gpr_zalloc(grpc_pollset_size()); + proxy->pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(proxy->pollset, &proxy->mu); grpc_tcp_server_start(&exec_ctx, proxy->server, &proxy->pollset, 1, on_accept, proxy); @@ -487,7 +488,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { static void destroy_pollset(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - grpc_pollset* pollset = arg; + grpc_pollset* pollset = (grpc_pollset*)arg; grpc_pollset_destroy(pollset); gpr_free(pollset); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index f0550db1d0..838625705d 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -55,7 +55,7 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems, size_t num_md, grpc_credentials_status status, const char *error_details) { - oauth2_request *request = user_data; + oauth2_request *request = (oauth2_request *)user_data; char *token = NULL; grpc_slice token_slice; if (status == GRPC_CREDENTIALS_ERROR) { @@ -63,7 +63,7 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, } else { GPR_ASSERT(num_md == 1); token_slice = md_elems[0].value; - token = gpr_malloc(GRPC_SLICE_LENGTH(token_slice) + 1); + token = (char *)gpr_malloc(GRPC_SLICE_LENGTH(token_slice) + 1); memcpy(token, GRPC_SLICE_START_PTR(token_slice), GRPC_SLICE_LENGTH(token_slice)); token[GRPC_SLICE_LENGTH(token_slice)] = '\0'; @@ -87,7 +87,7 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_closure do_nothing_closure; grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL}; - grpc_pollset *pollset = gpr_zalloc(grpc_pollset_size()); + grpc_pollset *pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(pollset, &request.mu); request.pops = grpc_polling_entity_create_from_pollset(pollset); request.is_done = 0; diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index c91219e98c..67e7c02535 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -563,7 +563,8 @@ static void BM_IsolatedFilter(benchmark::State &state) { } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t channel_size = grpc_channel_stack_size(&filters[0], filters.size()); + size_t channel_size = grpc_channel_stack_size( + filters.size() == 0 ? NULL : &filters[0], filters.size()); grpc_channel_stack *channel_stack = static_cast<grpc_channel_stack *>(gpr_zalloc(channel_size)); GPR_ASSERT(GRPC_LOG_IF_ERROR( diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index fc99b06dbb..6b9fa8b38d 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -53,7 +53,7 @@ DEFINE_int32( "Number of megabytes to pump before collecting flow control stats"); DEFINE_int32( warmup_iterations, 100, - "Number of megabytes to pump before collecting flow control stats"); + "Number of iterations to run before collecting flow control stats"); DEFINE_int32(warmup_max_time_seconds, 10, "Maximum number of seconds to run warmup loop"); @@ -77,13 +77,14 @@ static void write_csv(std::ostream* out, A0&& a0, Arg&&... arg) { class TrickledCHTTP2 : public EndpointPairFixture { public: - TrickledCHTTP2(Service* service, size_t message_size, - size_t kilobits_per_second) + TrickledCHTTP2(Service* service, bool streaming, size_t req_size, + size_t resp_size, size_t kilobits_per_second) : EndpointPairFixture(service, MakeEndpoints(kilobits_per_second), FixtureConfiguration()) { if (FLAGS_log) { std::ostringstream fn; - fn << "trickle." << message_size << "." << kilobits_per_second << ".csv"; + fn << "trickle." << (streaming ? "streaming" : "unary") << "." << req_size + << "." << resp_size << "." << kilobits_per_second << ".csv"; log_.reset(new std::ofstream(fn.str().c_str())); write_csv(log_.get(), "t", "iteration", "client_backlog", "server_backlog", "client_t_stall", "client_s_stall", @@ -242,8 +243,9 @@ static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok, static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { EchoTestService::AsyncService service; - std::unique_ptr<TrickledCHTTP2> fixture( - new TrickledCHTTP2(&service, state.range(0), state.range(1))); + std::unique_ptr<TrickledCHTTP2> fixture(new TrickledCHTTP2( + &service, true, state.range(0) /* req_size */, + state.range(0) /* resp_size */, state.range(1) /* bw in kbit/s */)); { EchoResponse send_response; EchoResponse recv_response; @@ -314,11 +316,7 @@ static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { state.SetBytesProcessed(state.range(0) * state.iterations()); } -/******************************************************************************* - * CONFIGURATIONS - */ - -static void TrickleArgs(benchmark::internal::Benchmark* b) { +static void StreamingTrickleArgs(benchmark::internal::Benchmark* b) { for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { for (int j = 64; j <= 128 * 1024 * 1024; j *= 8) { double expected_time = @@ -328,8 +326,111 @@ static void TrickleArgs(benchmark::internal::Benchmark* b) { } } } +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(StreamingTrickleArgs); -BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); +static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr<TrickledCHTTP2> fixture(new TrickledCHTTP2( + &service, true, state.range(0) /* req_size */, + state.range(1) /* resp_size */, state.range(2) /* bw in kbit/s */)); + EchoRequest send_request; + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + if (state.range(1) > 0) { + send_response.set_message(std::string(state.range(1), 'a')); + } + Status recv_status; + struct ServerEnv { + ServerContext ctx; + EchoRequest recv_request; + grpc::ServerAsyncResponseWriter<EchoResponse> response_writer; + ServerEnv() : response_writer(&ctx) {} + }; + uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; + ServerEnv* server_env[2] = { + reinterpret_cast<ServerEnv*>(server_env_buffer), + reinterpret_cast<ServerEnv*>(server_env_buffer + sizeof(ServerEnv))}; + new (server_env[0]) ServerEnv; + new (server_env[1]) ServerEnv; + service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, + &server_env[0]->response_writer, fixture->cq(), + fixture->cq(), tag(0)); + service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, + &server_env[1]->response_writer, fixture->cq(), + fixture->cq(), tag(1)); + std::unique_ptr<EchoTestService::Stub> stub( + EchoTestService::NewStub(fixture->channel())); + auto inner_loop = [&](bool in_warmup) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + recv_response.Clear(); + ClientContext cli_ctx; + std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); + void* t; + bool ok; + TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + GPR_ASSERT(ok); + GPR_ASSERT(t == tag(0) || t == tag(1)); + intptr_t slot = reinterpret_cast<intptr_t>(t); + ServerEnv* senv = server_env[slot]; + senv->response_writer.Finish(send_response, Status::OK, tag(3)); + response_reader->Finish(&recv_response, &recv_status, tag(4)); + for (int i = (1 << 3) | (1 << 4); i != 0;) { + TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + GPR_ASSERT(ok); + int tagnum = (int)reinterpret_cast<intptr_t>(t); + GPR_ASSERT(i & (1 << tagnum)); + i -= 1 << tagnum; + } + GPR_ASSERT(recv_status.ok()); + + senv->~ServerEnv(); + senv = new (senv) ServerEnv(); + service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, + fixture->cq(), fixture->cq(), tag(slot)); + }; + gpr_timespec warmup_start = gpr_now(GPR_CLOCK_MONOTONIC); + for (int i = 0; + i < GPR_MAX(FLAGS_warmup_iterations, FLAGS_warmup_megabytes * 1024 * + 1024 / (14 + state.range(0))); + i++) { + inner_loop(true); + if (gpr_time_cmp(gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), warmup_start), + gpr_time_from_seconds(FLAGS_warmup_max_time_seconds, + GPR_TIMESPAN)) > 0) { + break; + } + } + while (state.KeepRunning()) { + inner_loop(false); + } + fixture->Finish(state); + fixture.reset(); + server_env[0]->~ServerEnv(); + server_env[1]->~ServerEnv(); + state.SetBytesProcessed(state.range(0) * state.iterations() + + state.range(1) * state.iterations()); +} + +static void UnaryTrickleArgs(benchmark::internal::Benchmark* b) { + const int cli_1024k = 1024 * 1024; + const int cli_32M = 32 * 1024 * 1024; + const int svr_256k = 256 * 1024; + const int svr_4M = 4 * 1024 * 1024; + const int svr_64M = 64 * 1024 * 1024; + for (int bw = 64; bw <= 128 * 1024 * 1024; bw *= 16) { + b->Args({bw, cli_1024k, svr_256k}); + b->Args({bw, cli_1024k, svr_4M}); + b->Args({bw, cli_1024k, svr_64M}); + b->Args({bw, cli_32M, svr_256k}); + b->Args({bw, cli_32M, svr_4M}); + b->Args({bw, cli_32M, svr_64M}); + } +} +BENCHMARK(BM_PumpUnbalancedUnary_Trickle)->Apply(UnaryTrickleArgs); } } diff --git a/test/cpp/performance/writes_per_rpc_test.cc b/test/cpp/performance/writes_per_rpc_test.cc index 7a914c1547..12d8268330 100644 --- a/test/cpp/performance/writes_per_rpc_test.cc +++ b/test/cpp/performance/writes_per_rpc_test.cc @@ -254,8 +254,8 @@ TEST(WritesPerRpcTest, UnaryPingPong) { EXPECT_LT(UnaryPingPong(0, 0), 2.05); EXPECT_LT(UnaryPingPong(1, 0), 2.05); EXPECT_LT(UnaryPingPong(0, 1), 2.05); - EXPECT_LT(UnaryPingPong(4096, 0), 2.2); - EXPECT_LT(UnaryPingPong(0, 4096), 2.2); + EXPECT_LT(UnaryPingPong(4096, 0), 2.5); + EXPECT_LT(UnaryPingPong(0, 4096), 2.5); } } // namespace testing diff --git a/third_party/protobuf b/third_party/protobuf -Subproject 593e917c176b5bc5aafa57bf9f6030d749d91cd +Subproject a6189acd18b00611c1dc7042299ad75486f08a1 diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index c2aa6198b3..8a251f876a 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -29,8 +29,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] -PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/profile.pb.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/profile.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 211d442f17..43b60b142f 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -211,7 +211,7 @@ setuptools.setup( ext_modules=extension_modules(), packages=setuptools.find_packages('.'), install_requires=[ - 'protobuf>=3.2.0', + 'protobuf>=3.3.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ], package_data=package_data(), diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index f9e709dccb..0b21a22226 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index 4bb97c7aa9..d9dc272a1b 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index c4b710b5df..11ef52d1c0 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index bd742dff34..41d3b2b520 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index bc46b3055a..23d6fb8c41 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile index f7d7f542c1..1848e5e5c8 100644 --- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index b398b70b64..4200ba0b26 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index c1cce0a141..9b50d85e2f 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #================ # C# dependencies diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 4595aa6bea..deef892952 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + # Install Electron apt dependencies RUN apt-get update && apt-get install -y \ diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index 0e2c103afd..6057c2d6eb 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -75,6 +75,10 @@ RUN cd /var/local/git/php-src \ && make \ && make install +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index c6f3dde39a..1510c3649c 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 94c17078d3..cc69f4b5cd 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index 435a9fdc97..a105d334da 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index 679c8ff47a..0a5c9a633d 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 0da2a1914a..76923303ea 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + #==================== # Python dependencies diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh index d552343bde..f795980aa8 100755 --- a/tools/gce/linux_worker_init.sh +++ b/tools/gce/linux_worker_init.sh @@ -61,6 +61,10 @@ sudo usermod -aG docker jenkins # see https://github.com/grpc/grpc/issues/4988 printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json +# Install pip and Google API library to enable using GCP services +sudo apt-get install -y python-pip +sudo pip install google-api-python-client + # Install RVM # TODO(jtattermusch): why is RVM needed? gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 04702bacca..c90da066c3 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -202,6 +202,7 @@ class CSharpExtArtifact: 'EMBED_OPENSSL': 'true', 'EMBED_ZLIB': 'true', 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', 'LDFLAGS': ''} if self.platform == 'linux': return create_docker_jobspec(self.name, @@ -211,6 +212,7 @@ class CSharpExtArtifact: else: archflag = _ARCH_FLAG_MAP[self.arch] environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_csharp.sh'], diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index f10916d192..731bb02d21 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -79,7 +79,10 @@ docker run \ -e HOST_GIT_ROOT=$git_root \ -e LOCAL_GIT_ROOT=$docker_instance_git_root \ -e "BUILD_ID=$BUILD_ID" \ + -e "BUILD_URL=$BUILD_URL" \ + -e "JOB_BASE_NAME=$JOB_BASE_NAME" \ -i $TTY_FLAG \ + -v ~/.config/gcloud:/root/.config/gcloud \ -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a004e8d945..c77961c012 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4205,7 +4205,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4229,7 +4230,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4253,7 +4255,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4277,7 +4280,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4303,7 +4307,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4327,7 +4332,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4351,7 +4357,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4378,7 +4385,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4405,7 +4413,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4432,7 +4441,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4459,7 +4469,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4486,7 +4497,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4513,7 +4525,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4540,7 +4553,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4567,7 +4581,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4594,7 +4609,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4621,7 +4637,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4648,7 +4665,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4675,7 +4693,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4702,7 +4721,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4729,7 +4749,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4756,7 +4777,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4783,7 +4805,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4810,7 +4833,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4837,7 +4861,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4864,7 +4889,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4891,7 +4917,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4917,7 +4944,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4941,7 +4969,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4965,7 +4994,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4991,7 +5021,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5015,7 +5046,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5039,7 +5071,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5063,7 +5096,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5087,7 +5121,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5111,7 +5146,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5135,7 +5171,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5159,7 +5196,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5185,7 +5223,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5211,7 +5250,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5235,7 +5275,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5261,7 +5302,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5285,7 +5327,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5309,7 +5352,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5335,7 +5379,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5359,7 +5404,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5383,7 +5429,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5409,7 +5456,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5433,7 +5481,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5457,7 +5506,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5481,7 +5531,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5505,7 +5556,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5531,7 +5583,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5555,7 +5608,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5579,7 +5633,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5603,7 +5658,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5629,7 +5685,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5653,7 +5710,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5677,7 +5735,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5701,7 +5760,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5725,7 +5785,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5749,7 +5810,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5773,7 +5835,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5797,7 +5860,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5821,7 +5885,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index d01e82a76b..4e97128d46 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -222,7 +222,8 @@ class JobResult(object): self.num_failures = 0 self.retries = 0 self.message = '' - + self.cpu_estimated = 1 + self.cpu_measured = 0 class Job(object): """Manages one job.""" @@ -312,7 +313,9 @@ class Job(object): sys = float(m.group(3)) if real > 0.5: cores = (user + sys) / real - measurement = '; cpu_cost=%.01f; estimated=%.01f' % (cores, self._spec.cpu_cost) + self.result.cpu_measured = float('%.01f' % cores) + self.result.cpu_estimated = float('%.01f' % self._spec.cpu_cost) + measurement = '; cpu_cost=%.01f; estimated=%.01f' % (self.result.cpu_measured, self.result.cpu_estimated) if not self._quiet_success: message('PASSED', '%s [time=%.1fsec; retries=%d:%d%s]' % ( self._spec.shortname, elapsed, self._retries, self._timeout_retries, measurement), diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 502efc31f4..b8dd67e060 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -89,6 +89,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', tree = ET.ElementTree(root) tree.write(xml_report, encoding='UTF-8') + def render_interop_html_report( client_langs, server_langs, test_cases, auth_test_cases, http2_cases, http2_server_cases, resultset, diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py new file mode 100644 index 0000000000..d076d1e5a2 --- /dev/null +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Helper to upload Jenkins test results to BQ""" + +from __future__ import print_function + +import os +import six +import sys +import time +import uuid + +gcp_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gcp/utils')) +sys.path.append(gcp_utils_dir) +import big_query_utils + +_DATASET_ID = 'jenkins_test_results' +_DESCRIPTION = 'Test results from master job run on Jenkins' +_PROJECT_ID = 'grpc-testing' +_RESULTS_SCHEMA = [ + ('job_name', 'STRING', 'Name of Jenkins job'), + ('build_id', 'INTEGER', 'Build ID of Jenkins job'), + ('build_url', 'STRING', 'URL of Jenkins job'), + ('test_name', 'STRING', 'Individual test name'), + ('language', 'STRING', 'Language of test'), + ('platform', 'STRING', 'Platform used for test'), + ('config', 'STRING', 'Config used for test'), + ('compiler', 'STRING', 'Compiler used for test'), + ('iomgr_platform', 'STRING', 'Iomgr used for test'), + ('result', 'STRING', 'Test result: PASSED, TIMEOUT, FAILED, or SKIPPED'), + ('timestamp', 'TIMESTAMP', 'Timestamp of test run'), + ('elapsed_time', 'FLOAT', 'How long test took to run'), + ('cpu_estimated', 'FLOAT', 'Estimated CPU usage of test'), + ('cpu_measured', 'FLOAT', 'Actual CPU usage of test'), +] + + +def _get_build_metadata(test_results): + """Add Jenkins build metadata to test_results based on environment variables set by Jenkins.""" + build_id = os.getenv('BUILD_ID') + build_url = os.getenv('BUILD_URL') + job_name = os.getenv('JOB_BASE_NAME') + + if build_id: + test_results['build_id'] = build_id + if build_url: + test_results['build_url'] = build_url + if job_name: + test_results['job_name'] = job_name + +def upload_results_to_bq(resultset, bq_table, args, platform): + """Upload test results to a BQ table. + + Args: + resultset: dictionary generated by jobset.run + bq_table: string name of table to create/upload results to in BQ + args: args in run_tests.py, generated by argparse + platform: string name of platform tests were run on + """ + bq = big_query_utils.create_big_query() + big_query_utils.create_table(bq, _PROJECT_ID, _DATASET_ID, bq_table, _RESULTS_SCHEMA, _DESCRIPTION) + + for shortname, results in six.iteritems(resultset): + for result in results: + test_results = {} + _get_build_metadata(test_results) + test_results['compiler'] = args.compiler + test_results['config'] = args.config + test_results['cpu_estimated'] = result.cpu_estimated + test_results['cpu_measured'] = result.cpu_measured + test_results['elapsed_time'] = '%.2f' % result.elapsed_time + test_results['iomgr_platform'] = args.iomgr_platform + # args.language is a list, but will always have one element in the contexts + # this function is used. + test_results['language'] = args.language[0] + test_results['platform'] = platform + test_results['result'] = result.state + test_results['test_name'] = shortname + test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') + + row = big_query_utils.make_row(str(uuid.uuid4()), test_results) + if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, bq_table, [row]): + print('Error uploading result to bigquery.') + sys.exit(1) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 62e92c3d1d..22891c5c98 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -60,7 +60,10 @@ import python_utils.jobset as jobset import python_utils.report_utils as report_utils import python_utils.watch_dirs as watch_dirs import python_utils.start_port_server as start_port_server - +try: + from python_utils.upload_test_results import upload_results_to_bq +except (ImportError): + pass # It's ok to not import because this is only necessary to upload results to BQ. _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -1185,7 +1188,7 @@ argp.add_argument('--build_only', default=False, action='store_const', const=True, - help='Perform all the build steps but dont run any tests.') + help='Perform all the build steps but don\'t run any tests.') argp.add_argument('--measure_cpu_costs', default=False, action='store_const', const=True, help='Measure the cpu costs of tests') argp.add_argument('--update_submodules', default=[], nargs='*', @@ -1200,11 +1203,16 @@ argp.add_argument('--quiet_success', default=False, action='store_const', const=True, - help='Dont print anything when a test passes. Passing tests also will not be reported in XML report. ' + + help='Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' + 'Useful when running many iterations of each test (argument -n).') argp.add_argument('--force_default_poller', default=False, action='store_const', const=True, - help='Dont try to iterate over many polling strategies when they exist') + help='Don\'t try to iterate over many polling strategies when they exist') argp.add_argument('--max_time', default=-1, type=int, help='Maximum test runtime in seconds') +argp.add_argument('--bq_result_table', + default='', + type=str, + nargs='?', + help='Upload test results to a specified BQ table.') args = argp.parse_args() if args.force_default_poller: @@ -1503,6 +1511,8 @@ def _build_and_run( finally: for antagonist in antagonists: antagonist.kill() + if args.bq_result_table and resultset: + upload_results_to_bq(resultset, args.bq_result_table, args, platform_string()) if xml_report and resultset: report_utils.render_junit_xml_report(resultset, xml_report, suite_name=args.report_suite_name) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 474b56b5de..884d626469 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -208,7 +208,7 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) - for compiler in ['gcc4.8', 'gcc5.3', + for compiler in ['gcc4.8', 'gcc5.3', 'gcc_musl', 'clang3.5', 'clang3.6', 'clang3.7']: test_jobs += _generate_jobs(languages=['c++'], configs=['dbg'], @@ -267,6 +267,15 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) + test_jobs += _generate_jobs(languages=['python'], + configs=['dbg'], + platforms=['linux'], + arch='default', + compiler='python_alpine', + labels=['portability'], + extra_args=extra_args, + inner_jobs=inner_jobs) + test_jobs += _generate_jobs(languages=['csharp'], configs=['dbg'], platforms=['linux'], @@ -387,6 +396,11 @@ if __name__ == "__main__": const=True, help='Put reports into subdirectories to improve presentation of ' 'results by Internal CI.') + argp.add_argument('--bq_result_table', + default='', + type=str, + nargs='?', + help='Upload test results to a specified BQ table.') args = argp.parse_args() if args.internal_ci: @@ -403,6 +417,10 @@ if __name__ == "__main__": extra_args.append('--quiet_success') if args.max_time > 0: extra_args.extend(('--max_time', '%d' % args.max_time)) + if args.bq_result_table: + extra_args.append('--bq_result_table') + extra_args.append('%s' % args.bq_result_table) + extra_args.append('--measure_cpu_costs') all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 6be7a39d07..43d05212e8 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -46,7 +46,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7) 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0) ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) - 593e917c176b5bc5aafa57bf9f6030d749d91cd5 third_party/protobuf (v3.1.0-alpha-1-326-g593e917) + a6189acd18b00611c1dc7042299ad75486f08a1a third_party/protobuf (v3.3.0) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) 7691f773af79bf75a62d1863fd0f13ebf9dc51b1 third_party/cares/cares (1.12.0) EOF diff --git a/tools/ubsan_suppressions.txt b/tools/ubsan_suppressions.txt index 9869f98a22..f87ed18565 100644 --- a/tools/ubsan_suppressions.txt +++ b/tools/ubsan_suppressions.txt @@ -4,4 +4,6 @@ nonnull-attribute:CBB_add_bytes nonnull-attribute:rsa_blinding_get nonnull-attribute:ssl_copy_key_material alignment:CRYPTO_cbc128_encrypt +nonnull-attribute:google::protobuf::DescriptorBuilder::BuildFileImpl +nonnull-attribute:google::protobuf::TextFormat::Printer::TextGenerator::Write |