diff options
50 files changed, 2081 insertions, 34 deletions
diff --git a/doc/python/sphinx/conf.py b/doc/python/sphinx/conf.py index 1eb3a5de7f..307c3bdaf6 100644 --- a/doc/python/sphinx/conf.py +++ b/doc/python/sphinx/conf.py @@ -19,6 +19,7 @@ import sys PYTHON_FOLDER = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', '..', 'src', 'python') sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio')) +sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_channelz')) sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_health_checking')) sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_reflection')) sys.path.insert(0, os.path.join(PYTHON_FOLDER, 'grpcio_testing')) @@ -63,6 +64,8 @@ autodoc_default_options = { autodoc_mock_imports = [ 'grpc._cython', + 'grpc_channelz.v1.channelz_pb2', + 'grpc_channelz.v1.channelz_pb2_grpc', 'grpc_health.v1.health_pb2', 'grpc_health.v1.health_pb2_grpc', 'grpc_reflection.v1alpha.reflection_pb2', diff --git a/doc/python/sphinx/grpc_channelz.rst b/doc/python/sphinx/grpc_channelz.rst new file mode 100644 index 0000000000..f65793a071 --- /dev/null +++ b/doc/python/sphinx/grpc_channelz.rst @@ -0,0 +1,12 @@ +gRPC Channelz +==================== + +What is gRPC Channelz? +--------------------------------------------- + +Design Document `gRPC Channelz <https://github.com/grpc/proposal/blob/master/A14-channelz.md>`_ + +Module Contents +--------------- + +.. automodule:: grpc_channelz.v1.channelz diff --git a/doc/python/sphinx/index.rst b/doc/python/sphinx/index.rst index 322ca33e15..2f8a47a074 100644 --- a/doc/python/sphinx/index.rst +++ b/doc/python/sphinx/index.rst @@ -10,6 +10,7 @@ API Reference :caption: Contents: grpc + grpc_channelz grpc_health_checking grpc_reflection grpc_testing diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 17a43fab0f..d1cc6af04f 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -293,7 +293,7 @@ typedef struct { "grpc.max_channel_trace_event_memory_per_node" /** If non-zero, gRPC library will track stats and information at at per channel * level. Disabling channelz naturally disables channel tracing. The default - * is for channelz to be disabled. */ + * is for channelz to be enabled. */ #define GRPC_ARG_ENABLE_CHANNELZ "grpc.enable_channelz" /** If non-zero, Cronet transport will coalesce packets to fewer frames * when possible. */ diff --git a/include/grpcpp/generic/generic_stub.h b/include/grpcpp/generic/generic_stub.h index d509d9a520..eb014184e4 100644 --- a/include/grpcpp/generic/generic_stub.h +++ b/include/grpcpp/generic/generic_stub.h @@ -24,6 +24,7 @@ #include <grpcpp/support/async_stream.h> #include <grpcpp/support/async_unary_call.h> #include <grpcpp/support/byte_buffer.h> +#include <grpcpp/support/client_callback.h> #include <grpcpp/support/status.h> namespace grpc { @@ -76,6 +77,10 @@ class GenericStub final { const ByteBuffer* request, ByteBuffer* response, std::function<void(Status)> on_completion); + void PrepareBidiStreamingCall( + ClientContext* context, const grpc::string& method, + experimental::ClientBidiReactor<ByteBuffer, ByteBuffer>* reactor); + private: GenericStub* stub_; }; diff --git a/include/grpcpp/impl/codegen/callback_common.h b/include/grpcpp/impl/codegen/callback_common.h index 51367cf550..f7a24204dc 100644 --- a/include/grpcpp/impl/codegen/callback_common.h +++ b/include/grpcpp/impl/codegen/callback_common.h @@ -145,18 +145,19 @@ class CallbackWithSuccessTag // or on a tag that has been Set before unless the tag has been cleared. void Set(grpc_call* call, std::function<void(bool)> f, CompletionQueueTag* ops) { + GPR_CODEGEN_ASSERT(call_ == nullptr); + g_core_codegen_interface->grpc_call_ref(call); call_ = call; func_ = std::move(f); ops_ = ops; - g_core_codegen_interface->grpc_call_ref(call); functor_run = &CallbackWithSuccessTag::StaticRun; } void Clear() { if (call_ != nullptr) { - func_ = nullptr; grpc_call* call = call_; call_ = nullptr; + func_ = nullptr; g_core_codegen_interface->grpc_call_unref(call); } } @@ -182,10 +183,9 @@ class CallbackWithSuccessTag } void Run(bool ok) { void* ignored = ops_; - bool new_ok = ok; // Allow a "false" return value from FinalizeResult to silence the // callback, just as it silences a CQ tag in the async cases - bool do_callback = ops_->FinalizeResult(&ignored, &new_ok); + bool do_callback = ops_->FinalizeResult(&ignored, &ok); GPR_CODEGEN_ASSERT(ignored == ops_); if (do_callback) { diff --git a/include/grpcpp/impl/codegen/channel_interface.h b/include/grpcpp/impl/codegen/channel_interface.h index 6ec1ffb8c7..728a7b9049 100644 --- a/include/grpcpp/impl/codegen/channel_interface.h +++ b/include/grpcpp/impl/codegen/channel_interface.h @@ -53,6 +53,12 @@ template <class W, class R> class ClientAsyncReaderWriterFactory; template <class R> class ClientAsyncResponseReaderFactory; +template <class W, class R> +class ClientCallbackReaderWriterFactory; +template <class R> +class ClientCallbackReaderFactory; +template <class W> +class ClientCallbackWriterFactory; class InterceptedChannel; } // namespace internal @@ -106,6 +112,12 @@ class ChannelInterface { friend class ::grpc::internal::ClientAsyncReaderWriterFactory; template <class R> friend class ::grpc::internal::ClientAsyncResponseReaderFactory; + template <class W, class R> + friend class ::grpc::internal::ClientCallbackReaderWriterFactory; + template <class R> + friend class ::grpc::internal::ClientCallbackReaderFactory; + template <class W> + friend class ::grpc::internal::ClientCallbackWriterFactory; template <class InputMessage, class OutputMessage> friend class ::grpc::internal::BlockingUnaryCallImpl; template <class InputMessage, class OutputMessage> diff --git a/include/grpcpp/impl/codegen/client_callback.h b/include/grpcpp/impl/codegen/client_callback.h index 4baa819091..4d9579fd6a 100644 --- a/include/grpcpp/impl/codegen/client_callback.h +++ b/include/grpcpp/impl/codegen/client_callback.h @@ -22,6 +22,7 @@ #include <functional> #include <grpcpp/impl/codegen/call.h> +#include <grpcpp/impl/codegen/call_op_set.h> #include <grpcpp/impl/codegen/callback_common.h> #include <grpcpp/impl/codegen/channel_interface.h> #include <grpcpp/impl/codegen/config.h> @@ -88,6 +89,646 @@ class CallbackUnaryCallImpl { call.PerformOps(ops); } }; +} // namespace internal + +namespace experimental { + +// Forward declarations +template <class Request, class Response> +class ClientBidiReactor; +template <class Response> +class ClientReadReactor; +template <class Request> +class ClientWriteReactor; + +// NOTE: The streaming objects are not actually implemented in the public API. +// These interfaces are provided for mocking only. Typical applications +// will interact exclusively with the reactors that they define. +template <class Request, class Response> +class ClientCallbackReaderWriter { + public: + virtual ~ClientCallbackReaderWriter() {} + virtual void StartCall() = 0; + virtual void Write(const Request* req, WriteOptions options) = 0; + virtual void WritesDone() = 0; + virtual void Read(Response* resp) = 0; + + protected: + void BindReactor(ClientBidiReactor<Request, Response>* reactor) { + reactor->BindStream(this); + } +}; + +template <class Response> +class ClientCallbackReader { + public: + virtual ~ClientCallbackReader() {} + virtual void StartCall() = 0; + virtual void Read(Response* resp) = 0; + + protected: + void BindReactor(ClientReadReactor<Response>* reactor) { + reactor->BindReader(this); + } +}; + +template <class Request> +class ClientCallbackWriter { + public: + virtual ~ClientCallbackWriter() {} + virtual void StartCall() = 0; + void Write(const Request* req) { Write(req, WriteOptions()); } + virtual void Write(const Request* req, WriteOptions options) = 0; + void WriteLast(const Request* req, WriteOptions options) { + Write(req, options.set_last_message()); + } + virtual void WritesDone() = 0; + + protected: + void BindReactor(ClientWriteReactor<Request>* reactor) { + reactor->BindWriter(this); + } +}; + +// The user must implement this reactor interface with reactions to each event +// type that gets called by the library. An empty reaction is provided by +// default +template <class Request, class Response> +class ClientBidiReactor { + public: + virtual ~ClientBidiReactor() {} + virtual void OnDone(const Status& s) {} + virtual void OnReadInitialMetadataDone(bool ok) {} + virtual void OnReadDone(bool ok) {} + virtual void OnWriteDone(bool ok) {} + virtual void OnWritesDoneDone(bool ok) {} + + void StartCall() { stream_->StartCall(); } + void StartRead(Response* resp) { stream_->Read(resp); } + void StartWrite(const Request* req) { StartWrite(req, WriteOptions()); } + void StartWrite(const Request* req, WriteOptions options) { + stream_->Write(req, std::move(options)); + } + void StartWriteLast(const Request* req, WriteOptions options) { + StartWrite(req, std::move(options.set_last_message())); + } + void StartWritesDone() { stream_->WritesDone(); } + + private: + friend class ClientCallbackReaderWriter<Request, Response>; + void BindStream(ClientCallbackReaderWriter<Request, Response>* stream) { + stream_ = stream; + } + ClientCallbackReaderWriter<Request, Response>* stream_; +}; + +template <class Response> +class ClientReadReactor { + public: + virtual ~ClientReadReactor() {} + virtual void OnDone(const Status& s) {} + virtual void OnReadInitialMetadataDone(bool ok) {} + virtual void OnReadDone(bool ok) {} + + void StartCall() { reader_->StartCall(); } + void StartRead(Response* resp) { reader_->Read(resp); } + + private: + friend class ClientCallbackReader<Response>; + void BindReader(ClientCallbackReader<Response>* reader) { reader_ = reader; } + ClientCallbackReader<Response>* reader_; +}; + +template <class Request> +class ClientWriteReactor { + public: + virtual ~ClientWriteReactor() {} + virtual void OnDone(const Status& s) {} + virtual void OnReadInitialMetadataDone(bool ok) {} + virtual void OnWriteDone(bool ok) {} + virtual void OnWritesDoneDone(bool ok) {} + + void StartCall() { writer_->StartCall(); } + void StartWrite(const Request* req) { StartWrite(req, WriteOptions()); } + void StartWrite(const Request* req, WriteOptions options) { + writer_->Write(req, std::move(options)); + } + void StartWriteLast(const Request* req, WriteOptions options) { + StartWrite(req, std::move(options.set_last_message())); + } + void StartWritesDone() { writer_->WritesDone(); } + + private: + friend class ClientCallbackWriter<Request>; + void BindWriter(ClientCallbackWriter<Request>* writer) { writer_ = writer; } + ClientCallbackWriter<Request>* writer_; +}; + +} // namespace experimental + +namespace internal { + +// Forward declare factory classes for friendship +template <class Request, class Response> +class ClientCallbackReaderWriterFactory; +template <class Response> +class ClientCallbackReaderFactory; +template <class Request> +class ClientCallbackWriterFactory; + +template <class Request, class Response> +class ClientCallbackReaderWriterImpl + : public ::grpc::experimental::ClientCallbackReaderWriter<Request, + Response> { + public: + // always allocated against a call arena, no memory free required + static void operator delete(void* ptr, std::size_t size) { + assert(size == sizeof(ClientCallbackReaderWriterImpl)); + } + + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + + void MaybeFinish() { + if (--callbacks_outstanding_ == 0) { + reactor_->OnDone(finish_status_); + auto* call = call_.call(); + this->~ClientCallbackReaderWriterImpl(); + g_core_codegen_interface->grpc_call_unref(call); + } + } + + void StartCall() override { + // This call initiates two batches, plus any backlog, each with a callback + // 1. Send initial metadata (unless corked) + recv initial metadata + // 2. Any read backlog + // 3. Recv trailing metadata, on_completion callback + // 4. Any write backlog + started_ = true; + + start_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadInitialMetadataDone(ok); + MaybeFinish(); + }, + &start_ops_); + if (!start_corked_) { + start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + } + start_ops_.RecvInitialMetadata(context_); + start_ops_.set_core_cq_tag(&start_tag_); + call_.PerformOps(&start_ops_); + + // Also set up the read and write tags so that they don't have to be set up + // each time + write_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWriteDone(ok); + MaybeFinish(); + }, + &write_ops_); + write_ops_.set_core_cq_tag(&write_tag_); + + read_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadDone(ok); + MaybeFinish(); + }, + &read_ops_); + read_ops_.set_core_cq_tag(&read_tag_); + if (read_ops_at_start_) { + call_.PerformOps(&read_ops_); + } + + finish_tag_.Set(call_.call(), [this](bool ok) { MaybeFinish(); }, + &finish_ops_); + finish_ops_.ClientRecvStatus(context_, &finish_status_); + finish_ops_.set_core_cq_tag(&finish_tag_); + call_.PerformOps(&finish_ops_); + + if (write_ops_at_start_) { + call_.PerformOps(&write_ops_); + } + + if (writes_done_ops_at_start_) { + call_.PerformOps(&writes_done_ops_); + } + } + + void Read(Response* msg) override { + read_ops_.RecvMessage(msg); + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&read_ops_); + } else { + read_ops_at_start_ = true; + } + } + + void Write(const Request* msg, WriteOptions options) override { + if (start_corked_) { + write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + start_corked_ = false; + } + // TODO(vjpai): don't assert + GPR_CODEGEN_ASSERT(write_ops_.SendMessage(*msg).ok()); + + if (options.is_last_message()) { + options.set_buffer_hint(); + write_ops_.ClientSendClose(); + } + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&write_ops_); + } else { + write_ops_at_start_ = true; + } + } + void WritesDone() override { + if (start_corked_) { + writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + start_corked_ = false; + } + writes_done_ops_.ClientSendClose(); + writes_done_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWritesDoneDone(ok); + MaybeFinish(); + }, + &writes_done_ops_); + writes_done_ops_.set_core_cq_tag(&writes_done_tag_); + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&writes_done_ops_); + } else { + writes_done_ops_at_start_ = true; + } + } + + private: + friend class ClientCallbackReaderWriterFactory<Request, Response>; + + ClientCallbackReaderWriterImpl( + Call call, ClientContext* context, + ::grpc::experimental::ClientBidiReactor<Request, Response>* reactor) + : context_(context), + call_(call), + reactor_(reactor), + start_corked_(context_->initial_metadata_corked_) { + this->BindReactor(reactor); + } + + ClientContext* context_; + Call call_; + ::grpc::experimental::ClientBidiReactor<Request, Response>* reactor_; + + CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_; + CallbackWithSuccessTag start_tag_; + bool start_corked_; + + CallOpSet<CallOpClientRecvStatus> finish_ops_; + CallbackWithSuccessTag finish_tag_; + Status finish_status_; + + CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> + write_ops_; + CallbackWithSuccessTag write_tag_; + bool write_ops_at_start_{false}; + + CallOpSet<CallOpSendInitialMetadata, CallOpClientSendClose> writes_done_ops_; + CallbackWithSuccessTag writes_done_tag_; + bool writes_done_ops_at_start_{false}; + + CallOpSet<CallOpRecvMessage<Response>> read_ops_; + CallbackWithSuccessTag read_tag_; + bool read_ops_at_start_{false}; + + // Minimum of 2 outstanding callbacks to pre-register for start and finish + std::atomic_int callbacks_outstanding_{2}; + bool started_{false}; +}; + +template <class Request, class Response> +class ClientCallbackReaderWriterFactory { + public: + static void Create( + ChannelInterface* channel, const ::grpc::internal::RpcMethod& method, + ClientContext* context, + ::grpc::experimental::ClientBidiReactor<Request, Response>* reactor) { + Call call = channel->CreateCall(method, context, channel->CallbackCQ()); + + g_core_codegen_interface->grpc_call_ref(call.call()); + new (g_core_codegen_interface->grpc_call_arena_alloc( + call.call(), sizeof(ClientCallbackReaderWriterImpl<Request, Response>))) + ClientCallbackReaderWriterImpl<Request, Response>(call, context, + reactor); + } +}; + +template <class Response> +class ClientCallbackReaderImpl + : public ::grpc::experimental::ClientCallbackReader<Response> { + public: + // always allocated against a call arena, no memory free required + static void operator delete(void* ptr, std::size_t size) { + assert(size == sizeof(ClientCallbackReaderImpl)); + } + + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + + void MaybeFinish() { + if (--callbacks_outstanding_ == 0) { + reactor_->OnDone(finish_status_); + auto* call = call_.call(); + this->~ClientCallbackReaderImpl(); + g_core_codegen_interface->grpc_call_unref(call); + } + } + + void StartCall() override { + // This call initiates two batches, plus any backlog, each with a callback + // 1. Send initial metadata (unless corked) + recv initial metadata + // 2. Any backlog + // 3. Recv trailing metadata, on_completion callback + started_ = true; + + start_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadInitialMetadataDone(ok); + MaybeFinish(); + }, + &start_ops_); + start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + start_ops_.RecvInitialMetadata(context_); + start_ops_.set_core_cq_tag(&start_tag_); + call_.PerformOps(&start_ops_); + + // Also set up the read tag so it doesn't have to be set up each time + read_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadDone(ok); + MaybeFinish(); + }, + &read_ops_); + read_ops_.set_core_cq_tag(&read_tag_); + if (read_ops_at_start_) { + call_.PerformOps(&read_ops_); + } + + finish_tag_.Set(call_.call(), [this](bool ok) { MaybeFinish(); }, + &finish_ops_); + finish_ops_.ClientRecvStatus(context_, &finish_status_); + finish_ops_.set_core_cq_tag(&finish_tag_); + call_.PerformOps(&finish_ops_); + } + + void Read(Response* msg) override { + read_ops_.RecvMessage(msg); + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&read_ops_); + } else { + read_ops_at_start_ = true; + } + } + + private: + friend class ClientCallbackReaderFactory<Response>; + + template <class Request> + ClientCallbackReaderImpl( + Call call, ClientContext* context, Request* request, + ::grpc::experimental::ClientReadReactor<Response>* reactor) + : context_(context), call_(call), reactor_(reactor) { + this->BindReactor(reactor); + // TODO(vjpai): don't assert + GPR_CODEGEN_ASSERT(start_ops_.SendMessage(*request).ok()); + start_ops_.ClientSendClose(); + } + + ClientContext* context_; + Call call_; + ::grpc::experimental::ClientReadReactor<Response>* reactor_; + + CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose, + CallOpRecvInitialMetadata> + start_ops_; + CallbackWithSuccessTag start_tag_; + + CallOpSet<CallOpClientRecvStatus> finish_ops_; + CallbackWithSuccessTag finish_tag_; + Status finish_status_; + + CallOpSet<CallOpRecvMessage<Response>> read_ops_; + CallbackWithSuccessTag read_tag_; + bool read_ops_at_start_{false}; + + // Minimum of 2 outstanding callbacks to pre-register for start and finish + std::atomic_int callbacks_outstanding_{2}; + bool started_{false}; +}; + +template <class Response> +class ClientCallbackReaderFactory { + public: + template <class Request> + static void Create( + ChannelInterface* channel, const ::grpc::internal::RpcMethod& method, + ClientContext* context, const Request* request, + ::grpc::experimental::ClientReadReactor<Response>* reactor) { + Call call = channel->CreateCall(method, context, channel->CallbackCQ()); + + g_core_codegen_interface->grpc_call_ref(call.call()); + new (g_core_codegen_interface->grpc_call_arena_alloc( + call.call(), sizeof(ClientCallbackReaderImpl<Response>))) + ClientCallbackReaderImpl<Response>(call, context, request, reactor); + } +}; + +template <class Request> +class ClientCallbackWriterImpl + : public ::grpc::experimental::ClientCallbackWriter<Request> { + public: + // always allocated against a call arena, no memory free required + static void operator delete(void* ptr, std::size_t size) { + assert(size == sizeof(ClientCallbackWriterImpl)); + } + + // This operator should never be called as the memory should be freed as part + // of the arena destruction. It only exists to provide a matching operator + // delete to the operator new so that some compilers will not complain (see + // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this + // there are no tests catching the compiler warning. + static void operator delete(void*, void*) { assert(0); } + + void MaybeFinish() { + if (--callbacks_outstanding_ == 0) { + reactor_->OnDone(finish_status_); + auto* call = call_.call(); + this->~ClientCallbackWriterImpl(); + g_core_codegen_interface->grpc_call_unref(call); + } + } + + void StartCall() override { + // This call initiates two batches, plus any backlog, each with a callback + // 1. Send initial metadata (unless corked) + recv initial metadata + // 2. Recv trailing metadata, on_completion callback + // 3. Any backlog + started_ = true; + + start_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnReadInitialMetadataDone(ok); + MaybeFinish(); + }, + &start_ops_); + if (!start_corked_) { + start_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + } + start_ops_.RecvInitialMetadata(context_); + start_ops_.set_core_cq_tag(&start_tag_); + call_.PerformOps(&start_ops_); + + // Also set up the read and write tags so that they don't have to be set up + // each time + write_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWriteDone(ok); + MaybeFinish(); + }, + &write_ops_); + write_ops_.set_core_cq_tag(&write_tag_); + + finish_tag_.Set(call_.call(), [this](bool ok) { MaybeFinish(); }, + &finish_ops_); + finish_ops_.ClientRecvStatus(context_, &finish_status_); + finish_ops_.set_core_cq_tag(&finish_tag_); + call_.PerformOps(&finish_ops_); + + if (write_ops_at_start_) { + call_.PerformOps(&write_ops_); + } + + if (writes_done_ops_at_start_) { + call_.PerformOps(&writes_done_ops_); + } + } + + void Write(const Request* msg, WriteOptions options) override { + if (start_corked_) { + write_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + start_corked_ = false; + } + // TODO(vjpai): don't assert + GPR_CODEGEN_ASSERT(write_ops_.SendMessage(*msg).ok()); + + if (options.is_last_message()) { + options.set_buffer_hint(); + write_ops_.ClientSendClose(); + } + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&write_ops_); + } else { + write_ops_at_start_ = true; + } + } + void WritesDone() override { + if (start_corked_) { + writes_done_ops_.SendInitialMetadata(&context_->send_initial_metadata_, + context_->initial_metadata_flags()); + start_corked_ = false; + } + writes_done_ops_.ClientSendClose(); + writes_done_tag_.Set(call_.call(), + [this](bool ok) { + reactor_->OnWritesDoneDone(ok); + MaybeFinish(); + }, + &writes_done_ops_); + writes_done_ops_.set_core_cq_tag(&writes_done_tag_); + callbacks_outstanding_++; + if (started_) { + call_.PerformOps(&writes_done_ops_); + } else { + writes_done_ops_at_start_ = true; + } + } + + private: + friend class ClientCallbackWriterFactory<Request>; + + template <class Response> + ClientCallbackWriterImpl( + Call call, ClientContext* context, Response* response, + ::grpc::experimental::ClientWriteReactor<Request>* reactor) + : context_(context), + call_(call), + reactor_(reactor), + start_corked_(context_->initial_metadata_corked_) { + this->BindReactor(reactor); + finish_ops_.RecvMessage(response); + finish_ops_.AllowNoMessage(); + } + + ClientContext* context_; + Call call_; + ::grpc::experimental::ClientWriteReactor<Request>* reactor_; + + CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_; + CallbackWithSuccessTag start_tag_; + bool start_corked_; + + CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_; + CallbackWithSuccessTag finish_tag_; + Status finish_status_; + + CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> + write_ops_; + CallbackWithSuccessTag write_tag_; + bool write_ops_at_start_{false}; + + CallOpSet<CallOpSendInitialMetadata, CallOpClientSendClose> writes_done_ops_; + CallbackWithSuccessTag writes_done_tag_; + bool writes_done_ops_at_start_{false}; + + // Minimum of 2 outstanding callbacks to pre-register for start and finish + std::atomic_int callbacks_outstanding_{2}; + bool started_{false}; +}; + +template <class Request> +class ClientCallbackWriterFactory { + public: + template <class Response> + static void Create( + ChannelInterface* channel, const ::grpc::internal::RpcMethod& method, + ClientContext* context, Response* response, + ::grpc::experimental::ClientWriteReactor<Request>* reactor) { + Call call = channel->CreateCall(method, context, channel->CallbackCQ()); + + g_core_codegen_interface->grpc_call_ref(call.call()); + new (g_core_codegen_interface->grpc_call_arena_alloc( + call.call(), sizeof(ClientCallbackWriterImpl<Request>))) + ClientCallbackWriterImpl<Request>(call, context, response, reactor); + } +}; } // namespace internal } // namespace grpc diff --git a/include/grpcpp/impl/codegen/client_context.h b/include/grpcpp/impl/codegen/client_context.h index 75b955e760..6059c3c58a 100644 --- a/include/grpcpp/impl/codegen/client_context.h +++ b/include/grpcpp/impl/codegen/client_context.h @@ -71,6 +71,12 @@ template <class InputMessage, class OutputMessage> class BlockingUnaryCallImpl; template <class InputMessage, class OutputMessage> class CallbackUnaryCallImpl; +template <class Request, class Response> +class ClientCallbackReaderWriterImpl; +template <class Response> +class ClientCallbackReaderImpl; +template <class Request> +class ClientCallbackWriterImpl; } // namespace internal template <class R> @@ -394,6 +400,12 @@ class ClientContext { friend class ::grpc::internal::BlockingUnaryCallImpl; template <class InputMessage, class OutputMessage> friend class ::grpc::internal::CallbackUnaryCallImpl; + template <class Request, class Response> + friend class ::grpc::internal::ClientCallbackReaderWriterImpl; + template <class Response> + friend class ::grpc::internal::ClientCallbackReaderImpl; + template <class Request> + friend class ::grpc::internal::ClientCallbackWriterImpl; // Used by friend class CallOpClientRecvStatus void set_debug_error_string(const grpc::string& debug_error_string) { diff --git a/src/compiler/cpp_generator.cc b/src/compiler/cpp_generator.cc index 7986aca696..a368b47f01 100644 --- a/src/compiler/cpp_generator.cc +++ b/src/compiler/cpp_generator.cc @@ -132,6 +132,7 @@ grpc::string GetHeaderIncludes(grpc_generator::File* file, "grpcpp/impl/codegen/async_generic_service.h", "grpcpp/impl/codegen/async_stream.h", "grpcpp/impl/codegen/async_unary_call.h", + "grpcpp/impl/codegen/client_callback.h", "grpcpp/impl/codegen/method_handler_impl.h", "grpcpp/impl/codegen/proto_utils.h", "grpcpp/impl/codegen/rpc_method.h", @@ -580,11 +581,22 @@ void PrintHeaderClientMethodCallbackInterfaces( "const $Request$* request, $Response$* response, " "std::function<void(::grpc::Status)>) = 0;\n"); } else if (ClientOnlyStreaming(method)) { - // TODO(vjpai): Add support for client-side streaming + printer->Print(*vars, + "virtual void $Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::experimental::ClientWriteReactor< $Request$>* " + "reactor) = 0;\n"); } else if (ServerOnlyStreaming(method)) { - // TODO(vjpai): Add support for server-side streaming + printer->Print(*vars, + "virtual void $Method$(::grpc::ClientContext* context, " + "$Request$* request, " + "::grpc::experimental::ClientReadReactor< $Response$>* " + "reactor) = 0;\n"); } else if (method->BidiStreaming()) { - // TODO(vjpai): Add support for bidi streaming + printer->Print(*vars, + "virtual void $Method$(::grpc::ClientContext* context, " + "::grpc::experimental::ClientBidiReactor< " + "$Request$,$Response$>* reactor) = 0;\n"); } } @@ -631,11 +643,23 @@ void PrintHeaderClientMethodCallback(grpc_generator::Printer* printer, "const $Request$* request, $Response$* response, " "std::function<void(::grpc::Status)>) override;\n"); } else if (ClientOnlyStreaming(method)) { - // TODO(vjpai): Add support for client-side streaming + printer->Print(*vars, + "void $Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::experimental::ClientWriteReactor< $Request$>* " + "reactor) override;\n"); } else if (ServerOnlyStreaming(method)) { - // TODO(vjpai): Add support for server-side streaming + printer->Print(*vars, + "void $Method$(::grpc::ClientContext* context, " + "$Request$* request, " + "::grpc::experimental::ClientReadReactor< $Response$>* " + "reactor) override;\n"); + } else if (method->BidiStreaming()) { - // TODO(vjpai): Add support for bidi streaming + printer->Print(*vars, + "void $Method$(::grpc::ClientContext* context, " + "::grpc::experimental::ClientBidiReactor< " + "$Request$,$Response$>* reactor) override;\n"); } } @@ -1607,7 +1631,19 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "context, response);\n" "}\n\n"); - // TODO(vjpai): Add callback version + printer->Print( + *vars, + "void $ns$$Service$::" + "Stub::experimental_async::$Method$(::grpc::ClientContext* context, " + "$Response$* response, " + "::grpc::experimental::ClientWriteReactor< $Request$>* reactor) {\n"); + printer->Print(*vars, + " ::grpc::internal::ClientCallbackWriterFactory< " + "$Request$>::Create(" + "stub_->channel_.get(), " + "stub_->rpcmethod_$Method$_, " + "context, response, reactor);\n" + "}\n\n"); for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; @@ -1641,7 +1677,19 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "context, request);\n" "}\n\n"); - // TODO(vjpai): Add callback version + printer->Print( + *vars, + "void $ns$$Service$::Stub::experimental_async::$Method$(::grpc::" + "ClientContext* context, " + "$Request$* request, " + "::grpc::experimental::ClientReadReactor< $Response$>* reactor) {\n"); + printer->Print(*vars, + " ::grpc::internal::ClientCallbackReaderFactory< " + "$Response$>::Create(" + "stub_->channel_.get(), " + "stub_->rpcmethod_$Method$_, " + "context, request, reactor);\n" + "}\n\n"); for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; @@ -1675,7 +1723,19 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer, "context);\n" "}\n\n"); - // TODO(vjpai): Add callback version + printer->Print( + *vars, + "void $ns$$Service$::Stub::experimental_async::$Method$(::grpc::" + "ClientContext* context, " + "::grpc::experimental::ClientBidiReactor< $Request$,$Response$>* " + "reactor) {\n"); + printer->Print(*vars, + " ::grpc::internal::ClientCallbackReaderWriterFactory< " + "$Request$,$Response$>::Create(" + "stub_->channel_.get(), " + "stub_->rpcmethod_$Method$_, " + "context, reactor);\n" + "}\n\n"); for (auto async_prefix : async_prefixes) { (*vars)["AsyncPrefix"] = async_prefix.prefix; diff --git a/src/core/ext/transport/chttp2/transport/context_list.cc b/src/core/ext/transport/chttp2/transport/context_list.cc index 4acd0c9583..f30d41c332 100644 --- a/src/core/ext/transport/chttp2/transport/context_list.cc +++ b/src/core/ext/transport/chttp2/transport/context_list.cc @@ -32,6 +32,7 @@ void ContextList::Execute(void* arg, grpc_core::Timestamps* ts, while (head != nullptr) { if (error == GRPC_ERROR_NONE && ts != nullptr) { if (write_timestamps_callback_g) { + ts->byte_offset = static_cast<uint32_t>(head->byte_offset_); write_timestamps_callback_g(head->s_->context, ts); } } diff --git a/src/core/ext/transport/chttp2/transport/context_list.h b/src/core/ext/transport/chttp2/transport/context_list.h index 68d11e94d8..d870107749 100644 --- a/src/core/ext/transport/chttp2/transport/context_list.h +++ b/src/core/ext/transport/chttp2/transport/context_list.h @@ -50,6 +50,7 @@ class ContextList { /* Create a new element in the list and add it at the front */ ContextList* elem = grpc_core::New<ContextList>(); elem->s_ = s; + elem->byte_offset_ = s->byte_counter; elem->next_ = *head; *head = elem; } @@ -61,6 +62,7 @@ class ContextList { private: grpc_chttp2_stream* s_ = nullptr; ContextList* next_ = nullptr; + size_t byte_offset_ = 0; }; void grpc_http2_set_write_timestamps_callback( diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index aeaa4935ad..6aa68f5d4a 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -646,6 +646,8 @@ struct grpc_chttp2_stream { bool traced = false; /** gRPC header bytes that are already decompressed */ size_t decompressed_header_bytes = 0; + /** Byte counter for number of bytes written */ + size_t byte_counter = 0; }; /** Transport writing call flow: diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index 77320b496f..265d3365d3 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -363,6 +363,7 @@ class DataSendContext { grpc_chttp2_encode_data(s_->id, &s_->compressed_data_buffer, send_bytes, is_last_frame_, &s_->stats.outgoing, &t_->outbuf); s_->flow_control->SentData(send_bytes); + s_->byte_counter += send_bytes; if (s_->compressed_data_buffer.length == 0) { s_->sending_bytes += s_->uncompressed_data_size; } @@ -488,9 +489,6 @@ class StreamWriteContext { return; // early out: nothing to do } - if (s_->traced && grpc_endpoint_can_track_err(t_->ep)) { - grpc_core::ContextList::Append(&t_->cl, s_); - } while ((s_->flow_controlled_buffer.length > 0 || s_->compressed_data_buffer.length > 0) && data_send_context.max_outgoing() > 0) { @@ -500,6 +498,9 @@ class StreamWriteContext { data_send_context.CompressMoreBytes(); } } + if (s_->traced && grpc_endpoint_can_track_err(t_->ep)) { + grpc_core::ContextList::Append(&t_->cl, s_); + } write_context_->ResetPingClock(); if (data_send_context.is_last_frame()) { SentLastFrame(); diff --git a/src/core/lib/iomgr/buffer_list.cc b/src/core/lib/iomgr/buffer_list.cc index e20dab15b1..ace17a108d 100644 --- a/src/core/lib/iomgr/buffer_list.cc +++ b/src/core/lib/iomgr/buffer_list.cc @@ -35,6 +35,9 @@ void TracedBuffer::AddNewEntry(TracedBuffer** head, uint32_t seq_no, TracedBuffer* new_elem = New<TracedBuffer>(seq_no, arg); /* Store the current time as the sendmsg time. */ new_elem->ts_.sendmsg_time = gpr_now(GPR_CLOCK_REALTIME); + new_elem->ts_.scheduled_time = gpr_inf_past(GPR_CLOCK_REALTIME); + new_elem->ts_.sent_time = gpr_inf_past(GPR_CLOCK_REALTIME); + new_elem->ts_.acked_time = gpr_inf_past(GPR_CLOCK_REALTIME); if (*head == nullptr) { *head = new_elem; return; diff --git a/src/core/lib/iomgr/buffer_list.h b/src/core/lib/iomgr/buffer_list.h index 87d74f9ce2..627f1bde99 100644 --- a/src/core/lib/iomgr/buffer_list.h +++ b/src/core/lib/iomgr/buffer_list.h @@ -37,6 +37,8 @@ struct Timestamps { gpr_timespec scheduled_time; gpr_timespec sent_time; gpr_timespec acked_time; + + uint32_t byte_offset; /* byte offset relative to the start of the RPC */ }; /** TracedBuffer is a class to keep track of timestamps for a specific buffer in @@ -73,7 +75,7 @@ class TracedBuffer { private: GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_NEW - TracedBuffer(int seq_no, void* arg) + TracedBuffer(uint32_t seq_no, void* arg) : seq_no_(seq_no), arg_(arg), next_(nullptr) {} uint32_t seq_no_; /* The sequence number for the last byte in the buffer */ diff --git a/src/core/lib/iomgr/fork_posix.cc b/src/core/lib/iomgr/fork_posix.cc index e957bad73d..05ecd2a49b 100644 --- a/src/core/lib/iomgr/fork_posix.cc +++ b/src/core/lib/iomgr/fork_posix.cc @@ -60,7 +60,7 @@ void grpc_prefork() { } if (strcmp(grpc_get_poll_strategy_name(), "epoll1") != 0 && strcmp(grpc_get_poll_strategy_name(), "poll") != 0) { - gpr_log(GPR_ERROR, + gpr_log(GPR_INFO, "Fork support is only compatible with the epoll1 and poll polling " "strategies"); } diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index 606bfce6e7..cfcb190d60 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -634,7 +634,7 @@ static bool tcp_write_with_timestamps(grpc_tcp* tcp, struct msghdr* msg, if (sending_length == static_cast<size_t>(length)) { gpr_mu_lock(&tcp->tb_mu); grpc_core::TracedBuffer::AddNewEntry( - &tcp->tb_head, static_cast<int>(tcp->bytes_counter + length), + &tcp->tb_head, static_cast<uint32_t>(tcp->bytes_counter + length), tcp->outgoing_buffer_arg); gpr_mu_unlock(&tcp->tb_mu); tcp->outgoing_buffer_arg = nullptr; @@ -686,11 +686,9 @@ struct cmsghdr* process_timestamp(grpc_tcp* tcp, msghdr* msg, } /** For linux platforms, reads the socket's error queue and processes error - * messages from the queue. Returns true if all the errors processed were - * timestamps. Returns false if any of the errors were not timestamps. For - * non-linux platforms, error processing is not used/enabled currently. + * messages from the queue. */ -static bool process_errors(grpc_tcp* tcp) { +static void process_errors(grpc_tcp* tcp) { while (true) { struct iovec iov; iov.iov_base = nullptr; @@ -719,10 +717,10 @@ static bool process_errors(grpc_tcp* tcp) { } while (r < 0 && saved_errno == EINTR); if (r == -1 && saved_errno == EAGAIN) { - return true; /* No more errors to process */ + return; /* No more errors to process */ } if (r == -1) { - return false; + return; } if (grpc_tcp_trace.enabled()) { if ((msg.msg_flags & MSG_CTRUNC) == 1) { @@ -732,8 +730,9 @@ static bool process_errors(grpc_tcp* tcp) { if (msg.msg_controllen == 0) { /* There was no control message found. It was probably spurious. */ - return true; + return; } + bool seen = false; for (auto cmsg = CMSG_FIRSTHDR(&msg); cmsg && cmsg->cmsg_len; cmsg = CMSG_NXTHDR(&msg, cmsg)) { if (cmsg->cmsg_level != SOL_SOCKET || @@ -745,9 +744,13 @@ static bool process_errors(grpc_tcp* tcp) { "unknown control message cmsg_level:%d cmsg_type:%d", cmsg->cmsg_level, cmsg->cmsg_type); } - return false; + return; } cmsg = process_timestamp(tcp, &msg, cmsg); + seen = true; + } + if (!seen) { + return; } } } diff --git a/src/cpp/client/generic_stub.cc b/src/cpp/client/generic_stub.cc index 87902b26f0..f61c1b5317 100644 --- a/src/cpp/client/generic_stub.cc +++ b/src/cpp/client/generic_stub.cc @@ -72,4 +72,13 @@ void GenericStub::experimental_type::UnaryCall( context, request, response, std::move(on_completion)); } +void GenericStub::experimental_type::PrepareBidiStreamingCall( + ClientContext* context, const grpc::string& method, + experimental::ClientBidiReactor<ByteBuffer, ByteBuffer>* reactor) { + internal::ClientCallbackReaderWriterFactory<ByteBuffer, ByteBuffer>::Create( + stub_->channel_.get(), + internal::RpcMethod(method.c_str(), internal::RpcMethod::BIDI_STREAMING), + context, reactor); +} + } // namespace grpc diff --git a/src/proto/grpc/channelz/BUILD b/src/proto/grpc/channelz/BUILD index bdb03d5e2d..b6b485e3e8 100644 --- a/src/proto/grpc/channelz/BUILD +++ b/src/proto/grpc/channelz/BUILD @@ -24,3 +24,10 @@ grpc_proto_library( has_services = True, well_known_protos = True, ) + +filegroup( + name = "channelz_proto_file", + srcs = [ + "channelz.proto", + ], +) diff --git a/src/proto/grpc/health/v1/BUILD b/src/proto/grpc/health/v1/BUILD index 97642985c9..38a7d992e0 100644 --- a/src/proto/grpc/health/v1/BUILD +++ b/src/proto/grpc/health/v1/BUILD @@ -29,4 +29,3 @@ filegroup( "health.proto", ], ) - diff --git a/src/proto/grpc/testing/compiler_test.proto b/src/proto/grpc/testing/compiler_test.proto index db5ca48331..9fa5590a59 100644 --- a/src/proto/grpc/testing/compiler_test.proto +++ b/src/proto/grpc/testing/compiler_test.proto @@ -20,6 +20,9 @@ syntax = "proto3"; // Ignored detached comment +// The comments in this file are not meant for readability +// but rather to test to make sure that the code generator +// properly preserves comments on files, services, and RPCs // Ignored package leading comment package grpc.testing; diff --git a/src/python/grpcio/grpc/_cython/BUILD.bazel b/src/python/grpcio/grpc/_cython/BUILD.bazel index cfd3a51d9b..e318298d0a 100644 --- a/src/python/grpcio/grpc/_cython/BUILD.bazel +++ b/src/python/grpcio/grpc/_cython/BUILD.bazel @@ -12,6 +12,7 @@ pyx_library( "_cygrpc/grpc_string.pyx.pxi", "_cygrpc/arguments.pyx.pxi", "_cygrpc/call.pyx.pxi", + "_cygrpc/channelz.pyx.pxi", "_cygrpc/channel.pyx.pxi", "_cygrpc/credentials.pyx.pxi", "_cygrpc/completion_queue.pyx.pxi", diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channelz.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channelz.pyx.pxi new file mode 100644 index 0000000000..113f7976dd --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channelz.pyx.pxi @@ -0,0 +1,69 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def channelz_get_top_channels(start_channel_id): + cdef char *c_returned_str = grpc_channelz_get_top_channels( + start_channel_id, + ) + if c_returned_str == NULL: + raise ValueError('Failed to get top channels, please ensure your' \ + ' start_channel_id==%s is valid' % start_channel_id) + return c_returned_str + +def channelz_get_servers(start_server_id): + cdef char *c_returned_str = grpc_channelz_get_servers(start_server_id) + if c_returned_str == NULL: + raise ValueError('Failed to get servers, please ensure your' \ + ' start_server_id==%s is valid' % start_server_id) + return c_returned_str + +def channelz_get_server(server_id): + cdef char *c_returned_str = grpc_channelz_get_server(server_id) + if c_returned_str == NULL: + raise ValueError('Failed to get the server, please ensure your' \ + ' server_id==%s is valid' % server_id) + return c_returned_str + +def channelz_get_server_sockets(server_id, start_socket_id): + cdef char *c_returned_str = grpc_channelz_get_server_sockets( + server_id, + start_socket_id, + ) + if c_returned_str == NULL: + raise ValueError('Failed to get server sockets, please ensure your' \ + ' server_id==%s and start_socket_id==%s is valid' % + (server_id, start_socket_id)) + return c_returned_str + +def channelz_get_channel(channel_id): + cdef char *c_returned_str = grpc_channelz_get_channel(channel_id) + if c_returned_str == NULL: + raise ValueError('Failed to get the channel, please ensure your' \ + ' channel_id==%s is valid' % (channel_id)) + return c_returned_str + +def channelz_get_subchannel(subchannel_id): + cdef char *c_returned_str = grpc_channelz_get_subchannel(subchannel_id) + if c_returned_str == NULL: + raise ValueError('Failed to get the subchannel, please ensure your' \ + ' subchannel_id==%s is valid' % (subchannel_id)) + return c_returned_str + +def channelz_get_socket(socket_id): + cdef char *c_returned_str = grpc_channelz_get_socket(socket_id) + if c_returned_str == NULL: + raise ValueError('Failed to get the socket, please ensure your' \ + ' socket_id==%s is valid' % (socket_id)) + return c_returned_str diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 5aaf31e36c..5bbc10af25 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -13,6 +13,7 @@ # limitations under the License. cimport libc.time +from libc.stdint cimport intptr_t # Typedef types with approximately the same semantics to provide their names to @@ -384,6 +385,15 @@ cdef extern from "grpc/grpc.h": void grpc_server_cancel_all_calls(grpc_server *server) nogil void grpc_server_destroy(grpc_server *server) nogil + char* grpc_channelz_get_top_channels(intptr_t start_channel_id) + char* grpc_channelz_get_servers(intptr_t start_server_id) + char* grpc_channelz_get_server(intptr_t server_id) + char* grpc_channelz_get_server_sockets(intptr_t server_id, + intptr_t start_socket_id) + char* grpc_channelz_get_channel(intptr_t channel_id) + char* grpc_channelz_get_subchannel(intptr_t subchannel_id) + char* grpc_channelz_get_socket(intptr_t socket_id) + cdef extern from "grpc/grpc_security.h": diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index ae5c07bfc8..9ab919375c 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -35,6 +35,7 @@ include "_cygrpc/server.pyx.pxi" include "_cygrpc/tag.pyx.pxi" include "_cygrpc/time.pyx.pxi" include "_cygrpc/_hooks.pyx.pxi" +include "_cygrpc/channelz.pyx.pxi" include "_cygrpc/grpc_gevent.pyx.pxi" diff --git a/src/python/grpcio_channelz/.gitignore b/src/python/grpcio_channelz/.gitignore new file mode 100644 index 0000000000..0c5da6b5af --- /dev/null +++ b/src/python/grpcio_channelz/.gitignore @@ -0,0 +1,6 @@ +*.proto +*_pb2.py +*_pb2_grpc.py +build/ +grpcio_channelz.egg-info/ +dist/ diff --git a/src/python/grpcio_channelz/MANIFEST.in b/src/python/grpcio_channelz/MANIFEST.in new file mode 100644 index 0000000000..5597f375ba --- /dev/null +++ b/src/python/grpcio_channelz/MANIFEST.in @@ -0,0 +1,3 @@ +include grpc_version.py +recursive-include grpc_channelz *.py +global-exclude *.pyc diff --git a/src/python/grpcio_channelz/README.rst b/src/python/grpcio_channelz/README.rst new file mode 100644 index 0000000000..efeaa56064 --- /dev/null +++ b/src/python/grpcio_channelz/README.rst @@ -0,0 +1,9 @@ +gRPC Python Channelz package +============================== + +Channelz is a live debug tool in gRPC Python. + +Dependencies +------------ + +Depends on the `grpcio` package, available from PyPI via `pip install grpcio`. diff --git a/src/python/grpcio_channelz/channelz_commands.py b/src/python/grpcio_channelz/channelz_commands.py new file mode 100644 index 0000000000..e9ad355034 --- /dev/null +++ b/src/python/grpcio_channelz/channelz_commands.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Provides distutils command classes for the GRPC Python setup process.""" + +import os +import shutil + +import setuptools + +ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))) +CHANNELZ_PROTO = os.path.join(ROOT_DIR, + '../../proto/grpc/channelz/channelz.proto') + + +class CopyProtoModules(setuptools.Command): + """Command to copy proto modules from grpc/src/proto.""" + + description = '' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + if os.path.isfile(CHANNELZ_PROTO): + shutil.copyfile(CHANNELZ_PROTO, + os.path.join(ROOT_DIR, + 'grpc_channelz/v1/channelz.proto')) + + +class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + + description = 'build grpc protobuf modules' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + # due to limitations of the proto generator, we require that only *one* + # directory is provided as an 'include' directory. We assume it's the '' key + # to `self.distribution.package_dir` (and get a key error if it's not + # there). + from grpc_tools import command + command.build_package_protos(self.distribution.package_dir['']) diff --git a/src/python/grpcio_channelz/grpc_channelz/__init__.py b/src/python/grpcio_channelz/grpc_channelz/__init__.py new file mode 100644 index 0000000000..38fdfc9c5c --- /dev/null +++ b/src/python/grpcio_channelz/grpc_channelz/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel new file mode 100644 index 0000000000..aae8cedb76 --- /dev/null +++ b/src/python/grpcio_channelz/grpc_channelz/v1/BUILD.bazel @@ -0,0 +1,38 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_library") + +package(default_visibility = ["//visibility:public"]) + +genrule( + name = "mv_channelz_proto", + srcs = [ + "//src/proto/grpc/channelz:channelz_proto_file", + ], + outs = ["channelz.proto",], + cmd = "cp $< $@", +) + +py_proto_library( + name = "py_channelz_proto", + protos = ["mv_channelz_proto",], + imports = [ + "external/com_google_protobuf/src/", + ], + inputs = [ + "@com_google_protobuf//:well_known_protos", + ], + with_grpc = True, + deps = [ + requirement('protobuf'), + ], +) + +py_library( + name = "grpc_channelz", + srcs = ["channelz.py",], + deps = [ + ":py_channelz_proto", + "//src/python/grpcio/grpc:grpcio", + ], + imports=["../../",], +) diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/__init__.py b/src/python/grpcio_channelz/grpc_channelz/v1/__init__.py new file mode 100644 index 0000000000..38fdfc9c5c --- /dev/null +++ b/src/python/grpcio_channelz/grpc_channelz/v1/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py new file mode 100644 index 0000000000..573b9d0d5a --- /dev/null +++ b/src/python/grpcio_channelz/grpc_channelz/v1/channelz.py @@ -0,0 +1,141 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Channelz debug service implementation in gRPC Python.""" + +import grpc +from grpc._cython import cygrpc + +import grpc_channelz.v1.channelz_pb2 as _channelz_pb2 +import grpc_channelz.v1.channelz_pb2_grpc as _channelz_pb2_grpc + +from google.protobuf import json_format + + +class ChannelzServicer(_channelz_pb2_grpc.ChannelzServicer): + """Servicer handling RPCs for service statuses.""" + + @staticmethod + def GetTopChannels(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_top_channels(request.start_channel_id), + _channelz_pb2.GetTopChannelsResponse(), + ) + except (ValueError, json_format.ParseError) as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetServers(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_servers(request.start_server_id), + _channelz_pb2.GetServersResponse(), + ) + except (ValueError, json_format.ParseError) as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetServer(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_server(request.server_id), + _channelz_pb2.GetServerResponse(), + ) + except ValueError as e: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details(str(e)) + except json_format.ParseError as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetServerSockets(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_server_sockets(request.server_id, + request.start_socket_id), + _channelz_pb2.GetServerSocketsResponse(), + ) + except ValueError as e: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details(str(e)) + except json_format.ParseError as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetChannel(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_channel(request.channel_id), + _channelz_pb2.GetChannelResponse(), + ) + except ValueError as e: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details(str(e)) + except json_format.ParseError as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetSubchannel(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_subchannel(request.subchannel_id), + _channelz_pb2.GetSubchannelResponse(), + ) + except ValueError as e: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details(str(e)) + except json_format.ParseError as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + @staticmethod + def GetSocket(request, context): + try: + return json_format.Parse( + cygrpc.channelz_get_socket(request.socket_id), + _channelz_pb2.GetSocketResponse(), + ) + except ValueError as e: + context.set_code(grpc.StatusCode.NOT_FOUND) + context.set_details(str(e)) + except json_format.ParseError as e: + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + + +def add_channelz_servicer(server): + """Add Channelz servicer to a server. Channelz servicer is in charge of + pulling information from C-Core for entire process. It will allow the + server to response to Channelz queries. + + The Channelz statistic is enabled by default inside C-Core. Whether the + statistic is enabled or not is isolated from adding Channelz servicer. + That means you can query Channelz info with a Channelz-disabled channel, + and you can add Channelz servicer to a Channelz-disabled server. + + The Channelz statistic can be enabled or disabled by channel option + 'grpc.enable_channelz'. Set to 1 to enable, set to 0 to disable. + + This is an EXPERIMENTAL API. + + Args: + server: grpc.Server to which Channelz service will be added. + """ + _channelz_pb2_grpc.add_ChannelzServicer_to_server(ChannelzServicer(), + server) diff --git a/src/python/grpcio_channelz/grpc_version.py b/src/python/grpcio_channelz/grpc_version.py new file mode 100644 index 0000000000..16356ea402 --- /dev/null +++ b/src/python/grpcio_channelz/grpc_version.py @@ -0,0 +1,17 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! + +VERSION = '1.18.0.dev0' diff --git a/src/python/grpcio_channelz/setup.py b/src/python/grpcio_channelz/setup.py new file mode 100644 index 0000000000..a495052376 --- /dev/null +++ b/src/python/grpcio_channelz/setup.py @@ -0,0 +1,96 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Setup module for the GRPC Python package's Channelz.""" + +import os +import sys + +import setuptools + +# Ensure we're in the proper directory whether or not we're being used by pip. +os.chdir(os.path.dirname(os.path.abspath(__file__))) + +# Break import-style to ensure we can actually find our local modules. +import grpc_version + + +class _NoOpCommand(setuptools.Command): + """No-op command.""" + + description = '' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + pass + + +CLASSIFIERS = [ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: Apache Software License', +] + +PACKAGE_DIRECTORIES = { + '': '.', +} + +INSTALL_REQUIRES = ( + 'protobuf>=3.6.0', + 'grpcio>={version}'.format(version=grpc_version.VERSION), +) + +try: + import channelz_commands as _channelz_commands + # we are in the build environment, otherwise the above import fails + SETUP_REQUIRES = ( + 'grpcio-tools=={version}'.format(version=grpc_version.VERSION),) + COMMAND_CLASS = { + # Run preprocess from the repository *before* doing any packaging! + 'preprocess': _channelz_commands.CopyProtoModules, + 'build_package_protos': _channelz_commands.BuildPackageProtos, + } +except ImportError: + SETUP_REQUIRES = () + COMMAND_CLASS = { + # wire up commands to no-op not to break the external dependencies + 'preprocess': _NoOpCommand, + 'build_package_protos': _NoOpCommand, + } + +setuptools.setup( + name='grpcio-channelz', + version=grpc_version.VERSION, + license='Apache License 2.0', + description='Channel Level Live Debug Information Service for gRPC', + author='The gRPC Authors', + author_email='grpc-io@googlegroups.com', + classifiers=CLASSIFIERS, + url='https://grpc.io', + package_dir=PACKAGE_DIRECTORIES, + packages=setuptools.find_packages('.'), + install_requires=INSTALL_REQUIRES, + setup_requires=SETUP_REQUIRES, + cmdclass=COMMAND_CLASS) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index d163f6fb68..65e9a99950 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -132,7 +132,12 @@ class TestGevent(setuptools.Command): 'unit.beta._beta_features_test', # TODO(https://github.com/grpc/grpc/issues/15411) unpin gevent version # This test will stuck while running higher version of gevent - 'unit._auth_context_test.AuthContextTest.testSessionResumption') + 'unit._auth_context_test.AuthContextTest.testSessionResumption', + # TODO(https://github.com/grpc/grpc/issues/17330) enable these three tests + 'channelz._channelz_servicer_test.ChannelzServicerTest.test_many_subchannels', + 'channelz._channelz_servicer_test.ChannelzServicerTest.test_many_subchannels_and_sockets', + 'channelz._channelz_servicer_test.ChannelzServicerTest.test_streaming_rpc' + ) description = 'run tests with gevent. Assumes grpc/gevent are installed' user_options = [] diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 61c98fa038..f56425ac6d 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -39,6 +39,7 @@ PACKAGE_DIRECTORIES = { INSTALL_REQUIRES = ( 'coverage>=4.0', 'enum34>=1.0.4', 'grpcio>={version}'.format(version=grpc_version.VERSION), + 'grpcio-channelz>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', diff --git a/src/python/grpcio_tests/tests/channelz/BUILD.bazel b/src/python/grpcio_tests/tests/channelz/BUILD.bazel new file mode 100644 index 0000000000..63513616e7 --- /dev/null +++ b/src/python/grpcio_tests/tests/channelz/BUILD.bazel @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +py_test( + name = "channelz_servicer_test", + srcs = ["_channelz_servicer_test.py"], + main = "_channelz_servicer_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_channelz/grpc_channelz/v1:grpc_channelz", + "//src/python/grpcio_tests/tests/unit:test_common", + "//src/python/grpcio_tests/tests/unit/framework/common:common", + ], + imports = ["../../",], +) diff --git a/src/python/grpcio_tests/tests/channelz/__init__.py b/src/python/grpcio_tests/tests/channelz/__init__.py new file mode 100644 index 0000000000..38fdfc9c5c --- /dev/null +++ b/src/python/grpcio_tests/tests/channelz/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py new file mode 100644 index 0000000000..84f8594689 --- /dev/null +++ b/src/python/grpcio_tests/tests/channelz/_channelz_servicer_test.py @@ -0,0 +1,494 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests of grpc_channelz.v1.channelz.""" + +import unittest + +from concurrent import futures + +import grpc +from grpc_channelz.v1 import channelz +from grpc_channelz.v1 import channelz_pb2 +from grpc_channelz.v1 import channelz_pb2_grpc + +from tests.unit import test_common +from tests.unit.framework.common import test_constants + +_SUCCESSFUL_UNARY_UNARY = '/test/SuccessfulUnaryUnary' +_FAILED_UNARY_UNARY = '/test/FailedUnaryUnary' +_SUCCESSFUL_STREAM_STREAM = '/test/SuccessfulStreamStream' + +_REQUEST = b'\x00\x00\x00' +_RESPONSE = b'\x01\x01\x01' + +_DISABLE_REUSE_PORT = (('grpc.so_reuseport', 0),) +_ENABLE_CHANNELZ = (('grpc.enable_channelz', 1),) +_DISABLE_CHANNELZ = (('grpc.enable_channelz', 0),) + + +def _successful_unary_unary(request, servicer_context): + return _RESPONSE + + +def _failed_unary_unary(request, servicer_context): + servicer_context.set_code(grpc.StatusCode.INTERNAL) + servicer_context.set_details("Channelz Test Intended Failure") + + +def _successful_stream_stream(request_iterator, servicer_context): + for _ in request_iterator: + yield _RESPONSE + + +class _GenericHandler(grpc.GenericRpcHandler): + + def service(self, handler_call_details): + if handler_call_details.method == _SUCCESSFUL_UNARY_UNARY: + return grpc.unary_unary_rpc_method_handler(_successful_unary_unary) + elif handler_call_details.method == _FAILED_UNARY_UNARY: + return grpc.unary_unary_rpc_method_handler(_failed_unary_unary) + elif handler_call_details.method == _SUCCESSFUL_STREAM_STREAM: + return grpc.stream_stream_rpc_method_handler( + _successful_stream_stream) + else: + return None + + +class _ChannelServerPair(object): + + def __init__(self): + # Server will enable channelz service + # Bind as attribute, so its `del` can be called explicitly, during + # the destruction process. Otherwise, if the removal of server + # rely on gc cycle, the test will become non-deterministic. + self._server = grpc.server( + futures.ThreadPoolExecutor(max_workers=3), + options=_DISABLE_REUSE_PORT + _ENABLE_CHANNELZ) + port = self._server.add_insecure_port('[::]:0') + self._server.add_generic_rpc_handlers((_GenericHandler(),)) + self._server.start() + + # Channel will enable channelz service... + self.channel = grpc.insecure_channel('localhost:%d' % port, + _ENABLE_CHANNELZ) + + def __del__(self): + self._server.__del__() + self.channel.close() + + +def _generate_channel_server_pairs(n): + return [_ChannelServerPair() for i in range(n)] + + +def _clean_channel_server_pairs(pairs): + for pair in pairs: + pair.__del__() + + +class ChannelzServicerTest(unittest.TestCase): + + def _send_successful_unary_unary(self, idx): + _, r = self._pairs[idx].channel.unary_unary( + _SUCCESSFUL_UNARY_UNARY).with_call(_REQUEST) + self.assertEqual(r.code(), grpc.StatusCode.OK) + + def _send_failed_unary_unary(self, idx): + try: + self._pairs[idx].channel.unary_unary(_FAILED_UNARY_UNARY).with_call( + _REQUEST) + except grpc.RpcError: + return + else: + self.fail("This call supposed to fail") + + def _send_successful_stream_stream(self, idx): + response_iterator = self._pairs[idx].channel.stream_stream( + _SUCCESSFUL_STREAM_STREAM).__call__( + iter([_REQUEST] * test_constants.STREAM_LENGTH)) + cnt = 0 + for _ in response_iterator: + cnt += 1 + self.assertEqual(cnt, test_constants.STREAM_LENGTH) + + def _get_channel_id(self, idx): + """Channel id may not be consecutive""" + resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=0)) + self.assertGreater(len(resp.channel), idx) + return resp.channel[idx].ref.channel_id + + def setUp(self): + self._pairs = [] + # This server is for Channelz info fetching only + # It self should not enable Channelz + self._server = grpc.server( + futures.ThreadPoolExecutor(max_workers=3), + options=_DISABLE_REUSE_PORT + _DISABLE_CHANNELZ) + port = self._server.add_insecure_port('[::]:0') + channelz.add_channelz_servicer(self._server) + self._server.start() + + # This channel is used to fetch Channelz info only + # Channelz should not be enabled + self._channel = grpc.insecure_channel('localhost:%d' % port, + _DISABLE_CHANNELZ) + self._channelz_stub = channelz_pb2_grpc.ChannelzStub(self._channel) + + def tearDown(self): + self._server.__del__() + self._channel.close() + _clean_channel_server_pairs(self._pairs) + + def test_get_top_channels_basic(self): + self._pairs = _generate_channel_server_pairs(1) + resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=0)) + self.assertEqual(len(resp.channel), 1) + self.assertEqual(resp.end, True) + + def test_get_top_channels_high_start_id(self): + self._pairs = _generate_channel_server_pairs(1) + resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=10000)) + self.assertEqual(len(resp.channel), 0) + self.assertEqual(resp.end, True) + + def test_successful_request(self): + self._pairs = _generate_channel_server_pairs(1) + self._send_successful_unary_unary(0) + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(0))) + self.assertEqual(resp.channel.data.calls_started, 1) + self.assertEqual(resp.channel.data.calls_succeeded, 1) + self.assertEqual(resp.channel.data.calls_failed, 0) + + def test_failed_request(self): + self._pairs = _generate_channel_server_pairs(1) + self._send_failed_unary_unary(0) + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(0))) + self.assertEqual(resp.channel.data.calls_started, 1) + self.assertEqual(resp.channel.data.calls_succeeded, 0) + self.assertEqual(resp.channel.data.calls_failed, 1) + + def test_many_requests(self): + self._pairs = _generate_channel_server_pairs(1) + k_success = 7 + k_failed = 9 + for i in range(k_success): + self._send_successful_unary_unary(0) + for i in range(k_failed): + self._send_failed_unary_unary(0) + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(0))) + self.assertEqual(resp.channel.data.calls_started, k_success + k_failed) + self.assertEqual(resp.channel.data.calls_succeeded, k_success) + self.assertEqual(resp.channel.data.calls_failed, k_failed) + + def test_many_channel(self): + k_channels = 4 + self._pairs = _generate_channel_server_pairs(k_channels) + resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=0)) + self.assertEqual(len(resp.channel), k_channels) + + def test_many_requests_many_channel(self): + k_channels = 4 + self._pairs = _generate_channel_server_pairs(k_channels) + k_success = 11 + k_failed = 13 + for i in range(k_success): + self._send_successful_unary_unary(0) + self._send_successful_unary_unary(2) + for i in range(k_failed): + self._send_failed_unary_unary(1) + self._send_failed_unary_unary(2) + + # The first channel saw only successes + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(0))) + self.assertEqual(resp.channel.data.calls_started, k_success) + self.assertEqual(resp.channel.data.calls_succeeded, k_success) + self.assertEqual(resp.channel.data.calls_failed, 0) + + # The second channel saw only failures + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(1))) + self.assertEqual(resp.channel.data.calls_started, k_failed) + self.assertEqual(resp.channel.data.calls_succeeded, 0) + self.assertEqual(resp.channel.data.calls_failed, k_failed) + + # The third channel saw both successes and failures + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(2))) + self.assertEqual(resp.channel.data.calls_started, k_success + k_failed) + self.assertEqual(resp.channel.data.calls_succeeded, k_success) + self.assertEqual(resp.channel.data.calls_failed, k_failed) + + # The fourth channel saw nothing + resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(3))) + self.assertEqual(resp.channel.data.calls_started, 0) + self.assertEqual(resp.channel.data.calls_succeeded, 0) + self.assertEqual(resp.channel.data.calls_failed, 0) + + def test_many_subchannels(self): + k_channels = 4 + self._pairs = _generate_channel_server_pairs(k_channels) + k_success = 17 + k_failed = 19 + for i in range(k_success): + self._send_successful_unary_unary(0) + self._send_successful_unary_unary(2) + for i in range(k_failed): + self._send_failed_unary_unary(1) + self._send_failed_unary_unary(2) + + gtc_resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=0)) + self.assertEqual(len(gtc_resp.channel), k_channels) + for i in range(k_channels): + # If no call performed in the channel, there shouldn't be any subchannel + if gtc_resp.channel[i].data.calls_started == 0: + self.assertEqual(len(gtc_resp.channel[i].subchannel_ref), 0) + continue + + # Otherwise, the subchannel should exist + self.assertGreater(len(gtc_resp.channel[i].subchannel_ref), 0) + gsc_resp = self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest( + subchannel_id=gtc_resp.channel[i].subchannel_ref[ + 0].subchannel_id)) + self.assertEqual(gtc_resp.channel[i].data.calls_started, + gsc_resp.subchannel.data.calls_started) + self.assertEqual(gtc_resp.channel[i].data.calls_succeeded, + gsc_resp.subchannel.data.calls_succeeded) + self.assertEqual(gtc_resp.channel[i].data.calls_failed, + gsc_resp.subchannel.data.calls_failed) + + @unittest.skip('Servers in core are not guaranteed to be destroyed ' \ + 'immediately when the reference goes out of scope, so ' \ + 'servers from multiple test cases are not hermetic. ' \ + 'TODO(https://github.com/grpc/grpc/issues/17258)') + def test_server_basic(self): + self._pairs = _generate_channel_server_pairs(1) + resp = self._channelz_stub.GetServers( + channelz_pb2.GetServersRequest(start_server_id=0)) + self.assertEqual(len(resp.server), 1) + + @unittest.skip('Servers in core are not guaranteed to be destroyed ' \ + 'immediately when the reference goes out of scope, so ' \ + 'servers from multiple test cases are not hermetic. ' \ + 'TODO(https://github.com/grpc/grpc/issues/17258)') + def test_get_one_server(self): + self._pairs = _generate_channel_server_pairs(1) + gss_resp = self._channelz_stub.GetServers( + channelz_pb2.GetServersRequest(start_server_id=0)) + self.assertEqual(len(gss_resp.server), 1) + gs_resp = self._channelz_stub.GetServer( + channelz_pb2.GetServerRequest( + server_id=gss_resp.server[0].ref.server_id)) + self.assertEqual(gss_resp.server[0].ref.server_id, + gs_resp.server.ref.server_id) + + @unittest.skip('Servers in core are not guaranteed to be destroyed ' \ + 'immediately when the reference goes out of scope, so ' \ + 'servers from multiple test cases are not hermetic. ' \ + 'TODO(https://github.com/grpc/grpc/issues/17258)') + def test_server_call(self): + self._pairs = _generate_channel_server_pairs(1) + k_success = 23 + k_failed = 29 + for i in range(k_success): + self._send_successful_unary_unary(0) + for i in range(k_failed): + self._send_failed_unary_unary(0) + + resp = self._channelz_stub.GetServers( + channelz_pb2.GetServersRequest(start_server_id=0)) + self.assertEqual(len(resp.server), 1) + self.assertEqual(resp.server[0].data.calls_started, + k_success + k_failed) + self.assertEqual(resp.server[0].data.calls_succeeded, k_success) + self.assertEqual(resp.server[0].data.calls_failed, k_failed) + + def test_many_subchannels_and_sockets(self): + k_channels = 4 + self._pairs = _generate_channel_server_pairs(k_channels) + k_success = 3 + k_failed = 5 + for i in range(k_success): + self._send_successful_unary_unary(0) + self._send_successful_unary_unary(2) + for i in range(k_failed): + self._send_failed_unary_unary(1) + self._send_failed_unary_unary(2) + + gtc_resp = self._channelz_stub.GetTopChannels( + channelz_pb2.GetTopChannelsRequest(start_channel_id=0)) + self.assertEqual(len(gtc_resp.channel), k_channels) + for i in range(k_channels): + # If no call performed in the channel, there shouldn't be any subchannel + if gtc_resp.channel[i].data.calls_started == 0: + self.assertEqual(len(gtc_resp.channel[i].subchannel_ref), 0) + continue + + # Otherwise, the subchannel should exist + self.assertGreater(len(gtc_resp.channel[i].subchannel_ref), 0) + gsc_resp = self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest( + subchannel_id=gtc_resp.channel[i].subchannel_ref[ + 0].subchannel_id)) + self.assertEqual(len(gsc_resp.subchannel.socket_ref), 1) + + gs_resp = self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest( + socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + self.assertEqual(gsc_resp.subchannel.data.calls_started, + gs_resp.socket.data.streams_started) + self.assertEqual(gsc_resp.subchannel.data.calls_started, + gs_resp.socket.data.streams_succeeded) + # Calls started == messages sent, only valid for unary calls + self.assertEqual(gsc_resp.subchannel.data.calls_started, + gs_resp.socket.data.messages_sent) + # Only receive responses when the RPC was successful + self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, + gs_resp.socket.data.messages_received) + + def test_streaming_rpc(self): + self._pairs = _generate_channel_server_pairs(1) + # In C++, the argument for _send_successful_stream_stream is message length. + # Here the argument is still channel idx, to be consistent with the other two. + self._send_successful_stream_stream(0) + + gc_resp = self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=self._get_channel_id(0))) + self.assertEqual(gc_resp.channel.data.calls_started, 1) + self.assertEqual(gc_resp.channel.data.calls_succeeded, 1) + self.assertEqual(gc_resp.channel.data.calls_failed, 0) + # Subchannel exists + self.assertGreater(len(gc_resp.channel.subchannel_ref), 0) + + gsc_resp = self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest( + subchannel_id=gc_resp.channel.subchannel_ref[0].subchannel_id)) + self.assertEqual(gsc_resp.subchannel.data.calls_started, 1) + self.assertEqual(gsc_resp.subchannel.data.calls_succeeded, 1) + self.assertEqual(gsc_resp.subchannel.data.calls_failed, 0) + # Socket exists + self.assertEqual(len(gsc_resp.subchannel.socket_ref), 1) + + gs_resp = self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest( + socket_id=gsc_resp.subchannel.socket_ref[0].socket_id)) + self.assertEqual(gs_resp.socket.data.streams_started, 1) + self.assertEqual(gs_resp.socket.data.streams_succeeded, 1) + self.assertEqual(gs_resp.socket.data.streams_failed, 0) + self.assertEqual(gs_resp.socket.data.messages_sent, + test_constants.STREAM_LENGTH) + self.assertEqual(gs_resp.socket.data.messages_received, + test_constants.STREAM_LENGTH) + + @unittest.skip('Servers in core are not guaranteed to be destroyed ' \ + 'immediately when the reference goes out of scope, so ' \ + 'servers from multiple test cases are not hermetic. ' \ + 'TODO(https://github.com/grpc/grpc/issues/17258)') + def test_server_sockets(self): + self._pairs = _generate_channel_server_pairs(1) + self._send_successful_unary_unary(0) + self._send_failed_unary_unary(0) + + gs_resp = self._channelz_stub.GetServers( + channelz_pb2.GetServersRequest(start_server_id=0)) + self.assertEqual(len(gs_resp.server), 1) + self.assertEqual(gs_resp.server[0].data.calls_started, 2) + self.assertEqual(gs_resp.server[0].data.calls_succeeded, 1) + self.assertEqual(gs_resp.server[0].data.calls_failed, 1) + + gss_resp = self._channelz_stub.GetServerSockets( + channelz_pb2.GetServerSocketsRequest( + server_id=gs_resp.server[0].ref.server_id, start_socket_id=0)) + # If the RPC call failed, it will raise a grpc.RpcError + # So, if there is no exception raised, considered pass + + @unittest.skip('Servers in core are not guaranteed to be destroyed ' \ + 'immediately when the reference goes out of scope, so ' \ + 'servers from multiple test cases are not hermetic. ' \ + 'TODO(https://github.com/grpc/grpc/issues/17258)') + def test_server_listen_sockets(self): + self._pairs = _generate_channel_server_pairs(1) + + gss_resp = self._channelz_stub.GetServers( + channelz_pb2.GetServersRequest(start_server_id=0)) + self.assertEqual(len(gss_resp.server), 1) + self.assertEqual(len(gss_resp.server[0].listen_socket), 1) + + gs_resp = self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest( + socket_id=gss_resp.server[0].listen_socket[0].socket_id)) + # If the RPC call failed, it will raise a grpc.RpcError + # So, if there is no exception raised, considered pass + + def test_invalid_query_get_server(self): + try: + self._channelz_stub.GetServer( + channelz_pb2.GetServerRequest(server_id=10000)) + except BaseException as e: + self.assertIn('StatusCode.NOT_FOUND', str(e)) + else: + self.fail('Invalid query not detected') + + def test_invalid_query_get_channel(self): + try: + self._channelz_stub.GetChannel( + channelz_pb2.GetChannelRequest(channel_id=10000)) + except BaseException as e: + self.assertIn('StatusCode.NOT_FOUND', str(e)) + else: + self.fail('Invalid query not detected') + + def test_invalid_query_get_subchannel(self): + try: + self._channelz_stub.GetSubchannel( + channelz_pb2.GetSubchannelRequest(subchannel_id=10000)) + except BaseException as e: + self.assertIn('StatusCode.NOT_FOUND', str(e)) + else: + self.fail('Invalid query not detected') + + def test_invalid_query_get_socket(self): + try: + self._channelz_stub.GetSocket( + channelz_pb2.GetSocketRequest(socket_id=10000)) + except BaseException as e: + self.assertIn('StatusCode.NOT_FOUND', str(e)) + else: + self.fail('Invalid query not detected') + + def test_invalid_query_get_server_sockets(self): + try: + self._channelz_stub.GetServerSockets( + channelz_pb2.GetServerSocketsRequest( + server_id=10000, + start_socket_id=0, + )) + except BaseException as e: + self.assertIn('StatusCode.NOT_FOUND', str(e)) + else: + self.fail('Invalid query not detected') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index a3006d9afc..9cffd3df19 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -1,5 +1,6 @@ [ "_sanity._sanity_test.SanityTest", + "channelz._channelz_servicer_test.ChannelzServicerTest", "health_check._health_servicer_test.HealthServicerTest", "interop._insecure_intraop_test.InsecureIntraopTest", "interop._secure_intraop_test.SecureIntraopTest", diff --git a/templates/src/python/grpcio_channelz/grpc_version.py.template b/templates/src/python/grpcio_channelz/grpc_version.py.template new file mode 100644 index 0000000000..75d510cd17 --- /dev/null +++ b/templates/src/python/grpcio_channelz/grpc_version.py.template @@ -0,0 +1,19 @@ +%YAML 1.2 +--- | + # Copyright 2018 The gRPC Authors + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_channelz/grpc_version.py.template`!!! + + VERSION = '${settings.python_version.pep440()}' diff --git a/test/core/transport/chttp2/context_list_test.cc b/test/core/transport/chttp2/context_list_test.cc index e2100899d3..e64f1532ec 100644 --- a/test/core/transport/chttp2/context_list_test.cc +++ b/test/core/transport/chttp2/context_list_test.cc @@ -33,8 +33,12 @@ namespace grpc_core { namespace testing { namespace { + +const uint32_t kByteOffset = 123; + void TestExecuteFlushesListVerifier(void* arg, grpc_core::Timestamps* ts) { - GPR_ASSERT(arg != nullptr); + ASSERT_NE(arg, nullptr); + EXPECT_EQ(ts->byte_offset, kByteOffset); gpr_atm* done = reinterpret_cast<gpr_atm*>(arg); gpr_atm_rel_store(done, static_cast<gpr_atm>(1)); } @@ -43,7 +47,7 @@ void discard_write(grpc_slice slice) {} /** Tests that all ContextList elements in the list are flushed out on * execute. - * Also tests that arg is passed correctly. + * Also tests that arg and byte_counter are passed correctly. */ TEST(ContextList, ExecuteFlushesList) { grpc_core::ContextList* list = nullptr; @@ -68,14 +72,14 @@ TEST(ContextList, ExecuteFlushesList) { reinterpret_cast<grpc_stream*>(s[i]), &ref, nullptr, nullptr); s[i]->context = &verifier_called[i]; + s[i]->byte_counter = kByteOffset; gpr_atm_rel_store(&verifier_called[i], static_cast<gpr_atm>(0)); grpc_core::ContextList::Append(&list, s[i]); } grpc_core::Timestamps ts; grpc_core::ContextList::Execute(list, &ts, GRPC_ERROR_NONE); for (auto i = 0; i < kNumElems; i++) { - GPR_ASSERT(gpr_atm_acq_load(&verifier_called[i]) == - static_cast<gpr_atm>(1)); + EXPECT_EQ(gpr_atm_acq_load(&verifier_called[i]), static_cast<gpr_atm>(1)); grpc_transport_destroy_stream(reinterpret_cast<grpc_transport*>(t), reinterpret_cast<grpc_stream*>(s[i]), nullptr); diff --git a/test/cpp/codegen/compiler_test_golden b/test/cpp/codegen/compiler_test_golden index fdc67969d9..5f0eb6c35c 100644 --- a/test/cpp/codegen/compiler_test_golden +++ b/test/cpp/codegen/compiler_test_golden @@ -30,6 +30,7 @@ #include <grpcpp/impl/codegen/async_generic_service.h> #include <grpcpp/impl/codegen/async_stream.h> #include <grpcpp/impl/codegen/async_unary_call.h> +#include <grpcpp/impl/codegen/client_callback.h> #include <grpcpp/impl/codegen/method_handler_impl.h> #include <grpcpp/impl/codegen/proto_utils.h> #include <grpcpp/impl/codegen/rpc_method.h> @@ -117,10 +118,13 @@ class ServiceA final { // // Method A2 leading comment 1 // Method A2 leading comment 2 + virtual void MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::experimental::ClientWriteReactor< ::grpc::testing::Request>* reactor) = 0; // MethodA2 trailing comment 1 // Method A3 leading comment 1 + virtual void MethodA3(::grpc::ClientContext* context, ::grpc::testing::Request* request, ::grpc::experimental::ClientReadReactor< ::grpc::testing::Response>* reactor) = 0; // Method A3 trailing comment 1 // Method A4 leading comment 1 + virtual void MethodA4(::grpc::ClientContext* context, ::grpc::experimental::ClientBidiReactor< ::grpc::testing::Request,::grpc::testing::Response>* reactor) = 0; // Method A4 trailing comment 1 }; virtual class experimental_async_interface* experimental_async() { return nullptr; } @@ -178,6 +182,9 @@ class ServiceA final { public StubInterface::experimental_async_interface { public: void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override; + void MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::experimental::ClientWriteReactor< ::grpc::testing::Request>* reactor) override; + void MethodA3(::grpc::ClientContext* context, ::grpc::testing::Request* request, ::grpc::experimental::ClientReadReactor< ::grpc::testing::Response>* reactor) override; + void MethodA4(::grpc::ClientContext* context, ::grpc::experimental::ClientBidiReactor< ::grpc::testing::Request,::grpc::testing::Response>* reactor) override; private: friend class Stub; explicit experimental_async(Stub* stub): stub_(stub) { } diff --git a/test/cpp/end2end/client_callback_end2end_test.cc b/test/cpp/end2end/client_callback_end2end_test.cc index a35991396a..6c18703f06 100644 --- a/test/cpp/end2end/client_callback_end2end_test.cc +++ b/test/cpp/end2end/client_callback_end2end_test.cc @@ -182,6 +182,55 @@ class ClientCallbackEnd2endTest } } + void SendGenericEchoAsBidi(int num_rpcs) { + const grpc::string kMethodName("/grpc.testing.EchoTestService/Echo"); + grpc::string test_string(""); + for (int i = 0; i < num_rpcs; i++) { + test_string += "Hello world. "; + class Client : public grpc::experimental::ClientBidiReactor<ByteBuffer, + ByteBuffer> { + public: + Client(ClientCallbackEnd2endTest* test, const grpc::string& method_name, + const grpc::string& test_str) { + test->generic_stub_->experimental().PrepareBidiStreamingCall( + &cli_ctx_, method_name, this); + request_.set_message(test_str); + send_buf_ = SerializeToByteBuffer(&request_); + StartWrite(send_buf_.get()); + StartRead(&recv_buf_); + StartCall(); + } + void OnWriteDone(bool ok) override { StartWritesDone(); } + void OnReadDone(bool ok) override { + EchoResponse response; + EXPECT_TRUE(ParseFromByteBuffer(&recv_buf_, &response)); + EXPECT_EQ(request_.message(), response.message()); + }; + void OnDone(const Status& s) override { + EXPECT_TRUE(s.ok()); + std::unique_lock<std::mutex> l(mu_); + done_ = true; + cv_.notify_one(); + } + void Await() { + std::unique_lock<std::mutex> l(mu_); + while (!done_) { + cv_.wait(l); + } + } + + EchoRequest request_; + std::unique_ptr<ByteBuffer> send_buf_; + ByteBuffer recv_buf_; + ClientContext cli_ctx_; + std::mutex mu_; + std::condition_variable cv_; + bool done_ = false; + } rpc{this, kMethodName, test_string}; + + rpc.Await(); + } + } bool is_server_started_; std::shared_ptr<Channel> channel_; std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_; @@ -211,6 +260,11 @@ TEST_P(ClientCallbackEnd2endTest, SequentialGenericRpcs) { SendRpcsGeneric(10, false); } +TEST_P(ClientCallbackEnd2endTest, SequentialGenericRpcsAsBidi) { + ResetStub(); + SendGenericEchoAsBidi(10); +} + #if GRPC_ALLOW_EXCEPTIONS TEST_P(ClientCallbackEnd2endTest, ExceptingRpc) { ResetStub(); @@ -267,6 +321,170 @@ TEST_P(ClientCallbackEnd2endTest, CancelRpcBeforeStart) { } } +TEST_P(ClientCallbackEnd2endTest, RequestStream) { + // TODO(vjpai): test with callback server once supported + if (GetParam().callback_server) { + return; + } + + ResetStub(); + class Client : public grpc::experimental::ClientWriteReactor<EchoRequest> { + public: + explicit Client(grpc::testing::EchoTestService::Stub* stub) { + context_.set_initial_metadata_corked(true); + stub->experimental_async()->RequestStream(&context_, &response_, this); + StartCall(); + request_.set_message("Hello server."); + StartWrite(&request_); + } + void OnWriteDone(bool ok) override { + writes_left_--; + if (writes_left_ > 1) { + StartWrite(&request_); + } else if (writes_left_ == 1) { + StartWriteLast(&request_, WriteOptions()); + } + } + void OnDone(const Status& s) override { + EXPECT_TRUE(s.ok()); + EXPECT_EQ(response_.message(), "Hello server.Hello server.Hello server."); + std::unique_lock<std::mutex> l(mu_); + done_ = true; + cv_.notify_one(); + } + void Await() { + std::unique_lock<std::mutex> l(mu_); + while (!done_) { + cv_.wait(l); + } + } + + private: + EchoRequest request_; + EchoResponse response_; + ClientContext context_; + int writes_left_{3}; + std::mutex mu_; + std::condition_variable cv_; + bool done_ = false; + } test{stub_.get()}; + + test.Await(); +} + +TEST_P(ClientCallbackEnd2endTest, ResponseStream) { + // TODO(vjpai): test with callback server once supported + if (GetParam().callback_server) { + return; + } + + ResetStub(); + class Client : public grpc::experimental::ClientReadReactor<EchoResponse> { + public: + explicit Client(grpc::testing::EchoTestService::Stub* stub) { + request_.set_message("Hello client "); + stub->experimental_async()->ResponseStream(&context_, &request_, this); + StartCall(); + StartRead(&response_); + } + void OnReadDone(bool ok) override { + if (!ok) { + EXPECT_EQ(reads_complete_, kServerDefaultResponseStreamsToSend); + } else { + EXPECT_LE(reads_complete_, kServerDefaultResponseStreamsToSend); + EXPECT_EQ(response_.message(), + request_.message() + grpc::to_string(reads_complete_)); + reads_complete_++; + StartRead(&response_); + } + } + void OnDone(const Status& s) override { + EXPECT_TRUE(s.ok()); + std::unique_lock<std::mutex> l(mu_); + done_ = true; + cv_.notify_one(); + } + void Await() { + std::unique_lock<std::mutex> l(mu_); + while (!done_) { + cv_.wait(l); + } + } + + private: + EchoRequest request_; + EchoResponse response_; + ClientContext context_; + int reads_complete_{0}; + std::mutex mu_; + std::condition_variable cv_; + bool done_ = false; + } test{stub_.get()}; + + test.Await(); +} + +TEST_P(ClientCallbackEnd2endTest, BidiStream) { + // TODO(vjpai): test with callback server once supported + if (GetParam().callback_server) { + return; + } + ResetStub(); + class Client : public grpc::experimental::ClientBidiReactor<EchoRequest, + EchoResponse> { + public: + explicit Client(grpc::testing::EchoTestService::Stub* stub) { + request_.set_message("Hello fren "); + stub->experimental_async()->BidiStream(&context_, this); + StartCall(); + StartRead(&response_); + StartWrite(&request_); + } + void OnReadDone(bool ok) override { + if (!ok) { + EXPECT_EQ(reads_complete_, kServerDefaultResponseStreamsToSend); + } else { + EXPECT_LE(reads_complete_, kServerDefaultResponseStreamsToSend); + EXPECT_EQ(response_.message(), request_.message()); + reads_complete_++; + StartRead(&response_); + } + } + void OnWriteDone(bool ok) override { + EXPECT_TRUE(ok); + if (++writes_complete_ == kServerDefaultResponseStreamsToSend) { + StartWritesDone(); + } else { + StartWrite(&request_); + } + } + void OnDone(const Status& s) override { + EXPECT_TRUE(s.ok()); + std::unique_lock<std::mutex> l(mu_); + done_ = true; + cv_.notify_one(); + } + void Await() { + std::unique_lock<std::mutex> l(mu_); + while (!done_) { + cv_.wait(l); + } + } + + private: + EchoRequest request_; + EchoResponse response_; + ClientContext context_; + int reads_complete_{0}; + int writes_complete_{0}; + std::mutex mu_; + std::condition_variable cv_; + bool done_ = false; + } test{stub_.get()}; + + test.Await(); +} + TestScenario scenarios[] = {TestScenario{false}, TestScenario{true}}; INSTANTIATE_TEST_CASE_P(ClientCallbackEnd2endTest, ClientCallbackEnd2endTest, diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 605356724f..1726e87ea6 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -223,6 +223,7 @@ void CallbackTestServiceImpl::EchoNonDelayed( return; } + gpr_log(GPR_DEBUG, "Request message was %s", request->message().c_str()); response->set_message(request->message()); MaybeEchoDeadline(context, request, response); if (host_) { diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index 82a818cce5..d17eb9fdb8 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -20,6 +20,7 @@ cd "$(dirname "$0")/../.." DIRS=( 'src/python/grpcio/grpc' + 'src/python/grpcio_channelz/grpc_channelz' 'src/python/grpcio_health_checking/grpc_health' 'src/python/grpcio_reflection/grpc_reflection' 'src/python/grpcio_testing/grpc_testing' diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 65f2d1c765..bc6e558204 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -108,6 +108,11 @@ then ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_testing/setup.py sdist cp -r src/python/grpcio_testing/dist/* "$ARTIFACT_DIR" + # Build grpcio_channelz source distribution + ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_channelz/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_channelz/dist/* "$ARTIFACT_DIR" + # Build grpcio_health_checking source distribution ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \ preprocess build_package_protos sdist diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 8394f07e51..e43f1fb429 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -189,6 +189,11 @@ pip_install_dir "$ROOT" $VENV_PYTHON "$ROOT/tools/distrib/python/make_grpcio_tools.py" pip_install_dir "$ROOT/tools/distrib/python/grpcio_tools" +# Build/install Channelz +$VENV_PYTHON "$ROOT/src/python/grpcio_channelz/setup.py" preprocess +$VENV_PYTHON "$ROOT/src/python/grpcio_channelz/setup.py" build_package_protos +pip_install_dir "$ROOT/src/python/grpcio_channelz" + # Build/install health checking $VENV_PYTHON "$ROOT/src/python/grpcio_health_checking/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_health_checking/setup.py" build_package_protos |