diff options
48 files changed, 285 insertions, 133 deletions
diff --git a/.gitmodules b/.gitmodules index bb4b749bee..06f1394df6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -51,3 +51,6 @@ [submodule "third_party/protoc-gen-validate"] path = third_party/protoc-gen-validate url = https://github.com/lyft/protoc-gen-validate.git +[submodule "third_party/upb"] + path = third_party/upb + url = https://github.com/google/upb.git @@ -1,2 +1,3 @@ +Dropbox, Inc. Google Inc. WeWork Companies Inc. @@ -647,6 +647,7 @@ grpc_cc_library( "debug_location", "gpr_base", "grpc_trace", + "ref_counted", "ref_counted_ptr", ], ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8582d9af5..1d14d5e0e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,7 @@ How to get your contributions merged smoothly and quickly. copyright holder for the contribution (yourself, if you are signing the individual CLA, or your company, for corporate CLAs) in the same PR as your contribution. This needs to be done only once, for each company, or - individual. + individual. Please keep this file in alphabetical order. - Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 8626817855..e091e5abc0 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -9,6 +9,11 @@ def grpc_deps(): ) native.bind( + name = "upblib", + actual = "@upb//:upb", + ) + + native.bind( name = "absl-base", actual = "@com_google_absl//absl/base", ) @@ -184,6 +189,13 @@ def grpc_deps(): url = "https://github.com/census-instrumentation/opencensus-cpp/archive/fdf0f308b1631bb4a942e32ba5d22536a6170274.tar.gz", ) + if "upb" not in native.existing_rules(): + native.http_archive( + name = "upb", + strip_prefix = "upb-9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e3", + url = "https://github.com/google/upb/archive/9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e3.tar.gz", + ) + # TODO: move some dependencies from "grpc_deps" here? def grpc_test_only_deps(): diff --git a/include/grpc/impl/codegen/atm_gcc_sync.h b/include/grpc/impl/codegen/atm_gcc_sync.h index c0010a3469..728c3d5412 100644 --- a/include/grpc/impl/codegen/atm_gcc_sync.h +++ b/include/grpc/impl/codegen/atm_gcc_sync.h @@ -26,6 +26,8 @@ typedef intptr_t gpr_atm; #define GPR_ATM_MAX INTPTR_MAX #define GPR_ATM_MIN INTPTR_MIN +#define GPR_ATM_INC_CAS_THEN(blah) blah +#define GPR_ATM_INC_ADD_THEN(blah) blah #define GPR_ATM_COMPILE_BARRIER_() __asm__ __volatile__("" : : : "memory") diff --git a/include/grpc/impl/codegen/atm_windows.h b/include/grpc/impl/codegen/atm_windows.h index f6b27e5df7..c016b90095 100644 --- a/include/grpc/impl/codegen/atm_windows.h +++ b/include/grpc/impl/codegen/atm_windows.h @@ -25,6 +25,8 @@ typedef intptr_t gpr_atm; #define GPR_ATM_MAX INTPTR_MAX #define GPR_ATM_MIN INTPTR_MIN +#define GPR_ATM_INC_CAS_THEN(blah) blah +#define GPR_ATM_INC_ADD_THEN(blah) blah #define gpr_atm_full_barrier MemoryBarrier diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 4ca0f49adf..99c675f503 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -211,32 +211,23 @@ grpc_chttp2_transport::~grpc_chttp2_transport() { void grpc_chttp2_unref_transport(grpc_chttp2_transport* t, const char* reason, const char* file, int line) { if (grpc_trace_chttp2_refcount.enabled()) { - gpr_atm val = gpr_atm_no_barrier_load(&t->refs.count); + const grpc_core::RefCount::Value val = t->refs.get(); gpr_log(GPR_DEBUG, "chttp2:unref:%p %" PRIdPTR "->%" PRIdPTR " %s [%s:%d]", t, val, val - 1, reason, file, line); } - if (!gpr_unref(&t->refs)) return; - t->~grpc_chttp2_transport(); - gpr_free(t); + if (!t->refs.Unref()) return; + grpc_core::Delete(t); } void grpc_chttp2_ref_transport(grpc_chttp2_transport* t, const char* reason, const char* file, int line) { if (grpc_trace_chttp2_refcount.enabled()) { - gpr_atm val = gpr_atm_no_barrier_load(&t->refs.count); + const grpc_core::RefCount::Value val = t->refs.get(); gpr_log(GPR_DEBUG, "chttp2: ref:%p %" PRIdPTR "->%" PRIdPTR " %s [%s:%d]", t, val, val + 1, reason, file, line); } - gpr_ref(&t->refs); + t->refs.Ref(); } -#else -void grpc_chttp2_unref_transport(grpc_chttp2_transport* t) { - if (!gpr_unref(&t->refs)) return; - t->~grpc_chttp2_transport(); - gpr_free(t); -} - -void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { gpr_ref(&t->refs); } #endif static const grpc_transport_vtable* get_vtable(void); @@ -500,8 +491,6 @@ grpc_chttp2_transport::grpc_chttp2_transport( GPR_ASSERT(strlen(GRPC_CHTTP2_CLIENT_CONNECT_STRING) == GRPC_CHTTP2_CLIENT_CONNECT_STRLEN); base.vtable = get_vtable(); - /* one ref is for destroy */ - gpr_ref_init(&refs, 1); /* 8 is a random stab in the dark as to a good initial size: it's small enough that it shouldn't waste memory for infrequently used connections, yet large enough that the exponential growth should happen nicely when it's @@ -1401,6 +1390,7 @@ static void perform_stream_op_locked(void* stream_op, GRPC_STATS_INC_HTTP2_OP_BATCHES(); s->context = op->payload->context; + s->traced = op->is_traced; if (grpc_http_trace.enabled()) { char* str = grpc_transport_stream_op_batch_string(op); gpr_log(GPR_INFO, "perform_stream_op_locked: %s; on_complete = %p", str, @@ -2845,8 +2835,8 @@ Chttp2IncomingByteStream::Chttp2IncomingByteStream( : ByteStream(frame_size, flags), transport_(transport), stream_(stream), + refs_(2), remaining_bytes_(frame_size) { - gpr_ref_init(&refs_, 2); GRPC_ERROR_UNREF(stream->byte_stream_error); stream->byte_stream_error = GRPC_ERROR_NONE; } @@ -2871,14 +2861,6 @@ void Chttp2IncomingByteStream::Orphan() { GRPC_ERROR_NONE); } -void Chttp2IncomingByteStream::Unref() { - if (gpr_unref(&refs_)) { - Delete(this); - } -} - -void Chttp2IncomingByteStream::Ref() { gpr_ref(&refs_); } - void Chttp2IncomingByteStream::NextLocked(void* arg, grpc_error* error_ignored) { Chttp2IncomingByteStream* bs = static_cast<Chttp2IncomingByteStream*>(arg); @@ -3198,8 +3180,8 @@ intptr_t grpc_chttp2_transport_get_socket_uuid(grpc_transport* transport) { grpc_transport* grpc_create_chttp2_transport( const grpc_channel_args* channel_args, grpc_endpoint* ep, bool is_client, grpc_resource_user* resource_user) { - auto t = new (gpr_malloc(sizeof(grpc_chttp2_transport))) - grpc_chttp2_transport(channel_args, ep, is_client, resource_user); + auto t = grpc_core::New<grpc_chttp2_transport>(channel_args, ep, is_client, + resource_user); return &t->base; } diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 877b8aba77..aeaa4935ad 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -236,8 +236,12 @@ class Chttp2IncomingByteStream : public ByteStream { // alone for now. We can revisit this once we're able to link against // libc++, at which point we can eliminate New<> and Delete<> and // switch to std::shared_ptr<>. - void Ref(); - void Unref(); + void Ref() { refs_.Ref(); } + void Unref() { + if (refs_.Unref()) { + grpc_core::Delete(this); + } + } void PublishError(grpc_error* error); @@ -256,7 +260,7 @@ class Chttp2IncomingByteStream : public ByteStream { grpc_chttp2_transport* transport_; // Immutable. grpc_chttp2_stream* stream_; // Immutable. - gpr_refcount refs_; + grpc_core::RefCount refs_; /* Accessed only by transport thread when stream->pending_byte_stream == false * Accessed only by application thread when stream->pending_byte_stream == @@ -290,7 +294,7 @@ struct grpc_chttp2_transport { ~grpc_chttp2_transport(); grpc_transport base; /* must be first */ - gpr_refcount refs; + grpc_core::RefCount refs; grpc_endpoint* ep; char* peer_string; @@ -638,10 +642,10 @@ struct grpc_chttp2_stream { /** Whether bytes stored in unprocessed_incoming_byte_stream is decompressed */ bool unprocessed_incoming_frames_decompressed = false; - /** gRPC header bytes that are already decompressed */ - size_t decompressed_header_bytes = 0; /** Whether the bytes needs to be traced using Fathom */ bool traced = false; + /** gRPC header bytes that are already decompressed */ + size_t decompressed_header_bytes = 0; }; /** Transport writing call flow: @@ -793,8 +797,14 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport* t, const char* reason, #else #define GRPC_CHTTP2_REF_TRANSPORT(t, r) grpc_chttp2_ref_transport(t) #define GRPC_CHTTP2_UNREF_TRANSPORT(t, r) grpc_chttp2_unref_transport(t) -void grpc_chttp2_unref_transport(grpc_chttp2_transport* t); -void grpc_chttp2_ref_transport(grpc_chttp2_transport* t); +inline void grpc_chttp2_unref_transport(grpc_chttp2_transport* t) { + if (t->refs.Unref()) { + grpc_core::Delete(t); + } +} +inline void grpc_chttp2_ref_transport(grpc_chttp2_transport* t) { + t->refs.Ref(); +} #endif void grpc_chttp2_ack_ping(grpc_chttp2_transport* t, uint64_t id); diff --git a/src/core/lib/gprpp/orphanable.h b/src/core/lib/gprpp/orphanable.h index 3123e3f5a3..3eb510165e 100644 --- a/src/core/lib/gprpp/orphanable.h +++ b/src/core/lib/gprpp/orphanable.h @@ -31,6 +31,7 @@ #include "src/core/lib/gprpp/abstract.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" namespace grpc_core { @@ -89,8 +90,8 @@ class InternallyRefCounted : public Orphanable { template <typename T> friend class RefCountedPtr; - InternallyRefCounted() { gpr_ref_init(&refs_, 1); } - virtual ~InternallyRefCounted() {} + InternallyRefCounted() = default; + virtual ~InternallyRefCounted() = default; RefCountedPtr<Child> Ref() GRPC_MUST_USE_RESULT { IncrementRefCount(); @@ -98,15 +99,15 @@ class InternallyRefCounted : public Orphanable { } void Unref() { - if (gpr_unref(&refs_)) { + if (refs_.Unref()) { Delete(static_cast<Child*>(this)); } } private: - void IncrementRefCount() { gpr_ref(&refs_); } + void IncrementRefCount() { refs_.Ref(); } - gpr_refcount refs_; + grpc_core::RefCount refs_; }; // An alternative version of the InternallyRefCounted base class that @@ -137,16 +138,14 @@ class InternallyRefCountedWithTracing : public Orphanable { : InternallyRefCountedWithTracing(static_cast<TraceFlag*>(nullptr)) {} explicit InternallyRefCountedWithTracing(TraceFlag* trace_flag) - : trace_flag_(trace_flag) { - gpr_ref_init(&refs_, 1); - } + : trace_flag_(trace_flag) {} #ifdef NDEBUG explicit InternallyRefCountedWithTracing(DebugOnlyTraceFlag* trace_flag) : InternallyRefCountedWithTracing() {} #endif - virtual ~InternallyRefCountedWithTracing() {} + virtual ~InternallyRefCountedWithTracing() = default; RefCountedPtr<Child> Ref() GRPC_MUST_USE_RESULT { IncrementRefCount(); @@ -156,7 +155,7 @@ class InternallyRefCountedWithTracing : public Orphanable { RefCountedPtr<Child> Ref(const DebugLocation& location, const char* reason) GRPC_MUST_USE_RESULT { if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - gpr_atm old_refs = gpr_atm_no_barrier_load(&refs_.count); + const grpc_core::RefCount::Value old_refs = refs_.get(); gpr_log(GPR_INFO, "%s:%p %s:%d ref %" PRIdPTR " -> %" PRIdPTR " %s", trace_flag_->name(), this, location.file(), location.line(), old_refs, old_refs + 1, reason); @@ -170,14 +169,14 @@ class InternallyRefCountedWithTracing : public Orphanable { // friend of this class. void Unref() { - if (gpr_unref(&refs_)) { + if (refs_.Unref()) { Delete(static_cast<Child*>(this)); } } void Unref(const DebugLocation& location, const char* reason) { if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - gpr_atm old_refs = gpr_atm_no_barrier_load(&refs_.count); + const grpc_core::RefCount::Value old_refs = refs_.get(); gpr_log(GPR_INFO, "%s:%p %s:%d unref %" PRIdPTR " -> %" PRIdPTR " %s", trace_flag_->name(), this, location.file(), location.line(), old_refs, old_refs - 1, reason); @@ -186,10 +185,10 @@ class InternallyRefCountedWithTracing : public Orphanable { } private: - void IncrementRefCount() { gpr_ref(&refs_); } + void IncrementRefCount() { refs_.Ref(); } TraceFlag* trace_flag_ = nullptr; - gpr_refcount refs_; + grpc_core::RefCount refs_; }; } // namespace grpc_core diff --git a/src/core/lib/gprpp/ref_counted.h b/src/core/lib/gprpp/ref_counted.h index e366445bff..98de1a3653 100644 --- a/src/core/lib/gprpp/ref_counted.h +++ b/src/core/lib/gprpp/ref_counted.h @@ -21,6 +21,7 @@ #include <grpc/support/port_platform.h> +#include <grpc/support/atm.h> #include <grpc/support/log.h> #include <grpc/support/sync.h> @@ -76,12 +77,15 @@ class RefCount { constexpr explicit RefCount(Value init = 1) : value_(init) {} // Increases the ref-count by `n`. - void Ref(Value n = 1) { value_.fetch_add(n, std::memory_order_relaxed); } + void Ref(Value n = 1) { + GPR_ATM_INC_ADD_THEN(value_.fetch_add(n, std::memory_order_relaxed)); + } // Similar to Ref() with an assert on the ref-count being non-zero. void RefNonZero() { #ifndef NDEBUG - const Value prior = value_.fetch_add(1, std::memory_order_relaxed); + const Value prior = + GPR_ATM_INC_ADD_THEN(value_.fetch_add(1, std::memory_order_relaxed)); assert(prior > 0); #else Ref(); @@ -90,7 +94,8 @@ class RefCount { // Decrements the ref-count and returns true if the ref-count reaches 0. bool Unref() { - const Value prior = value_.fetch_sub(1, std::memory_order_acq_rel); + const Value prior = + GPR_ATM_INC_ADD_THEN(value_.fetch_sub(1, std::memory_order_acq_rel)); GPR_DEBUG_ASSERT(prior > 0); return prior == 1; } diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 38571b1957..4b8c891e9b 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -1242,6 +1242,8 @@ static void pollset_set_del_pollset_set(grpc_pollset_set* bag, * Event engine binding */ +static void shutdown_background_closure(void) {} + static void shutdown_engine(void) { fd_global_shutdown(); pollset_global_shutdown(); @@ -1255,6 +1257,7 @@ static void shutdown_engine(void) { static const grpc_event_engine_vtable vtable = { sizeof(grpc_pollset), true, + false, fd_create, fd_wrapped_fd, @@ -1284,6 +1287,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 06a382c556..7a4870db78 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -1604,6 +1604,8 @@ static void pollset_set_del_pollset_set(grpc_pollset_set* bag, * Event engine binding */ +static void shutdown_background_closure(void) {} + static void shutdown_engine(void) { fd_global_shutdown(); pollset_global_shutdown(); @@ -1612,6 +1614,7 @@ static void shutdown_engine(void) { static const grpc_event_engine_vtable vtable = { sizeof(grpc_pollset), true, + false, fd_create, fd_wrapped_fd, @@ -1641,6 +1644,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index 16562538a6..67cbfbbd02 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -1782,6 +1782,8 @@ static void global_cv_fd_table_shutdown() { * event engine binding */ +static void shutdown_background_closure(void) {} + static void shutdown_engine(void) { pollset_global_shutdown(); if (grpc_cv_wakeup_fds_enabled()) { @@ -1796,6 +1798,7 @@ static void shutdown_engine(void) { static const grpc_event_engine_vtable vtable = { sizeof(grpc_pollset), false, + false, fd_create, fd_wrapped_fd, @@ -1825,6 +1828,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 8a7dc7b004..32d1b6c43e 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/ev_epoll1_linux.h" #include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" +#include "src/core/lib/iomgr/internal_errqueue.h" grpc_core::TraceFlag grpc_polling_trace(false, "polling"); /* Disabled by default */ @@ -236,19 +237,22 @@ void grpc_event_engine_shutdown(void) { } bool grpc_event_engine_can_track_errors(void) { -/* Only track errors if platform supports errqueue. */ -#ifdef GRPC_LINUX_ERRQUEUE - return g_event_engine->can_track_err; -#else + /* Only track errors if platform supports errqueue. */ + if (grpc_core::kernel_supports_errqueue()) { + return g_event_engine->can_track_err; + } return false; -#endif /* GRPC_LINUX_ERRQUEUE */ +} + +bool grpc_event_engine_run_in_background(void) { + return g_event_engine->run_in_background; } grpc_fd* grpc_fd_create(int fd, const char* name, bool track_err) { GRPC_POLLING_API_TRACE("fd_create(%d, %s, %d)", fd, name, track_err); GRPC_FD_TRACE("fd_create(%d, %s, %d)", fd, name, track_err); - return g_event_engine->fd_create(fd, name, - track_err && g_event_engine->can_track_err); + return g_event_engine->fd_create( + fd, name, track_err && grpc_event_engine_can_track_errors()); } int grpc_fd_wrapped_fd(grpc_fd* fd) { @@ -395,4 +399,8 @@ void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { g_event_engine->pollset_set_del_fd(pollset_set, fd); } +void grpc_shutdown_background_closure(void) { + g_event_engine->shutdown_background_closure(); +} + #endif // GRPC_POSIX_SOCKET_EV diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index b8fb8f534b..812c7a0f0f 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -42,6 +42,7 @@ typedef struct grpc_fd grpc_fd; typedef struct grpc_event_engine_vtable { size_t pollset_size; bool can_track_err; + bool run_in_background; grpc_fd* (*fd_create)(int fd, const char* name, bool track_err); int (*fd_wrapped_fd)(grpc_fd* fd); @@ -79,6 +80,7 @@ typedef struct grpc_event_engine_vtable { void (*pollset_set_add_fd)(grpc_pollset_set* pollset_set, grpc_fd* fd); void (*pollset_set_del_fd)(grpc_pollset_set* pollset_set, grpc_fd* fd); + void (*shutdown_background_closure)(void); void (*shutdown_engine)(void); } grpc_event_engine_vtable; @@ -101,6 +103,11 @@ const char* grpc_get_poll_strategy_name(); */ bool grpc_event_engine_can_track_errors(); +/* Returns true if polling engine runs in the background, false otherwise. + * Currently only 'epollbg' runs in the background. + */ +bool grpc_event_engine_run_in_background(); + /* Create a wrapped file descriptor. Requires fd is a non-blocking file descriptor. \a track_err if true means that error events would be tracked separately @@ -174,6 +181,9 @@ void grpc_pollset_add_fd(grpc_pollset* pollset, struct grpc_fd* fd); void grpc_pollset_set_add_fd(grpc_pollset_set* pollset_set, grpc_fd* fd); void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd); +/* Shut down all the closures registered in the background poller. */ +void grpc_shutdown_background_closure(); + /* override to allow tests to hook poll() usage */ typedef int (*grpc_poll_function_type)(struct pollfd*, nfds_t, int); extern grpc_poll_function_type grpc_poll_function; diff --git a/src/core/lib/iomgr/internal_errqueue.cc b/src/core/lib/iomgr/internal_errqueue.cc index 99c22e9055..982d709f09 100644 --- a/src/core/lib/iomgr/internal_errqueue.cc +++ b/src/core/lib/iomgr/internal_errqueue.cc @@ -20,17 +20,50 @@ #include "src/core/lib/iomgr/port.h" +#include <grpc/impl/codegen/log.h> #include "src/core/lib/iomgr/internal_errqueue.h" #ifdef GRPC_POSIX_SOCKET_TCP -bool kernel_supports_errqueue() { +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#include <sys/utsname.h> + +namespace grpc_core { +static bool errqueue_supported = false; + +bool kernel_supports_errqueue() { return errqueue_supported; } + +void grpc_errqueue_init() { +/* Both-compile time and run-time linux kernel versions should be atleast 4.0.0 + */ #ifdef LINUX_VERSION_CODE #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) - return true; + struct utsname buffer; + if (uname(&buffer) != 0) { + gpr_log(GPR_ERROR, "uname: %s", strerror(errno)); + return; + } + char* release = buffer.release; + if (release == nullptr) { + return; + } + + if (strtol(release, nullptr, 10) >= 4) { + errqueue_supported = true; + } else { + gpr_log(GPR_DEBUG, "ERRQUEUE support not enabled"); + } #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(4, 0, 0) */ #endif /* LINUX_VERSION_CODE */ - return false; } +} /* namespace grpc_core */ + +#else + +namespace grpc_core { +void grpc_errqueue_init() {} +} /* namespace grpc_core */ #endif /* GRPC_POSIX_SOCKET_TCP */ diff --git a/src/core/lib/iomgr/internal_errqueue.h b/src/core/lib/iomgr/internal_errqueue.h index 9d122808f9..f8644c2536 100644 --- a/src/core/lib/iomgr/internal_errqueue.h +++ b/src/core/lib/iomgr/internal_errqueue.h @@ -76,8 +76,14 @@ constexpr uint32_t kTimestampingRecordingOptions = * Currently allowing only linux kernels above 4.0.0 */ bool kernel_supports_errqueue(); -} // namespace grpc_core + +} /* namespace grpc_core */ #endif /* GRPC_POSIX_SOCKET_TCP */ +namespace grpc_core { +/* Initializes errqueue support */ +void grpc_errqueue_init(); +} /* namespace grpc_core */ + #endif /* GRPC_CORE_LIB_IOMGR_INTERNAL_ERRQUEUE_H */ diff --git a/src/core/lib/iomgr/iomgr.cc b/src/core/lib/iomgr/iomgr.cc index 30b68db4df..eb29973514 100644 --- a/src/core/lib/iomgr/iomgr.cc +++ b/src/core/lib/iomgr/iomgr.cc @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/buffer_list.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/internal_errqueue.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/network_status_tracker.h" #include "src/core/lib/iomgr/timer.h" @@ -58,6 +59,7 @@ void grpc_iomgr_init() { g_root_object.name = (char*)"root"; grpc_network_status_init(); grpc_iomgr_platform_init(); + grpc_core::grpc_errqueue_init(); } void grpc_iomgr_start() { grpc_timer_manager_init(); } @@ -155,6 +157,10 @@ void grpc_iomgr_shutdown() { gpr_cv_destroy(&g_rcv); } +void grpc_iomgr_shutdown_background_closure() { + grpc_iomgr_platform_shutdown_background_closure(); +} + void grpc_iomgr_register_object(grpc_iomgr_object* obj, const char* name) { obj->name = gpr_strdup(name); gpr_mu_lock(&g_mu); diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 537ef8a6ff..8ea9289e06 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -35,6 +35,10 @@ void grpc_iomgr_start(); * exec_ctx. */ void grpc_iomgr_shutdown(); +/** Signals the intention to shutdown all the closures registered in the + * background poller. */ +void grpc_iomgr_shutdown_background_closure(); + /* Exposed only for testing */ size_t grpc_iomgr_count_objects_for_testing(); diff --git a/src/core/lib/iomgr/iomgr_custom.cc b/src/core/lib/iomgr/iomgr_custom.cc index d34c8e7cd1..4b112c9097 100644 --- a/src/core/lib/iomgr/iomgr_custom.cc +++ b/src/core/lib/iomgr/iomgr_custom.cc @@ -40,9 +40,11 @@ static void iomgr_platform_init(void) { } static void iomgr_platform_flush(void) {} static void iomgr_platform_shutdown(void) { grpc_pollset_global_shutdown(); } +static void iomgr_platform_shutdown_background_closure(void) {} static grpc_iomgr_platform_vtable vtable = { - iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; + iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, + iomgr_platform_shutdown_background_closure}; void grpc_custom_iomgr_init(grpc_socket_vtable* socket, grpc_custom_resolver_vtable* resolver, diff --git a/src/core/lib/iomgr/iomgr_internal.cc b/src/core/lib/iomgr/iomgr_internal.cc index 32dbabb79d..b6c9211865 100644 --- a/src/core/lib/iomgr/iomgr_internal.cc +++ b/src/core/lib/iomgr/iomgr_internal.cc @@ -41,3 +41,7 @@ void grpc_iomgr_platform_init() { iomgr_platform_vtable->init(); } void grpc_iomgr_platform_flush() { iomgr_platform_vtable->flush(); } void grpc_iomgr_platform_shutdown() { iomgr_platform_vtable->shutdown(); } + +void grpc_iomgr_platform_shutdown_background_closure() { + iomgr_platform_vtable->shutdown_background_closure(); +} diff --git a/src/core/lib/iomgr/iomgr_internal.h b/src/core/lib/iomgr/iomgr_internal.h index b011d9c7b1..bca7409907 100644 --- a/src/core/lib/iomgr/iomgr_internal.h +++ b/src/core/lib/iomgr/iomgr_internal.h @@ -35,6 +35,7 @@ typedef struct grpc_iomgr_platform_vtable { void (*init)(void); void (*flush)(void); void (*shutdown)(void); + void (*shutdown_background_closure)(void); } grpc_iomgr_platform_vtable; void grpc_iomgr_register_object(grpc_iomgr_object* obj, const char* name); @@ -52,6 +53,9 @@ void grpc_iomgr_platform_flush(void); /** tear down all platform specific global iomgr structures */ void grpc_iomgr_platform_shutdown(void); +/** shut down all the closures registered in the background poller */ +void grpc_iomgr_platform_shutdown_background_closure(void); + bool grpc_iomgr_abort_on_leaks(void); #endif /* GRPC_CORE_LIB_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/lib/iomgr/iomgr_posix.cc b/src/core/lib/iomgr/iomgr_posix.cc index ca7334c9a4..9386adf060 100644 --- a/src/core/lib/iomgr/iomgr_posix.cc +++ b/src/core/lib/iomgr/iomgr_posix.cc @@ -51,8 +51,13 @@ static void iomgr_platform_shutdown(void) { grpc_wakeup_fd_global_destroy(); } +static void iomgr_platform_shutdown_background_closure(void) { + grpc_shutdown_background_closure(); +} + static grpc_iomgr_platform_vtable vtable = { - iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; + iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, + iomgr_platform_shutdown_background_closure}; void grpc_set_default_iomgr_platform() { grpc_set_tcp_client_impl(&grpc_posix_tcp_client_vtable); diff --git a/src/core/lib/iomgr/iomgr_posix_cfstream.cc b/src/core/lib/iomgr/iomgr_posix_cfstream.cc index 235a9e0712..552ef4309c 100644 --- a/src/core/lib/iomgr/iomgr_posix_cfstream.cc +++ b/src/core/lib/iomgr/iomgr_posix_cfstream.cc @@ -54,8 +54,13 @@ static void iomgr_platform_shutdown(void) { grpc_wakeup_fd_global_destroy(); } +static void iomgr_platform_shutdown_background_closure(void) { + grpc_shutdown_background_closure(); +} + static grpc_iomgr_platform_vtable vtable = { - iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; + iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, + iomgr_platform_shutdown_background_closure}; void grpc_set_default_iomgr_platform() { char* enable_cfstream = getenv(grpc_cfstream_env_var); diff --git a/src/core/lib/iomgr/iomgr_windows.cc b/src/core/lib/iomgr/iomgr_windows.cc index cdef89cbf0..24ef0dba7b 100644 --- a/src/core/lib/iomgr/iomgr_windows.cc +++ b/src/core/lib/iomgr/iomgr_windows.cc @@ -71,8 +71,11 @@ static void iomgr_platform_shutdown(void) { winsock_shutdown(); } +static void iomgr_platform_shutdown_background_closure(void) {} + static grpc_iomgr_platform_vtable vtable = { - iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; + iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, + iomgr_platform_shutdown_background_closure}; void grpc_set_default_iomgr_platform() { grpc_set_tcp_client_impl(&grpc_windows_tcp_client_vtable); diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index bf56a7298d..c8046b21dc 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -62,8 +62,7 @@ #define GRPC_HAVE_UNIX_SOCKET 1 #ifdef LINUX_VERSION_CODE #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) -/* TODO(yashykt): Re-enable once Fathom changes are commited. -#define GRPC_LINUX_ERRQUEUE 1 */ +#define GRPC_LINUX_ERRQUEUE 1 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0) */ #endif /* LINUX_VERSION_CODE */ #define GRPC_LINUX_MULTIPOLL_WITH_EPOLL 1 diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc index cb4c9db7a6..606bfce6e7 100644 --- a/src/core/lib/iomgr/tcp_posix.cc +++ b/src/core/lib/iomgr/tcp_posix.cc @@ -260,10 +260,17 @@ static void notify_on_write(grpc_tcp* tcp) { if (grpc_tcp_trace.enabled()) { gpr_log(GPR_INFO, "TCP:%p notify_on_write", tcp); } - cover_self(tcp); - GRPC_CLOSURE_INIT(&tcp->write_done_closure, - tcp_drop_uncovered_then_handle_write, tcp, - grpc_schedule_on_exec_ctx); + if (grpc_event_engine_run_in_background()) { + // If there is a polling engine always running in the background, there is + // no need to run the backup poller. + GRPC_CLOSURE_INIT(&tcp->write_done_closure, tcp_handle_write, tcp, + grpc_schedule_on_exec_ctx); + } else { + cover_self(tcp); + GRPC_CLOSURE_INIT(&tcp->write_done_closure, + tcp_drop_uncovered_then_handle_write, tcp, + grpc_schedule_on_exec_ctx); + } grpc_fd_notify_on_write(tcp->em_fd, &tcp->write_done_closure); } @@ -740,7 +747,7 @@ static bool process_errors(grpc_tcp* tcp) { } return false; } - process_timestamp(tcp, &msg, cmsg); + cmsg = process_timestamp(tcp, &msg, cmsg); } } } @@ -761,13 +768,11 @@ static void tcp_handle_error(void* arg /* grpc_tcp */, grpc_error* error) { /* We are still interested in collecting timestamps, so let's try reading * them. */ - if (!process_errors(tcp)) { - /* This was not a timestamps error. This was an actual error. Set the - * read and write closures to be ready. - */ - grpc_fd_set_readable(tcp->em_fd); - grpc_fd_set_writable(tcp->em_fd); - } + process_errors(tcp); + /* This might not a timestamps error. Set the read and write closures to be + * ready. */ + grpc_fd_set_readable(tcp->em_fd); + grpc_fd_set_writable(tcp->em_fd); GRPC_CLOSURE_INIT(&tcp->error_closure, tcp_handle_error, tcp, grpc_schedule_on_exec_ctx); grpc_fd_notify_on_error(tcp->em_fd, &tcp->error_closure); diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index c6198b8ae7..67cf5d89bf 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -161,6 +161,7 @@ void grpc_shutdown(void) { if (--g_initializations == 0) { { grpc_core::ExecCtx exec_ctx(0); + grpc_iomgr_shutdown_background_closure(); { grpc_timer_manager_set_threading( false); // shutdown timer_manager thread diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index cbdb77c844..b32f9c6ec1 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -27,6 +27,7 @@ #include <grpc/support/log.h> #include <grpc/support/sync.h> +#include "src/core/lib/gpr/alloc.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/slice/slice_internal.h" @@ -149,7 +150,7 @@ void grpc_transport_move_stats(grpc_transport_stream_stats* from, } size_t grpc_transport_stream_size(grpc_transport* transport) { - return transport->vtable->sizeof_stream; + return GPR_ROUND_UP_TO_ALIGNMENT_SIZE(transport->vtable->sizeof_stream); } void grpc_transport_destroy(grpc_transport* transport) { diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index edfa7030d1..5ce568834e 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -129,7 +129,8 @@ struct grpc_transport_stream_op_batch { recv_initial_metadata(false), recv_message(false), recv_trailing_metadata(false), - cancel_stream(false) {} + cancel_stream(false), + is_traced(false) {} /** Should be scheduled when all of the non-recv operations in the batch are complete. @@ -167,6 +168,9 @@ struct grpc_transport_stream_op_batch { /** Cancel this stream with the provided error */ bool cancel_stream : 1; + /** Is this stream traced */ + bool is_traced : 1; + /*************************************************************************** * remaining fields are initialized and used at the discretion of the * current handler of the op */ diff --git a/src/php/ext/grpc/php_grpc.c b/src/php/ext/grpc/php_grpc.c index 492325b1e8..111c6f4867 100644 --- a/src/php/ext/grpc/php_grpc.c +++ b/src/php/ext/grpc/php_grpc.c @@ -171,11 +171,13 @@ void prefork() { } void postfork_child() { + TSRMLS_FETCH(); + // loop through persistant list and destroy all underlying grpc_channel objs destroy_grpc_channels(); // clear completion queue - grpc_php_shutdown_completion_queue(); + grpc_php_shutdown_completion_queue(TSRMLS_C); // clean-up grpc_core grpc_shutdown(); @@ -187,7 +189,7 @@ void postfork_child() { // restart grpc_core grpc_init(); - grpc_php_init_completion_queue(); + grpc_php_init_completion_queue(TSRMLS_C); // re-create grpc_channel and point wrapped to it // unlock wrapped grpc channel mutex diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index 63048e8da0..ff523fb256 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -129,6 +129,8 @@ cdef class SSLSessionCacheLRU: cdef class SSLChannelCredentials(ChannelCredentials): def __cinit__(self, pem_root_certificates, private_key, certificate_chain): + if pem_root_certificates is not None and not isinstance(pem_root_certificates, bytes): + raise TypeError('expected certificate to be bytes, got %s' % (type(pem_root_certificates))) self._pem_root_certificates = pem_root_certificates self._private_key = private_key self._certificate_chain = certificate_chain diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 23428f0b0c..5aaf31e36c 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -121,7 +121,6 @@ cdef extern from "grpc/grpc.h": GRPC_STATUS_DATA_LOSS GRPC_STATUS__DO_NOT_USE - const char *GRPC_ARG_PRIMARY_USER_AGENT_STRING const char *GRPC_ARG_ENABLE_CENSUS const char *GRPC_ARG_MAX_CONCURRENT_STREAMS const char *GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH @@ -190,12 +189,6 @@ cdef extern from "grpc/grpc.h": size_t arguments_length "num_args" grpc_arg *arguments "args" - ctypedef enum grpc_compression_level: - GRPC_COMPRESS_LEVEL_NONE - GRPC_COMPRESS_LEVEL_LOW - GRPC_COMPRESS_LEVEL_MED - GRPC_COMPRESS_LEVEL_HIGH - ctypedef enum grpc_stream_compression_level: GRPC_STREAM_COMPRESS_LEVEL_NONE GRPC_STREAM_COMPRESS_LEVEL_LOW diff --git a/src/python/grpcio/grpc/_interceptor.py b/src/python/grpcio/grpc/_interceptor.py index 4345114026..2a8ddd8ce4 100644 --- a/src/python/grpcio/grpc/_interceptor.py +++ b/src/python/grpcio/grpc/_interceptor.py @@ -232,8 +232,8 @@ class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): credentials=new_credentials, wait_for_ready=new_wait_for_ready) return _UnaryOutcome(response, call) - except grpc.RpcError: - raise + except grpc.RpcError as rpc_error: + return rpc_error except Exception as exception: # pylint:disable=broad-except return _FailureOutcome(exception, sys.exc_info()[2]) @@ -354,8 +354,8 @@ class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): credentials=new_credentials, wait_for_ready=new_wait_for_ready) return _UnaryOutcome(response, call) - except grpc.RpcError: - raise + except grpc.RpcError as rpc_error: + return rpc_error except Exception as exception: # pylint:disable=broad-except return _FailureOutcome(exception, sys.exc_info()[2]) diff --git a/src/python/grpcio_tests/tests/unit/_credentials_test.py b/src/python/grpcio_tests/tests/unit/_credentials_test.py index be7378ecbc..187a6f0388 100644 --- a/src/python/grpcio_tests/tests/unit/_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_credentials_test.py @@ -15,6 +15,7 @@ import unittest import logging +import six import grpc @@ -53,6 +54,16 @@ class CredentialsTest(unittest.TestCase): self.assertIsInstance(channel_first_second_and_third, grpc.ChannelCredentials) + @unittest.skipIf(six.PY2, 'only invalid in Python3') + def test_invalid_string_certificate(self): + self.assertRaises( + TypeError, + grpc.ssl_channel_credentials, + root_certificates='A Certificate', + private_key=None, + certificate_chain=None, + ) + if __name__ == '__main__': logging.basicConfig() diff --git a/src/ruby/lib/grpc/generic/service.rb b/src/ruby/lib/grpc/generic/service.rb index 4764217406..169a62f11d 100644 --- a/src/ruby/lib/grpc/generic/service.rb +++ b/src/ruby/lib/grpc/generic/service.rb @@ -95,7 +95,7 @@ module GRPC rpc_descs[name] = RpcDesc.new(name, input, output, marshal_class_method, unmarshal_class_method) - define_method(GenericService.underscore(name.to_s).to_sym) do |_, _| + define_method(GenericService.underscore(name.to_s).to_sym) do |*| fail GRPC::BadStatus.new_status_exception( GRPC::Core::StatusCodes::UNIMPLEMENTED) end diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index 44a6134086..924d747a79 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -342,6 +342,28 @@ describe GRPC::RpcServer do t.join end + it 'should return UNIMPLEMENTED on unimplemented ' \ + 'methods for client_streamer', server: true do + @srv.handle(EchoService) + t = Thread.new { @srv.run } + @srv.wait_till_running + blk = proc do + stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts) + requests = [EchoMsg.new, EchoMsg.new] + stub.a_client_streaming_rpc_unimplemented(requests) + end + + begin + expect(&blk).to raise_error do |error| + expect(error).to be_a(GRPC::BadStatus) + expect(error.code).to eq(GRPC::Core::StatusCodes::UNIMPLEMENTED) + end + ensure + @srv.stop # should be call not to crash + t.join + end + end + it 'should handle multiple sequential requests', server: true do @srv.handle(EchoService) t = Thread.new { @srv.run } diff --git a/src/ruby/spec/support/services.rb b/src/ruby/spec/support/services.rb index 6e693f1cde..438459dfd7 100644 --- a/src/ruby/spec/support/services.rb +++ b/src/ruby/spec/support/services.rb @@ -33,6 +33,7 @@ class EchoService rpc :a_client_streaming_rpc, stream(EchoMsg), EchoMsg rpc :a_server_streaming_rpc, EchoMsg, stream(EchoMsg) rpc :a_bidi_rpc, stream(EchoMsg), stream(EchoMsg) + rpc :a_client_streaming_rpc_unimplemented, stream(EchoMsg), EchoMsg attr_reader :received_md def initialize(**kw) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template index bf28796de3..f584a2378e 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template @@ -14,15 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM debian:jessie - - <%include file="../../apt_get_basic.include"/> - <%include file="../../python_deps.include"/> - # Install pip and virtualenv for Python 3.4 - RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 - RUN python3.4 -m pip install virtualenv + <%include file="../../python_stretch.include"/> - <%include file="../../run_tests_addons.include"/> - # Define the default command. - CMD ["bash"] - + RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 85767c8758..dca97c85b1 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -94,6 +94,7 @@ static const grpc_event_engine_vtable* init_engine_vtable(bool) { g_vtable.pollset_destroy = pollset_destroy; g_vtable.pollset_work = pollset_work; g_vtable.pollset_kick = pollset_kick; + g_vtable.shutdown_background_closure = [] {}; g_vtable.shutdown_engine = [] {}; return &g_vtable; diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc index 2b81cca2d6..1d67e79b86 100644 --- a/test/cpp/qps/qps_json_driver.cc +++ b/test/cpp/qps/qps_json_driver.cc @@ -91,7 +91,7 @@ ConstructPerWorkerCredentialTypesMap() { // a map. std::string remaining = FLAGS_per_worker_credential_types; std::map<std::string, std::string> out; - while (remaining.size() > 0) { + while (!remaining.empty()) { size_t next_semicolon = remaining.find(';'); std::string next_entry = remaining.substr(0, next_semicolon); if (next_semicolon == std::string::npos) { diff --git a/third_party/upb b/third_party/upb new file mode 160000 +Subproject 9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index dadd856740..acad500f90 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:jessie - +FROM debian:stretch + # Install Git and basic packages. RUN apt-get update && apt-get install -y \ autoconf \ @@ -49,27 +49,24 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client oauth2client -# Install Python packages from PyPI -RUN pip install --upgrade pip==10.0.1 -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 +# Install Python 2.7 +RUN apt-get update && apt-get install -y python2.7 python-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 -# Install pip and virtualenv for Python 3.4 -RUN curl https://bootstrap.pypa.io/get-pip.py | python3.4 -RUN python3.4 -m pip install virtualenv +# Add Debian 'testing' repository +RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list +RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local RUN mkdir /var/local/jenkins # Define the default command. CMD ["bash"] + + +RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh index ee042d40c8..468aa20e3f 100755 --- a/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_python/build_interop.sh @@ -28,5 +28,5 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -# interop tests only run using python3.4 currently (and python build is slow) -tools/run_tests/run_tests.py -l python --compiler python3.4 -c opt --build_only +# interop tests only run using python3.7 currently (and python build is slow) +tools/run_tests/run_tests.py -l python --compiler python3.7 -c opt --build_only diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 1983220463..021f21dcd4 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -545,13 +545,13 @@ class PythonLanguage: def client_cmd(self, args): return [ - 'py34_native/bin/python', 'src/python/grpcio_tests/setup.py', + 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 'run_interop', '--client', '--args="{}"'.format(' '.join(args)) ] def client_cmd_http2interop(self, args): return [ - 'py34_native/bin/python', + 'py37_native/bin/python', 'src/python/grpcio_tests/tests/http2/negative_http2_client.py', ] + args @@ -560,7 +560,7 @@ class PythonLanguage: def server_cmd(self, args): return [ - 'py34_native/bin/python', 'src/python/grpcio_tests/setup.py', + 'py37_native/bin/python', 'src/python/grpcio_tests/setup.py', 'run_interop', '--server', '--args="{}"'.format(' '.join(args)) ] diff --git a/tools/run_tests/sanity/check_bazel_workspace.py b/tools/run_tests/sanity/check_bazel_workspace.py index d562fffc8a..35da88d70e 100755 --- a/tools/run_tests/sanity/check_bazel_workspace.py +++ b/tools/run_tests/sanity/check_bazel_workspace.py @@ -42,6 +42,7 @@ _ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME = 'com_github_zopefoundation_zope_interf _TWISTED_CONSTANTLY_DEP_NAME = 'com_github_twisted_constantly' _GRPC_DEP_NAMES = [ + 'upb', 'boringssl', 'com_github_madler_zlib', 'com_google_protobuf', diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index fa2628f18e..f1103596d5 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -40,6 +40,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) e143189bf6f37b3957fb31743df6a1bcf4a8c685 third_party/protoc-gen-validate (v0.0.10) + 9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e3 third_party/upb (heads/upbc-cpp) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF |