diff options
36 files changed, 312 insertions, 559 deletions
diff --git a/bazel/cc_grpc_library.bzl b/bazel/cc_grpc_library.bzl index 3288565714..6bfcd653f5 100644 --- a/bazel/cc_grpc_library.bzl +++ b/bazel/cc_grpc_library.bzl @@ -1,6 +1,6 @@ """Generates and compiles C++ grpc stubs from proto_library rules.""" -load("//:bazel/generate_cc.bzl", "generate_cc") +load("//bazel:generate_cc.bzl", "generate_cc") def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, generate_mocks = False, use_external = False, **kwargs): """Generates C++ grpc classes from a .proto file. diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index ae747aa42c..2f14071f92 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -83,7 +83,7 @@ _generate_cc = rule( attrs = { "srcs": attr.label_list( mandatory = True, - non_empty = True, + allow_empty = False, providers = ["proto"], ), "plugin": attr.label( diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 65fe5a10aa..caeafc76b6 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -23,6 +23,8 @@ # each change must be ported from one to the other. # +load("//bazel:cc_grpc_library.bzl", "cc_grpc_library") + # The set of pollers to test against if a test exercises polling POLLERS = ["epollex", "epoll1", "poll", "poll-cv"] @@ -111,7 +113,6 @@ def grpc_proto_plugin(name, srcs = [], deps = []): deps = deps, ) -load("//:bazel/cc_grpc_library.bzl", "cc_grpc_library") def grpc_proto_library( name, diff --git a/src/core/ext/filters/client_channel/client_channel_factory.cc b/src/core/ext/filters/client_channel/client_channel_factory.cc index 172e9f03c7..130bbe0418 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.cc +++ b/src/core/ext/filters/client_channel/client_channel_factory.cc @@ -30,7 +30,7 @@ void grpc_client_channel_factory_unref(grpc_client_channel_factory* factory) { } grpc_subchannel* grpc_client_channel_factory_create_subchannel( - grpc_client_channel_factory* factory, const grpc_subchannel_args* args) { + grpc_client_channel_factory* factory, const grpc_channel_args* args) { return factory->vtable->create_subchannel(factory, args); } diff --git a/src/core/ext/filters/client_channel/client_channel_factory.h b/src/core/ext/filters/client_channel/client_channel_factory.h index 601ec46b2a..91dec12282 100644 --- a/src/core/ext/filters/client_channel/client_channel_factory.h +++ b/src/core/ext/filters/client_channel/client_channel_factory.h @@ -49,7 +49,7 @@ struct grpc_client_channel_factory_vtable { void (*ref)(grpc_client_channel_factory* factory); void (*unref)(grpc_client_channel_factory* factory); grpc_subchannel* (*create_subchannel)(grpc_client_channel_factory* factory, - const grpc_subchannel_args* args); + const grpc_channel_args* args); grpc_channel* (*create_client_channel)(grpc_client_channel_factory* factory, const char* target, grpc_client_channel_type type, @@ -61,7 +61,7 @@ void grpc_client_channel_factory_unref(grpc_client_channel_factory* factory); /** Create a new grpc_subchannel */ grpc_subchannel* grpc_client_channel_factory_create_subchannel( - grpc_client_channel_factory* factory, const grpc_subchannel_args* args); + grpc_client_channel_factory* factory, const grpc_channel_args* args); /** Create a new grpc_channel */ grpc_channel* grpc_client_channel_factory_create_channel( diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 6f31a643c1..1d0ecbe3f6 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -509,12 +509,10 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList( GRPC_ARG_SERVER_ADDRESS_LIST, GRPC_ARG_INHIBIT_HEALTH_CHECKING}; // Create a subchannel for each address. - grpc_subchannel_args sc_args; for (size_t i = 0; i < addresses.size(); i++) { // If there were any balancer addresses, we would have chosen grpclb // policy, which does not use a SubchannelList. GPR_ASSERT(!addresses[i].IsBalancer()); - memset(&sc_args, 0, sizeof(grpc_subchannel_args)); InlinedVector<grpc_arg, 4> args_to_add; args_to_add.emplace_back( grpc_create_subchannel_address_arg(&addresses[i].address())); @@ -527,9 +525,8 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList( &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), args_to_add.data(), args_to_add.size()); gpr_free(args_to_add[0].value.string); - sc_args.args = new_args; grpc_subchannel* subchannel = grpc_client_channel_factory_create_subchannel( - client_channel_factory, &sc_args); + client_channel_factory, new_args); grpc_channel_args_destroy(new_args); if (subchannel == nullptr) { // Subchannel could not be created. diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index dff213efc6..640a052e91 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -537,7 +537,7 @@ struct HealthCheckParams { } // namespace grpc_core grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, - const grpc_subchannel_args* args) { + const grpc_channel_args* args) { grpc_subchannel_key* key = grpc_subchannel_key_create(args); grpc_subchannel* c = grpc_subchannel_index_find(key); if (c) { @@ -554,11 +554,10 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, c->pollset_set = grpc_pollset_set_create(); grpc_resolved_address* addr = static_cast<grpc_resolved_address*>(gpr_malloc(sizeof(*addr))); - grpc_get_subchannel_address_arg(args->args, addr); + grpc_get_subchannel_address_arg(args, addr); grpc_resolved_address* new_address = nullptr; grpc_channel_args* new_args = nullptr; - if (grpc_proxy_mappers_map_address(addr, args->args, &new_address, - &new_args)) { + if (grpc_proxy_mappers_map_address(addr, args, &new_address, &new_args)) { GPR_ASSERT(new_address != nullptr); gpr_free(addr); addr = new_address; @@ -567,7 +566,7 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, grpc_arg new_arg = grpc_create_subchannel_address_arg(addr); gpr_free(addr); c->args = grpc_channel_args_copy_and_add_and_remove( - new_args != nullptr ? new_args : args->args, keys_to_remove, + new_args != nullptr ? new_args : args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove), &new_arg, 1); gpr_free(new_arg.value.string); if (new_args != nullptr) grpc_channel_args_destroy(new_args); @@ -580,7 +579,7 @@ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, grpc_connectivity_state_init(&c->state_and_health_tracker, GRPC_CHANNEL_IDLE, "subchannel"); grpc_core::BackOff::Options backoff_options; - parse_args_for_backoff_values(args->args, &backoff_options, + parse_args_for_backoff_values(args, &backoff_options, &c->min_connect_timeout_ms); c->backoff.Init(backoff_options); gpr_mu_init(&c->mu); diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index d0c0a672fa..8c994c64f5 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -38,7 +38,6 @@ address. Provides a target for load balancing. */ typedef struct grpc_subchannel grpc_subchannel; typedef struct grpc_subchannel_call grpc_subchannel_call; -typedef struct grpc_subchannel_args grpc_subchannel_args; typedef struct grpc_subchannel_key grpc_subchannel_key; #ifndef NDEBUG @@ -186,16 +185,9 @@ void grpc_subchannel_call_set_cleanup_closure( grpc_call_stack* grpc_subchannel_call_get_call_stack( grpc_subchannel_call* subchannel_call); -struct grpc_subchannel_args { - /* When updating this struct, also update subchannel_index.c */ - - /** Channel arguments to be supplied to the newly created channel */ - const grpc_channel_args* args; -}; - /** create a subchannel given a connector */ grpc_subchannel* grpc_subchannel_create(grpc_connector* connector, - const grpc_subchannel_args* args); + const grpc_channel_args* args); /// Sets \a addr from \a args. void grpc_get_subchannel_address_arg(const grpc_channel_args* args, diff --git a/src/core/ext/filters/client_channel/subchannel_index.cc b/src/core/ext/filters/client_channel/subchannel_index.cc index 0ae7898c5a..d0ceda8312 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.cc +++ b/src/core/ext/filters/client_channel/subchannel_index.cc @@ -39,38 +39,37 @@ static gpr_mu g_mu; static gpr_refcount g_refcount; struct grpc_subchannel_key { - grpc_subchannel_args args; + grpc_channel_args* args; }; static bool g_force_creation = false; static grpc_subchannel_key* create_key( - const grpc_subchannel_args* args, + const grpc_channel_args* args, grpc_channel_args* (*copy_channel_args)(const grpc_channel_args* args)) { grpc_subchannel_key* k = static_cast<grpc_subchannel_key*>(gpr_malloc(sizeof(*k))); - k->args.args = copy_channel_args(args->args); + k->args = copy_channel_args(args); return k; } -grpc_subchannel_key* grpc_subchannel_key_create( - const grpc_subchannel_args* args) { +grpc_subchannel_key* grpc_subchannel_key_create(const grpc_channel_args* args) { return create_key(args, grpc_channel_args_normalize); } static grpc_subchannel_key* subchannel_key_copy(grpc_subchannel_key* k) { - return create_key(&k->args, grpc_channel_args_copy); + return create_key(k->args, grpc_channel_args_copy); } int grpc_subchannel_key_compare(const grpc_subchannel_key* a, const grpc_subchannel_key* b) { // To pretend the keys are different, return a non-zero value. if (GPR_UNLIKELY(g_force_creation)) return 1; - return grpc_channel_args_compare(a->args.args, b->args.args); + return grpc_channel_args_compare(a->args, b->args); } void grpc_subchannel_key_destroy(grpc_subchannel_key* k) { - grpc_channel_args_destroy(const_cast<grpc_channel_args*>(k->args.args)); + grpc_channel_args_destroy(k->args); gpr_free(k); } diff --git a/src/core/ext/filters/client_channel/subchannel_index.h b/src/core/ext/filters/client_channel/subchannel_index.h index c135613d26..429634bd54 100644 --- a/src/core/ext/filters/client_channel/subchannel_index.h +++ b/src/core/ext/filters/client_channel/subchannel_index.h @@ -27,8 +27,7 @@ shared amongst channels */ /** Create a key that can be used to uniquely identify a subchannel */ -grpc_subchannel_key* grpc_subchannel_key_create( - const grpc_subchannel_args* args); +grpc_subchannel_key* grpc_subchannel_key_create(const grpc_channel_args* args); /** Destroy a subchannel key */ void grpc_subchannel_key_destroy(grpc_subchannel_key* key); diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.cc b/src/core/ext/transport/chttp2/client/insecure/channel_create.cc index e6c8c38260..a5bf1bf21d 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.cc +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.cc @@ -40,14 +40,12 @@ static void client_channel_factory_unref( grpc_client_channel_factory* cc_factory) {} static grpc_subchannel* client_channel_factory_create_subchannel( - grpc_client_channel_factory* cc_factory, const grpc_subchannel_args* args) { - grpc_subchannel_args final_sc_args; - memcpy(&final_sc_args, args, sizeof(*args)); - final_sc_args.args = grpc_default_authority_add_if_not_present(args->args); + grpc_client_channel_factory* cc_factory, const grpc_channel_args* args) { + grpc_channel_args* new_args = grpc_default_authority_add_if_not_present(args); grpc_connector* connector = grpc_chttp2_connector_create(); - grpc_subchannel* s = grpc_subchannel_create(connector, &final_sc_args); + grpc_subchannel* s = grpc_subchannel_create(connector, new_args); grpc_connector_unref(connector); - grpc_channel_args_destroy(const_cast<grpc_channel_args*>(final_sc_args.args)); + grpc_channel_args_destroy(new_args); return s; } diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc index 9612698e96..ddd538faa8 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc @@ -46,10 +46,10 @@ static void client_channel_factory_ref( static void client_channel_factory_unref( grpc_client_channel_factory* cc_factory) {} -static grpc_subchannel_args* get_secure_naming_subchannel_args( - const grpc_subchannel_args* args) { +static grpc_channel_args* get_secure_naming_channel_args( + const grpc_channel_args* args) { grpc_channel_credentials* channel_credentials = - grpc_channel_credentials_find_in_args(args->args); + grpc_channel_credentials_find_in_args(args); if (channel_credentials == nullptr) { gpr_log(GPR_ERROR, "Can't create subchannel: channel credentials missing for secure " @@ -57,7 +57,7 @@ static grpc_subchannel_args* get_secure_naming_subchannel_args( return nullptr; } // Make sure security connector does not already exist in args. - if (grpc_security_connector_find_in_args(args->args) != nullptr) { + if (grpc_security_connector_find_in_args(args) != nullptr) { gpr_log(GPR_ERROR, "Can't create subchannel: security connector already present in " "channel args."); @@ -65,19 +65,18 @@ static grpc_subchannel_args* get_secure_naming_subchannel_args( } // To which address are we connecting? By default, use the server URI. const grpc_arg* server_uri_arg = - grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI); + grpc_channel_args_find(args, GRPC_ARG_SERVER_URI); const char* server_uri_str = grpc_channel_arg_get_string(server_uri_arg); GPR_ASSERT(server_uri_str != nullptr); grpc_uri* server_uri = grpc_uri_parse(server_uri_str, true /* supress errors */); GPR_ASSERT(server_uri != nullptr); const grpc_core::TargetAuthorityTable* target_authority_table = - grpc_core::FindTargetAuthorityTableInArgs(args->args); + grpc_core::FindTargetAuthorityTableInArgs(args); grpc_core::UniquePtr<char> authority; if (target_authority_table != nullptr) { // Find the authority for the target. - const char* target_uri_str = - grpc_get_subchannel_address_uri_arg(args->args); + const char* target_uri_str = grpc_get_subchannel_address_uri_arg(args); grpc_uri* target_uri = grpc_uri_parse(target_uri_str, false /* suppress errors */); GPR_ASSERT(target_uri != nullptr); @@ -100,15 +99,14 @@ static grpc_subchannel_args* get_secure_naming_subchannel_args( } grpc_arg args_to_add[2]; size_t num_args_to_add = 0; - if (grpc_channel_args_find(args->args, GRPC_ARG_DEFAULT_AUTHORITY) == - nullptr) { + if (grpc_channel_args_find(args, GRPC_ARG_DEFAULT_AUTHORITY) == nullptr) { // If the channel args don't already contain GRPC_ARG_DEFAULT_AUTHORITY, add // the arg, setting it to the value just obtained. args_to_add[num_args_to_add++] = grpc_channel_arg_string_create( const_cast<char*>(GRPC_ARG_DEFAULT_AUTHORITY), authority.get()); } grpc_channel_args* args_with_authority = - grpc_channel_args_copy_and_add(args->args, args_to_add, num_args_to_add); + grpc_channel_args_copy_and_add(args, args_to_add, num_args_to_add); grpc_uri_destroy(server_uri); // Create the security connector using the credentials and target name. grpc_channel_args* new_args_from_connector = nullptr; @@ -137,29 +135,21 @@ static grpc_subchannel_args* get_secure_naming_subchannel_args( grpc_channel_args_destroy(new_args_from_connector); } grpc_channel_args_destroy(args_with_authority); - grpc_subchannel_args* final_sc_args = - static_cast<grpc_subchannel_args*>(gpr_malloc(sizeof(*final_sc_args))); - memcpy(final_sc_args, args, sizeof(*args)); - final_sc_args->args = new_args; - return final_sc_args; + return new_args; } static grpc_subchannel* client_channel_factory_create_subchannel( - grpc_client_channel_factory* cc_factory, const grpc_subchannel_args* args) { - grpc_subchannel_args* subchannel_args = - get_secure_naming_subchannel_args(args); - if (subchannel_args == nullptr) { - gpr_log( - GPR_ERROR, - "Failed to create subchannel arguments during subchannel creation."); + grpc_client_channel_factory* cc_factory, const grpc_channel_args* args) { + grpc_channel_args* new_args = get_secure_naming_channel_args(args); + if (new_args == nullptr) { + gpr_log(GPR_ERROR, + "Failed to create channel args during subchannel creation."); return nullptr; } grpc_connector* connector = grpc_chttp2_connector_create(); - grpc_subchannel* s = grpc_subchannel_create(connector, subchannel_args); + grpc_subchannel* s = grpc_subchannel_create(connector, new_args); grpc_connector_unref(connector); - grpc_channel_args_destroy( - const_cast<grpc_channel_args*>(subchannel_args->args)); - gpr_free(subchannel_args); + grpc_channel_args_destroy(new_args); return s; } diff --git a/src/core/lib/iomgr/combiner.cc b/src/core/lib/iomgr/combiner.cc index 7c0062eb4e..402f8904ea 100644 --- a/src/core/lib/iomgr/combiner.cc +++ b/src/core/lib/iomgr/combiner.cc @@ -29,6 +29,7 @@ #include "src/core/lib/debug/stats.h" #include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/profiling/timers.h" grpc_core::DebugOnlyTraceFlag grpc_combiner_trace(false, "combiner"); @@ -228,8 +229,14 @@ bool grpc_combiner_continue_exec_ctx() { grpc_core::ExecCtx::Get()->IsReadyToFinish(), lock->time_to_execute_final_list)); + // offload only if all the following conditions are true: + // 1. the combiner is contended and has more than one closure to execute + // 2. the current execution context needs to finish as soon as possible + // 3. the DEFAULT executor is threaded + // 4. the current thread is not a worker for any background poller if (contended && grpc_core::ExecCtx::Get()->IsReadyToFinish() && - grpc_executor_is_threaded()) { + grpc_executor_is_threaded() && + !grpc_iomgr_is_any_background_poller_thread()) { GPR_TIMER_MARK("offload_from_finished_exec_ctx", 0); // this execution context wants to move on: schedule remaining work to be // picked up on the executor diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 4b8c891e9b..9eb4c089d8 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -1242,6 +1242,8 @@ static void pollset_set_del_pollset_set(grpc_pollset_set* bag, * Event engine binding */ +static bool is_any_background_poller_thread(void) { return false; } + static void shutdown_background_closure(void) {} static void shutdown_engine(void) { @@ -1287,6 +1289,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + is_any_background_poller_thread, shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 7a4870db78..0a0891013a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -1604,6 +1604,8 @@ static void pollset_set_del_pollset_set(grpc_pollset_set* bag, * Event engine binding */ +static bool is_any_background_poller_thread(void) { return false; } + static void shutdown_background_closure(void) {} static void shutdown_engine(void) { @@ -1644,6 +1646,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + is_any_background_poller_thread, shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index 67cbfbbd02..c479206410 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -1782,6 +1782,8 @@ static void global_cv_fd_table_shutdown() { * event engine binding */ +static bool is_any_background_poller_thread(void) { return false; } + static void shutdown_background_closure(void) {} static void shutdown_engine(void) { @@ -1828,6 +1830,7 @@ static const grpc_event_engine_vtable vtable = { pollset_set_add_fd, pollset_set_del_fd, + is_any_background_poller_thread, shutdown_background_closure, shutdown_engine, }; diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 32d1b6c43e..fb2e70eee4 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -399,6 +399,10 @@ void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd) { g_event_engine->pollset_set_del_fd(pollset_set, fd); } +bool grpc_is_any_background_poller_thread(void) { + return g_event_engine->is_any_background_poller_thread(); +} + void grpc_shutdown_background_closure(void) { g_event_engine->shutdown_background_closure(); } diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 812c7a0f0f..94ac9fdba6 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -80,6 +80,7 @@ typedef struct grpc_event_engine_vtable { void (*pollset_set_add_fd)(grpc_pollset_set* pollset_set, grpc_fd* fd); void (*pollset_set_del_fd)(grpc_pollset_set* pollset_set, grpc_fd* fd); + bool (*is_any_background_poller_thread)(void); void (*shutdown_background_closure)(void); void (*shutdown_engine)(void); } grpc_event_engine_vtable; @@ -181,6 +182,9 @@ void grpc_pollset_add_fd(grpc_pollset* pollset, struct grpc_fd* fd); void grpc_pollset_set_add_fd(grpc_pollset_set* pollset_set, grpc_fd* fd); void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd); +/* Returns true if the caller is a worker thread for any background poller. */ +bool grpc_is_any_background_poller_thread(); + /* Shut down all the closures registered in the background poller. */ void grpc_shutdown_background_closure(); diff --git a/src/core/lib/iomgr/iomgr.cc b/src/core/lib/iomgr/iomgr.cc index eb29973514..a492146857 100644 --- a/src/core/lib/iomgr/iomgr.cc +++ b/src/core/lib/iomgr/iomgr.cc @@ -161,6 +161,10 @@ void grpc_iomgr_shutdown_background_closure() { grpc_iomgr_platform_shutdown_background_closure(); } +bool grpc_iomgr_is_any_background_poller_thread() { + return grpc_iomgr_platform_is_any_background_poller_thread(); +} + void grpc_iomgr_register_object(grpc_iomgr_object* obj, const char* name) { obj->name = gpr_strdup(name); gpr_mu_lock(&g_mu); diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 8ea9289e06..6261aa550c 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -39,6 +39,9 @@ void grpc_iomgr_shutdown(); * background poller. */ void grpc_iomgr_shutdown_background_closure(); +/** Returns true if the caller is a worker thread for any background poller. */ +bool grpc_iomgr_is_any_background_poller_thread(); + /* Exposed only for testing */ size_t grpc_iomgr_count_objects_for_testing(); diff --git a/src/core/lib/iomgr/iomgr_custom.cc b/src/core/lib/iomgr/iomgr_custom.cc index 4b112c9097..e1cd8f7310 100644 --- a/src/core/lib/iomgr/iomgr_custom.cc +++ b/src/core/lib/iomgr/iomgr_custom.cc @@ -41,10 +41,14 @@ static void iomgr_platform_init(void) { static void iomgr_platform_flush(void) {} static void iomgr_platform_shutdown(void) { grpc_pollset_global_shutdown(); } static void iomgr_platform_shutdown_background_closure(void) {} +static bool iomgr_platform_is_any_background_poller_thread(void) { + return false; +} static grpc_iomgr_platform_vtable vtable = { iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, - iomgr_platform_shutdown_background_closure}; + iomgr_platform_shutdown_background_closure, + iomgr_platform_is_any_background_poller_thread}; void grpc_custom_iomgr_init(grpc_socket_vtable* socket, grpc_custom_resolver_vtable* resolver, diff --git a/src/core/lib/iomgr/iomgr_internal.cc b/src/core/lib/iomgr/iomgr_internal.cc index b6c9211865..e68b1cf581 100644 --- a/src/core/lib/iomgr/iomgr_internal.cc +++ b/src/core/lib/iomgr/iomgr_internal.cc @@ -45,3 +45,7 @@ void grpc_iomgr_platform_shutdown() { iomgr_platform_vtable->shutdown(); } void grpc_iomgr_platform_shutdown_background_closure() { iomgr_platform_vtable->shutdown_background_closure(); } + +bool grpc_iomgr_platform_is_any_background_poller_thread() { + return iomgr_platform_vtable->is_any_background_poller_thread(); +} diff --git a/src/core/lib/iomgr/iomgr_internal.h b/src/core/lib/iomgr/iomgr_internal.h index bca7409907..2250ad9a18 100644 --- a/src/core/lib/iomgr/iomgr_internal.h +++ b/src/core/lib/iomgr/iomgr_internal.h @@ -36,6 +36,7 @@ typedef struct grpc_iomgr_platform_vtable { void (*flush)(void); void (*shutdown)(void); void (*shutdown_background_closure)(void); + bool (*is_any_background_poller_thread)(void); } grpc_iomgr_platform_vtable; void grpc_iomgr_register_object(grpc_iomgr_object* obj, const char* name); @@ -56,6 +57,9 @@ void grpc_iomgr_platform_shutdown(void); /** shut down all the closures registered in the background poller */ void grpc_iomgr_platform_shutdown_background_closure(void); +/** return true is the caller is a worker thread for any background poller */ +bool grpc_iomgr_platform_is_any_background_poller_thread(void); + bool grpc_iomgr_abort_on_leaks(void); #endif /* GRPC_CORE_LIB_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/lib/iomgr/iomgr_posix.cc b/src/core/lib/iomgr/iomgr_posix.cc index 9386adf060..278c8de688 100644 --- a/src/core/lib/iomgr/iomgr_posix.cc +++ b/src/core/lib/iomgr/iomgr_posix.cc @@ -55,9 +55,14 @@ static void iomgr_platform_shutdown_background_closure(void) { grpc_shutdown_background_closure(); } +static bool iomgr_platform_is_any_background_poller_thread(void) { + return grpc_is_any_background_poller_thread(); +} + static grpc_iomgr_platform_vtable vtable = { iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, - iomgr_platform_shutdown_background_closure}; + iomgr_platform_shutdown_background_closure, + iomgr_platform_is_any_background_poller_thread}; void grpc_set_default_iomgr_platform() { grpc_set_tcp_client_impl(&grpc_posix_tcp_client_vtable); diff --git a/src/core/lib/iomgr/iomgr_posix_cfstream.cc b/src/core/lib/iomgr/iomgr_posix_cfstream.cc index 552ef4309c..462ac41fcd 100644 --- a/src/core/lib/iomgr/iomgr_posix_cfstream.cc +++ b/src/core/lib/iomgr/iomgr_posix_cfstream.cc @@ -58,9 +58,14 @@ static void iomgr_platform_shutdown_background_closure(void) { grpc_shutdown_background_closure(); } +static bool iomgr_platform_is_any_background_poller_thread(void) { + return grpc_is_any_background_poller_thread(); +} + static grpc_iomgr_platform_vtable vtable = { iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, - iomgr_platform_shutdown_background_closure}; + iomgr_platform_shutdown_background_closure, + iomgr_platform_is_any_background_poller_thread}; void grpc_set_default_iomgr_platform() { char* enable_cfstream = getenv(grpc_cfstream_env_var); diff --git a/src/core/lib/iomgr/iomgr_windows.cc b/src/core/lib/iomgr/iomgr_windows.cc index 24ef0dba7b..0579e16aa7 100644 --- a/src/core/lib/iomgr/iomgr_windows.cc +++ b/src/core/lib/iomgr/iomgr_windows.cc @@ -73,9 +73,14 @@ static void iomgr_platform_shutdown(void) { static void iomgr_platform_shutdown_background_closure(void) {} +static bool iomgr_platform_is_any_background_poller_thread(void) { + return false; +} + static grpc_iomgr_platform_vtable vtable = { iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown, - iomgr_platform_shutdown_background_closure}; + iomgr_platform_shutdown_background_closure, + iomgr_platform_is_any_background_poller_thread}; void grpc_set_default_iomgr_platform() { grpc_set_tcp_client_impl(&grpc_windows_tcp_client_vtable); diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 70d7618e05..8613bc501f 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -24,7 +24,7 @@ from grpc._cython import cygrpc as _cygrpc logging.getLogger(__name__).addHandler(logging.NullHandler()) try: - from ._grpcio_metadata import __version__ + from grpc._grpcio_metadata import __version__ except ImportError: __version__ = "dev0" diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index d72648a35d..ef74f61e04 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -149,6 +149,6 @@ cdef class Server: grpc_server_destroy(self.c_server) self.c_server = NULL - def __dealloc(self): + def __dealloc__(self): if self.c_server == NULL: grpc_shutdown() diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index 5acf269988..7daabd5052 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -254,7 +254,6 @@ grpc_cc_test( name = "tcp_server_posix_test", srcs = ["tcp_server_posix_test.cc"], language = "C++", - tags = ["manual"], # TODO(adelez): Remove once this works on Foundry. deps = [ "//:gpr", "//:grpc", diff --git a/test/core/iomgr/tcp_server_posix_test.cc b/test/core/iomgr/tcp_server_posix_test.cc index 2c66cdec77..81e26b20cd 100644 --- a/test/core/iomgr/tcp_server_posix_test.cc +++ b/test/core/iomgr/tcp_server_posix_test.cc @@ -439,6 +439,11 @@ int main(int argc, char** argv) { static_cast<test_addrs*>(gpr_zalloc(sizeof(*dst_addrs))); grpc::testing::TestEnvironment env(argc, argv); grpc_init(); + // wait a few seconds to make sure IPv6 link-local addresses can be bound + // if we are running under docker container that has just started. + // See https://github.com/moby/moby/issues/38491 + // See https://github.com/grpc/grpc/issues/15610 + gpr_sleep_until(grpc_timeout_seconds_to_deadline(4)); { grpc_core::ExecCtx exec_ctx; g_pollset = static_cast<grpc_pollset*>(gpr_zalloc(grpc_pollset_size())); diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 8d12606434..125b1ce5c4 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -326,7 +326,7 @@ class FakeClientChannelFactory : public grpc_client_channel_factory { static void NoRef(grpc_client_channel_factory* factory) {} static void NoUnref(grpc_client_channel_factory* factory) {} static grpc_subchannel* CreateSubchannel(grpc_client_channel_factory* factory, - const grpc_subchannel_args* args) { + const grpc_channel_args* args) { return nullptr; } static grpc_channel* CreateClientChannel(grpc_client_channel_factory* factory, diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index dca97c85b1..7aa197b597 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -94,6 +94,7 @@ static const grpc_event_engine_vtable* init_engine_vtable(bool) { g_vtable.pollset_destroy = pollset_destroy; g_vtable.pollset_work = pollset_work; g_vtable.pollset_kick = pollset_kick; + g_vtable.is_any_background_poller_thread = [] { return false; }; g_vtable.shutdown_background_closure = [] {}; g_vtable.shutdown_engine = [] {}; diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index db84d9b454..6676f5d470 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -1,6 +1,6 @@ # Overview -This directory contains scripts that facilitate building and running gRPC tests for combinations of language/runtimes (known as matrix). +This directory contains scripts that facilitate building and running gRPC interoperability tests for combinations of language/runtimes (known as matrix). The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack from specific releases/tag, are used to test version compatiblity between gRPC release versions. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 4964fd6167..12051e70a0 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -15,6 +15,8 @@ # Defines languages, runtimes and releases for backward compatibility testing +from collections import OrderedDict + def get_github_repo(lang): return { @@ -27,23 +29,16 @@ def get_github_repo(lang): def get_release_tags(lang): - return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang]) - - -def get_release_tag_name(release_info): - assert len(release_info.keys()) == 1 - return release_info.keys()[0] + """Returns list of known releases for given language.""" + return list(LANG_RELEASE_MATRIX[lang].keys()) def get_runtimes_for_lang_release(lang, release): """Get list of valid runtimes for given release of lang.""" runtimes_to_skip = [] - # see if any the lang release has "skip_runtime" annotation. - for release_info in LANG_RELEASE_MATRIX[lang]: - if get_release_tag_name(release_info) == release: - if release_info[release] is not None: - runtimes_to_skip = release_info[release].get('skip_runtime', []) - break + release_info = LANG_RELEASE_MATRIX[lang][release] + if release_info: + runtimes_to_skip = release_info.skip_runtime return [ runtime for runtime in LANG_RUNTIME_MATRIX[lang] if runtime not in runtimes_to_skip @@ -51,6 +46,9 @@ def get_runtimes_for_lang_release(lang, release): def should_build_docker_interop_image_from_release_tag(lang): + # All dockerfile definitions live in grpc/grpc repository. + # For language that have a separate repo, we need to use + # dockerfile definitions from head of grpc/grpc. if lang in ['go', 'java', 'node']: return False return True @@ -68,465 +66,183 @@ LANG_RUNTIME_MATRIX = { 'csharp': ['csharp', 'csharpcoreclr'], } + +class ReleaseInfo: + """Info about a single release of a language""" + + def __init__(self, patch=[], skip_runtime=[], testcases_file=None): + self.patch = patch + self.skip_runtime = skip_runtime + self.testcases_file = None + + # Dictionary of known releases for given language. LANG_RELEASE_MATRIX = { - 'cxx': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'go': [ - { - 'v1.0.5': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.2.1': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.3.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.4.2': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.5.2': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.6.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.7.4': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.8.2': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.9.2': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.10.1': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.11.3': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.12.2': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.13.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.14.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.15.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.16.0': { - 'skip_runtime': ['go1.11'] - } - }, - { - 'v1.17.0': { - 'skip_runtime': ['go1.7', 'go1.8'] - } - }, - ], - 'java': [ - { - 'v1.0.3': None - }, - { - 'v1.1.2': None - }, - { - 'v1.2.0': None - }, - { - 'v1.3.1': None - }, - { - 'v1.4.0': None - }, - { - 'v1.5.0': None - }, - { - 'v1.6.1': None - }, - { - 'v1.7.0': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.0': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.1': None - }, - { - 'v1.14.0': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.1': None - }, - { - 'v1.17.1': None - }, - ], - 'python': [ - { - 'v1.0.x': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.1': None # first python 1.8 release is 1.8.1 - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'node': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, + 'cxx': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'go': + OrderedDict([ + ('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])), + ]), + 'java': + OrderedDict([ + ('v1.0.3', ReleaseInfo()), + ('v1.1.2', ReleaseInfo()), + ('v1.2.0', ReleaseInfo()), + ('v1.3.1', ReleaseInfo()), + ('v1.4.0', ReleaseInfo()), + ('v1.5.0', ReleaseInfo()), + ('v1.6.1', ReleaseInfo()), + ('v1.7.0', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.0', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.1', ReleaseInfo()), + ('v1.14.0', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.1', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'python': + OrderedDict([ + ('v1.0.x', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.1', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'node': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), # TODO: https://github.com/grpc/grpc-node/issues/235. - #{ - # 'v1.7.2': None - #}, - { - 'v1.8.4': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.0': None - }, - { - 'v1.11.3': None - }, - { - 'v1.12.4': None - }, - ], - 'ruby': [ - { - 'v1.0.1': { - 'patch': [ - 'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile', - 'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh', - ] - } - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'php': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'csharp': [ - { - 'v1.0.1': { - 'patch': [ - 'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile', - 'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile', - ] - } - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], + # ('v1.7.2', ReleaseInfo()), + ('v1.8.4', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.0', ReleaseInfo()), + ('v1.11.3', ReleaseInfo()), + ('v1.12.4', ReleaseInfo()), + ]), + 'ruby': + OrderedDict([ + ('v1.0.1', + ReleaseInfo(patch=[ + 'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile', + 'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh', + ])), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'php': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'csharp': + OrderedDict([ + ('v1.0.1', + ReleaseInfo(patch=[ + 'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile', + 'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile', + ])), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), } # This matrix lists the version of testcases to use for a release. As new @@ -535,6 +251,8 @@ LANG_RELEASE_MATRIX = { # particular version in some cases. If not specified, xxx__master file will be # used. For example, all java versions will run the commands in java__master. # The testcases files exist under the testcases directory. +# TODO(jtattermusch): make this data part of LANG_RELEASE_MATRIX, +# there is no reason for this to be a separate data structure. TESTCASES_VERSION_MATRIX = { 'node_v1.0.1': 'node__v1.0.1', 'node_v1.1.4': 'node__v1.1.4', diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py index cf61d46248..31a0e1c7ba 100755 --- a/tools/interop_matrix/create_matrix_images.py +++ b/tools/interop_matrix/create_matrix_images.py @@ -39,10 +39,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() # All gRPC release tags, flattened, deduped and sorted. _RELEASES = sorted( list( - set( - client_matrix.get_release_tag_name(info) - for lang in client_matrix.LANG_RELEASE_MATRIX.values() - for info in lang))) + set(release + for release_dict in client_matrix.LANG_RELEASE_MATRIX.values() + for release in release_dict.keys()))) # Destination directory inside docker image to keep extra info from build time. _BUILD_INFO = '/var/local/build_info' @@ -260,11 +259,10 @@ atexit.register(cleanup) def maybe_apply_patches_on_git_tag(stack_base, lang, release): files_to_patch = [] - for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]: - if client_matrix.get_release_tag_name(release_info) == release: - if release_info[release] is not None: - files_to_patch = release_info[release].get('patch') - break + + release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release] + if release_info: + files_to_patch = release_info.patch if not files_to_patch: return patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release) diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index dabb486523..c855de3b1e 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -44,10 +44,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() # All gRPC release tags, flattened, deduped and sorted. _RELEASES = sorted( list( - set( - client_matrix.get_release_tag_name(info) - for lang in client_matrix.LANG_RELEASE_MATRIX.values() - for info in lang))) + set(release + for release_dict in client_matrix.LANG_RELEASE_MATRIX.values() + for release in release_dict.keys()))) argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |