diff options
44 files changed, 1203 insertions, 407 deletions
@@ -993,6 +993,7 @@ grpc_cc_library( "src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c", ], hdrs = [ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d2418c5a5..a60786458b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1131,6 +1131,7 @@ add_library(grpc src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c src/core/ext/filters/load_reporting/load_reporting.c @@ -2006,6 +2007,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 0000000000..9d4213ebca --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,3 @@ +## Community Code of Conduct + +gRPC follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). @@ -3114,6 +3114,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c \ src/core/ext/filters/load_reporting/load_reporting.c \ @@ -3958,6 +3959,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c \ diff --git a/PATENTS b/PATENTS deleted file mode 100644 index 619f9dbfe6..0000000000 --- a/PATENTS +++ /dev/null @@ -1,22 +0,0 @@ -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the GRPC project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of GRPC, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of GRPC. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of GRPC or any code incorporated within this -implementation of GRPC constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of GRPC -shall terminate as of the date such litigation is filed. diff --git a/binding.gyp b/binding.gyp index 9b1a56692a..d5bb27f6da 100644 --- a/binding.gyp +++ b/binding.gyp @@ -862,6 +862,7 @@ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c', 'src/core/ext/filters/load_reporting/load_reporting.c', diff --git a/build.yaml b/build.yaml index 56d7907891..7ab1cee263 100644 --- a/build.yaml +++ b/build.yaml @@ -589,6 +589,7 @@ filegroups: - src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c plugin: grpc_resolver_dns_ares uses: - grpc_base @@ -309,6 +309,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c \ src/core/ext/filters/load_reporting/load_reporting.c \ diff --git a/config.w32 b/config.w32 index 3e21815e82..919587dc58 100644 --- a/config.w32 +++ b/config.w32 @@ -286,6 +286,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\dns_resolver_ares.c " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_posix.c " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper.c " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_fallback.c " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\native\\dns_resolver.c " + "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.c " + "src\\core\\ext\\filters\\load_reporting\\load_reporting.c " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index f1d2190c15..277f97944c 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -685,6 +685,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c', 'src/core/ext/filters/load_reporting/load_reporting.c', diff --git a/grpc.gemspec b/grpc.gemspec index 31f72201a7..a4e22dfbdf 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -616,6 +616,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c ) s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c ) s.files += %w( src/core/ext/filters/load_reporting/load_reporting.c ) diff --git a/include/grpc++/impl/codegen/async_unary_call.h b/include/grpc++/impl/codegen/async_unary_call.h index 66843d0da3..4aa4e25a9e 100644 --- a/include/grpc++/impl/codegen/async_unary_call.h +++ b/include/grpc++/impl/codegen/async_unary_call.h @@ -139,7 +139,7 @@ class ClientAsyncResponseReader final // disable operator new static void* operator new(std::size_t size); - static void* operator new(std::size_t size, void* p) { return p; }; + static void* operator new(std::size_t size, void* p) { return p; } SneakyCallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose> diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 0fef12c091..ca757e2a9c 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -167,7 +167,7 @@ class CompletionQueue : private GrpcLibraryCodegen { void RegisterAvalanching() { gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_, static_cast<gpr_atm>(1)); - }; + } void CompleteAvalanching(); protected: diff --git a/package.xml b/package.xml index 36e77699dc..18b1d639b9 100644 --- a/package.xml +++ b/package.xml @@ -630,6 +630,7 @@ <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c" role="src" /> <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c" role="src" /> <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c" role="src" /> + <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c" role="src" /> <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c" role="src" /> <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c" role="src" /> <file baseinstalldir="/" name="src/core/ext/filters/load_reporting/load_reporting.c" role="src" /> diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c index ebdba2835f..92f060b422 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c @@ -65,6 +65,8 @@ typedef struct { grpc_combiner *combiner; /** are we currently resolving? */ bool resolving; + /** the pending resolving request */ + grpc_ares_request *pending_request; /** which version of the result have we published? */ int published_version; /** which version of the result is current? */ @@ -82,7 +84,7 @@ typedef struct { gpr_backoff backoff_state; /** currently resolving addresses */ - grpc_resolved_addresses *addresses; + grpc_lb_addresses *lb_addresses; } ares_dns_resolver; static void dns_ares_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *r); @@ -109,6 +111,9 @@ static void dns_ares_shutdown_locked(grpc_exec_ctx *exec_ctx, if (r->have_retry_timer) { grpc_timer_cancel(exec_ctx, &r->retry_timer); } + if (r->pending_request != NULL) { + grpc_cancel_ares_request(exec_ctx, r->pending_request); + } if (r->next_completion != NULL) { *r->target_result = NULL; grpc_closure_sched( @@ -145,19 +150,11 @@ static void dns_ares_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_channel_args *result = NULL; GPR_ASSERT(r->resolving); r->resolving = false; - if (r->addresses != NULL) { - grpc_lb_addresses *addresses = grpc_lb_addresses_create( - r->addresses->naddrs, NULL /* user_data_vtable */); - for (size_t i = 0; i < r->addresses->naddrs; ++i) { - grpc_lb_addresses_set_address( - addresses, i, &r->addresses->addrs[i].addr, - r->addresses->addrs[i].len, false /* is_balancer */, - NULL /* balancer_name */, NULL /* user_data */); - } - grpc_arg new_arg = grpc_lb_addresses_create_channel_arg(addresses); + r->pending_request = NULL; + if (r->lb_addresses != NULL) { + grpc_arg new_arg = grpc_lb_addresses_create_channel_arg(r->lb_addresses); result = grpc_channel_args_copy_and_add(r->channel_args, &new_arg, 1); - grpc_resolved_addresses_destroy(r->addresses); - grpc_lb_addresses_destroy(exec_ctx, addresses); + grpc_lb_addresses_destroy(exec_ctx, r->lb_addresses); } else { const char *msg = grpc_error_string(error); gpr_log(GPR_DEBUG, "dns resolution failed: %s", msg); @@ -209,10 +206,11 @@ static void dns_ares_start_resolving_locked(grpc_exec_ctx *exec_ctx, GRPC_RESOLVER_REF(&r->base, "dns-resolving"); GPR_ASSERT(!r->resolving); r->resolving = true; - r->addresses = NULL; - grpc_dns_lookup_ares(exec_ctx, r->dns_server, r->name_to_resolve, - r->default_port, r->interested_parties, - &r->dns_ares_on_resolved_locked, &r->addresses); + r->lb_addresses = NULL; + r->pending_request = grpc_dns_lookup_ares( + exec_ctx, r->dns_server, r->name_to_resolve, r->default_port, + r->interested_parties, &r->dns_ares_on_resolved_locked, &r->lb_addresses, + true /* check_grpclb */); } static void dns_ares_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx, @@ -222,6 +220,7 @@ static void dns_ares_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx, *r->target_result = r->resolved_result == NULL ? NULL : grpc_channel_args_copy(r->resolved_result); + gpr_log(GPR_DEBUG, "dns_ares_maybe_finish_next_locked"); grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE); r->next_completion = NULL; r->published_version = r->resolved_version; @@ -245,10 +244,10 @@ static void dns_ares_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) { static grpc_resolver *dns_ares_create(grpc_exec_ctx *exec_ctx, grpc_resolver_args *args, const char *default_port) { - // Get name from args. + /* Get name from args. */ const char *path = args->uri->path; if (path[0] == '/') ++path; - // Create resolver. + /* Create resolver. */ ares_dns_resolver *r = gpr_zalloc(sizeof(ares_dns_resolver)); grpc_resolver_init(&r->base, &dns_ares_resolver_vtable, args->combiner); if (0 != strcmp(args->uri->authority, "")) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h index ff3904e252..eb1a8ab011 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h @@ -47,5 +47,9 @@ grpc_error *grpc_ares_ev_driver_create(grpc_ares_ev_driver **ev_driver, of ARES_ECANCELLED. */ void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver *ev_driver); +/* Shutdown all the grpc_fds used by \a ev_driver */ +void grpc_ares_ev_driver_shutdown(grpc_exec_ctx *exec_ctx, + grpc_ares_ev_driver *ev_driver); + #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_EV_DRIVER_H \ */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c index 03ce303616..293e6b0252 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c @@ -99,9 +99,12 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) { GPR_ASSERT(!fdn->writable_registered); gpr_mu_destroy(&fdn->mu); grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->grpc_fd); - grpc_fd_shutdown(exec_ctx, fdn->grpc_fd, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("fd node destroyed")); - grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, NULL, "c-ares query finished"); + /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up + immediately by another thread, and should not be closed by the following + grpc_fd_orphan. To prevent this fd from being closed by grpc_fd_orphan, + a fd pointer is provided. */ + int fd; + grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, &fd, "c-ares query finished"); gpr_free(fdn); } @@ -140,6 +143,20 @@ void grpc_ares_ev_driver_destroy(grpc_ares_ev_driver *ev_driver) { grpc_ares_ev_driver_unref(ev_driver); } +void grpc_ares_ev_driver_shutdown(grpc_exec_ctx *exec_ctx, + grpc_ares_ev_driver *ev_driver) { + gpr_mu_lock(&ev_driver->mu); + ev_driver->shutting_down = true; + fd_node *fn = ev_driver->fds; + while (fn != NULL) { + grpc_fd_shutdown( + exec_ctx, fn->grpc_fd, + GRPC_ERROR_CREATE_FROM_STATIC_STRING("grpc_ares_ev_driver_shutdown")); + fn = fn->next; + } + gpr_mu_unlock(&ev_driver->mu); +} + // Search fd in the fd_node list head. This is an O(n) search, the max possible // value of n is ARES_GETSOCK_MAXNUM (16). n is typically 1 - 2 in our tests. static fd_node *pop_fd_node(fd_node **head, int fd) { diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c index 4ee07e170a..153287c30e 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c @@ -33,6 +33,7 @@ #include <grpc/support/string_util.h> #include <grpc/support/time.h> #include <grpc/support/useful.h> +#include <nameser.h> #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" @@ -45,20 +46,14 @@ static gpr_once g_basic_init = GPR_ONCE_INIT; static gpr_mu g_init_mu; -typedef struct grpc_ares_request { +struct grpc_ares_request { /** indicates the DNS server to use, if specified */ struct ares_addr_port_node dns_server_addr; /** following members are set in grpc_resolve_address_ares_impl */ - /** host to resolve, parsed from the name to resolve */ - char *host; - /** port to fill in sockaddr_in, parsed from the name to resolve */ - char *port; - /** default port to use */ - char *default_port; /** closure to call when the request completes */ grpc_closure *on_done; /** the pointer to receive the resolved addresses */ - grpc_resolved_addresses **addrs_out; + grpc_lb_addresses **lb_addrs_out; /** the evernt driver used by this request */ grpc_ares_ev_driver *ev_driver; /** number of ongoing queries */ @@ -70,7 +65,19 @@ typedef struct grpc_ares_request { bool success; /** the errors explaining the request failure, set in on_done_cb */ grpc_error *error; -} grpc_ares_request; +}; + +typedef struct grpc_ares_hostbyname_request { + /** following members are set in create_hostbyname_request */ + /** the top-level request instance */ + grpc_ares_request *parent_request; + /** host to resolve, parsed from the name to resolve */ + char *host; + /** port to fill in sockaddr_in, parsed from the name to resolve */ + uint16_t port; + /** is it a grpclb address */ + bool is_balancer; +} grpc_ares_hostbyname_request; static void do_basic_init(void) { gpr_mu_init(&g_init_mu); } @@ -83,6 +90,10 @@ static uint16_t strhtons(const char *port) { return htons((unsigned short)atoi(port)); } +static void grpc_ares_request_ref(grpc_ares_request *r) { + gpr_ref(&r->pending_queries); +} + static void grpc_ares_request_unref(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) { /* If there are no pending queries, invoke on_done callback and destroy the @@ -103,67 +114,95 @@ static void grpc_ares_request_unref(grpc_exec_ctx *exec_ctx, } gpr_mu_destroy(&r->mu); grpc_ares_ev_driver_destroy(r->ev_driver); - gpr_free(r->host); - gpr_free(r->port); - gpr_free(r->default_port); gpr_free(r); } } -static void on_done_cb(void *arg, int status, int timeouts, - struct hostent *hostent) { - grpc_ares_request *r = (grpc_ares_request *)arg; +static grpc_ares_hostbyname_request *create_hostbyname_request( + grpc_ares_request *parent_request, char *host, uint16_t port, + bool is_balancer) { + grpc_ares_hostbyname_request *hr = + gpr_zalloc(sizeof(grpc_ares_hostbyname_request)); + hr->parent_request = parent_request; + hr->host = gpr_strdup(host); + hr->port = port; + hr->is_balancer = is_balancer; + grpc_ares_request_ref(parent_request); + return hr; +} + +static void destroy_hostbyname_request(grpc_exec_ctx *exec_ctx, + grpc_ares_hostbyname_request *hr) { + grpc_ares_request_unref(exec_ctx, hr->parent_request); + gpr_free(hr->host); + gpr_free(hr); +} + +static void on_hostbyname_done_cb(void *arg, int status, int timeouts, + struct hostent *hostent) { + grpc_ares_hostbyname_request *hr = (grpc_ares_hostbyname_request *)arg; + grpc_ares_request *r = hr->parent_request; gpr_mu_lock(&r->mu); if (status == ARES_SUCCESS) { GRPC_ERROR_UNREF(r->error); r->error = GRPC_ERROR_NONE; r->success = true; - grpc_resolved_addresses **addresses = r->addrs_out; - if (*addresses == NULL) { - *addresses = gpr_malloc(sizeof(grpc_resolved_addresses)); - (*addresses)->naddrs = 0; - (*addresses)->addrs = NULL; + grpc_lb_addresses **lb_addresses = r->lb_addrs_out; + if (*lb_addresses == NULL) { + *lb_addresses = grpc_lb_addresses_create(0, NULL); } - size_t prev_naddr = (*addresses)->naddrs; + size_t prev_naddr = (*lb_addresses)->num_addresses; size_t i; for (i = 0; hostent->h_addr_list[i] != NULL; i++) { } - (*addresses)->naddrs += i; - (*addresses)->addrs = - gpr_realloc((*addresses)->addrs, - sizeof(grpc_resolved_address) * (*addresses)->naddrs); - for (i = prev_naddr; i < (*addresses)->naddrs; i++) { - memset(&(*addresses)->addrs[i], 0, sizeof(grpc_resolved_address)); - if (hostent->h_addrtype == AF_INET6) { - (*addresses)->addrs[i].len = sizeof(struct sockaddr_in6); - struct sockaddr_in6 *addr = - (struct sockaddr_in6 *)&(*addresses)->addrs[i].addr; - addr->sin6_family = (sa_family_t)hostent->h_addrtype; - addr->sin6_port = strhtons(r->port); - - char output[INET6_ADDRSTRLEN]; - memcpy(&addr->sin6_addr, hostent->h_addr_list[i - prev_naddr], - sizeof(struct in6_addr)); - ares_inet_ntop(AF_INET6, &addr->sin6_addr, output, INET6_ADDRSTRLEN); - gpr_log(GPR_DEBUG, - "c-ares resolver gets a AF_INET6 result: \n" - " addr: %s\n port: %s\n sin6_scope_id: %d\n", - output, r->port, addr->sin6_scope_id); - } else { - (*addresses)->addrs[i].len = sizeof(struct sockaddr_in); - struct sockaddr_in *addr = - (struct sockaddr_in *)&(*addresses)->addrs[i].addr; - memcpy(&addr->sin_addr, hostent->h_addr_list[i - prev_naddr], - sizeof(struct in_addr)); - addr->sin_family = (sa_family_t)hostent->h_addrtype; - addr->sin_port = strhtons(r->port); - - char output[INET_ADDRSTRLEN]; - ares_inet_ntop(AF_INET, &addr->sin_addr, output, INET_ADDRSTRLEN); - gpr_log(GPR_DEBUG, - "c-ares resolver gets a AF_INET result: \n" - " addr: %s\n port: %s\n", - output, r->port); + (*lb_addresses)->num_addresses += i; + (*lb_addresses)->addresses = + gpr_realloc((*lb_addresses)->addresses, + sizeof(grpc_lb_address) * (*lb_addresses)->num_addresses); + for (i = prev_naddr; i < (*lb_addresses)->num_addresses; i++) { + switch (hostent->h_addrtype) { + case AF_INET6: { + size_t addr_len = sizeof(struct sockaddr_in6); + struct sockaddr_in6 addr; + memset(&addr, 0, addr_len); + memcpy(&addr.sin6_addr, hostent->h_addr_list[i - prev_naddr], + sizeof(struct in6_addr)); + addr.sin6_family = (sa_family_t)hostent->h_addrtype; + addr.sin6_port = hr->port; + grpc_lb_addresses_set_address( + *lb_addresses, i, &addr, addr_len, + hr->is_balancer /* is_balancer */, + hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + NULL /* user_data */); + char output[INET6_ADDRSTRLEN]; + ares_inet_ntop(AF_INET6, &addr.sin6_addr, output, INET6_ADDRSTRLEN); + gpr_log(GPR_DEBUG, + "c-ares resolver gets a AF_INET6 result: \n" + " addr: %s\n port: %d\n sin6_scope_id: %d\n", + output, ntohs(hr->port), addr.sin6_scope_id); + break; + } + case AF_INET: { + size_t addr_len = sizeof(struct sockaddr_in); + struct sockaddr_in addr; + memset(&addr, 0, addr_len); + memcpy(&addr.sin_addr, hostent->h_addr_list[i - prev_naddr], + sizeof(struct in_addr)); + addr.sin_family = (sa_family_t)hostent->h_addrtype; + addr.sin_port = hr->port; + grpc_lb_addresses_set_address( + *lb_addresses, i, &addr, addr_len, + hr->is_balancer /* is_balancer */, + hr->is_balancer ? strdup(hr->host) : NULL /* balancer_name */, + NULL /* user_data */); + char output[INET_ADDRSTRLEN]; + ares_inet_ntop(AF_INET, &addr.sin_addr, output, INET_ADDRSTRLEN); + gpr_log(GPR_DEBUG, + "c-ares resolver gets a AF_INET result: \n" + " addr: %s\n port: %d\n", + output, ntohs(hr->port)); + break; + } } } } else if (!r->success) { @@ -179,14 +218,58 @@ static void on_done_cb(void *arg, int status, int timeouts, } } gpr_mu_unlock(&r->mu); - grpc_ares_request_unref(NULL, r); + destroy_hostbyname_request(NULL, hr); +} + +static void on_srv_query_done_cb(void *arg, int status, int timeouts, + unsigned char *abuf, int alen) { + grpc_ares_request *r = (grpc_ares_request *)arg; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_log(GPR_DEBUG, "on_query_srv_done_cb"); + if (status == ARES_SUCCESS) { + gpr_log(GPR_DEBUG, "on_query_srv_done_cb ARES_SUCCESS"); + struct ares_srv_reply *reply; + const int parse_status = ares_parse_srv_reply(abuf, alen, &reply); + if (parse_status == ARES_SUCCESS) { + ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver); + for (struct ares_srv_reply *srv_it = reply; srv_it != NULL; + srv_it = srv_it->next) { + if (grpc_ipv6_loopback_available()) { + grpc_ares_hostbyname_request *hr = create_hostbyname_request( + r, srv_it->host, srv_it->port, true /* is_balancer */); + ares_gethostbyname(*channel, hr->host, AF_INET6, + on_hostbyname_done_cb, hr); + } + grpc_ares_hostbyname_request *hr = create_hostbyname_request( + r, srv_it->host, srv_it->port, true /* is_balancer */); + ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, + hr); + grpc_ares_ev_driver_start(&exec_ctx, r->ev_driver); + } + } + if (reply != NULL) { + ares_free_data(reply); + } + } else if (!r->success) { + char *error_msg; + gpr_asprintf(&error_msg, "C-ares status is not ARES_SUCCESS: %s", + ares_strerror(status)); + grpc_error *error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); + gpr_free(error_msg); + if (r->error == GRPC_ERROR_NONE) { + r->error = error; + } else { + r->error = grpc_error_add_child(error, r->error); + } + } + grpc_ares_request_unref(&exec_ctx, r); + grpc_exec_ctx_finish(&exec_ctx); } -void grpc_dns_lookup_ares(grpc_exec_ctx *exec_ctx, const char *dns_server, - const char *name, const char *default_port, - grpc_pollset_set *interested_parties, - grpc_closure *on_done, - grpc_resolved_addresses **addrs) { +static grpc_ares_request *grpc_dns_lookup_ares_impl( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) { grpc_error *error = GRPC_ERROR_NONE; /* TODO(zyc): Enable tracing after #9603 is checked in */ /* if (grpc_dns_trace) { @@ -221,10 +304,7 @@ void grpc_dns_lookup_ares(grpc_exec_ctx *exec_ctx, const char *dns_server, gpr_mu_init(&r->mu); r->ev_driver = ev_driver; r->on_done = on_done; - r->addrs_out = addrs; - r->default_port = gpr_strdup(default_port); - r->port = port; - r->host = host; + r->lb_addrs_out = addrs; r->success = false; r->error = GRPC_ERROR_NONE; ares_channel *channel = grpc_ares_ev_driver_get_channel(r->ev_driver); @@ -248,6 +328,7 @@ void grpc_dns_lookup_ares(grpc_exec_ctx *exec_ctx, const char *dns_server, error = grpc_error_set_str( GRPC_ERROR_CREATE_FROM_STATIC_STRING("cannot parse authority"), GRPC_ERROR_STR_TARGET_ADDRESS, grpc_slice_from_copied_string(name)); + gpr_free(r); goto error_cleanup; } int status = ares_set_servers_ports(*channel, &r->dns_server_addr); @@ -257,41 +338,55 @@ void grpc_dns_lookup_ares(grpc_exec_ctx *exec_ctx, const char *dns_server, ares_strerror(status)); error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg); gpr_free(error_msg); + gpr_free(r); goto error_cleanup; } } // An extra reference is put here to avoid destroying the request in // on_done_cb before calling grpc_ares_ev_driver_start. - gpr_ref_init(&r->pending_queries, 2); + gpr_ref_init(&r->pending_queries, 1); if (grpc_ipv6_loopback_available()) { - gpr_ref(&r->pending_queries); - ares_gethostbyname(*channel, r->host, AF_INET6, on_done_cb, r); + grpc_ares_hostbyname_request *hr = create_hostbyname_request( + r, host, strhtons(port), false /* is_balancer */); + ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_cb, hr); + } + grpc_ares_hostbyname_request *hr = create_hostbyname_request( + r, host, strhtons(port), false /* is_balancer */); + ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb, hr); + if (check_grpclb) { + /* Query the SRV record */ + grpc_ares_request_ref(r); + char *service_name; + gpr_asprintf(&service_name, "_grpclb._tcp.%s", host); + ares_query(*channel, service_name, ns_c_in, ns_t_srv, on_srv_query_done_cb, + r); + gpr_free(service_name); } - ares_gethostbyname(*channel, r->host, AF_INET, on_done_cb, r); /* TODO(zyc): Handle CNAME records here. */ grpc_ares_ev_driver_start(exec_ctx, r->ev_driver); grpc_ares_request_unref(exec_ctx, r); - return; + gpr_free(host); + gpr_free(port); + return r; error_cleanup: grpc_closure_sched(exec_ctx, on_done, error); gpr_free(host); gpr_free(port); + return NULL; } -void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, const char *name, - const char *default_port, - grpc_pollset_set *interested_parties, - grpc_closure *on_done, - grpc_resolved_addresses **addrs) { - grpc_dns_lookup_ares(exec_ctx, NULL /* dns_server */, name, default_port, - interested_parties, on_done, addrs); -} +grpc_ares_request *(*grpc_dns_lookup_ares)( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addrs, + bool check_grpclb) = grpc_dns_lookup_ares_impl; -void (*grpc_resolve_address_ares)( - grpc_exec_ctx *exec_ctx, const char *name, const char *default_port, - grpc_pollset_set *interested_parties, grpc_closure *on_done, - grpc_resolved_addresses **addrs) = grpc_resolve_address_ares_impl; +void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) { + if (grpc_dns_lookup_ares == grpc_dns_lookup_ares_impl) { + grpc_ares_ev_driver_shutdown(exec_ctx, r->ev_driver); + } +} grpc_error *grpc_ares_init(void) { gpr_once_init(&g_basic_init, do_basic_init); @@ -316,4 +411,66 @@ void grpc_ares_cleanup(void) { gpr_mu_unlock(&g_init_mu); } +/* + * grpc_resolve_address_ares related structs and functions + */ + +typedef struct grpc_resolve_address_ares_request { + /** the pointer to receive the resolved addresses */ + grpc_resolved_addresses **addrs_out; + /** currently resolving lb addresses */ + grpc_lb_addresses *lb_addrs; + /** closure to call when the resolve_address_ares request completes */ + grpc_closure *on_resolve_address_done; + /** a closure wrapping on_dns_lookup_done_cb, which should be invoked when the + grpc_dns_lookup_ares operation is done. */ + grpc_closure on_dns_lookup_done; +} grpc_resolve_address_ares_request; + +static void on_dns_lookup_done_cb(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_resolve_address_ares_request *r = + (grpc_resolve_address_ares_request *)arg; + grpc_resolved_addresses **resolved_addresses = r->addrs_out; + if (r->lb_addrs == NULL || r->lb_addrs->num_addresses == 0) { + *resolved_addresses = NULL; + } else { + *resolved_addresses = gpr_zalloc(sizeof(grpc_resolved_addresses)); + (*resolved_addresses)->naddrs = r->lb_addrs->num_addresses; + (*resolved_addresses)->addrs = gpr_zalloc(sizeof(grpc_resolved_address) * + (*resolved_addresses)->naddrs); + for (size_t i = 0; i < (*resolved_addresses)->naddrs; i++) { + GPR_ASSERT(!r->lb_addrs->addresses[i].is_balancer); + memcpy(&(*resolved_addresses)->addrs[i], + &r->lb_addrs->addresses[i].address, sizeof(grpc_resolved_address)); + } + } + grpc_closure_sched(exec_ctx, r->on_resolve_address_done, + GRPC_ERROR_REF(error)); + grpc_lb_addresses_destroy(exec_ctx, r->lb_addrs); + gpr_free(r); +} + +static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, + const char *name, + const char *default_port, + grpc_pollset_set *interested_parties, + grpc_closure *on_done, + grpc_resolved_addresses **addrs) { + grpc_resolve_address_ares_request *r = + gpr_zalloc(sizeof(grpc_resolve_address_ares_request)); + r->addrs_out = addrs; + r->on_resolve_address_done = on_done; + grpc_closure_init(&r->on_dns_lookup_done, on_dns_lookup_done_cb, r, + grpc_schedule_on_exec_ctx); + grpc_dns_lookup_ares(exec_ctx, NULL /* dns_server */, name, default_port, + interested_parties, &r->on_dns_lookup_done, &r->lb_addrs, + false /* check_grpclb */); +} + +void (*grpc_resolve_address_ares)( + grpc_exec_ctx *exec_ctx, const char *name, const char *default_port, + grpc_pollset_set *interested_parties, grpc_closure *on_done, + grpc_resolved_addresses **addrs) = grpc_resolve_address_ares_impl; + #endif /* GRPC_ARES == 1 && !defined(GRPC_UV) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 08efeea04a..5d2d6c993b 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -19,11 +19,14 @@ #ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_WRAPPER_H #define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_WRAPPER_H +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/iomgr/resolve_address.h" +typedef struct grpc_ares_request grpc_ares_request; + /* Asynchronously resolve addr. Use \a default_port if a port isn't designated in addr, otherwise use the port in addr. grpc_ares_init() must be called at least once before this function. \a on_done may be called directly in this @@ -36,11 +39,21 @@ extern void (*grpc_resolve_address_ares)(grpc_exec_ctx *exec_ctx, grpc_closure *on_done, grpc_resolved_addresses **addresses); -void grpc_dns_lookup_ares(grpc_exec_ctx *exec_ctx, const char *dns_server, - const char *addr, const char *default_port, - grpc_pollset_set *interested_parties, - grpc_closure *on_done, - grpc_resolved_addresses **addresses); +/* Asynchronously resolve addr. It will try to resolve grpclb SRV records in + addition to the normal address records. For normal address records, it uses + \a default_port if a port isn't designated in \a addr, otherwise it uses the + port in \a addr. grpc_ares_init() must be called at least once before this + function. \a on_done may be called directly in this function without being + scheduled with \a exec_ctx, it must not try to acquire locks that are being + held by the caller. */ +extern grpc_ares_request *(*grpc_dns_lookup_ares)( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb); + +/* Cancel the pending grpc_ares_request \a request */ +void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, + grpc_ares_request *request); /* Initialize gRPC ares wrapper. Must be called at least once before grpc_resolve_address_ares(). */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c new file mode 100644 index 0000000000..5c15e3bdf5 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c @@ -0,0 +1,74 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <grpc/support/port_platform.h> +#if GRPC_ARES != 1 || defined(GRPC_UV) + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + +struct grpc_ares_request { + char val; +}; + +static grpc_ares_request *grpc_dns_lookup_ares_impl( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addrs, bool check_grpclb) { + return NULL; +} + +grpc_ares_request *(*grpc_dns_lookup_ares)( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *name, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addrs, + bool check_grpclb) = grpc_dns_lookup_ares_impl; + +void grpc_cancel_ares_request(grpc_exec_ctx *exec_ctx, grpc_ares_request *r) {} + +grpc_error *grpc_ares_init(void) { return GRPC_ERROR_NONE; } + +void grpc_ares_cleanup(void) {} + +static void grpc_resolve_address_ares_impl(grpc_exec_ctx *exec_ctx, + const char *name, + const char *default_port, + grpc_pollset_set *interested_parties, + grpc_closure *on_done, + grpc_resolved_addresses **addrs) {} + +void (*grpc_resolve_address_ares)( + grpc_exec_ctx *exec_ctx, const char *name, const char *default_port, + grpc_pollset_set *interested_parties, grpc_closure *on_done, + grpc_resolved_addresses **addrs) = grpc_resolve_address_ares_impl; + +#endif /* GRPC_ARES != 1 || defined(GRPC_UV) */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 006262cd27..29a86f73ff 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -904,7 +904,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, unhappy. */ PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); } - GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + if (error != GRPC_ERROR_NONE) { + const char *msg = grpc_error_string(error); + gpr_log(GPR_DEBUG, "fd_orphan: %s", msg); + } GRPC_ERROR_UNREF(error); } diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index f11d883517..48782174a7 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -285,6 +285,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c', 'src/core/ext/filters/load_reporting/load_reporting.c', diff --git a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c index 03753da88a..43dc7e9084 100644 --- a/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c +++ b/test/core/client_channel/resolvers/dns_resolver_connectivity_test.c @@ -21,7 +21,9 @@ #include <grpc/grpc.h> #include <grpc/support/alloc.h> +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" #include "src/core/ext/filters/client_channel/resolver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/combiner.h" @@ -55,6 +57,26 @@ static void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, grpc_closure_sched(exec_ctx, on_done, error); } +static grpc_ares_request *my_dns_lookup_ares( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + gpr_mu_lock(&g_mu); + GPR_ASSERT(0 == strcmp("test", addr)); + grpc_error *error = GRPC_ERROR_NONE; + if (g_fail_resolution) { + g_fail_resolution = false; + gpr_mu_unlock(&g_mu); + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Forced Failure"); + } else { + gpr_mu_unlock(&g_mu); + *lb_addrs = grpc_lb_addresses_create(1, NULL); + grpc_lb_addresses_set_address(*lb_addrs, 0, NULL, 0, false, NULL, NULL); + } + grpc_closure_sched(exec_ctx, on_done, error); + return NULL; +} + static grpc_resolver *create_resolver(grpc_exec_ctx *exec_ctx, const char *name) { grpc_resolver_factory *factory = grpc_resolver_factory_lookup("dns"); @@ -124,6 +146,7 @@ int main(int argc, char **argv) { gpr_mu_init(&g_mu); g_combiner = grpc_combiner_create(); grpc_resolve_address = my_resolve_address; + grpc_dns_lookup_ares = my_dns_lookup_ares; grpc_channel_args *result = (grpc_channel_args *)1; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; diff --git a/test/core/client_channel/resolvers/dns_resolver_test.c b/test/core/client_channel/resolvers/dns_resolver_test.c index e3451d0682..a14926f173 100644 --- a/test/core/client_channel/resolvers/dns_resolver_test.c +++ b/test/core/client_channel/resolvers/dns_resolver_test.c @@ -20,6 +20,7 @@ #include <grpc/support/log.h> +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/lib/iomgr/combiner.h" #include "test/core/util/test_config.h" @@ -73,7 +74,11 @@ int main(int argc, char **argv) { test_succeeds(dns, "dns:10.2.1.1"); test_succeeds(dns, "dns:10.2.1.1:1234"); test_succeeds(dns, "ipv4:www.google.com"); - test_fails(dns, "ipv4://8.8.8.8/8.8.8.8:8888"); + if (grpc_resolve_address == grpc_resolve_address_ares) { + test_succeeds(dns, "ipv4://8.8.8.8/8.8.8.8:8888"); + } else { + test_fails(dns, "ipv4://8.8.8.8/8.8.8.8:8888"); + } grpc_resolver_factory_unref(dns); { diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 59c82837c7..1e9e7e6194 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -24,6 +24,8 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/executor.h" @@ -363,6 +365,7 @@ typedef struct addr_req { char *addr; grpc_closure *on_done; grpc_resolved_addresses **addrs; + grpc_lb_addresses **lb_addrs; } addr_req; static void finish_resolve(grpc_exec_ctx *exec_ctx, void *arg, @@ -370,11 +373,17 @@ static void finish_resolve(grpc_exec_ctx *exec_ctx, void *arg, addr_req *r = arg; if (error == GRPC_ERROR_NONE && 0 == strcmp(r->addr, "server")) { - grpc_resolved_addresses *addrs = gpr_malloc(sizeof(*addrs)); - addrs->naddrs = 1; - addrs->addrs = gpr_malloc(sizeof(*addrs->addrs)); - addrs->addrs[0].len = 0; - *r->addrs = addrs; + if (r->addrs != NULL) { + grpc_resolved_addresses *addrs = gpr_malloc(sizeof(*addrs)); + addrs->naddrs = 1; + addrs->addrs = gpr_malloc(sizeof(*addrs->addrs)); + addrs->addrs[0].len = 0; + *r->addrs = addrs; + } else if (r->lb_addrs != NULL) { + grpc_lb_addresses *lb_addrs = grpc_lb_addresses_create(1, NULL); + grpc_lb_addresses_set_address(lb_addrs, 0, NULL, 0, NULL, NULL, NULL); + *r->lb_addrs = lb_addrs; + } grpc_closure_sched(exec_ctx, r->on_done, GRPC_ERROR_NONE); } else { grpc_closure_sched(exec_ctx, r->on_done, @@ -395,11 +404,29 @@ void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, r->addr = gpr_strdup(addr); r->on_done = on_done; r->addrs = addresses; + r->lb_addrs = NULL; + grpc_timer_init( + exec_ctx, &r->timer, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), + gpr_time_from_seconds(1, GPR_TIMESPAN)), + grpc_closure_create(finish_resolve, r, grpc_schedule_on_exec_ctx), + gpr_now(GPR_CLOCK_MONOTONIC)); +} + +grpc_ares_request *my_dns_lookup_ares( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + addr_req *r = gpr_malloc(sizeof(*r)); + r->addr = gpr_strdup(addr); + r->on_done = on_done; + r->addrs = NULL; + r->lb_addrs = lb_addrs; grpc_timer_init( exec_ctx, &r->timer, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(1, GPR_TIMESPAN)), grpc_closure_create(finish_resolve, r, grpc_schedule_on_exec_ctx), gpr_now(GPR_CLOCK_MONOTONIC)); + return NULL; } //////////////////////////////////////////////////////////////////////////////// @@ -716,6 +743,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_exec_ctx_finish(&exec_ctx); } grpc_resolve_address = my_resolve_address; + grpc_dns_lookup_ares = my_dns_lookup_ares; GPR_ASSERT(g_channel == NULL); GPR_ASSERT(g_server == NULL); diff --git a/test/core/end2end/goaway_server_test.c b/test/core/end2end/goaway_server_test.c index 49378665a0..91a377e6b9 100644 --- a/test/core/end2end/goaway_server_test.c +++ b/test/core/end2end/goaway_server_test.c @@ -27,6 +27,8 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> #include <string.h> +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/sockaddr.h" #include "test/core/end2end/cq_verifier.h" @@ -43,6 +45,11 @@ static void (*iomgr_resolve_address)(grpc_exec_ctx *exec_ctx, const char *addr, grpc_closure *on_done, grpc_resolved_addresses **addresses); +static grpc_ares_request *(*iomgr_dns_lookup_ares)( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **addresses, bool check_grpclb); + static void set_resolve_port(int port) { gpr_mu_lock(&g_mu); g_resolve_port = port; @@ -80,6 +87,36 @@ static void my_resolve_address(grpc_exec_ctx *exec_ctx, const char *addr, grpc_closure_sched(exec_ctx, on_done, error); } +static grpc_ares_request *my_dns_lookup_ares( + grpc_exec_ctx *exec_ctx, const char *dns_server, const char *addr, + const char *default_port, grpc_pollset_set *interested_parties, + grpc_closure *on_done, grpc_lb_addresses **lb_addrs, bool check_grpclb) { + if (0 != strcmp(addr, "test")) { + return iomgr_dns_lookup_ares(exec_ctx, dns_server, addr, default_port, + interested_parties, on_done, lb_addrs, + check_grpclb); + } + + grpc_error *error = GRPC_ERROR_NONE; + gpr_mu_lock(&g_mu); + if (g_resolve_port < 0) { + gpr_mu_unlock(&g_mu); + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Forced Failure"); + } else { + *lb_addrs = grpc_lb_addresses_create(1, NULL); + struct sockaddr_in *sa = gpr_zalloc(sizeof(struct sockaddr_in)); + sa->sin_family = AF_INET; + sa->sin_addr.s_addr = htonl(0x7f000001); + sa->sin_port = htons((uint16_t)g_resolve_port); + grpc_lb_addresses_set_address(*lb_addrs, 0, sa, sizeof(*sa), false, NULL, + NULL); + gpr_free(sa); + gpr_mu_unlock(&g_mu); + } + grpc_closure_sched(exec_ctx, on_done, error); + return NULL; +} + int main(int argc, char **argv) { grpc_completion_queue *cq; cq_verifier *cqv; @@ -91,7 +128,9 @@ int main(int argc, char **argv) { gpr_mu_init(&g_mu); grpc_init(); iomgr_resolve_address = grpc_resolve_address; + iomgr_dns_lookup_ares = grpc_dns_lookup_ares; grpc_resolve_address = my_resolve_address; + grpc_dns_lookup_ares = my_dns_lookup_ares; int was_cancelled1; int was_cancelled2; diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 89e0aca79c..e5a2ecd517 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -92,6 +92,7 @@ void grpc_free_port_using_server(int port) { grpc_schedule_on_exec_ctx), &rsp); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(pr.mu); while (!pr.done) { grpc_pollset_worker *worker = NULL; @@ -224,6 +225,7 @@ int grpc_pick_port_using_server(void) { grpc_closure_create(got_port_from_server, &pr, grpc_schedule_on_exec_ctx), &pr.response); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 70e332970f..766c20f59b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -948,6 +948,7 @@ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c \ src/core/ext/filters/client_channel/resolver/dns/native/README.md \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.c \ diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index fc61b3b900..177b076650 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -23,4 +23,4 @@ BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN +tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN diff --git a/tools/profiling/microbenchmarks/README.md b/tools/profiling/microbenchmarks/README.md new file mode 100644 index 0000000000..035888ee18 --- /dev/null +++ b/tools/profiling/microbenchmarks/README.md @@ -0,0 +1,4 @@ +Microbenchmarks +==== + +This directory contains helper scripts for the microbenchmark suites. diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py deleted file mode 100755 index 3de5c663ce..0000000000 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import json -import bm_json -import tabulate -import argparse -from scipy import stats -import subprocess -import multiprocessing -import collections -import pipes -import os -sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) -import comment_on_pr -import jobset -import itertools -import speedup -import random -import shutil -import errno - -_INTERESTING = ( - 'cpu_time', - 'real_time', - 'locks_per_iteration', - 'allocs_per_iteration', - 'writes_per_iteration', - 'atm_cas_per_iteration', - 'atm_add_per_iteration', - 'cli_transport_stalls_per_iteration', - 'cli_stream_stalls_per_iteration', - 'svr_transport_stalls_per_iteration', - 'svr_stream_stalls_per_iteration' - 'nows_per_iteration', -) - -def changed_ratio(n, o): - if float(o) <= .0001: o = 0 - if float(n) <= .0001: n = 0 - if o == 0 and n == 0: return 0 - if o == 0: return 100 - return (float(n)-float(o))/float(o) - -def median(ary): - ary = sorted(ary) - n = len(ary) - if n%2 == 0: - return (ary[n/2] + ary[n/2+1]) / 2.0 - else: - return ary[n/2] - -def min_change(pct): - return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 - -_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', - 'bm_closure', - 'bm_cq', - 'bm_call_create', - 'bm_error', - 'bm_chttp2_hpack', - 'bm_chttp2_transport', - 'bm_pollset', - 'bm_metadata', - 'bm_fullstack_trickle'] - -argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') -argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING), - nargs='+', - default=sorted(_INTERESTING), - help='Which metrics to track') -argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) -argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=1) -argp.add_argument('-l', '--loops', type=int, default=20) -argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) -args = argp.parse_args() - -assert args.diff_base - -def avg(lst): - sum = 0.0 - n = 0.0 - for el in lst: - sum += el - n += 1 - return sum / n - -def make_cmd(cfg): - return ['make'] + args.benchmarks + [ - 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] - -def build(dest): - shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True) - subprocess.check_call(['git', 'submodule', 'update']) - try: - subprocess.check_call(make_cmd('opt')) - subprocess.check_call(make_cmd('counters')) - except subprocess.CalledProcessError, e: - subprocess.check_call(['make', 'clean']) - subprocess.check_call(make_cmd('opt')) - subprocess.check_call(make_cmd('counters')) - os.rename('bins', 'bm_diff_%s' % dest) - -def collect1(bm, cfg, ver, idx): - cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm), - '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), - '--benchmark_out_format=json', - '--benchmark_repetitions=%d' % (args.repetitions) - ] - return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops), - verbose_success=True, timeout_seconds=None) - -build('new') - -where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() -subprocess.check_call(['git', 'checkout', args.diff_base]) -try: - build('old') -finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) - -jobs = [] -for loop in range(0, args.loops): - jobs.extend(x for x in itertools.chain( - (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), - )) -random.shuffle(jobs, random.SystemRandom().random) - -jobset.run(jobs, maxjobs=args.jobs) - -class Benchmark: - - def __init__(self): - self.samples = { - True: collections.defaultdict(list), - False: collections.defaultdict(list) - } - self.final = {} - - def add_sample(self, data, new): - for f in args.track: - if f in data: - self.samples[new][f].append(float(data[f])) - - def process(self): - for f in sorted(args.track): - new = self.samples[True][f] - old = self.samples[False][f] - if not new or not old: continue - mdn_diff = abs(median(new) - median(old)) - print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) - s = speedup.speedup(new, old) - if abs(s) > 3 and mdn_diff > 0.5: - self.final[f] = '%+d%%' % s - return self.final.keys() - - def skip(self): - return not self.final - - def row(self, flds): - return [self.final[f] if f in self.final else '' for f in flds] - - -def eintr_be_gone(fn): - """Run fn until it doesn't stop because of EINTR""" - while True: - try: - return fn() - except IOError, e: - if e.errno != errno.EINTR: - raise - - -def read_json(filename): - try: - with open(filename) as f: return json.loads(f.read()) - except ValueError, e: - return None - - -def finalize(): - benchmarks = collections.defaultdict(Benchmark) - - for bm in args.benchmarks: - for loop in range(0, args.loops): - js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) - js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) - js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) - js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) - - if js_new_ctr: - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - if js_old_ctr: - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) - - really_interesting = set() - for name, bm in benchmarks.items(): - print name - really_interesting.update(bm.process()) - fields = [f for f in args.track if f in really_interesting] - - headers = ['Benchmark'] + fields - rows = [] - for name in sorted(benchmarks.keys()): - if benchmarks[name].skip(): continue - rows.append([name] + benchmarks[name].row(fields)) - if rows: - text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') - else: - text = 'No significant performance differences' - print text - comment_on_pr.comment_on_pr('```\n%s\n```' % text) - - -eintr_be_gone(finalize) diff --git a/tools/profiling/microbenchmarks/bm_diff/README.md b/tools/profiling/microbenchmarks/bm_diff/README.md new file mode 100644 index 0000000000..caa4770229 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/README.md @@ -0,0 +1,116 @@ +The bm_diff Family +==== + +This family of python scripts can be incredibly useful for fast iteration over +different performance tweaks. The tools allow you to save performance data from +a baseline commit, then quickly compare data from your working branch to that +baseline data to see if you have made any performance wins. + +The tools operate with three concrete steps, which can be invoked separately, +or all together via the driver script, bm_main.py. This readme will describe +the typical workflow for these scripts, then it will include sections on the +details of every script for advanced usage. + +## Normal Workflow + +Let's say you are working on a performance optimization for grpc_error. You have +made some significant changes and want to see some data. From your branch, run +(ensure everything is committed first): + +`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -d master` + +This will build the `bm_error` binary on your branch, and then it will checkout +master and build it there too. It will then run these benchmarks 5 times each. +Lastly it will compute the statistically significant performance differences +between the two branches. This should show the nice performance wins your +changes have made. + +If you have already invoked bm_main with `-d master`, you should instead use +`-o` for subsequent runs. This allows the script to skip re-building and +re-running the unchanged master branch. For example: + +`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -o` + +This will only build and run `bm_error` on your branch. It will then compare +the output to the saved runs from master. + +## Advanced Workflow + +If you have a deeper knowledge of these scripts, you can use them to do more +fine tuned benchmark comparisons. For example, you could build, run, and save +the benchmark output from two different base branches. Then you could diff both +of these baselines against your working branch to see how the different metrics +change. The rest of this doc goes over the details of what each of the +individual modules accomplishes. + +## bm_build.py + +This scrips builds the benchmarks. It takes in a name parameter, and will +store the binaries based on that. Both `opt` and `counter` configurations +will be used. The `opt` is used to get cpu_time and real_time, and the +`counters` build is used to track other metrics like allocs, atomic adds, +etc etc etc. + +For example, if you were to invoke (we assume everything is run from the +root of the repo): + +`tools/profiling/microbenchmarks/bm_diff/bm_build.py -b bm_error -n baseline` + +then the microbenchmark binaries will show up under +`bm_diff_baseline/{opt,counters}/bm_error` + +## bm_run.py + +This script runs the benchmarks. It takes a name parameter that must match the +name that was passed to `bm_build.py`. The script then runs the benchmark +multiple times (default is 20, can be toggled via the loops parameter). The +output is saved as `<benchmark name>.<config>.<name>.<loop idx>.json` + +For example, if you were to run: + +`tools/profiling/microbenchmarks/bm_diff/bm_run.py -b bm_error -b baseline -l 5` + +Then an example output file would be `bm_error.opt.baseline.0.json` + +## bm_diff.py + +This script takes in the output from two benchmark runs, computes the diff +between them, and prints any significant improvements or regressions. It takes +in two name parameters, old and new. These must have previously been built and +run. + +For example, assuming you had already built and run a 'baseline' microbenchmark +from master, and then you also built and ran a 'current' microbenchmark from +the branch you were working on, you could invoke: + +`tools/profiling/microbenchmarks/bm_diff/bm_diff.py -b bm_error -o baseline -n current -l 5` + +This would output the percent difference between your branch and master. + +## bm_main.py + +This is the driver script. It uses the previous three modules and does +everything for you. You pass in the benchmarks to be run, the number of loops, +number of CPUs to use, and the commit to compare to. Then the script will: +* Build the benchmarks at head, then checkout the branch to compare to and + build the benchmarks there +* Run both sets of microbenchmarks +* Run bm_diff.py to compare the two, outputs the difference. + +For example, one might run: + +`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -d master` + +This would compare the current branch's error benchmarks to master. + +This script is invoked by our infrastructure on every PR to protect against +regressions and demonstrate performance wins. + +However, if you are iterating over different performance tweaks quickly, it is +unnecessary to build and run the baseline commit every time. That is why we +provide a different flag in case you are sure that the baseline benchmark has +already been built and run. In that case use the --old flag to pass in the name +of the baseline. This will only build and run the current branch. For example: + +`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -o old` + diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py new file mode 100755 index 0000000000..650ccdc2b2 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Python utility to build opt and counters benchmarks """ + +import bm_constants + +import argparse +import subprocess +import multiprocessing +import os +import shutil + + +def _args(): + argp = argparse.ArgumentParser(description='Builds microbenchmarks') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to build') + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='How many CPUs to dedicate to this task') + argp.add_argument( + '-n', + '--name', + type=str, + help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' + ) + args = argp.parse_args() + assert args.name + return args + + +def _make_cmd(cfg, benchmarks, jobs): + return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs] + + +def build(name, benchmarks, jobs): + shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) + subprocess.check_call(['git', 'submodule', 'update']) + try: + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + os.rename( + 'bins', + 'bm_diff_%s' % name,) + + +if __name__ == '__main__': + args = _args() + build(args.name, args.benchmarks, args.jobs) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py new file mode 100644 index 0000000000..2bbd987b20 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Configurable constants for the bm_*.py family """ + +_AVAILABLE_BENCHMARK_TESTS = [ + 'bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', + 'bm_error', 'bm_chttp2_hpack', 'bm_chttp2_transport', 'bm_pollset', + 'bm_metadata', 'bm_fullstack_trickle' +] + +_INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration', + 'allocs_per_iteration', 'writes_per_iteration', + 'atm_cas_per_iteration', 'atm_add_per_iteration', + 'nows_per_iteration',) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py new file mode 100755 index 0000000000..881e157ccd --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Computes the diff between two bm runs and outputs significant results """ + +import bm_constants +import bm_speedup + +import sys +import os + +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) +import bm_json + +import json +import tabulate +import argparse +import collections +import subprocess + +verbose = False + + +def _median(ary): + assert (len(ary)) + ary = sorted(ary) + n = len(ary) + if n % 2 == 0: + return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 + else: + return ary[n / 2] + + +def _args(): + argp = argparse.ArgumentParser( + description='Perform diff on microbenchmarks') + argp.add_argument( + '-t', + '--track', + choices=sorted(bm_constants._INTERESTING), + nargs='+', + default=sorted(bm_constants._INTERESTING), + help='Which metrics to track') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to run') + argp.add_argument( + '-l', + '--loops', + type=int, + default=20, + help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' + ) + argp.add_argument('-n', '--new', type=str, help='New benchmark name') + argp.add_argument('-o', '--old', type=str, help='Old benchmark name') + argp.add_argument( + '-v', '--verbose', type=bool, help='Print details of before/after') + args = argp.parse_args() + global verbose + if args.verbose: verbose = True + assert args.new + assert args.old + return args + + +def _maybe_print(str): + if verbose: print str + + +class Benchmark: + + def __init__(self): + self.samples = { + True: collections.defaultdict(list), + False: collections.defaultdict(list) + } + self.final = {} + + def add_sample(self, track, data, new): + for f in track: + if f in data: + self.samples[new][f].append(float(data[f])) + + def process(self, track, new_name, old_name): + for f in sorted(track): + new = self.samples[True][f] + old = self.samples[False][f] + if not new or not old: continue + mdn_diff = abs(_median(new) - _median(old)) + _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % + (f, new_name, new, old_name, old, mdn_diff)) + s = bm_speedup.speedup(new, old) + if abs(s) > 3 and mdn_diff > 0.5: + self.final[f] = '%+d%%' % s + return self.final.keys() + + def skip(self): + return not self.final + + def row(self, flds): + return [self.final[f] if f in self.final else '' for f in flds] + + +def _read_json(filename, badjson_files, nonexistant_files): + stripped = ".".join(filename.split(".")[:-2]) + try: + with open(filename) as f: + return json.loads(f.read()) + except IOError, e: + if stripped in nonexistant_files: + nonexistant_files[stripped] += 1 + else: + nonexistant_files[stripped] = 1 + return None + except ValueError, e: + if stripped in badjson_files: + badjson_files[stripped] += 1 + else: + badjson_files[stripped] = 1 + return None + + +def diff(bms, loops, track, old, new): + benchmarks = collections.defaultdict(Benchmark) + + badjson_files = {} + nonexistant_files = {} + for bm in bms: + for loop in range(0, loops): + for line in subprocess.check_output( + ['bm_diff_%s/opt/%s' % (old, bm), + '--benchmark_list_tests']).splitlines(): + stripped_line = line.strip().replace("/", "_").replace( + "<", "_").replace(">", "_").replace(", ", "_") + js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + + if js_new_ctr: + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): + continue + benchmarks[name].add_sample(track, row, True) + if js_old_ctr: + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): + continue + benchmarks[name].add_sample(track, row, False) + + really_interesting = set() + for name, bm in benchmarks.items(): + _maybe_print(name) + really_interesting.update(bm.process(track, new, old)) + fields = [f for f in track if f in really_interesting] + + headers = ['Benchmark'] + fields + rows = [] + for name in sorted(benchmarks.keys()): + if benchmarks[name].skip(): continue + rows.append([name] + benchmarks[name].row(fields)) + note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str( + badjson_files) + note += '\n\nMissing files (new benchmark) = %s' % str(nonexistant_files) + if rows: + return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note + else: + return None, note + + +if __name__ == '__main__': + args = _args() + diff, note = diff(args.benchmarks, args.loops, args.track, args.old, + args.new) + print('%s\n%s' % (note, diff if diff else "No performance differences")) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py new file mode 100755 index 0000000000..0136c6aa57 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Runs the entire bm_*.py pipeline, and possible comments on the PR """ + +import bm_constants +import bm_build +import bm_run +import bm_diff + +import sys +import os +import argparse +import multiprocessing +import subprocess + +sys.path.append( + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) +import comment_on_pr + + +def _args(): + argp = argparse.ArgumentParser( + description='Perform diff on microbenchmarks') + argp.add_argument( + '-t', + '--track', + choices=sorted(bm_constants._INTERESTING), + nargs='+', + default=sorted(bm_constants._INTERESTING), + help='Which metrics to track') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to run') + argp.add_argument( + '-d', + '--diff_base', + type=str, + help='Commit or branch to compare the current one to') + argp.add_argument( + '-o', + '--old', + default='old', + type=str, + help='Name of baseline run to compare to. Ususally just called "old"') + argp.add_argument( + '-r', + '--repetitions', + type=int, + default=1, + help='Number of repetitions to pass to the benchmarks') + argp.add_argument( + '-l', + '--loops', + type=int, + default=20, + help='Number of times to loops the benchmarks. More loops cuts down on noise' + ) + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + args = argp.parse_args() + assert args.diff_base or args.old, "One of diff_base or old must be set!" + if args.loops < 3: + print "WARNING: This run will likely be noisy. Increase loops." + return args + + +def eintr_be_gone(fn): + """Run fn until it doesn't stop because of EINTR""" + + def inner(*args): + while True: + try: + return fn(*args) + except IOError, e: + if e.errno != errno.EINTR: + raise + + return inner + + +def main(args): + + bm_build.build('new', args.benchmarks, args.jobs) + + old = args.old + if args.diff_base: + old = 'old' + where_am_i = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + try: + bm_build.build('old', args.benchmarks, args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + + bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions) + bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions) + + diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old, + 'new') + if diff: + text = 'Performance differences noted:\n' + diff + else: + text = 'No significant performance differences' + print('%s\n%s' % (note, text)) + comment_on_pr.comment_on_pr('```\n%s\n\n%s\n```' % (note, text)) + + +if __name__ == '__main__': + args = _args() + main(args) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py new file mode 100755 index 0000000000..3457af916b --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Python utility to run opt and counters benchmarks and save json output """ + +import bm_constants + +import argparse +import subprocess +import multiprocessing +import random +import itertools +import sys +import os + +sys.path.append( + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', + 'python_utils')) +import jobset + + +def _args(): + argp = argparse.ArgumentParser(description='Runs microbenchmarks') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Benchmarks to run') + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + argp.add_argument( + '-n', + '--name', + type=str, + help='Unique name of the build to run. Needs to match the handle passed to bm_build.py' + ) + argp.add_argument( + '-r', + '--repetitions', + type=int, + default=1, + help='Number of repetitions to pass to the benchmarks') + argp.add_argument( + '-l', + '--loops', + type=int, + default=20, + help='Number of times to loops the benchmarks. More loops cuts down on noise' + ) + args = argp.parse_args() + assert args.name + if args.loops < 3: + print "WARNING: This run will likely be noisy. Increase loops to at least 3." + return args + + +def _collect_bm_data(bm, cfg, name, reps, idx, loops): + jobs_list = [] + for line in subprocess.check_output( + ['bm_diff_%s/%s/%s' % (name, cfg, bm), + '--benchmark_list_tests']).splitlines(): + stripped_line = line.strip().replace("/", "_").replace( + "<", "_").replace(">", "_").replace(", ", "_") + cmd = [ + 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_filter=^%s$' % + line, '--benchmark_out=%s.%s.%s.%s.%d.json' % + (bm, stripped_line, cfg, name, idx), '--benchmark_out_format=json', + '--benchmark_repetitions=%d' % (reps) + ] + jobs_list.append( + jobset.JobSpec( + cmd, + shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, + loops), + verbose_success=True, + timeout_seconds=60 * 2)) + return jobs_list + + +def run(name, benchmarks, jobs, loops, reps): + jobs_list = [] + for loop in range(0, loops): + for bm in benchmarks: + jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops) + jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop, + loops) + random.shuffle(jobs_list, random.SystemRandom().random) + jobset.run(jobs_list, maxjobs=jobs) + + +if __name__ == '__main__': + args = _args() + run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py index 3f45cc38fb..3d126efa62 100644..100755 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python2.7 +# # Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +19,17 @@ import math _THRESHOLD = 1e-10 + def scale(a, mul): - return [x*mul for x in a] + return [x * mul for x in a] + def cmp(a, b): return stats.ttest_ind(a, b) + def speedup(new, old): + if (len(set(new))) == 1 and new == old: return 0 s0, p0 = cmp(new, old) if math.isnan(p0): return 0 if s0 == 0: return 0 @@ -31,7 +37,7 @@ def speedup(new, old): if s0 < 0: pct = 1 while pct < 101: - sp, pp = cmp(new, scale(old, 1 - pct/100.0)) + sp, pp = cmp(new, scale(old, 1 - pct / 100.0)) if sp > 0: break if pp > _THRESHOLD: break pct += 1 @@ -39,14 +45,15 @@ def speedup(new, old): else: pct = 1 while pct < 100000: - sp, pp = cmp(new, scale(old, 1 + pct/100.0)) + sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) if sp < 0: break if pp > _THRESHOLD: break pct += 1 return pct - 1 + if __name__ == "__main__": - new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] - old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + new = [1.0, 1.0, 1.0, 1.0] + old = [2.0, 2.0, 2.0, 2.0] print speedup(new, old) print speedup(old, new) diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index bfcdc6060a..16ff417a27 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8518,7 +8518,8 @@ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c", - "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.c" ], "third_party": false, "type": "filegroup" diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 3eecd8eff4..312abd59c4 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -23,18 +23,8 @@ import argparse import python_utils.jobset as jobset import python_utils.start_port_server as start_port_server -_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', - 'bm_closure', - 'bm_cq', - 'bm_call_create', - 'bm_error', - 'bm_chttp2_hpack', - 'bm_chttp2_transport', - 'bm_pollset', - 'bm_metadata', - 'bm_fullstack_trickle'] +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff')) +import bm_constants flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') @@ -199,8 +189,8 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - choices=_AVAILABLE_BENCHMARK_TESTS, - default=_AVAILABLE_BENCHMARK_TESTS, + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, nargs='+', type=str, help='Which microbenchmarks should be run') diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 3728887edb..f9badbbd35 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -967,6 +967,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper_fallback.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\native\dns_resolver.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\sockaddr\sockaddr_resolver.c"> diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index ada4a37bbc..f3e997e4e0 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -673,6 +673,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper.c"> <Filter>src\core\ext\filters\client_channel\resolver\dns\c_ares</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper_fallback.c"> + <Filter>src\core\ext\filters\client_channel\resolver\dns\c_ares</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\native\dns_resolver.c"> <Filter>src\core\ext\filters\client_channel\resolver\dns\native</Filter> </ClCompile> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index b252a01647..3a865f3e81 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -850,6 +850,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper_fallback.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\native\dns_resolver.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\sockaddr\sockaddr_resolver.c"> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index c73870e5cc..0310ebc012 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -550,6 +550,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper.c"> <Filter>src\core\ext\filters\client_channel\resolver\dns\c_ares</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\c_ares\grpc_ares_wrapper_fallback.c"> + <Filter>src\core\ext\filters\client_channel\resolver\dns\c_ares</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\ext\filters\client_channel\resolver\dns\native\dns_resolver.c"> <Filter>src\core\ext\filters\client_channel\resolver\dns\native</Filter> </ClCompile> |