diff options
Diffstat (limited to 'tools')
31 files changed, 1750 insertions, 172 deletions
diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index a4373ed350..9b5e2113af 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -35,6 +35,7 @@ gen_build_yaml_dirs=" \ src/benchmark \ src/proto \ src/zlib \ + src/c-ares \ test/core/bad_client \ test/core/bad_ssl \ test/core/end2end \ diff --git a/tools/buildgen/plugins/expand_bin_attrs.py b/tools/buildgen/plugins/expand_bin_attrs.py index dc72bf3b9d..f52168a9d3 100755 --- a/tools/buildgen/plugins/expand_bin_attrs.py +++ b/tools/buildgen/plugins/expand_bin_attrs.py @@ -52,9 +52,11 @@ def mako_plugin(dictionary): tgt['ci_platforms'] = sorted(tgt.get('ci_platforms', tgt['platforms'])) tgt['boringssl'] = tgt.get('boringssl', False) tgt['zlib'] = tgt.get('zlib', False) + tgt['ares'] = tgt.get('ares', False) tgt['gtest'] = tgt.get('gtest', False) libs = dictionary.get('libs') for lib in libs: lib['boringssl'] = lib.get('boringssl', False) lib['zlib'] = lib.get('zlib', False) + lib['ares'] = lib.get('ares', False) diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py index 46739f8f10..9eaba463ec 100755 --- a/tools/buildgen/plugins/expand_filegroups.py +++ b/tools/buildgen/plugins/expand_filegroups.py @@ -57,6 +57,7 @@ FILEGROUP_DEFAULTS = { 'language': 'c', 'boringssl': False, 'zlib': False, + 'ares': False, } diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index 611a9f40ae..710e8709e9 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -111,6 +111,8 @@ _EXEMPT = frozenset(( # An older file originally from outside gRPC. 'src/php/tests/bootstrap.php', + # census.proto copied from github + 'tools/grpcz/census.proto', )) @@ -125,7 +127,9 @@ RE_LICENSE = dict( if args.precommit: FILE_LIST_COMMAND = 'git status -z | grep -Poz \'(?<=^[MARC][MARCD ] )[^\s]+\'' else: - FILE_LIST_COMMAND = 'git ls-tree -r --name-only -r HEAD | grep -v ^third_party/' + FILE_LIST_COMMAND = 'git ls-tree -r --name-only -r HEAD | ' \ + 'grep -v ^third_party/ |' \ + 'grep -v "\(ares_config.h\|ares_build.h\)"' def load(name): with open(name) as f: diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index d3bf071c72..d7a0b1786b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index acc984acb0..094a4e096d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 hyper +RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile index bbf7de7f91..c099f339ae 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index fc24745869..4b0fb700bb 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -950,6 +950,11 @@ src/core/ext/load_reporting/load_reporting.h \ src/core/ext/load_reporting/load_reporting_filter.c \ src/core/ext/load_reporting/load_reporting_filter.h \ src/core/ext/resolver/README.md \ +src/core/ext/resolver/dns/c_ares/dns_resolver_ares.c \ +src/core/ext/resolver/dns/c_ares/grpc_ares_ev_driver.h \ +src/core/ext/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c \ +src/core/ext/resolver/dns/c_ares/grpc_ares_wrapper.c \ +src/core/ext/resolver/dns/c_ares/grpc_ares_wrapper.h \ src/core/ext/resolver/dns/native/README.md \ src/core/ext/resolver/dns/native/dns_resolver.c \ src/core/ext/resolver/sockaddr/README.md \ @@ -1356,16 +1361,16 @@ src/core/lib/transport/transport.c \ src/core/lib/transport/transport.h \ src/core/lib/transport/transport_impl.h \ src/core/lib/transport/transport_op_string.c \ -src/core/lib/tsi/README.md \ -src/core/lib/tsi/fake_transport_security.c \ -src/core/lib/tsi/fake_transport_security.h \ -src/core/lib/tsi/ssl_transport_security.c \ -src/core/lib/tsi/ssl_transport_security.h \ -src/core/lib/tsi/ssl_types.h \ -src/core/lib/tsi/transport_security.c \ -src/core/lib/tsi/transport_security.h \ -src/core/lib/tsi/transport_security_interface.h \ src/core/plugin_registry/grpc_plugin_registry.c \ +src/core/tsi/README.md \ +src/core/tsi/fake_transport_security.c \ +src/core/tsi/fake_transport_security.h \ +src/core/tsi/ssl_transport_security.c \ +src/core/tsi/ssl_transport_security.h \ +src/core/tsi/ssl_types.h \ +src/core/tsi/transport_security.c \ +src/core/tsi/transport_security.h \ +src/core/tsi/transport_security_interface.h \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_common.h \ diff --git a/tools/grpcz/BUILD b/tools/grpcz/BUILD new file mode 100644 index 0000000000..5e1faf7064 --- /dev/null +++ b/tools/grpcz/BUILD @@ -0,0 +1,63 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +licenses(["notice"]) # 3-clause BSD + +package(default_visibility = ["//visibility:public"]) + +load("//:bazel/grpc_build_system.bzl", "grpc_proto_library") + +grpc_proto_library ( + name = "monitoring_proto", + srcs = [ + "monitoring.proto", + ], + deps = [ + ":census_proto", + ], + well_known_protos = "@submodule_protobuf//:well_known_protos", +) + +grpc_proto_library ( + name = "census_proto", + srcs = [ + "census.proto", + ], + well_known_protos = "@submodule_protobuf//:well_known_protos", +) + +cc_binary( + name = "grpcz_client", + srcs = ["grpcz_client.cc",], + deps = [ + "//external:gflags", + "monitoring_proto", + "@mongoose_repo//:mongoose_lib", + ], +) diff --git a/tools/grpcz/census.proto b/tools/grpcz/census.proto new file mode 100644 index 0000000000..d1ff69400b --- /dev/null +++ b/tools/grpcz/census.proto @@ -0,0 +1,318 @@ +// Copyright 2017, Google Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//TODO(ericgribkoff) Depend on this directly from the instrumentation-proto +//repository. + +syntax = "proto3"; + +package google.instrumentation; + +option java_package = "com.google.instrumentation.stats.proto"; +option java_outer_classname = "CensusProto"; + +// All the census protos. +// +// Nomenclature notes: +// * Capitalized names below (like View) are protos. +// * Protos which describe types are named with a Descriptor suffix (e.g. +// MesurementDescriptor). +// +// Census lets you define the type and description of the data being measured +// (e.g. the latency of an RPC or the number of CPU cycles spent on an +// operation using MeasurementDescriptor. As individual measurements (a double +// value) for are recorded, they are aggregated together into an +// Aggregation. There are two Aggregation types available: Distribution +// (describes the distribution of all measurements, possibly with a histogram) +// and IntervalStats (the count and mean of measurements across specified time +// periods). An Aggregation is described by an AggregationDescriptor. +// +// You can define how your measurements (described by a MeasurementDescriptor) +// are broken down by Tag values and which Aggregations to use through a +// ViewDescriptor. The output (all measurements broken down by tag values into +// specific Aggregations) is called a View. + + +// The following two types are copied from +// google/protobuf/{duration,timestamp}.proto. Ideally, we would be able to +// import them, but this causes compilation issues on C-based systems +// (e.g. https://koti.kapsi.fi/jpa/nanopb/), which cannot process the C++ +// headers generated from the standard protobuf distribution. See the relevant +// proto files for full documentation of these types. + +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} + +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} + +// MeasurementDescriptor describes a data point (measurement) type. +message MeasurementDescriptor { + // A descriptive name, e.g. rpc_latency, cpu. Must be unique. + string name = 1; + + // More detailed description of the resource, used in documentation. + string description = 2; + + // Fundamental units of measurement supported by Census + // TODO(aveitch): expand this to include other S.I. units? + enum BasicUnit { + UNKNOWN = 0; // Implementations should not use this + SCALAR = 1; // Dimensionless + BITS = 2; // A single bit + BYTES = 3; // An 8-bit byte + SECONDS = 4; // S.I. unit + CORES = 5; // CPU core usage + MAX_UNITS = 6; // Last defined value; implementations should only use + // this for validation. + } + + // MeasurementUnit lets you build compound units of the form + // 10^n * (A * B * ...) / (X * Y * ...), + // where the elements in the numerator and denominator are all BasicUnits. A + // MeasurementUnit must have at least one BasicUnit in its numerator. + // + // To specify multiplication in the numerator or denominator, simply specify + // multiple numerator or denominator fields. For example: + // + // - byte-seconds (i.e. bytes * seconds): + // numerator: BYTES + // numerator: SECS + // + // - events/sec^2 (i.e. rate of change of events/sec): + // numerator: SCALAR + // denominator: SECS + // denominator: SECS + // + // To specify multiples (in power of 10) of units, specify a non-zero + // 'power10' value, for example: + // + // - MB/s (i.e. megabytes / s): + // power10: 6 + // numerator: BYTES + // denominator: SECS + // + // - nanoseconds + // power10: -9 + // numerator: SECS + message MeasurementUnit { + int32 power10 = 1; + repeated BasicUnit numerators = 2; + repeated BasicUnit denominators = 3; + } + + // The units used by this type of measurement. + MeasurementUnit unit = 3; +} + +// An aggregation summarizes a series of individual measurements. There are +// two types of aggregation (IntervalAggregation and DistributionAggregation), +// unique types of each can be set using descriptors for each. + +// DistributionAggregation contains summary statistics for a population of +// values and, optionally, a histogram representing the distribution of those +// values across a specified set of histogram buckets, as defined in +// DistributionAggregationDescriptor.bucket_bounds. +// +// The summary statistics are the count, mean, minimum, and the maximum of the +// set of population of values. +// +// Although it is not forbidden, it is generally a bad idea to include +// non-finite values (infinities or NaNs) in the population of values, as this +// will render the `mean` field meaningless. +message DistributionAggregation { + // The number of values in the population. Must be non-negative. + int64 count = 1; + + // The arithmetic mean of the values in the population. If `count` is zero + // then this field must be zero. + double mean = 2; + + // The sum of the values in the population. If `count` is zero then this + // field must be zero. + double sum = 3; + + // Describes a range of population values. + message Range { + // The minimum of the population values. + double min = 1; + // The maximum of the population values. + double max = 2; + } + + // The range of the population values. If `count` is zero, this field will not + // be defined. + Range range = 4; + + // A Distribution may optionally contain a histogram of the values in the + // population. The histogram is given in `bucket_count` as counts of values + // that fall into one of a sequence of non-overlapping buckets, as described + // by `DistributionAggregationDescriptor.bucket_boundaries`. The sum of the + // values in `bucket_counts` must equal the value in `count`. + // + // Bucket counts are given in order under the numbering scheme described + // above (the underflow bucket has number 0; the finite buckets, if any, + // have numbers 1 through N-2; the overflow bucket has number N-1). + // + // The size of `bucket_count` must be no greater than N as defined in + // `bucket_boundaries`. + // + // Any suffix of trailing zero bucket_count fields may be omitted. + repeated int64 bucket_counts = 5; + + // Tags associated with this DistributionAggregation. These will be filled + // in based on the View specification. + repeated Tag tags = 6; +} + +message DistributionAggregationDescriptor { + // A Distribution may optionally contain a histogram of the values in the + // population. The bucket boundaries for that histogram are described by + // `bucket_bounds`. This defines `size(bucket_bounds) + 1` (= N) + // buckets. The boundaries for bucket index i are: + // + // [-infinity, bucket_bounds[i]) for i == 0 + // [bucket_bounds[i-1], bucket_bounds[i]) for 0 < i < N-2 + // [bucket_bounds[i-1], +infinity) for i == N-1 + // + // i.e. an underflow bucket (number 0), zero or more finite buckets (1 + // through N - 2, and an overflow bucket (N - 1), with inclusive lower + // bounds and exclusive upper bounds. + // + // If `bucket_bounds` has no elements (zero size), then there is no + // histogram associated with the Distribution. If `bucket_bounds` has only + // one element, there are no finite buckets, and that single element is the + // common boundary of the overflow and underflow buckets. The values must + // be monotonically increasing. + repeated double bucket_bounds = 1; +} + +// An IntervalAggreation records summary stats over various time +// windows. These stats are approximate, with the degree of accuracy +// controlled by setting the n_sub_intervals parameter in the +// IntervalAggregationDescriptor. +message IntervalAggregation { + // Summary statistic over a single time interval. + message Interval { + // The interval duration. Must be positive. + Duration interval_size = 1; + // Approximate number of measurements recorded in this interval. + double count = 2; + // The cumulative sum of measurements in this interval. + double sum = 3; + } + + // Full set of intervals for this aggregation. + repeated Interval intervals = 1; + + // Tags associated with this IntervalAggregation. These will be filled in + // based on the View specification. + repeated Tag tags = 2; +} + +// An IntervalAggreationDescriptor specifies time intervals for an +// IntervalAggregation. +message IntervalAggregationDescriptor { + // Number of internal sub-intervals to use when collecting stats for each + // interval. The max error in interval measurements will be approximately + // 1/n_sub_intervals (although in practice, this will only be approached in + // the presence of very large and bursty workload changes), and underlying + // memory usage will be roughly proportional to the value of this + // field. Must be in the range [2, 20]. A value of 5 will be used if this is + // unspecified. + int32 n_sub_intervals = 1; + + // The size of each interval, as a time duration. Must have at least one + // element. + repeated Duration interval_sizes = 2; +} + +// A Tag: key-value pair. +message Tag { + string key = 1; + string value = 2; +} + +// A ViewDescriptor specifies an AggregationDescriptor and a set of tag +// keys. Views instantiated from this descriptor will contain Aggregations +// broken down by the unique set of matching tag values for each measurement. +message ViewDescriptor { + // Name of view. Must be unique. + string name = 1; + + // More detailed description, for documentation purposes. + string description = 2; + + // Name of a MeasurementDescriptor to be used for this view. + string measurement_descriptor_name = 3; + + // Aggregation type to associate with View. + oneof aggregation { + IntervalAggregationDescriptor interval_aggregation = 4; + DistributionAggregationDescriptor distribution_aggregation = 5; + } + + // Tag keys to match with a given measurement. If no keys are specified, + // then all stats are recorded. Keys must be unique. + repeated string tag_keys = 6; +} + +// DistributionView contains all aggregations for a view specified using a +// DistributionAggregationDescriptor. +message DistributionView { + // Aggregations - each will have a unique set of tag values for the tag_keys + // associated with the corresponding View. + repeated DistributionAggregation aggregations = 1; + + // Start and end timestamps over which aggregations was accumulated. + Timestamp start = 2; + Timestamp end = 3; +} + +// IntervalView contains all aggregations for a view specified using a +// IntervalAggregationDescriptor. +message IntervalView { + // Aggregations - each will have a unique set of tag values for the tag_keys + // associated with the corresponding View. + repeated IntervalAggregation aggregations = 1; +} + +// A View contains the aggregations based on a ViewDescriptor. +message View { + // ViewDescriptor name associated with this set of View. + string view_name = 1; + + oneof view { + DistributionView distribution_view = 2; + IntervalView interval_view = 3; + } +} diff --git a/tools/grpcz/grpcz_client.cc b/tools/grpcz/grpcz_client.cc new file mode 100644 index 0000000000..47eec8dfc3 --- /dev/null +++ b/tools/grpcz/grpcz_client.cc @@ -0,0 +1,181 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include <string> + +#include <google/protobuf/util/json_util.h> +#include <grpc++/grpc++.h> +#include <grpc/support/log.h> + +#include "gflags/gflags.h" +#include "mongoose.h" + +// TODO (makdharma): remove local copies of these protos +#include "tools/grpcz/census.grpc.pb.h" +#include "tools/grpcz/monitoring.grpc.pb.h" + +DEFINE_string( + grpcz_server, "127.0.0.1:8080", + "Unix domain socket path (e.g. unix://tmp/grpcz.sock) or IP address" + "(host:port) where grpcz server is running."); +DEFINE_string(http_port, "8000", + "Port id for accessing the HTTP server that renders /grpcz page"); +DEFINE_bool(print_to_console, false, + "print the JSON retreived from grpcz server and quit"); + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; + +using ::grpc::instrumentation::v1alpha::CanonicalRpcStats; +using ::grpc::instrumentation::v1alpha::Monitoring; + +static const std::string static_html_header = + "<!DOCTYPE html> <html> <head> <style> \ +table { border-collapse: collapse; width: 100%; } \ +table, td, th { border: 1px solid black; } \ +</style> </head> <body>\ +<div id='stats' data-stats='"; + +static const std::string static_html_footer = + "' class='hidden'></div>\ +<h1>GRPCZ Statistics</h1> <div id='table'> </div> \ +<script> \ + var canonical_stats = JSON.parse(\ + document.getElementById('stats').getAttribute('data-stats')); \ + var table = document.createElement('table'); \ + if (canonical_stats['Error Message'] != undefined) { \ + document.getElementById('table').innerHTML = canonical_stats['Error Message']; } \ + else {\ + for (var key in canonical_stats) { \ + name = canonical_stats[key]['view']['viewName']; \ + distribution = canonical_stats[key]['view']['distributionView']; \ + interval = canonical_stats[key]['view']['intervalView']; \ + value = (interval == undefined) ? \ + JSON.stringify(distribution, null, ' ') : \ + JSON.stringify(interval, null, ' '); \ + var row = table.insertRow(-1); \ + var col1 = row.insertCell(0); \ + var col2 = row.insertCell(1); \ + col1.innerHTML = name; \ + col2.innerHTML = '<pre>' + value + '</pre>'; \ + } \ + document.getElementById('table').appendChild(table); \ + }\ +</script> </body> </html>"; + +class GrpczClient { + public: + GrpczClient(std::shared_ptr<Channel> channel) + : stub_(Monitoring::NewStub(channel)) {} + + std::string GetStatsAsJson() { + const ::google::protobuf::Empty request; + CanonicalRpcStats reply; + ClientContext context; + Status status = stub_->GetCanonicalRpcStats(&context, request, &reply); + + if (status.ok()) { + std::string json_str; + ::google::protobuf::util::MessageToJsonString(reply, &json_str); + return json_str; + } else { + static const std::string error_message_json = + "{\"Error Message\":\"" + status.error_message() + "\"}"; + gpr_log(GPR_DEBUG, "%d: %s", status.error_code(), + status.error_message().c_str()); + return error_message_json; + } + } + + private: + std::unique_ptr<Monitoring::Stub> stub_; +}; + +static struct mg_serve_http_opts s_http_server_opts; +std::unique_ptr<GrpczClient> g_grpcz_client; + +static void ev_handler(struct mg_connection *nc, int ev, void *p) { + if (ev == MG_EV_HTTP_REQUEST) { + mg_serve_http(nc, (struct http_message *)p, s_http_server_opts); + } +} + +static void grpcz_handler(struct mg_connection *nc, int ev, void *ev_data) { + (void)ev; + (void)ev_data; + gpr_log(GPR_INFO, "fetching grpcz stats from %s", FLAGS_grpcz_server.c_str()); + std::string json_str = g_grpcz_client->GetStatsAsJson(); + std::string rendered_html = + static_html_header + json_str + static_html_footer; + mg_printf(nc, "HTTP/1.0 200 OK\r\n\r\n%s", rendered_html.c_str()); + nc->flags |= MG_F_SEND_AND_CLOSE; +} + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + + // Create a client + g_grpcz_client.reset(new GrpczClient(grpc::CreateChannel( + FLAGS_grpcz_server, grpc::InsecureChannelCredentials()))); + if (FLAGS_print_to_console) { + // using GPR_ERROR since this is the default verbosity. _DEBUG or _INFO + // won't print unless GRPC_VERBOSITY env var is set appropriately, which + // might confuse users of this utility. + gpr_log(GPR_ERROR, "%s\n", g_grpcz_client->GetStatsAsJson().c_str()); + return 0; + } + + // Set up a mongoose webserver handler + struct mg_mgr mgr; + mg_mgr_init(&mgr, NULL); + gpr_log(GPR_INFO, "Starting grpcz web server on port %s\n", + FLAGS_http_port.c_str()); + + struct mg_connection *nc = mg_bind(&mgr, FLAGS_http_port.c_str(), ev_handler); + if (nc == NULL) { + gpr_log(GPR_ERROR, "Failed to create listener on port %s\n", + FLAGS_http_port.c_str()); + return -1; + } + mg_register_http_endpoint(nc, "/grpcz", grpcz_handler); + mg_set_protocol_http_websocket(nc); + + // Poll in a loop and serve /grpcz pages + for (;;) { + static const int k_sleep_millis = 100; + mg_mgr_poll(&mgr, k_sleep_millis); + } + mg_mgr_free(&mgr); + return 0; +} diff --git a/tools/grpcz/monitoring.proto b/tools/grpcz/monitoring.proto new file mode 100644 index 0000000000..fefcd7d22f --- /dev/null +++ b/tools/grpcz/monitoring.proto @@ -0,0 +1,156 @@ +// Copyright 2017, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. +syntax = "proto3"; + +// TODO(ericgribkoff) Figure out how to manage the external Census proto +// dependency. +import "tools/grpcz/census.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/empty.proto"; + +package grpc.instrumentation.v1alpha; + +option java_multiple_files = true; +option java_package = "io.grpc.instrumentation.v1alpha"; +option java_outer_classname = "MonitoringProto"; + +service Monitoring { + // Return canonical RPC stats + rpc GetCanonicalRpcStats(google.protobuf.Empty) returns (CanonicalRpcStats) { + } + + // Query the server for specific stats + rpc GetStats(StatsRequest) returns (StatsResponse) { + } + + // Request the server to stream back snapshots of the requested stats + rpc WatchStats(StatsRequest) returns (stream StatsResponse) { + } + + + // Return request traces. + rpc GetRequestTraces(TraceRequest) returns(TraceResponse) { + // TODO(aveitch): Please define the messages here + } + + // Return application-defined groups of monitoring data. + // This is a low level facility to allow extension of the monitoring API to + // application-specific monitoring data. Frameworks may use this to define + // additional groups of monitoring data made available by servers. + rpc GetCustomMonitoringData(MonitoringDataGroup) + returns (CustomMonitoringData) { + } + +} + +// Canonical RPC stats exported by gRPC. +message CanonicalRpcStats { + StatsResponse rpc_client_errors = 1; + StatsResponse rpc_client_completed_rpcs = 2; + StatsResponse rpc_client_started_rpcs = 3; + StatsResponse rpc_client_elapsed_time = 4; + StatsResponse rpc_client_server_elapsed_time = 5; + StatsResponse rpc_client_request_bytes = 6; + StatsResponse rpc_client_response_bytes = 7; + StatsResponse rpc_client_request_count = 8; + StatsResponse rpc_client_response_count = 9; + StatsResponse rpc_server_errors = 10; + StatsResponse rpc_server_completed_rpcs = 11; + StatsResponse rpc_server_server_elapsed_time = 12; + StatsResponse rpc_server_request_bytes = 13; + StatsResponse rpc_server_response_bytes = 14; + StatsResponse rpc_server_request_count = 15; + StatsResponse rpc_server_response_count = 16; + StatsResponse rpc_server_elapsed_time = 17; + //TODO(ericgribkoff) Add minute-hour interval stats. +} + +// This message is sent when requesting a set of stats (Census Views) from +// a client system, as part of the MonitoringService API's. +message StatsRequest { + // An optional set of ViewDescriptor names. Only Views using these + // descriptors will be sent back in the response. If no names are provided, + // then all Views present in the client system will be included in every + // response. If measurement_names is also provided, then Views matching the + // intersection of the two are returned. + // TODO(aveitch): Consider making this a list of regexes or prefix matches in + // the future. + repeated string view_names = 1; + + // An optional set of MeasurementDescriptor names. Only Views using these + // descriptors will be sent back in the response. If no names are provided, + // then all Views present in the client system will be included in every + // response. If view_names is also provided, then Views matching the + // intersection of the two are returned. + // TODO(aveitch): Consider making this a list of regexes or prefix matches in + // the future. + repeated string measurement_names = 2; + + // By default, the MeasurementDescriptors and ViewDescriptors corresponding to + // the Views that are returned in a StatsResponse will be included in the + // first such response. Set this to true to have them not sent. + bool dont_include_descriptors_in_first_response = 3; +} + +// This message contains all information relevant to a single View. It is the +// return type for GetStats and WatchStats, and used in CanonicalRpcStats. +message StatsResponse { + // A StatsResponse can optionally contain the MeasurementDescriptor and + // ViewDescriptor for the View. These will be sent in the first WatchStats + // response, or all GetStats and GetCanonicalRpcStats responses. These will + // not be set for {Get,Watch}Stats if + // dont_include_descriptors_in_first_response is set to true in the + // StatsRequest. + google.instrumentation.MeasurementDescriptor measurement_descriptor = 1; + google.instrumentation.ViewDescriptor view_descriptor = 2; + + // The View data. + google.instrumentation.View view = 3; +} + +message TraceRequest { + // TODO(aveitch): Complete definition of this type +} + +message TraceResponse { + // TODO(aveitch): Complete definition of this type +} + +message MonitoringDataGroup { + string name = 1; // name of a group of monitoring data +} + +// The wrapper for custom monitoring data. +message CustomMonitoringData { + // can be any application specific monitoring data + google.protobuf.Any contents = 1; +} diff --git a/tools/http2_interop/http2interop_test.go b/tools/http2_interop/http2interop_test.go index 305125f0c1..fb314da196 100644 --- a/tools/http2_interop/http2interop_test.go +++ b/tools/http2_interop/http2interop_test.go @@ -49,7 +49,7 @@ func InteropCtx(t *testing.T) *HTTP2InteropCtx { if ctx.UseTestCa { // It would be odd if useTestCa was true, but not useTls. meh - certData, err := ioutil.ReadFile("src/core/lib/tsi/test_creds/ca.pem") + certData, err := ioutil.ReadFile("src/core/tsi/test_creds/ca.pem") if err != nil { t.Fatal(err) } diff --git a/tools/jenkins/run_line_count.sh b/tools/jenkins/run_line_count.sh new file mode 100755 index 0000000000..a2bde53413 --- /dev/null +++ b/tools/jenkins/run_line_count.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and counts the number of lines in the +# project. +set -ex + +cd $(dirname $0)/../.. +tools/line_count/collect-now.sh diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 99b920f6a0..544e31dcbd 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -31,7 +31,10 @@ # This script is invoked by Jenkins and runs performance smoke test. set -ex +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + # Enter the gRPC repo root cd $(dirname $0)/../.. -tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest +tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py new file mode 100755 index 0000000000..4c1bf73b1e --- /dev/null +++ b/tools/line_count/collect-history.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import subprocess +import datetime + +# this script is only of historical interest: it's the script that was used to +# bootstrap the dataset + +def daterange(start, end): + for n in range(int((end - start).days)): + yield start + datetime.timedelta(n) + +start_date = datetime.date(2017, 3, 26) +end_date = datetime.date(2017, 3, 29) + +for dt in daterange(start_date, end_date): + dmy = dt.strftime('%Y-%m-%d') + sha1 = subprocess.check_output(['git', 'rev-list', '-n', '1', + '--before=%s' % dmy, + 'master']).strip() + subprocess.check_call(['git', 'checkout', sha1]) + subprocess.check_call(['git', 'submodule', 'update']) + subprocess.check_call(['git', 'clean', '-f', '-x', '-d']) + subprocess.check_call(['cloc', '--vcs=git', '--by-file', '--yaml', '--out=../count/%s.yaml' % dmy, '.']) + diff --git a/tools/line_count/collect-now.sh b/tools/line_count/collect-now.sh new file mode 100755 index 0000000000..44f4b4ed31 --- /dev/null +++ b/tools/line_count/collect-now.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cloc --vcs=git --by-file --yaml --out=cloc.yaml . +tools/line_count/yaml2csv.py -i cloc.yaml -d `date +%Y-%m-%d` -o cloc.csv +bq load line_counts.grpc cloc.csv + diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py new file mode 100755 index 0000000000..cb6d570f66 --- /dev/null +++ b/tools/line_count/summarize-history.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import subprocess +import datetime + +# this script is only of historical interest: it's the script that was used to +# bootstrap the dataset + +def daterange(start, end): + for n in range(int((end - start).days)): + yield start + datetime.timedelta(n) + +start_date = datetime.date(2017, 3, 26) +end_date = datetime.date(2017, 3, 29) + +for dt in daterange(start_date, end_date): + dmy = dt.strftime('%Y-%m-%d') + print dmy + subprocess.check_call(['tools/line_count/yaml2csv.py', '-i', '../count/%s.yaml' % dmy, '-d', dmy, '-o', '../count/%s.csv' % dmy]) + diff --git a/tools/line_count/yaml2csv.py b/tools/line_count/yaml2csv.py new file mode 100755 index 0000000000..9bda09fd08 --- /dev/null +++ b/tools/line_count/yaml2csv.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import yaml +import argparse +import datetime +import csv + +argp = argparse.ArgumentParser(description='Convert cloc yaml to bigquery csv') +argp.add_argument('-i', '--input', type=str) +argp.add_argument('-d', '--date', type=str, default=datetime.date.today().strftime('%Y-%m-%d')) +argp.add_argument('-o', '--output', type=str, default='out.csv') +args = argp.parse_args() + +data = yaml.load(open(args.input).read()) +with open(args.output, 'w') as outf: + writer = csv.DictWriter(outf, ['date', 'name', 'language', 'code', 'comment', 'blank']) + for key, value in data.iteritems(): + if key == 'header': continue + if key == 'SUM': continue + if key.startswith('third_party/'): continue + row = {'name': key, 'date': args.date} + row.update(value) + writer.writerow(row) + diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 14ea24f65e..1dff3335ee 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -33,6 +33,14 @@ import json import bm_json import tabulate import argparse +from scipy import stats +import subprocess +import multiprocessing +import collections +import pipes +import os +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) +import comment_on_pr def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -41,78 +49,193 @@ def changed_ratio(n, o): if o == 0: return 100 return (float(n)-float(o))/float(o) +def median(ary): + ary = sorted(ary) + n = len(ary) + if n%2 == 0: + return (ary[n/2] + ary[n/2+1]) / 2.0 + else: + return ary[n/2] + def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 +nanos = { + 'abs_diff': 5, + 'pct_diff': 5, +} +counter = { + 'abs_diff': 0.5, + 'pct_diff': 5, +} + _INTERESTING = { - 'cpu_time': min_change(10), - 'real_time': min_change(10), - 'locks_per_iteration': min_change(5), - 'allocs_per_iteration': min_change(5), - 'writes_per_iteration': min_change(5), - 'atm_cas_per_iteration': min_change(1), - 'atm_add_per_iteration': min_change(5), + 'cpu_time': nanos, + 'real_time': nanos, + 'locks_per_iteration': counter, + 'allocs_per_iteration': counter, + 'writes_per_iteration': counter, + 'atm_cas_per_iteration': counter, + 'atm_add_per_iteration': counter, } + +_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', + 'bm_closure', + 'bm_cq', + 'bm_call_create', + 'bm_error', + 'bm_chttp2_hpack', + 'bm_chttp2_transport', + 'bm_pollset', + 'bm_metadata', + 'bm_fullstack_trickle'] + argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', choices=sorted(_INTERESTING.keys()), nargs='+', default=sorted(_INTERESTING.keys()), help='Which metrics to track') -argp.add_argument('files', metavar='bm_file.json', type=str, nargs=4, - help='files to diff. ') +argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) +argp.add_argument('-d', '--diff_base', type=str) +argp.add_argument('-r', '--repetitions', type=int, default=4) +argp.add_argument('-p', '--p_threshold', type=float, default=0.03) args = argp.parse_args() -with open(args.files[0]) as f: - js_new_ctr = json.loads(f.read()) -with open(args.files[1]) as f: - js_new_opt = json.loads(f.read()) -with open(args.files[2]) as f: - js_old_ctr = json.loads(f.read()) -with open(args.files[3]) as f: - js_old_opt = json.loads(f.read()) - -new = {} -old = {} - -for row in bm_json.expand_json(js_new_ctr, js_new_opt): - new[row['cpp_name']] = row -for row in bm_json.expand_json(js_old_ctr, js_old_opt): - old[row['cpp_name']] = row - -changed = [] -for fld in args.track: - chk = _INTERESTING[fld] - for bm in new.keys(): - if bm not in old: continue - n = new[bm] - o = old[bm] - if fld not in n or fld not in o: continue - if chk(n[fld], o[fld]): - changed.append((fld, chk)) - break - -headers = ['Benchmark'] + [c[0] for c in changed] + ['Details'] +assert args.diff_base + +def avg(lst): + sum = 0.0 + n = 0.0 + for el in lst: + sum += el + n += 1 + return sum / n + +def make_cmd(cfg): + return ['make'] + args.benchmarks + [ + 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] + +def build(): + subprocess.check_call(['git', 'submodule', 'update']) + try: + subprocess.check_call(make_cmd('opt')) + subprocess.check_call(make_cmd('counters')) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(make_cmd('opt')) + subprocess.check_call(make_cmd('counters')) + +def collect1(bm, cfg, ver): + cmd = ['bins/%s/%s' % (cfg, bm), + '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), + '--benchmark_out_format=json', + '--benchmark_repetitions=%d' % (args.repetitions) + ] + print cmd + subprocess.check_call(cmd) + +build() +for bm in args.benchmarks: + collect1(bm, 'opt', 'new') + collect1(bm, 'counters', 'new') + +where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() +subprocess.check_call(['git', 'checkout', args.diff_base]) + +try: + build() + comparables = [] + for bm in args.benchmarks: + try: + collect1(bm, 'opt', 'old') + collect1(bm, 'counters', 'old') + comparables.append(bm) + except subprocess.CalledProcessError, e: + pass +finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + + +class Benchmark: + + def __init__(self): + self.samples = { + True: collections.defaultdict(list), + False: collections.defaultdict(list) + } + self.final = {} + + def add_sample(self, data, new): + for f in args.track: + if f in data: + self.samples[new][f].append(float(data[f])) + + def process(self): + for f in sorted(args.track): + new = self.samples[True][f] + old = self.samples[False][f] + if not new or not old: continue + p = stats.ttest_ind(new, old)[1] + new_mdn = median(new) + old_mdn = median(old) + delta = new_mdn - old_mdn + ratio = changed_ratio(new_mdn, old_mdn) + print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % ( + f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p + ) + if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0: + self.final[f] = delta + return self.final.keys() + + def skip(self): + return not self.final + + def row(self, flds): + return [self.final[f] if f in self.final else '' for f in flds] + + +benchmarks = collections.defaultdict(Benchmark) + +for bm in comparables: + with open('%s.counters.new.json' % bm) as f: + js_new_ctr = json.loads(f.read()) + with open('%s.opt.new.json' % bm) as f: + js_new_opt = json.loads(f.read()) + with open('%s.counters.old.json' % bm) as f: + js_old_ctr = json.loads(f.read()) + with open('%s.opt.old.json' % bm) as f: + js_old_opt = json.loads(f.read()) + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) + +really_interesting = set() +for name, bm in benchmarks.items(): + print name + really_interesting.update(bm.process()) +fields = [f for f in args.track if f in args.track] + +headers = ['Benchmark'] + fields rows = [] -for bm in sorted(new.keys()): - if bm not in old: continue - row = [bm] - any_changed = False - n = new[bm] - o = old[bm] - details = '' - for fld in args.track: - chk = _INTERESTING[fld] - if fld not in n or fld not in o: continue - if chk(n[fld], o[fld]): - row.append(changed_ratio(n[fld], o[fld])) - if details: details += ', ' - details += '%s:%r-->%r' % (fld, float(o[fld]), float(n[fld])) - any_changed = True - else: - row.append('') - if any_changed: - row.append(details) - rows.append(row) -print tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') +for name in sorted(benchmarks.keys()): + if benchmarks[name].skip(): continue + rows.append([name] + benchmarks[name].row(fields)) +if rows: + text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') +else: + text = 'No significant performance differences' +comment_on_pr.comment_on_pr('```\n%s\n```' % text) +print text diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index ca0af414a3..e885444f41 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -113,6 +113,14 @@ _BM_SPECS = { 'BM_TransportStreamRecv': { 'tpl': [], 'dyn': ['request_size'], + }, + 'BM_StreamingPingPongWithCoalescingApi': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count', 'end_of_stream'], + }, + 'BM_Base16SomeStuff': { + 'tpl': [], + 'dyn': ['request_size'], } } @@ -171,6 +179,7 @@ def parse_name(name): def expand_json(js, js2 = None): for bm in js['benchmarks']: + if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue context = js['context'] if 'label' in bm: labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] @@ -189,8 +198,9 @@ def expand_json(js, js2 = None): row.update(labels) if js2: for bm2 in js2['benchmarks']: - if bm['name'] == bm2['name']: + if bm['name'] == bm2['name'] and 'already_used' not in bm2: row['cpu_time'] = bm2['cpu_time'] row['real_time'] = bm2['real_time'] row['iterations'] = bm2['iterations'] + bm2['already_used'] = True yield row diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index 48a216a124..3385738f9c 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -71,7 +71,13 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +# on OSX use md5 instead of sha1sum +if which sha1sum > /dev/null; +then + BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +else + BASE_IMAGE=${BASE_NAME}_base:`md5 -r tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +fi # Make sure base docker image has been built. Should be instantaneous if so. docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/interoptest/$BASE_NAME || exit $? diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 69e0f44774..93dd6fb3d4 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -3,6 +3,26 @@ "config": "opt" }, { + "config": "asan-trace-cmp", + "environ": { + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + } + }, + { + "config": "dbg" + }, + { + "config": "asan", + "environ": { + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + } + }, + { + "config": "msan" + }, + { "config": "basicprof" }, { @@ -19,14 +39,16 @@ } }, { - "config": "asan-trace-cmp", + "config": "ubsan", "environ": { - "ASAN_OPTIONS": "detect_leaks=1:color=always", - "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" } }, { - "config": "dbg" + "config": "tsan", + "environ": { + "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" + } }, { "config": "stapprof" @@ -43,26 +65,7 @@ ] }, { - "config": "asan", - "environ": { - "ASAN_OPTIONS": "detect_leaks=1:color=always", - "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" - } - }, - { - "config": "tsan", - "environ": { - "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" - } - }, - { - "config": "ubsan", - "environ": { - "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" - } - }, - { - "config": "msan" + "config": "lto" }, { "config": "mutrace" diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 71aee1b05d..6557ecb769 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3810,7 +3810,6 @@ "test/cpp/interop/stress_interop_client.cc", "test/cpp/interop/stress_interop_client.h", "test/cpp/interop/stress_test.cc", - "test/cpp/util/create_test_channel.cc", "test/cpp/util/create_test_channel.h", "test/cpp/util/metrics_server.cc", "test/cpp/util/metrics_server.h" @@ -5577,6 +5576,7 @@ "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", + "grpc_resolver_dns_ares", "grpc_resolver_dns_native", "grpc_resolver_sockaddr", "grpc_secure", @@ -5678,6 +5678,7 @@ "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", + "grpc_resolver_dns_ares", "grpc_resolver_dns_native", "grpc_resolver_sockaddr", "grpc_transport_chttp2_client_insecure", @@ -6188,6 +6189,7 @@ "src/proto/grpc/testing/services.pb.h", "src/proto/grpc/testing/stats.grpc.pb.h", "src/proto/grpc/testing/stats.pb.h", + "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/driver.h", "test/cpp/qps/histogram.h", @@ -6197,13 +6199,14 @@ "test/cpp/qps/report.h", "test/cpp/qps/server.h", "test/cpp/qps/stats.h", - "test/cpp/qps/usage_timer.h", - "test/cpp/util/benchmark_config.h" + "test/cpp/qps/usage_timer.h" ], "is_filegroup": false, "language": "c++", "name": "qps", "src": [ + "test/cpp/qps/benchmark_config.cc", + "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/client_async.cc", "test/cpp/qps/client_sync.cc", @@ -6222,9 +6225,7 @@ "test/cpp/qps/server_sync.cc", "test/cpp/qps/stats.h", "test/cpp/qps/usage_timer.cc", - "test/cpp/qps/usage_timer.h", - "test/cpp/util/benchmark_config.cc", - "test/cpp/util/benchmark_config.h" + "test/cpp/qps/usage_timer.h" ], "third_party": false, "type": "lib" @@ -7023,6 +7024,41 @@ "type": "lib" }, { + "deps": [], + "headers": [ + "third_party/cares/ares_build.h", + "third_party/cares/cares/ares.h", + "third_party/cares/cares/ares_data.h", + "third_party/cares/cares/ares_dns.h", + "third_party/cares/cares/ares_getenv.h", + "third_party/cares/cares/ares_getopt.h", + "third_party/cares/cares/ares_inet_net_pton.h", + "third_party/cares/cares/ares_iphlpapi.h", + "third_party/cares/cares/ares_ipv6.h", + "third_party/cares/cares/ares_library_init.h", + "third_party/cares/cares/ares_llist.h", + "third_party/cares/cares/ares_nowarn.h", + "third_party/cares/cares/ares_platform.h", + "third_party/cares/cares/ares_private.h", + "third_party/cares/cares/ares_rules.h", + "third_party/cares/cares/ares_setup.h", + "third_party/cares/cares/ares_strcasecmp.h", + "third_party/cares/cares/ares_strdup.h", + "third_party/cares/cares/ares_version.h", + "third_party/cares/cares/bitncmp.h", + "third_party/cares/cares/config-win32.h", + "third_party/cares/cares/setup_once.h", + "third_party/cares/config_darwin/ares_config.h", + "third_party/cares/config_linux/ares_config.h" + ], + "is_filegroup": false, + "language": "c", + "name": "ares", + "src": [], + "third_party": false, + "type": "lib" + }, + { "deps": [ "gpr", "gpr_test_util", @@ -7757,6 +7793,7 @@ "src/core/lib/json/json_string.c", "src/core/lib/json/json_writer.c", "src/core/lib/json/json_writer.h", + "src/core/lib/security/util/b64.c", "src/core/lib/slice/percent_encoding.c", "src/core/lib/slice/percent_encoding.h", "src/core/lib/slice/slice.c", @@ -8046,6 +8083,29 @@ "grpc_base", "grpc_client_channel" ], + "headers": [ + "src/core/ext/resolver/dns/c_ares/grpc_ares_ev_driver.h", + "src/core/ext/resolver/dns/c_ares/grpc_ares_wrapper.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_resolver_dns_ares", + "src": [ + "src/core/ext/resolver/dns/c_ares/dns_resolver_ares.c", + "src/core/ext/resolver/dns/c_ares/grpc_ares_ev_driver.h", + "src/core/ext/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c", + "src/core/ext/resolver/dns/c_ares/grpc_ares_wrapper.c", + "src/core/ext/resolver/dns/c_ares/grpc_ares_wrapper.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", + "grpc_client_channel" + ], "headers": [], "is_filegroup": true, "language": "c", @@ -8464,24 +8524,24 @@ "gpr" ], "headers": [ - "src/core/lib/tsi/fake_transport_security.h", - "src/core/lib/tsi/ssl_transport_security.h", - "src/core/lib/tsi/ssl_types.h", - "src/core/lib/tsi/transport_security.h", - "src/core/lib/tsi/transport_security_interface.h" + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" ], "is_filegroup": true, "language": "c", "name": "tsi", "src": [ - "src/core/lib/tsi/fake_transport_security.c", - "src/core/lib/tsi/fake_transport_security.h", - "src/core/lib/tsi/ssl_transport_security.c", - "src/core/lib/tsi/ssl_transport_security.h", - "src/core/lib/tsi/ssl_types.h", - "src/core/lib/tsi/transport_security.c", - "src/core/lib/tsi/transport_security.h", - "src/core/lib/tsi/transport_security_interface.h" + "src/core/tsi/fake_transport_security.c", + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.c", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.c", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" ], "third_party": false, "type": "filegroup" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 0279010d04..6c103e1022 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -40776,6 +40776,56 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_secure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41105,6 +41155,56 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41400,6 +41500,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41436,6 +41537,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41456,6 +41558,80 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "lto", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_secure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "lto", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41472,6 +41648,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41508,6 +41685,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41546,6 +41724,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41584,6 +41763,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41620,6 +41800,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41656,6 +41837,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41692,6 +41874,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41728,6 +41911,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41764,6 +41948,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41800,6 +41985,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41836,6 +42022,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41872,6 +42059,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41908,6 +42096,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41928,6 +42117,80 @@ { "args": [ "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "lto", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "lto", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, @@ -41944,6 +42207,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41980,6 +42244,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42018,6 +42283,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42056,6 +42322,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42092,6 +42359,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42128,6 +42396,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42164,6 +42433,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42200,6 +42470,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42236,6 +42507,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42272,6 +42544,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -42308,6 +42581,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -83290,6 +83564,29 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5965570207907840" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "mac", + "linux" + ], + "uses_polling": false + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-6520142139752448" ], "ci_platforms": [ @@ -84325,6 +84622,29 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/crash-59a56fa18034a104fb9f16cd58071b6ff93b8756" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "mac", + "linux" + ], + "uses_polling": false + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/crash-59b587d15c0bcdb985417cd7a133cecfcc232698" ], "ci_platforms": [ @@ -95411,6 +95731,29 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/poc-c726ee220e980ed6ad17809fd9efe2844ee61555ac08e4f88afd8901cc2dd53a" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "mac", + "linux" + ], + "uses_polling": false + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/timeout-0fa0559576ad2a45b06d0bfb84115963d7d48206" ], "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 1d91b61ba4..1870a738ca 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -215,6 +215,29 @@ class CXXLanguage: categories=smoketest_categories+[SCALABLE]) yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, + rpc_type='STREAMING', + req_size=1024*1024, + resp_size=1024*1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, + secure=secure, + categories=smoketest_categories+[SCALABLE], + channels=1, outstanding=100) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % secstr, + rpc_type='STREAMING', + req_size=64*1024, + resp_size=64*1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, + secure=secure, + categories=smoketest_categories+[SCALABLE]) + + yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', diff --git a/tools/run_tests/python_utils/comment_on_pr.py b/tools/run_tests/python_utils/comment_on_pr.py new file mode 100644 index 0000000000..9708d0ffb4 --- /dev/null +++ b/tools/run_tests/python_utils/comment_on_pr.py @@ -0,0 +1,49 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import json +import urllib2 + +def comment_on_pr(text): + if 'JENKINS_OAUTH_TOKEN' not in os.environ: + print 'Missing JENKINS_OAUTH_TOKEN env var: not commenting' + return + if 'ghprbPullId' not in os.environ: + print 'Missing ghprbPullId env var: not commenting' + return + req = urllib2.Request( + url = 'https://api.github.com/repos/grpc/grpc/issues/%s/comments' % + os.environ['ghprbPullId'], + data = json.dumps({'body': text}), + headers = { + 'Authorization': 'token %s' % os.environ['JENKINS_OAUTH_TOKEN'], + 'Content-Type': 'application/json', + }) + print urllib2.urlopen(req).read() diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 6bedc54941..17b156c78f 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -38,6 +38,19 @@ import argparse import python_utils.jobset as jobset import python_utils.start_port_server as start_port_server +_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', + 'bm_closure', + 'bm_cq', + 'bm_call_create', + 'bm_error', + 'bm_chttp2_hpack', + 'bm_chttp2_transport', + 'bm_pollset', + 'bm_metadata', + 'bm_fullstack_trickle'] + flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -201,26 +214,11 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', - 'bm_closure', - 'bm_cq', - 'bm_call_create', - 'bm_error', - 'bm_chttp2_hpack', - 'bm_chttp2_transport', - 'bm_pollset', - 'bm_metadata', - 'bm_fullstack_trickle', - ], + choices=_AVAILABLE_BENCHMARK_TESTS, + default=_AVAILABLE_BENCHMARK_TESTS, nargs='+', type=str, help='Which microbenchmarks should be run') -argp.add_argument('--diff_perf', - default=None, - type=str, - help='Diff microbenchmarks against this git revision') argp.add_argument('--bigquery_upload', default=False, action='store_const', @@ -236,35 +234,9 @@ try: for collect in args.collect: for bm_name in args.benchmarks: collectors[collect](bm_name, args) - if args.diff_perf: - if 'summary' not in args.collect: - for bm_name in args.benchmarks: - run_summary(bm_name, 'opt', bm_name) - run_summary(bm_name, 'counters', bm_name) - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_perf]) - comparables = [] - subprocess.check_call(['make', 'clean']) - try: - for bm_name in args.benchmarks: - try: - run_summary(bm_name, 'opt', '%s.old' % bm_name) - run_summary(bm_name, 'counters', '%s.old' % bm_name) - comparables.append(bm_name) - except subprocess.CalledProcessError, e: - pass - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - for bm_name in comparables: - diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', - '%s.counters.json' % bm_name, - '%s.opt.json' % bm_name, - '%s.old.counters.json' % bm_name, - '%s.old.opt.json' % bm_name]).strip() - if diff: - heading('Performance diff: %s' % bm_name) - text(diff) finally: + if not os.path.exists('reports'): + os.makedirs('reports') index_html += "</body>\n</html>\n" with open('reports/index.html', 'w') as f: f.write(index_html) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 5e7c30e6f1..00dd4ca605 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -247,9 +247,12 @@ class CLanguage(object): polling_strategies = ['all'] for polling_strategy in polling_strategies: env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - _ROOT + '/src/core/lib/tsi/test_creds/ca.pem', + _ROOT + '/src/core/tsi/test_creds/ca.pem', 'GRPC_POLL_STRATEGY': polling_strategy, 'GRPC_VERBOSITY': 'DEBUG'} + resolver = os.environ.get('GRPC_DNS_RESOLVER', None); + if resolver: + env['GRPC_DNS_RESOLVER'] = resolver shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy timeout_scaling = 1 if polling_strategy == 'poll-cv': diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index b9bf6e5037..6c1d4bd15d 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -55,7 +55,8 @@ _DEFAULT_INNER_JOBS = 2 _REPORT_SUFFIX = 'sponge_log.xml' -def _docker_jobspec(name, runtests_args=[], inner_jobs=_DEFAULT_INNER_JOBS): +def _docker_jobspec(name, runtests_args=[], runtests_envs={}, + inner_jobs=_DEFAULT_INNER_JOBS): """Run a single instance of run_tests.py in a docker container""" test_job = jobset.JobSpec( cmdline=['python', 'tools/run_tests/run_tests.py', @@ -64,16 +65,19 @@ def _docker_jobspec(name, runtests_args=[], inner_jobs=_DEFAULT_INNER_JOBS): '-j', str(inner_jobs), '-x', 'report_%s_%s' % (name, _REPORT_SUFFIX), '--report_suite_name', '%s' % name] + runtests_args, + environ=runtests_envs, shortname='run_tests_%s' % name, timeout_seconds=_RUNTESTS_TIMEOUT) return test_job -def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_DEFAULT_INNER_JOBS): +def _workspace_jobspec(name, runtests_args=[], workspace_name=None, + runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS): """Run a single instance of run_tests.py in a separate workspace""" if not workspace_name: workspace_name = 'workspace_%s' % name env = {'WORKSPACE_NAME': workspace_name} + env.update(runtests_envs) test_job = jobset.JobSpec( cmdline=['bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh', @@ -89,7 +93,7 @@ def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', arch=None, compiler=None, - labels=[], extra_args=[], + labels=[], extra_args=[], extra_envs={}, inner_jobs=_DEFAULT_INNER_JOBS): result = [] for language in languages: @@ -103,11 +107,16 @@ def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native', name += '_%s_%s' % (arch, compiler) runtests_args += ['--arch', arch, '--compiler', compiler] + for extra_env in extra_envs: + name += '_%s_%s' % (extra_env, extra_envs[extra_env]) + runtests_args += extra_args if platform == 'linux': - job = _docker_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) + job = _docker_jobspec(name=name, runtests_args=runtests_args, + runtests_envs=extra_envs, inner_jobs=inner_jobs) else: - job = _workspace_jobspec(name=name, runtests_args=runtests_args, inner_jobs=inner_jobs) + job = _workspace_jobspec(name=name, runtests_args=runtests_args, + runtests_envs=extra_envs, inner_jobs=inner_jobs) job.labels = [platform, config, language, iomgr_platform] + labels result.append(job) @@ -212,6 +221,21 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) + # C and C++ with the c-ares DNS resolver on Linux + test_jobs += _generate_jobs(languages=['c', 'c++'], + configs=['dbg'], platforms=['linux'], + labels=['portability'], + extra_args=extra_args, + extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) + + # TODO(zyc): Turn on this test after adding c-ares support on windows. + # C with the c-ares DNS resolver on Windonws + # test_jobs += _generate_jobs(languages=['c'], + # configs=['dbg'], platforms=['windows'], + # labels=['portability'], + # extra_args=extra_args, + # extra_envs={'GRPC_DNS_RESOLVER': 'ares'}) + # cmake build for C and C++ # TODO(jtattermusch): some of the tests are failing, so we force --build_only # to make sure it's buildable at least. diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 3c5ba16b93..38dfe277ae 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -49,6 +49,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 593e917c176b5bc5aafa57bf9f6030d749d91cd5 third_party/protobuf (v3.1.0-alpha-1-326-g593e917) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) + 7691f773af79bf75a62d1863fd0f13ebf9dc51b1 third_party/cares/cares (1.12.0) EOF diff -u $submodules $want_submodules |