diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/codegen/core/gen_stats_data.py | 31 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core | 2 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 10 | ||||
| -rw-r--r-- | tools/internal_ci/linux/grpc_sanity_webhook_test.cfg | 30 | ||||
| -rw-r--r-- | tools/run_tests/generated/sources_and_headers.json | 69 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 25 |
6 files changed, 121 insertions, 46 deletions
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index df6cd5a6bd..8e4ef594af 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -19,13 +19,30 @@ import ctypes import math import sys import yaml +import json with open('src/core/lib/debug/stats_data.yaml') as f: attrs = yaml.load(f.read()) +REQUIRED_FIELDS = ['name', 'doc'] + +def make_type(name, fields): + return (collections.namedtuple(name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), []) + +def c_str(s, encoding='ascii'): + if isinstance(s, unicode): + s = s.encode(encoding) + result = '' + for c in s: + if not (32 <= ord(c) < 127) or c in ('\\', '"'): + result += '\\%03o' % ord(c) + else: + result += c + return '"' + result + '"' + types = ( - (collections.namedtuple('Counter', 'name'), []), - (collections.namedtuple('Histogram', 'name max buckets'), []), + make_type('Counter', []), + make_type('Histogram', ['max', 'buckets']), ) inst_map = dict((t[0].__name__, t[1]) for t in types) @@ -200,6 +217,8 @@ with open('src/core/lib/debug/stats_data.h', 'w') as H: print >>H, "} grpc_stats_%ss;" % (typename.lower()) print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % ( typename.lower(), typename.upper()) + print >>H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % ( + typename.lower(), typename.upper()) histo_start = [] histo_buckets = [] @@ -269,8 +288,14 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C: print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % ( typename.lower(), typename.upper()) for inst in instances: - print >>C, " \"%s\"," % inst.name + print >>C, " %s," % c_str(inst.name) print >>C, "};" + print >>C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % ( + typename.lower(), typename.upper()) + for inst in instances: + print >>C, " %s," % c_str(inst.doc) + print >>C, "};" + for i, tbl in enumerate(static_tables): print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % ( tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1])) diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index e4cc1c7461..632735342b 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 4.0.0-dev +PROJECT_NUMBER = 5.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index c2daf900fb..e352cb78f1 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 4.0.0-dev +PROJECT_NUMBER = 5.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -979,10 +979,10 @@ src/core/ext/filters/http/message_compress/message_compress_filter.c \ src/core/ext/filters/http/message_compress/message_compress_filter.h \ src/core/ext/filters/http/server/http_server_filter.c \ src/core/ext/filters/http/server/http_server_filter.h \ -src/core/ext/filters/load_reporting/load_reporting.c \ -src/core/ext/filters/load_reporting/load_reporting.h \ -src/core/ext/filters/load_reporting/load_reporting_filter.c \ -src/core/ext/filters/load_reporting/load_reporting_filter.h \ +src/core/ext/filters/load_reporting/server_load_reporting_filter.c \ +src/core/ext/filters/load_reporting/server_load_reporting_filter.h \ +src/core/ext/filters/load_reporting/server_load_reporting_plugin.c \ +src/core/ext/filters/load_reporting/server_load_reporting_plugin.h \ src/core/ext/filters/max_age/max_age_filter.c \ src/core/ext/filters/max_age/max_age_filter.h \ src/core/ext/filters/message_size/message_size_filter.c \ diff --git a/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg b/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg new file mode 100644 index 0000000000..24e7984f3a --- /dev/null +++ b/tools/internal_ci/linux/grpc_sanity_webhook_test.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" +timeout_mins: 20 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests linux sanity opt --inner_jobs 16 -j 1 --internal_ci" +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index dd9a171268..2f6e34bfb3 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2790,12 +2790,15 @@ "grpc_test_util_unsecure", "grpc_unsecure" ], - "headers": [], + "headers": [ + "test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h" + ], "is_filegroup": false, "language": "c++", "name": "bm_fullstack_streaming_ping_pong", "src": [ - "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc" + "test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc", + "test/cpp/microbenchmarks/fullstack_streaming_ping_pong.h" ], "third_party": false, "type": "target" @@ -2811,12 +2814,15 @@ "grpc_test_util_unsecure", "grpc_unsecure" ], - "headers": [], + "headers": [ + "test/cpp/microbenchmarks/fullstack_streaming_pump.h" + ], "is_filegroup": false, "language": "c++", "name": "bm_fullstack_streaming_pump", "src": [ - "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc" + "test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc", + "test/cpp/microbenchmarks/fullstack_streaming_pump.h" ], "third_party": false, "type": "target" @@ -2854,12 +2860,15 @@ "grpc_test_util_unsecure", "grpc_unsecure" ], - "headers": [], + "headers": [ + "test/cpp/microbenchmarks/fullstack_unary_ping_pong.h" + ], "is_filegroup": false, "language": "c++", "name": "bm_fullstack_unary_ping_pong", "src": [ - "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc" + "test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc", + "test/cpp/microbenchmarks/fullstack_unary_ping_pong.h" ], "third_party": false, "type": "target" @@ -5890,7 +5899,6 @@ "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", - "grpc_load_reporting", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -5899,6 +5907,7 @@ "grpc_resolver_sockaddr", "grpc_secure", "grpc_server_backward_compatibility", + "grpc_server_load_reporting", "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_insecure", @@ -5920,7 +5929,7 @@ "deps": [ "gpr", "grpc_base", - "grpc_load_reporting", + "grpc_server_load_reporting", "grpc_transport_chttp2_client_secure", "grpc_transport_cronet_client_secure" ], @@ -5997,7 +6006,6 @@ "grpc_lb_policy_grpclb", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", - "grpc_load_reporting", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -6005,6 +6013,7 @@ "grpc_resolver_fake", "grpc_resolver_sockaddr", "grpc_server_backward_compatibility", + "grpc_server_load_reporting", "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_server_insecure", "grpc_transport_inproc", @@ -8565,27 +8574,6 @@ "grpc_base" ], "headers": [ - "src/core/ext/filters/load_reporting/load_reporting.h", - "src/core/ext/filters/load_reporting/load_reporting_filter.h" - ], - "is_filegroup": true, - "language": "c", - "name": "grpc_load_reporting", - "src": [ - "src/core/ext/filters/load_reporting/load_reporting.c", - "src/core/ext/filters/load_reporting/load_reporting.h", - "src/core/ext/filters/load_reporting/load_reporting_filter.c", - "src/core/ext/filters/load_reporting/load_reporting_filter.h" - ], - "third_party": false, - "type": "filegroup" - }, - { - "deps": [ - "gpr", - "grpc_base" - ], - "headers": [ "src/core/ext/filters/max_age/max_age_filter.h" ], "is_filegroup": true, @@ -8793,6 +8781,27 @@ { "deps": [ "gpr", + "grpc_base" + ], + "headers": [ + "src/core/ext/filters/load_reporting/server_load_reporting_filter.h", + "src/core/ext/filters/load_reporting/server_load_reporting_plugin.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_server_load_reporting", + "src": [ + "src/core/ext/filters/load_reporting/server_load_reporting_filter.c", + "src/core/ext/filters/load_reporting/server_load_reporting_filter.h", + "src/core/ext/filters/load_reporting/server_load_reporting_plugin.c", + "src/core/ext/filters/load_reporting/server_load_reporting_plugin.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", "gpr_test_util", "grpc_base", "grpc_client_channel", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4311e53839..b66c5f7f71 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -69,17 +69,22 @@ _POLLING_STRATEGIES = { } -def get_flaky_tests(limit=None): +BigQueryTestData = collections.namedtuple('BigQueryTestData', 'name flaky cpu') + + +def get_bqtest_data(limit=None): import big_query_utils bq = big_query_utils.create_big_query() query = """ SELECT filtered_test_name, + SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, + MAX(cpu_measured) as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, - result + result, cpu_measured FROM [grpc-testing:jenkins_test_results.aggregate_results] WHERE @@ -89,15 +94,15 @@ SELECT GROUP BY filtered_test_name HAVING - SUM(result != 'PASSED' AND result != 'SKIPPED') > 0""" + flaky OR cpu > 0""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) page = bq.jobs().getQueryResults( pageToken=None, **query_job['jobReference']).execute(num_retries=3) - flake_names = [row['f'][0]['v'] for row in page['rows']] - return flake_names + test_data = [BigQueryTestData(row['f'][0]['v'], row['f'][1]['v'] == 'true', float(row['f'][2]['v'])) for row in page['rows']] + return test_data def platform_string(): @@ -141,6 +146,9 @@ class Config(object): if not flaky and shortname and shortname in flaky_tests: print('Setting %s to flaky' % shortname) flaky = True + if shortname in shortname_to_cpu: + print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, shortname_to_cpu[shortname])) + cpu_cost = shortname_to_cpu[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1254,9 +1262,12 @@ argp.add_argument('--disable_auto_set_flakes', default=False, const=True, action args = argp.parse_args() flaky_tests = set() +shortname_to_cpu = {} if not args.disable_auto_set_flakes: try: - flaky_tests = set(get_flaky_tests()) + for test in get_bqtest_data(): + if test.flaky: flaky_tests.add(test.name) + if test.cpu > 0: shortname_to_cpu[test.name] = test.cpu except: print("Unexpected error getting flaky tests:", sys.exc_info()[0]) @@ -1516,7 +1527,7 @@ def _build_and_run( # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis and args.max_time <= 0: - massaged_one_run = sorted(one_run, key=lambda x: x.shortname) + massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. |