From 9992bdb74ec2a39b7d8cd867ee4f3b3f96d151bb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 6 Sep 2017 15:00:31 -0700 Subject: Query CPU cost of tests and feed that into test runner --- tools/run_tests/run_tests.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4311e53839..6bb61c0b70 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -69,17 +69,20 @@ _POLLING_STRATEGIES = { } -def get_flaky_tests(limit=None): +BigQueryTestData = collections.namedtuple('BigQueryTestData', 'name flaky cpu') + + +def get_bqtest_data(limit=None): import big_query_utils bq = big_query_utils.create_big_query() query = """ SELECT - filtered_test_name, + filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, AVG(cpu_measured) as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, - result + result, cpu_measured FROM [grpc-testing:jenkins_test_results.aggregate_results] WHERE @@ -89,15 +92,15 @@ SELECT GROUP BY filtered_test_name HAVING - SUM(result != 'PASSED' AND result != 'SKIPPED') > 0""" + flaky OR cpu > 0""" if limit: query += " limit {}".format(limit) query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) page = bq.jobs().getQueryResults( pageToken=None, **query_job['jobReference']).execute(num_retries=3) - flake_names = [row['f'][0]['v'] for row in page['rows']] - return flake_names + test_data = [BigQueryTestData(row['f'][0]['v'], row['f'][1]['v'] == 'true', float(row['f'][2]['v'])) for row in page['rows']] + return test_data def platform_string(): @@ -141,6 +144,9 @@ class Config(object): if not flaky and shortname and shortname in flaky_tests: print('Setting %s to flaky' % shortname) flaky = True + if shortname in test_times: + print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, test_times[shortname])) + cpu_cost = test_times[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1254,9 +1260,12 @@ argp.add_argument('--disable_auto_set_flakes', default=False, const=True, action args = argp.parse_args() flaky_tests = set() +test_times = {} if not args.disable_auto_set_flakes: try: - flaky_tests = set(get_flaky_tests()) + for test in get_bqtest_data(): + if test.flaky: flaky_tests.add(test.name) + if test.cpu > 0: test_times[test.name] = test.cpu except: print("Unexpected error getting flaky tests:", sys.exc_info()[0]) -- cgit v1.2.3 From 120d4fda0715b157ed3a19d2a557906dac29f4b0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 12:25:25 -0700 Subject: Use max cpu_measured instead of average (for safety) --- tools/run_tests/run_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 6bb61c0b70..3edd28fd5e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -78,7 +78,9 @@ def get_bqtest_data(limit=None): bq = big_query_utils.create_big_query() query = """ SELECT - filtered_test_name, SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, AVG(cpu_measured) as cpu + filtered_test_name, + SUM(result != 'PASSED' AND result != 'SKIPPED') > 0 as flaky, + MAX(cpu_measured) as cpu FROM ( SELECT REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, -- cgit v1.2.3 From 0b86d03611204ef3b4f445c489eb4d8f041f6479 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 13:47:45 -0700 Subject: Better variable name --- tools/run_tests/run_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 6bb61c0b70..8b3c629d5f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -144,9 +144,9 @@ class Config(object): if not flaky and shortname and shortname in flaky_tests: print('Setting %s to flaky' % shortname) flaky = True - if shortname in test_times: - print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, test_times[shortname])) - cpu_cost = test_times[shortname] + if shortname in shortname_to_cpu: + print('Update CPU cost for %s: %f -> %f' % (shortname, cpu_cost, shortname_to_cpu[shortname])) + cpu_cost = shortname_to_cpu[shortname] return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, shortname=shortname, environ=actual_environ, @@ -1260,12 +1260,12 @@ argp.add_argument('--disable_auto_set_flakes', default=False, const=True, action args = argp.parse_args() flaky_tests = set() -test_times = {} +shortname_to_cpu = {} if not args.disable_auto_set_flakes: try: for test in get_bqtest_data(): if test.flaky: flaky_tests.add(test.name) - if test.cpu > 0: test_times[test.name] = test.cpu + if test.cpu > 0: shortname_to_cpu[test.name] = test.cpu except: print("Unexpected error getting flaky tests:", sys.exc_info()[0]) -- cgit v1.2.3 From 8d0fef20890d6bddc3739e5648910decfb0dabd5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 7 Sep 2017 13:59:47 -0700 Subject: Sort tests by cpu usage to better bin-pack --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 72ded5694a..b66c5f7f71 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1527,7 +1527,7 @@ def _build_and_run( # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis and args.max_time <= 0: - massaged_one_run = sorted(one_run, key=lambda x: x.shortname) + massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. -- cgit v1.2.3