aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Matt Kwong <mattkwong@google.com>2016-10-20 17:11:18 -0700
committerGravatar Matt Kwong <mattkwong@google.com>2016-10-31 14:26:22 -0700
commit5c691c634de144b50fc1f2e166303219721c19d0 (patch)
tree7894b00e8a03d8b2029dd4f1bca0b9c93098952a
parentccc6a9cbf264655ae6b60727cd86b987a62977c9 (diff)
Enable pull request test filtering
-rw-r--r--tools/run_tests/filter_pull_request_tests.py9
-rwxr-xr-xtools/run_tests/jobset.py13
-rw-r--r--tools/run_tests/report_utils.py2
-rwxr-xr-xtools/run_tests/run_tests_matrix.py35
4 files changed, 42 insertions, 17 deletions
diff --git a/tools/run_tests/filter_pull_request_tests.py b/tools/run_tests/filter_pull_request_tests.py
index e2027a2340..b7ebe2085a 100644
--- a/tools/run_tests/filter_pull_request_tests.py
+++ b/tools/run_tests/filter_pull_request_tests.py
@@ -77,6 +77,7 @@ _ALL_TEST_SUITES = [_SANITY_TEST_SUITE, _CORE_TEST_SUITE, _CPP_TEST_SUITE,
# and the value is a list of tests that should be run. An empty list means that
# the changed files should not trigger any tests. Any changed file that does not
# match any of these regexes will trigger all tests
+# DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING (be careful even if you do)
_WHITELIST_DICT = {
'^doc/': [],
'^examples/': [],
@@ -174,9 +175,13 @@ def filter_tests(tests, base_branch):
print("Finding file differences between gRPC %s branch and pull request...\n" % base_branch)
changed_files = _get_changed_files(base_branch)
for changed_file in changed_files:
- print(changed_file)
+ print(" %s" % changed_file)
print
+ # todo(mattkwong): Remove this
+ # Faking changed files to test test filtering on Jenkins
+ changed_files = ['src/node/something', 'src/python/something']
+
# Regex that combines all keys in _WHITELIST_DICT
all_triggers = "(" + ")|(".join(_WHITELIST_DICT.keys()) + ")"
# Check if all tests have to be run
@@ -188,7 +193,7 @@ def filter_tests(tests, base_branch):
for test_suite in _ALL_TEST_SUITES:
if _can_skip_tests(changed_files, test_suite.triggers):
for label in test_suite.labels:
- print(" Filtering %s tests" % label)
+ print(" %s tests safe to skip" % label)
skippable_labels.append(label)
tests = _remove_irrelevant_tests(tests, skippable_labels)
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index b6fb6318e0..b84eb3b5d7 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -96,6 +96,7 @@ _COLORS = {
'lightgray': [ 37, 0],
'gray': [ 30, 1 ],
'purple': [ 35, 0 ],
+ 'cyan': [ 36, 0 ]
}
@@ -114,6 +115,7 @@ _TAG_COLOR = {
'WAITING': 'yellow',
'SUCCESS': 'green',
'IDLE': 'gray',
+ 'SKIPPED': 'cyan'
}
@@ -450,7 +452,16 @@ def run(cmdlines,
travis=False,
infinite_runs=False,
stop_on_failure=False,
- add_env={}):
+ add_env={},
+ skip_jobs=False):
+ if skip_jobs:
+ results = {}
+ skipped_job_result = JobResult()
+ skipped_job_result.state = 'SKIPPED'
+ for job in cmdlines:
+ message('SKIPPED', job.shortname, do_newline=True)
+ results[job.shortname] = [skipped_job_result]
+ return results
js = Jobset(check_cancelled,
maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
newline_on_success, travis, stop_on_failure, add_env)
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 3e18f36510..90055e3530 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -74,6 +74,8 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
ET.SubElement(xml_test, 'failure', message='Failure')
elif result.state == 'TIMEOUT':
ET.SubElement(xml_test, 'error', message='Timeout')
+ elif result.state == 'SKIPPED':
+ ET.SubElement(xml_test, 'skipped', message='Skipped')
tree = ET.ElementTree(root)
tree.write(xml_report, encoding='UTF-8')
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 2656f1ac5d..ae8cb8518e 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -292,28 +292,29 @@ print('IMPORTANT: The changes you are testing need to be locally committed')
print('because only the committed changes in the current branch will be')
print('copied to the docker environment or into subworkspaces.')
-print
-print 'Will run these tests:'
-for job in jobs:
- if args.dry_run:
- print ' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))
- else:
- print ' %s' % job.shortname
-print
-
+skipped_jobs = []
if args.filter_pr_tests:
- print 'IMPORTANT: Test filtering is not active; this is only for testing.'
+ print 'Looking for irrelevant tests to skip...'
relevant_jobs = filter_tests(jobs, args.base_branch)
- # todo(mattkwong): add skipped tests to report.xml
print
if len(relevant_jobs) == len(jobs):
- print '(TESTING) No tests will be skipped.'
+ print 'No tests will be skipped.'
else:
- print '(TESTING) These tests will be skipped:'
- for job in list(set(jobs) - set(relevant_jobs)):
+ print 'These tests will be skipped:'
+ skipped_jobs = set(jobs) - set(relevant_jobs)
+ for job in list(skipped_jobs):
print ' %s' % job.shortname
+ jobs = relevant_jobs
print
+print 'Will run these tests:'
+for job in jobs:
+ if args.dry_run:
+ print ' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))
+ else:
+ print ' %s' % job.shortname
+print
+
if args.dry_run:
print '--dry_run was used, exiting'
sys.exit(1)
@@ -323,9 +324,15 @@ num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
travis=True,
maxjobs=args.jobs)
+# Merge skipped tests into results to show skipped tests on report.xml
+if skipped_jobs:
+ skipped_results = jobset.run(skipped_jobs,
+ skip_jobs=True)
+ resultset.update(skipped_results)
report_utils.render_junit_xml_report(resultset, 'report.xml',
suite_name='aggregate_tests')
+
if num_failures == 0:
jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
do_newline=True)