diff options
| -rw-r--r-- | tools/interop_matrix/README.md | 12 | ||||
| -rwxr-xr-x | tools/interop_matrix/create_matrix_images.py | 1 | ||||
| -rwxr-xr-x | tools/interop_matrix/run_interop_matrix_tests.py | 189 | ||||
| -rw-r--r-- | tools/run_tests/python_utils/report_utils.py | 26 |
4 files changed, 215 insertions, 13 deletions
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 8493099d1a..14fe82b9a1 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -11,7 +11,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r - `--git_checkout` enables git checkout grpc release branch/tag. - `--release` specifies a git release tag. Make sure it is a valid tag in the grpc github rep. - `--language` specifies a language. - For examle, To build all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/create_matrix_images.py --git_checkout --release=all`. + For example, To build all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/create_matrix_images.py --git_checkout --release=all`. - Verify the newly created docker images are uploaded to GCR. For example: - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_go1.7` show tags for a image repo. @@ -30,11 +30,17 @@ from specific releases/tag, are used to test version compatiblity between gRPC r - `LANG=go KEEP_IMAGE=1 ./create_testcases.sh` will generate `./testcases/go__master` and keep the local docker image so it can be invoked simply via `./testcases/go__master`. Note: remove local docker images manually afterwards with `docker rmi <image_id>`. - Stage and commit the generated test case file `./testcases/<lang>__<release>`. -## Instructions for running test cases against a GCR image +## Instructions for running test cases against GCR images +- Run `tools/interop_matrix/run_interop_matrix_tests.py`. Useful options: + - `--release` specifies a git release tag. Defaults to `--release=master`. Make sure the GCR images with the tag have been created using `create_matrix_images.py` above. + - `--language` specifies a language. Defaults to `--language=all`. + For example, To test all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/run_interop_matrix_test.py --release=all`. +- The output for all the test cases is recorded in a junit style xml file (default to 'report.xml'). + +## Instructions for running test cases against a GCR image manually - Run test cases by specifying `docker_image` variable inline with the test case script created above. For example: - `docker_image=gcr.io/grpc-testing/grpc_interop_go1.7:master ./testcases/go__master` will run go__master test cases against `go1.7` with gRPC release `master` docker image in GCR. - Note: - File path starting with `tools/` or `template/` are relative to the grpc repo root dir. File path starting with `./` are relative to current directory (`tools/interop_matrix`). diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py index 214ae0c3fd..119cbd2e88 100755 --- a/tools/interop_matrix/create_matrix_images.py +++ b/tools/interop_matrix/create_matrix_images.py @@ -253,5 +253,4 @@ for lang in languages: # docker image name must be in the format <gcr_path>/<image>:<gcr_tag> assert image.startswith(args.gcr_path) and image.find(':') != -1 - # subprocess.call(['gcloud', 'docker', '--', 'push', image]) subprocess.call(['gcloud', 'docker', '--', 'push', image]) diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py new file mode 100755 index 0000000000..3273c3d759 --- /dev/null +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python2.7 +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run tests using docker images in Google Container Registry per matrix.""" + +from __future__ import print_function + +import argparse +import atexit +import json +import multiprocessing +import os +import re +import subprocess +import sys +import uuid + +# Langauage Runtime Matrix +import client_matrix + +python_util_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../run_tests/python_utils')) +sys.path.append(python_util_dir) +import dockerjob +import jobset +import report_utils + +_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() +# All gRPC release tags, flattened, deduped and sorted. +_RELEASES = sorted(list(set( + i for l in client_matrix.LANG_RELEASE_MATRIX.values() for i in l))) +_TEST_TIMEOUT = 30 + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) +argp.add_argument('--gcr_path', + default='gcr.io/grpc-testing', + help='Path of docker images in Google Container Registry') + +argp.add_argument('--release', + default='master', + choices=['all', 'master'] + _RELEASES, + help='Release tags to test. When testing all ' + 'releases defined in client_matrix.py, use "all".') + +argp.add_argument('-l', '--language', + choices=['all'] + sorted(_LANGUAGES), + nargs='+', + default=['all'], + help='Languages to test') + +argp.add_argument('--keep', + action='store_true', + help='keep the created local images after finishing the tests.') + +argp.add_argument('--report_file', + default='report.xml', + help='The result file to create.') + +args = argp.parse_args() + + +def find_all_images_for_lang(lang): + """Find docker images for a language across releases and runtimes. + + Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime. + """ + # Find all defined releases. + if args.release == 'all': + releases = ['master'] + client_matrix.LANG_RELEASE_MATRIX[lang] + else: + # Look for a particular release. + if args.release not in ['master'] + client_matrix.LANG_RELEASE_MATRIX[lang]: + jobset.message('SKIPPED', + '%s for %s is not defined' % (args.release, lang), + do_newline=True) + return [] + releases = [args.release] + + # Images tuples keyed by runtime. + images = {} + for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: + image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime) + output = subprocess.check_output(['gcloud', 'beta', 'container', 'images', + 'list-tags', '--format=json', image_path]) + docker_image_list = json.loads(output) + # All images should have a single tag or no tag. + tags = [i['tags'][0] for i in docker_image_list if i['tags']] + jobset.message('START', 'Found images for %s: %s' % (image_path, tags), + do_newline=True) + skipped = len(docker_image_list) - len(tags) + jobset.message('START', 'Skipped images (no-tag/unknown-tag): %d' % skipped, + do_newline=True) + # Filter tags based on the releases. + images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if + tag in releases] + return images + +# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime. +_loaded_testcases = {} +def find_test_cases(lang, release): + """Returns the list of test cases from testcase files per lang/release.""" + file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s') + if not os.path.exists(file_tmpl % (lang, release)): + release = 'master' + testcases = file_tmpl % (lang, release) + if lang in _loaded_testcases.keys() and release in _loaded_testcases[lang].keys(): + return _loaded_testcases[lang][release] + + job_spec_list=[] + try: + with open(testcases) as f: + # Only line start with 'docker run' are test cases. + for line in f.readlines(): + if line.startswith('docker run'): + m = re.search('--test_case=(.*)"', line) + shortname = m.group(1) if m else 'unknown_test' + spec = jobset.JobSpec(cmdline=line, + shortname=shortname, + timeout_seconds=_TEST_TIMEOUT, + shell=True) + job_spec_list.append(spec) + jobset.message('START', + 'Loaded %s tests from %s' % (len(job_spec_list), testcases), + do_newline=True) + except IOError as err: + jobset.message('FAILED', err, do_newline=True) + if lang not in _loaded_testcases.keys(): + _loaded_testcases[lang] = {} + _loaded_testcases[lang][release]=job_spec_list + return job_spec_list + +_xml_report_tree = None +def run_tests_for_lang(lang, runtime, images): + """Find and run all test cases for a language. + + images is a list of (<release-tag>, <image-full-path>) tuple. + """ + for image_tuple in images: + release, image = image_tuple + jobset.message('START', 'Testing %s' % image, do_newline=True) + _docker_images_cleanup.append(image) + job_spec_list = find_test_cases(lang,release) + num_failures, resultset = jobset.run(job_spec_list, + newline_on_success=True, + add_env={'docker_image':image}, + maxjobs=args.jobs) + if num_failures: + jobset.message('FAILED', 'Some tests failed', do_newline=True) + else: + jobset.message('SUCCESS', 'All tests passed', do_newline=True) + + # Required, otherwise _xml_report_tree will be shadowed by local (undefined) + # reference in the next line. + global _xml_report_tree + _xml_report_tree = report_utils.add_junit_xml_results( + resultset, + 'grpc_interop_matrix', + '%s__%s:%s'%(lang,runtime,release), + str(uuid.uuid4()), + _xml_report_tree) + +_docker_images_cleanup = [] +def cleanup(): + if not args.keep: + for image in _docker_images_cleanup: + dockerjob.remove_image(image, skip_nonexistent=True) + +atexit.register(cleanup) + +languages = args.language if args.language != ['all'] else _LANGUAGES +for lang in languages: + docker_images = find_all_images_for_lang(lang) + for runtime in sorted(docker_images.keys()): + run_tests_for_lang(lang, runtime, docker_images[runtime]) + +report_utils.create_xml_report_file(args.report_file, _xml_report_tree) diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 4b4c50a03e..9e602102ff 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -46,8 +46,22 @@ def _filter_msg(msg, output_format): def render_junit_xml_report(resultset, xml_report, suite_package='grpc', suite_name='tests'): """Generate JUnit-like XML report.""" - root = ET.Element('testsuites') - testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package, + tree = add_junit_xml_results(resultset, suite_package, suite_name, '1') + create_xml_report_file(xml_report, tree) + +def create_xml_report_file(xml_report, tree): + """Generate JUnit-like report file from xml tree .""" + # ensure the report directory exists + report_dir = os.path.dirname(os.path.abspath(xml_report)) + if not os.path.exists(report_dir): + os.makedirs(report_dir) + tree.write(xml_report, encoding='UTF-8') + +def add_junit_xml_results(resultset, suite_package, suite_name, id, + old_tree=None): + """Returns a JUnit-like XML report tree with added test results.""" + root = ET.Element('testsuites') if not old_tree else old_tree.getroot() + testsuite = ET.SubElement(root, 'testsuite', id=id, package=suite_package, name=suite_name) failure_count = 0 error_count = 0 @@ -67,13 +81,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', ET.SubElement(xml_test, 'skipped', message='Skipped') testsuite.set('failures', str(failure_count)) testsuite.set('errors', str(error_count)) - # ensure the report directory exists - report_dir = os.path.dirname(os.path.abspath(xml_report)) - if not os.path.exists(report_dir): - os.makedirs(report_dir) - tree = ET.ElementTree(root) - tree.write(xml_report, encoding='UTF-8') - + return ET.ElementTree(root) def render_interop_html_report( client_langs, server_langs, test_cases, auth_test_cases, http2_cases, |