diff options
Diffstat (limited to 'tools/interop_matrix')
| -rw-r--r-- | tools/interop_matrix/README.md | 2 | ||||
| -rw-r--r-- | tools/interop_matrix/client_matrix.py | 618 | ||||
| -rwxr-xr-x | tools/interop_matrix/create_matrix_images.py | 18 | ||||
| -rwxr-xr-x | tools/interop_matrix/run_interop_matrix_tests.py | 23 |
4 files changed, 216 insertions, 445 deletions
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index db84d9b454..6676f5d470 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -1,6 +1,6 @@ # Overview -This directory contains scripts that facilitate building and running gRPC tests for combinations of language/runtimes (known as matrix). +This directory contains scripts that facilitate building and running gRPC interoperability tests for combinations of language/runtimes (known as matrix). The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack from specific releases/tag, are used to test version compatiblity between gRPC release versions. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index c0b08a59b2..12051e70a0 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -15,6 +15,8 @@ # Defines languages, runtimes and releases for backward compatibility testing +from collections import OrderedDict + def get_github_repo(lang): return { @@ -27,15 +29,26 @@ def get_github_repo(lang): def get_release_tags(lang): - return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang]) + """Returns list of known releases for given language.""" + return list(LANG_RELEASE_MATRIX[lang].keys()) -def get_release_tag_name(release_info): - assert len(release_info.keys()) == 1 - return release_info.keys()[0] +def get_runtimes_for_lang_release(lang, release): + """Get list of valid runtimes for given release of lang.""" + runtimes_to_skip = [] + release_info = LANG_RELEASE_MATRIX[lang][release] + if release_info: + runtimes_to_skip = release_info.skip_runtime + return [ + runtime for runtime in LANG_RUNTIME_MATRIX[lang] + if runtime not in runtimes_to_skip + ] def should_build_docker_interop_image_from_release_tag(lang): + # All dockerfile definitions live in grpc/grpc repository. + # For language that have a separate repo, we need to use + # dockerfile definitions from head of grpc/grpc. if lang in ['go', 'java', 'node']: return False return True @@ -44,7 +57,7 @@ def should_build_docker_interop_image_from_release_tag(lang): # Dictionary of runtimes per language LANG_RUNTIME_MATRIX = { 'cxx': ['cxx'], # This is actually debian8. - 'go': ['go1.7', 'go1.8'], + 'go': ['go1.7', 'go1.8', 'go1.11'], 'java': ['java_oracle8'], 'python': ['python'], 'node': ['node'], @@ -53,428 +66,183 @@ LANG_RUNTIME_MATRIX = { 'csharp': ['csharp', 'csharpcoreclr'], } + +class ReleaseInfo: + """Info about a single release of a language""" + + def __init__(self, patch=[], skip_runtime=[], testcases_file=None): + self.patch = patch + self.skip_runtime = skip_runtime + self.testcases_file = None + + # Dictionary of known releases for given language. LANG_RELEASE_MATRIX = { - 'cxx': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'go': [ - { - 'v1.0.5': None - }, - { - 'v1.2.1': None - }, - { - 'v1.3.0': None - }, - { - 'v1.4.2': None - }, - { - 'v1.5.2': None - }, - { - 'v1.6.0': None - }, - { - 'v1.7.4': None - }, - { - 'v1.8.2': None - }, - { - 'v1.9.2': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.3': None - }, - { - 'v1.12.2': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.0': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - ], - 'java': [ - { - 'v1.0.3': None - }, - { - 'v1.1.2': None - }, - { - 'v1.2.0': None - }, - { - 'v1.3.1': None - }, - { - 'v1.4.0': None - }, - { - 'v1.5.0': None - }, - { - 'v1.6.1': None - }, - { - 'v1.7.0': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.0': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.1': None - }, - { - 'v1.14.0': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.1': None - }, - { - 'v1.17.1': None - }, - ], - 'python': [ - { - 'v1.0.x': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.1': None # first python 1.8 release is 1.8.1 - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'node': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, + 'cxx': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'go': + OrderedDict([ + ('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])), + ('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])), + ]), + 'java': + OrderedDict([ + ('v1.0.3', ReleaseInfo()), + ('v1.1.2', ReleaseInfo()), + ('v1.2.0', ReleaseInfo()), + ('v1.3.1', ReleaseInfo()), + ('v1.4.0', ReleaseInfo()), + ('v1.5.0', ReleaseInfo()), + ('v1.6.1', ReleaseInfo()), + ('v1.7.0', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.0', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.1', ReleaseInfo()), + ('v1.14.0', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.1', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'python': + OrderedDict([ + ('v1.0.x', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.1', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'node': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), # TODO: https://github.com/grpc/grpc-node/issues/235. - #{ - # 'v1.7.2': None - #}, - { - 'v1.8.4': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.0': None - }, - { - 'v1.11.3': None - }, - { - 'v1.12.4': None - }, - ], - 'ruby': [ - { - 'v1.0.1': { - 'patch': [ - 'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile', - 'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh', - ] - } - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'php': [ - { - 'v1.0.1': None - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], - 'csharp': [ - { - 'v1.0.1': { - 'patch': [ - 'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile', - 'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile', - ] - } - }, - { - 'v1.1.4': None - }, - { - 'v1.2.5': None - }, - { - 'v1.3.9': None - }, - { - 'v1.4.2': None - }, - { - 'v1.6.6': None - }, - { - 'v1.7.2': None - }, - { - 'v1.8.0': None - }, - { - 'v1.9.1': None - }, - { - 'v1.10.1': None - }, - { - 'v1.11.1': None - }, - { - 'v1.12.0': None - }, - { - 'v1.13.0': None - }, - { - 'v1.14.1': None - }, - { - 'v1.15.0': None - }, - { - 'v1.16.0': None - }, - { - 'v1.17.1': None - }, - ], + # ('v1.7.2', ReleaseInfo()), + ('v1.8.4', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.0', ReleaseInfo()), + ('v1.11.3', ReleaseInfo()), + ('v1.12.4', ReleaseInfo()), + ]), + 'ruby': + OrderedDict([ + ('v1.0.1', + ReleaseInfo(patch=[ + 'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile', + 'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh', + ])), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'php': + OrderedDict([ + ('v1.0.1', ReleaseInfo()), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), + 'csharp': + OrderedDict([ + ('v1.0.1', + ReleaseInfo(patch=[ + 'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile', + 'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile', + ])), + ('v1.1.4', ReleaseInfo()), + ('v1.2.5', ReleaseInfo()), + ('v1.3.9', ReleaseInfo()), + ('v1.4.2', ReleaseInfo()), + ('v1.6.6', ReleaseInfo()), + ('v1.7.2', ReleaseInfo()), + ('v1.8.0', ReleaseInfo()), + ('v1.9.1', ReleaseInfo()), + ('v1.10.1', ReleaseInfo()), + ('v1.11.1', ReleaseInfo()), + ('v1.12.0', ReleaseInfo()), + ('v1.13.0', ReleaseInfo()), + ('v1.14.1', ReleaseInfo()), + ('v1.15.0', ReleaseInfo()), + ('v1.16.0', ReleaseInfo()), + ('v1.17.1', ReleaseInfo()), + ]), } # This matrix lists the version of testcases to use for a release. As new @@ -483,6 +251,8 @@ LANG_RELEASE_MATRIX = { # particular version in some cases. If not specified, xxx__master file will be # used. For example, all java versions will run the commands in java__master. # The testcases files exist under the testcases directory. +# TODO(jtattermusch): make this data part of LANG_RELEASE_MATRIX, +# there is no reason for this to be a separate data structure. TESTCASES_VERSION_MATRIX = { 'node_v1.0.1': 'node__v1.0.1', 'node_v1.1.4': 'node__v1.1.4', diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py index c2568efba0..31a0e1c7ba 100755 --- a/tools/interop_matrix/create_matrix_images.py +++ b/tools/interop_matrix/create_matrix_images.py @@ -39,10 +39,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() # All gRPC release tags, flattened, deduped and sorted. _RELEASES = sorted( list( - set( - client_matrix.get_release_tag_name(info) - for lang in client_matrix.LANG_RELEASE_MATRIX.values() - for info in lang))) + set(release + for release_dict in client_matrix.LANG_RELEASE_MATRIX.values() + for release in release_dict.keys()))) # Destination directory inside docker image to keep extra info from build time. _BUILD_INFO = '/var/local/build_info' @@ -217,7 +216,7 @@ def build_all_images_for_release(lang, release): }.get(lang, 'GRPC_ROOT') env[var] = stack_base - for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: + for runtime in client_matrix.get_runtimes_for_lang_release(lang, release): job = build_image_jobspec(runtime, env, release, stack_base) docker_images.append(job.tag) build_jobs.append(job) @@ -260,11 +259,10 @@ atexit.register(cleanup) def maybe_apply_patches_on_git_tag(stack_base, lang, release): files_to_patch = [] - for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]: - if client_matrix.get_release_tag_name(release_info) == release: - if release_info[release] is not None: - files_to_patch = release_info[release].get('patch') - break + + release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release] + if release_info: + files_to_patch = release_info.patch if not files_to_patch: return patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release) diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index d47b0ee0f7..c855de3b1e 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -44,10 +44,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() # All gRPC release tags, flattened, deduped and sorted. _RELEASES = sorted( list( - set( - client_matrix.get_release_tag_name(info) - for lang in client_matrix.LANG_RELEASE_MATRIX.values() - for info in lang))) + set(release + for release_dict in client_matrix.LANG_RELEASE_MATRIX.values() + for release in release_dict.keys()))) argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) @@ -113,13 +112,17 @@ def _get_test_images_for_lang(lang, release_arg, image_path_prefix): return {} releases = [release_arg] - # Images tuples keyed by runtime. + # Image tuples keyed by runtime. images = {} - for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: - image_path = '%s/grpc_interop_%s' % (image_path_prefix, runtime) - images[runtime] = [ - (tag, '%s:%s' % (image_path, tag)) for tag in releases - ] + for tag in releases: + for runtime in client_matrix.get_runtimes_for_lang_release(lang, tag): + image_name = '%s/grpc_interop_%s:%s' % (image_path_prefix, runtime, + tag) + image_tuple = (tag, image_name) + + if not images.has_key(runtime): + images[runtime] = [] + images[runtime].append(image_tuple) return images |