Merge pull request #17660 from jtattermusch/interop_matrix_lang_release_cleanup

interop_matrix: refactor LANG_RELEASE_MATRIX

4 files changed, 197 insertions, 482 deletions

diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index db84d9b454..6676f5d470 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -1,6 +1,6 @@
 # Overview
 
-This directory contains scripts that facilitate building and running gRPC tests for combinations of language/runtimes (known as matrix).
+This directory contains scripts that facilitate building and running gRPC interoperability tests for combinations of language/runtimes (known as matrix).
 
 The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack
 from specific releases/tag, are used to test version compatiblity between gRPC release versions.
diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index 4964fd6167..12051e70a0 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -15,6 +15,8 @@
 
 # Defines languages, runtimes and releases for backward compatibility testing
 
+from collections import OrderedDict
+
 
 def get_github_repo(lang):
     return {
@@ -27,23 +29,16 @@ def get_github_repo(lang):
 
 
 def get_release_tags(lang):
-    return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang])
-
-
-def get_release_tag_name(release_info):
-    assert len(release_info.keys()) == 1
-    return release_info.keys()[0]
+    """Returns list of known releases for given language."""
+    return list(LANG_RELEASE_MATRIX[lang].keys())
 
 
 def get_runtimes_for_lang_release(lang, release):
     """Get list of valid runtimes for given release of lang."""
     runtimes_to_skip = []
-    # see if any the lang release has "skip_runtime" annotation.
-    for release_info in LANG_RELEASE_MATRIX[lang]:
-        if get_release_tag_name(release_info) == release:
-            if release_info[release] is not None:
-                runtimes_to_skip = release_info[release].get('skip_runtime', [])
-                break
+    release_info = LANG_RELEASE_MATRIX[lang][release]
+    if release_info:
+        runtimes_to_skip = release_info.skip_runtime
     return [
         runtime for runtime in LANG_RUNTIME_MATRIX[lang]
         if runtime not in runtimes_to_skip
@@ -51,6 +46,9 @@ def get_runtimes_for_lang_release(lang, release):
 
 
 def should_build_docker_interop_image_from_release_tag(lang):
+    # All dockerfile definitions live in grpc/grpc repository.
+    # For language that have a separate repo, we need to use
+    # dockerfile definitions from head of grpc/grpc.
     if lang in ['go', 'java', 'node']:
         return False
     return True
@@ -68,465 +66,183 @@ LANG_RUNTIME_MATRIX = {
     'csharp': ['csharp', 'csharpcoreclr'],
 }
 
+
+class ReleaseInfo:
+    """Info about a single release of a language"""
+
+    def __init__(self, patch=[], skip_runtime=[], testcases_file=None):
+        self.patch = patch
+        self.skip_runtime = skip_runtime
+        self.testcases_file = None
+
+
 # Dictionary of known releases for given language.
 LANG_RELEASE_MATRIX = {
-    'cxx': [
-        {
-            'v1.0.1': None
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
-        {
-            'v1.7.2': None
-        },
-        {
-            'v1.8.0': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.1': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.0': None
-        },
-        {
-            'v1.14.1': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.0': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
-    'go': [
-        {
-            'v1.0.5': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.2.1': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.3.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.4.2': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.5.2': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.6.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.7.4': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.8.2': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.9.2': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.10.1': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.11.3': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.12.2': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.13.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.14.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.15.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.16.0': {
-                'skip_runtime': ['go1.11']
-            }
-        },
-        {
-            'v1.17.0': {
-                'skip_runtime': ['go1.7', 'go1.8']
-            }
-        },
-    ],
-    'java': [
-        {
-            'v1.0.3': None
-        },
-        {
-            'v1.1.2': None
-        },
-        {
-            'v1.2.0': None
-        },
-        {
-            'v1.3.1': None
-        },
-        {
-            'v1.4.0': None
-        },
-        {
-            'v1.5.0': None
-        },
-        {
-            'v1.6.1': None
-        },
-        {
-            'v1.7.0': None
-        },
-        {
-            'v1.8.0': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.0': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.1': None
-        },
-        {
-            'v1.14.0': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.1': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
-    'python': [
-        {
-            'v1.0.x': None
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
-        {
-            'v1.7.2': None
-        },
-        {
-            'v1.8.1': None  # first python 1.8 release is 1.8.1
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.1': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.0': None
-        },
-        {
-            'v1.14.1': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.0': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
-    'node': [
-        {
-            'v1.0.1': None
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
+    'cxx':
+    OrderedDict([
+        ('v1.0.1', ReleaseInfo()),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
+        ('v1.7.2', ReleaseInfo()),
+        ('v1.8.0', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.1', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.0', ReleaseInfo()),
+        ('v1.14.1', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.0', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
+    'go':
+    OrderedDict([
+        ('v1.0.5', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.2.1', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.3.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.4.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.5.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.6.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.7.4', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.8.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.9.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.10.1', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.11.3', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.12.2', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.13.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.14.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.15.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.16.0', ReleaseInfo(skip_runtime=['go1.11'])),
+        ('v1.17.0', ReleaseInfo(skip_runtime=['go1.7', 'go1.8'])),
+    ]),
+    'java':
+    OrderedDict([
+        ('v1.0.3', ReleaseInfo()),
+        ('v1.1.2', ReleaseInfo()),
+        ('v1.2.0', ReleaseInfo()),
+        ('v1.3.1', ReleaseInfo()),
+        ('v1.4.0', ReleaseInfo()),
+        ('v1.5.0', ReleaseInfo()),
+        ('v1.6.1', ReleaseInfo()),
+        ('v1.7.0', ReleaseInfo()),
+        ('v1.8.0', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.0', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.1', ReleaseInfo()),
+        ('v1.14.0', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.1', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
+    'python':
+    OrderedDict([
+        ('v1.0.x', ReleaseInfo()),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
+        ('v1.7.2', ReleaseInfo()),
+        ('v1.8.1', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.1', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.0', ReleaseInfo()),
+        ('v1.14.1', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.0', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
+    'node':
+    OrderedDict([
+        ('v1.0.1', ReleaseInfo()),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
         # TODO: https://github.com/grpc/grpc-node/issues/235.
-        #{
-        #    'v1.7.2': None
-        #},
-        {
-            'v1.8.4': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.0': None
-        },
-        {
-            'v1.11.3': None
-        },
-        {
-            'v1.12.4': None
-        },
-    ],
-    'ruby': [
-        {
-            'v1.0.1': {
-                'patch': [
-                    'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
-                    'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
-                ]
-            }
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
-        {
-            'v1.7.2': None
-        },
-        {
-            'v1.8.0': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.1': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.0': None
-        },
-        {
-            'v1.14.1': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.0': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
-    'php': [
-        {
-            'v1.0.1': None
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
-        {
-            'v1.7.2': None
-        },
-        {
-            'v1.8.0': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.1': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.0': None
-        },
-        {
-            'v1.14.1': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.0': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
-    'csharp': [
-        {
-            'v1.0.1': {
-                'patch': [
-                    'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile',
-                    'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile',
-                ]
-            }
-        },
-        {
-            'v1.1.4': None
-        },
-        {
-            'v1.2.5': None
-        },
-        {
-            'v1.3.9': None
-        },
-        {
-            'v1.4.2': None
-        },
-        {
-            'v1.6.6': None
-        },
-        {
-            'v1.7.2': None
-        },
-        {
-            'v1.8.0': None
-        },
-        {
-            'v1.9.1': None
-        },
-        {
-            'v1.10.1': None
-        },
-        {
-            'v1.11.1': None
-        },
-        {
-            'v1.12.0': None
-        },
-        {
-            'v1.13.0': None
-        },
-        {
-            'v1.14.1': None
-        },
-        {
-            'v1.15.0': None
-        },
-        {
-            'v1.16.0': None
-        },
-        {
-            'v1.17.1': None
-        },
-    ],
+        # ('v1.7.2', ReleaseInfo()),
+        ('v1.8.4', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.0', ReleaseInfo()),
+        ('v1.11.3', ReleaseInfo()),
+        ('v1.12.4', ReleaseInfo()),
+    ]),
+    'ruby':
+    OrderedDict([
+        ('v1.0.1',
+         ReleaseInfo(patch=[
+             'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
+             'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
+         ])),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
+        ('v1.7.2', ReleaseInfo()),
+        ('v1.8.0', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.1', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.0', ReleaseInfo()),
+        ('v1.14.1', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.0', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
+    'php':
+    OrderedDict([
+        ('v1.0.1', ReleaseInfo()),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
+        ('v1.7.2', ReleaseInfo()),
+        ('v1.8.0', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.1', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.0', ReleaseInfo()),
+        ('v1.14.1', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.0', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
+    'csharp':
+    OrderedDict([
+        ('v1.0.1',
+         ReleaseInfo(patch=[
+             'tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile',
+             'tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile',
+         ])),
+        ('v1.1.4', ReleaseInfo()),
+        ('v1.2.5', ReleaseInfo()),
+        ('v1.3.9', ReleaseInfo()),
+        ('v1.4.2', ReleaseInfo()),
+        ('v1.6.6', ReleaseInfo()),
+        ('v1.7.2', ReleaseInfo()),
+        ('v1.8.0', ReleaseInfo()),
+        ('v1.9.1', ReleaseInfo()),
+        ('v1.10.1', ReleaseInfo()),
+        ('v1.11.1', ReleaseInfo()),
+        ('v1.12.0', ReleaseInfo()),
+        ('v1.13.0', ReleaseInfo()),
+        ('v1.14.1', ReleaseInfo()),
+        ('v1.15.0', ReleaseInfo()),
+        ('v1.16.0', ReleaseInfo()),
+        ('v1.17.1', ReleaseInfo()),
+    ]),
 }
 
 # This matrix lists the version of testcases to use for a release. As new
@@ -535,6 +251,8 @@ LANG_RELEASE_MATRIX = {
 # particular version in some cases. If not specified, xxx__master file will be
 # used. For example, all java versions will run the commands in java__master.
 # The testcases files exist under the testcases directory.
+# TODO(jtattermusch): make this data part of LANG_RELEASE_MATRIX,
+# there is no reason for this to be a separate data structure.
 TESTCASES_VERSION_MATRIX = {
     'node_v1.0.1': 'node__v1.0.1',
     'node_v1.1.4': 'node__v1.1.4',
diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py
index cf61d46248..31a0e1c7ba 100755
--- a/tools/interop_matrix/create_matrix_images.py
+++ b/tools/interop_matrix/create_matrix_images.py
@@ -39,10 +39,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
 # All gRPC release tags, flattened, deduped and sorted.
 _RELEASES = sorted(
     list(
-        set(
-            client_matrix.get_release_tag_name(info)
-            for lang in client_matrix.LANG_RELEASE_MATRIX.values()
-            for info in lang)))
+        set(release
+            for release_dict in client_matrix.LANG_RELEASE_MATRIX.values()
+            for release in release_dict.keys())))
 
 # Destination directory inside docker image to keep extra info from build time.
 _BUILD_INFO = '/var/local/build_info'
@@ -260,11 +259,10 @@ atexit.register(cleanup)
 
 def maybe_apply_patches_on_git_tag(stack_base, lang, release):
     files_to_patch = []
-    for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:
-        if client_matrix.get_release_tag_name(release_info) == release:
-            if release_info[release] is not None:
-                files_to_patch = release_info[release].get('patch')
-                break
+
+    release_info = client_matrix.LANG_RELEASE_MATRIX[lang][release]
+    if release_info:
+        files_to_patch = release_info.patch
     if not files_to_patch:
         return
     patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
index dabb486523..c855de3b1e 100755
--- a/tools/interop_matrix/run_interop_matrix_tests.py
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -44,10 +44,9 @@ _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
 # All gRPC release tags, flattened, deduped and sorted.
 _RELEASES = sorted(
     list(
-        set(
-            client_matrix.get_release_tag_name(info)
-            for lang in client_matrix.LANG_RELEASE_MATRIX.values()
-            for info in lang)))
+        set(release
+            for release_dict in client_matrix.LANG_RELEASE_MATRIX.values()
+            for release in release_dict.keys())))
 
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)