yapf tools/interop_matrix

3 files changed, 627 insertions, 446 deletions

diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index 7281307363..f32bdf8b15 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -15,29 +15,34 @@
 
 # Dictionaries used for client matrix testing.
 
+
 def get_github_repo(lang):
-  return {
-      'go': 'git@github.com:grpc/grpc-go.git',
-      'java': 'git@github.com:grpc/grpc-java.git',
-      'node': 'git@github.com:grpc/grpc-node.git',
-      # all other languages use the grpc.git repo.
-  }.get(lang, 'git@github.com:grpc/grpc.git')
+    return {
+        'go': 'git@github.com:grpc/grpc-go.git',
+        'java': 'git@github.com:grpc/grpc-java.git',
+        'node': 'git@github.com:grpc/grpc-node.git',
+        # all other languages use the grpc.git repo.
+    }.get(lang, 'git@github.com:grpc/grpc.git')
+
 
 def get_release_tags(lang):
-  return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang])
+    return map(lambda r: get_release_tag_name(r), LANG_RELEASE_MATRIX[lang])
+
 
 def get_release_tag_name(release_info):
-  assert len(release_info.keys()) == 1
-  return release_info.keys()[0]
+    assert len(release_info.keys()) == 1
+    return release_info.keys()[0]
+
 
 def should_build_docker_interop_image_from_release_tag(lang):
-  if lang in ['go', 'java', 'node']:
-    return False
-  return True
+    if lang in ['go', 'java', 'node']:
+        return False
+    return True
+
 
 # Dictionary of runtimes per language
 LANG_RUNTIME_MATRIX = {
-    'cxx': ['cxx'],             # This is actually debian8.
+    'cxx': ['cxx'],  # This is actually debian8.
     'go': ['go1.7', 'go1.8'],
     'java': ['java_oracle8'],
     'python': ['python'],
@@ -51,81 +56,197 @@ LANG_RUNTIME_MATRIX = {
 # a release tag pointing to the latest build of the branch.
 LANG_RELEASE_MATRIX = {
     'cxx': [
-        {'v1.0.1': None},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
-        {'v1.7.2': None},
+        {
+            'v1.0.1': None
+        },
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
+        {
+            'v1.7.2': None
+        },
     ],
     'go': [
-        {'v1.0.5': None},
-        {'v1.2.1': None},
-        {'v1.3.0': None},
-        {'v1.4.2': None},
-        {'v1.5.2': None},
-        {'v1.6.0': None},
-        {'v1.7.4': None},
-        {'v1.8.1': None},
+        {
+            'v1.0.5': None
+        },
+        {
+            'v1.2.1': None
+        },
+        {
+            'v1.3.0': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.5.2': None
+        },
+        {
+            'v1.6.0': None
+        },
+        {
+            'v1.7.4': None
+        },
+        {
+            'v1.8.1': None
+        },
     ],
     'java': [
-        {'v1.0.3': None},
-        {'v1.1.2': None},
-        {'v1.2.0': None},
-        {'v1.3.1': None},
-        {'v1.4.0': None},
-        {'v1.5.0': None},
-        {'v1.6.1': None},
-        {'v1.7.0': None},
-        {'v1.8.0': None},
+        {
+            'v1.0.3': None
+        },
+        {
+            'v1.1.2': None
+        },
+        {
+            'v1.2.0': None
+        },
+        {
+            'v1.3.1': None
+        },
+        {
+            'v1.4.0': None
+        },
+        {
+            'v1.5.0': None
+        },
+        {
+            'v1.6.1': None
+        },
+        {
+            'v1.7.0': None
+        },
+        {
+            'v1.8.0': None
+        },
     ],
     'python': [
-        {'v1.0.x': None},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
-        {'v1.7.2': None},
+        {
+            'v1.0.x': None
+        },
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
+        {
+            'v1.7.2': None
+        },
     ],
     'node': [
-        {'v1.0.1': None},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
+        {
+            'v1.0.1': None
+        },
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
         #{'v1.7.1': None}, Failing tests
     ],
     'ruby': [
-        {'v1.0.1': {'patch': [
-            'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
-            'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
-        ]}},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
-        {'v1.7.2': None},
+        {
+            'v1.0.1': {
+                'patch': [
+                    'tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile',
+                    'tools/dockerfile/interoptest/grpc_interop_ruby/build_interop.sh',
+                ]
+            }
+        },
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
+        {
+            'v1.7.2': None
+        },
     ],
     'php': [
-        {'v1.0.1': None},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
-        {'v1.7.2': None},
+        {
+            'v1.0.1': None
+        },
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
+        {
+            'v1.7.2': None
+        },
     ],
-   'csharp': [
+    'csharp': [
         #{'v1.0.1': None},
-        {'v1.1.4': None},
-        {'v1.2.5': None},
-        {'v1.3.9': None},
-        {'v1.4.2': None},
-        {'v1.6.6': None},
-        {'v1.7.2': None},
+        {
+            'v1.1.4': None
+        },
+        {
+            'v1.2.5': None
+        },
+        {
+            'v1.3.9': None
+        },
+        {
+            'v1.4.2': None
+        },
+        {
+            'v1.6.6': None
+        },
+        {
+            'v1.7.2': None
+        },
     ],
 }
diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py
index a292368131..05af3a5632 100755
--- a/tools/interop_matrix/create_matrix_images.py
+++ b/tools/interop_matrix/create_matrix_images.py
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Build and upload docker images to Google Container Registry per matrix."""
 
 from __future__ import print_function
@@ -29,8 +28,8 @@ import tempfile
 # Langauage Runtime Matrix
 import client_matrix
 
-python_util_dir = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../run_tests/python_utils'))
+python_util_dir = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
 sys.path.append(python_util_dir)
 import dockerjob
 import jobset
@@ -38,267 +37,304 @@ import jobset
 _IMAGE_BUILDER = 'tools/run_tests/dockerize/build_interop_image.sh'
 _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
 # All gRPC release tags, flattened, deduped and sorted.
-_RELEASES = sorted(list(set(
-    client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang)))
+_RELEASES = sorted(
+    list(
+        set(
+            client_matrix.get_release_tag_name(info)
+            for lang in client_matrix.LANG_RELEASE_MATRIX.values()
+            for info in lang)))
 
 # Destination directory inside docker image to keep extra info from build time.
 _BUILD_INFO = '/var/local/build_info'
 
 argp = argparse.ArgumentParser(description='Run interop tests.')
-argp.add_argument('--gcr_path',
-                  default='gcr.io/grpc-testing',
-                  help='Path of docker images in Google Container Registry')
-
-argp.add_argument('--release',
-                  default='master',
-                  choices=['all', 'master'] + _RELEASES,
-                  help='github commit tag to checkout.  When building all '
-                  'releases defined in client_matrix.py, use "all". Valid only '
-                  'with --git_checkout.')
-
-argp.add_argument('-l', '--language',
-                  choices=['all'] + sorted(_LANGUAGES),
-                  nargs='+',
-                  default=['all'],
-                  help='Test languages to build docker images for.')
-
-argp.add_argument('--git_checkout',
-                  action='store_true',
-                  help='Use a separate git clone tree for building grpc stack. '
-                  'Required when using --release flag.  By default, current'
-                  'tree and the sibling will be used for building grpc stack.')
-
-argp.add_argument('--git_checkout_root',
-                  default='/export/hda3/tmp/grpc_matrix',
-                  help='Directory under which grpc-go/java/main repo will be '
-                  'cloned.  Valid only with --git_checkout.')
-
-argp.add_argument('--keep',
-                  action='store_true',
-                  help='keep the created local images after uploading to GCR')
-
-argp.add_argument('--reuse_git_root',
-                  default=False,
-                  action='store_const',
-                  const=True,                  
-                  help='reuse the repo dir. If False, the existing git root '
-                  'directory will removed before a clean checkout, because '
-                  'reusing the repo can cause git checkout error if you switch '
-                  'between releases.')
-
+argp.add_argument(
+    '--gcr_path',
+    default='gcr.io/grpc-testing',
+    help='Path of docker images in Google Container Registry')
+
+argp.add_argument(
+    '--release',
+    default='master',
+    choices=['all', 'master'] + _RELEASES,
+    help='github commit tag to checkout.  When building all '
+    'releases defined in client_matrix.py, use "all". Valid only '
+    'with --git_checkout.')
+
+argp.add_argument(
+    '-l',
+    '--language',
+    choices=['all'] + sorted(_LANGUAGES),
+    nargs='+',
+    default=['all'],
+    help='Test languages to build docker images for.')
+
+argp.add_argument(
+    '--git_checkout',
+    action='store_true',
+    help='Use a separate git clone tree for building grpc stack. '
+    'Required when using --release flag.  By default, current'
+    'tree and the sibling will be used for building grpc stack.')
+
+argp.add_argument(
+    '--git_checkout_root',
+    default='/export/hda3/tmp/grpc_matrix',
+    help='Directory under which grpc-go/java/main repo will be '
+    'cloned.  Valid only with --git_checkout.')
+
+argp.add_argument(
+    '--keep',
+    action='store_true',
+    help='keep the created local images after uploading to GCR')
+
+argp.add_argument(
+    '--reuse_git_root',
+    default=False,
+    action='store_const',
+    const=True,
+    help='reuse the repo dir. If False, the existing git root '
+    'directory will removed before a clean checkout, because '
+    'reusing the repo can cause git checkout error if you switch '
+    'between releases.')
 
 args = argp.parse_args()
 
+
 def add_files_to_image(image, with_files, label=None):
-  """Add files to a docker image.
+    """Add files to a docker image.
 
   image: docker image name, i.e. grpc_interop_java:26328ad8
   with_files: additional files to include in the docker image.
   label: label string to attach to the image.
   """
-  tag_idx = image.find(':')
-  if tag_idx == -1:
-    jobset.message('FAILED', 'invalid docker image %s' % image, do_newline=True)
-    sys.exit(1)
-  orig_tag = '%s_' % image
-  subprocess.check_output(['docker', 'tag', image, orig_tag])
-
-  lines = ['FROM ' + orig_tag]
-  if label:
-    lines.append('LABEL %s' % label)
-
-  temp_dir = tempfile.mkdtemp()
-  atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir]))
-
-  # Copy with_files inside the tmp directory, which will be the docker build
-  # context.
-  for f in with_files:
-    shutil.copy(f, temp_dir)
-    lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO))
-
-  # Create a Dockerfile.
-  with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f:
-    f.write('\n'.join(lines))
-
-  jobset.message('START', 'Repackaging %s' % image, do_newline=True)
-  build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir]
-  subprocess.check_output(build_cmd)
-  dockerjob.remove_image(orig_tag, skip_nonexistent=True)
+    tag_idx = image.find(':')
+    if tag_idx == -1:
+        jobset.message(
+            'FAILED', 'invalid docker image %s' % image, do_newline=True)
+        sys.exit(1)
+    orig_tag = '%s_' % image
+    subprocess.check_output(['docker', 'tag', image, orig_tag])
+
+    lines = ['FROM ' + orig_tag]
+    if label:
+        lines.append('LABEL %s' % label)
+
+    temp_dir = tempfile.mkdtemp()
+    atexit.register(lambda: subprocess.call(['rm', '-rf', temp_dir]))
+
+    # Copy with_files inside the tmp directory, which will be the docker build
+    # context.
+    for f in with_files:
+        shutil.copy(f, temp_dir)
+        lines.append('COPY %s %s/' % (os.path.basename(f), _BUILD_INFO))
+
+    # Create a Dockerfile.
+    with open(os.path.join(temp_dir, 'Dockerfile'), 'w') as f:
+        f.write('\n'.join(lines))
+
+    jobset.message('START', 'Repackaging %s' % image, do_newline=True)
+    build_cmd = ['docker', 'build', '--rm', '--tag', image, temp_dir]
+    subprocess.check_output(build_cmd)
+    dockerjob.remove_image(orig_tag, skip_nonexistent=True)
+
 
 def build_image_jobspec(runtime, env, gcr_tag, stack_base):
-  """Build interop docker image for a language with runtime.
+    """Build interop docker image for a language with runtime.
 
   runtime: a <lang><version> string, for example go1.8.
   env:     dictionary of env to passed to the build script.
   gcr_tag: the tag for the docker image (i.e. v1.3.0).
   stack_base: the local gRPC repo path.
   """
-  basename = 'grpc_interop_%s' % runtime
-  tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag)
-  build_env = {
-      'INTEROP_IMAGE': tag,
-      'BASE_NAME': basename,
-      'TTY_FLAG': '-t'
-  }
-  build_env.update(env)
-  image_builder_path = _IMAGE_BUILDER
-  if client_matrix.should_build_docker_interop_image_from_release_tag(lang):
-    image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER)
-  build_job = jobset.JobSpec(
-          cmdline=[image_builder_path],
-          environ=build_env,
-          shortname='build_docker_%s' % runtime,
-          timeout_seconds=30*60)
-  build_job.tag = tag
-  return build_job
+    basename = 'grpc_interop_%s' % runtime
+    tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag)
+    build_env = {'INTEROP_IMAGE': tag, 'BASE_NAME': basename, 'TTY_FLAG': '-t'}
+    build_env.update(env)
+    image_builder_path = _IMAGE_BUILDER
+    if client_matrix.should_build_docker_interop_image_from_release_tag(lang):
+        image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER)
+    build_job = jobset.JobSpec(
+        cmdline=[image_builder_path],
+        environ=build_env,
+        shortname='build_docker_%s' % runtime,
+        timeout_seconds=30 * 60)
+    build_job.tag = tag
+    return build_job
+
 
 def build_all_images_for_lang(lang):
-  """Build all docker images for a language across releases and runtimes."""
-  if not args.git_checkout:
-    if args.release != 'master':
-      print('WARNING: --release is set but will be ignored\n')
-    releases = ['master']
-  else:
-    if args.release == 'all':
-      releases = client_matrix.get_release_tags(lang)
+    """Build all docker images for a language across releases and runtimes."""
+    if not args.git_checkout:
+        if args.release != 'master':
+            print('WARNING: --release is set but will be ignored\n')
+        releases = ['master']
     else:
-      # Build a particular release.
-      if args.release not in ['master'] + client_matrix.get_release_tags(lang):
-        jobset.message('SKIPPED',
-                       '%s for %s is not defined' % (args.release, lang),
-                       do_newline=True)
-        return []
-      releases = [args.release]
-
-  images = []
-  for release in releases:
-    images += build_all_images_for_release(lang, release)
-  jobset.message('SUCCESS',
-                 'All docker images built for %s at %s.' % (lang, releases),
-                 do_newline=True)
-  return images
+        if args.release == 'all':
+            releases = client_matrix.get_release_tags(lang)
+        else:
+            # Build a particular release.
+            if args.release not in ['master'] + client_matrix.get_release_tags(
+                    lang):
+                jobset.message(
+                    'SKIPPED',
+                    '%s for %s is not defined' % (args.release, lang),
+                    do_newline=True)
+                return []
+            releases = [args.release]
+
+    images = []
+    for release in releases:
+        images += build_all_images_for_release(lang, release)
+    jobset.message(
+        'SUCCESS',
+        'All docker images built for %s at %s.' % (lang, releases),
+        do_newline=True)
+    return images
+
 
 def build_all_images_for_release(lang, release):
-  """Build all docker images for a release across all runtimes."""
-  docker_images = []
-  build_jobs = []
-
-  env = {}
-  # If we not using current tree or the sibling for grpc stack, do checkout.
-  stack_base = ''
-  if args.git_checkout:
-    stack_base = checkout_grpc_stack(lang, release)
-    var ={'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT'}.get(lang, 'GRPC_ROOT')
-    env[var] = stack_base
-
-  for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
-    job = build_image_jobspec(runtime, env, release, stack_base)
-    docker_images.append(job.tag)
-    build_jobs.append(job)
-
-  jobset.message('START', 'Building interop docker images.', do_newline=True)
-  print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
-
-  num_failures, _ = jobset.run(
-      build_jobs, newline_on_success=True, maxjobs=multiprocessing.cpu_count())
-  if num_failures:
-    jobset.message('FAILED', 'Failed to build interop docker images.',
-                   do_newline=True)
-    docker_images_cleanup.extend(docker_images)
-    sys.exit(1)
-
-  jobset.message('SUCCESS',
-                 'All docker images built for %s at %s.' % (lang, release),
-                 do_newline=True)
-
-  if release != 'master':
-    commit_log = os.path.join(stack_base, 'commit_log')
-    if os.path.exists(commit_log):
-      for image in docker_images:
-        add_files_to_image(image, [commit_log], 'release=%s' % release)
-  return docker_images
+    """Build all docker images for a release across all runtimes."""
+    docker_images = []
+    build_jobs = []
+
+    env = {}
+    # If we not using current tree or the sibling for grpc stack, do checkout.
+    stack_base = ''
+    if args.git_checkout:
+        stack_base = checkout_grpc_stack(lang, release)
+        var = {
+            'go': 'GRPC_GO_ROOT',
+            'java': 'GRPC_JAVA_ROOT',
+            'node': 'GRPC_NODE_ROOT'
+        }.get(lang, 'GRPC_ROOT')
+        env[var] = stack_base
+
+    for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
+        job = build_image_jobspec(runtime, env, release, stack_base)
+        docker_images.append(job.tag)
+        build_jobs.append(job)
+
+    jobset.message('START', 'Building interop docker images.', do_newline=True)
+    print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
+
+    num_failures, _ = jobset.run(
+        build_jobs,
+        newline_on_success=True,
+        maxjobs=multiprocessing.cpu_count())
+    if num_failures:
+        jobset.message(
+            'FAILED', 'Failed to build interop docker images.', do_newline=True)
+        docker_images_cleanup.extend(docker_images)
+        sys.exit(1)
+
+    jobset.message(
+        'SUCCESS',
+        'All docker images built for %s at %s.' % (lang, release),
+        do_newline=True)
+
+    if release != 'master':
+        commit_log = os.path.join(stack_base, 'commit_log')
+        if os.path.exists(commit_log):
+            for image in docker_images:
+                add_files_to_image(image, [commit_log], 'release=%s' % release)
+    return docker_images
+
 
 def cleanup():
-  if not args.keep:
-    for image in docker_images_cleanup:
-      dockerjob.remove_image(image, skip_nonexistent=True)
+    if not args.keep:
+        for image in docker_images_cleanup:
+            dockerjob.remove_image(image, skip_nonexistent=True)
+
 
 docker_images_cleanup = []
 atexit.register(cleanup)
 
+
 def maybe_apply_patches_on_git_tag(stack_base, lang, release):
-  files_to_patch = []
-  for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:
-    if client_matrix.get_release_tag_name(release_info) == release:
-      files_to_patch = release_info[release].get('patch')
-      break
-  if not files_to_patch:
-    return
-  patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)
-  patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                            patch_file_relative_path))
-  if not os.path.exists(patch_file):
-    jobset.message('FAILED', 'expected patch file |%s| to exist' % patch_file)
-    sys.exit(1)
-  subprocess.check_output(
-      ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)
-  for repo_relative_path in files_to_patch:
+    files_to_patch = []
+    for release_info in client_matrix.LANG_RELEASE_MATRIX[lang]:
+        if client_matrix.get_release_tag_name(release_info) == release:
+            files_to_patch = release_info[release].get('patch')
+            break
+    if not files_to_patch:
+        return
+    patch_file_relative_path = 'patches/%s_%s/git_repo.patch' % (lang, release)
+    patch_file = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), patch_file_relative_path))
+    if not os.path.exists(patch_file):
+        jobset.message('FAILED', 'expected patch file |%s| to exist' %
+                       patch_file)
+        sys.exit(1)
+    subprocess.check_output(
+        ['git', 'apply', patch_file], cwd=stack_base, stderr=subprocess.STDOUT)
+    for repo_relative_path in files_to_patch:
+        subprocess.check_output(
+            ['git', 'add', repo_relative_path],
+            cwd=stack_base,
+            stderr=subprocess.STDOUT)
     subprocess.check_output(
-        ['git', 'add', repo_relative_path],
+        [
+            'git', 'commit', '-m',
+            ('Hack performed on top of %s git '
+             'tag in order to build and run the %s '
+             'interop tests on that tag.' % (lang, release))
+        ],
         cwd=stack_base,
         stderr=subprocess.STDOUT)
-  subprocess.check_output(
-      ['git', 'commit', '-m', ('Hack performed on top of %s git '
-                               'tag in order to build and run the %s '
-                               'interop tests on that tag.' % (lang, release))],
-      cwd=stack_base, stderr=subprocess.STDOUT)
+
 
 def checkout_grpc_stack(lang, release):
-  """Invokes 'git check' for the lang/release and returns directory created."""
-  assert args.git_checkout and args.git_checkout_root
-
-  if not os.path.exists(args.git_checkout_root):
-    os.makedirs(args.git_checkout_root)
-
-  repo = client_matrix.get_github_repo(lang)
-  # Get the subdir name part of repo
-  # For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'.
-  repo_dir = os.path.splitext(os.path.basename(repo))[0]
-  stack_base = os.path.join(args.git_checkout_root, repo_dir)
-
-  # Clean up leftover repo dir if necessary.
-  if not args.reuse_git_root and os.path.exists(stack_base):
-    jobset.message('START', 'Removing git checkout root.', do_newline=True)
-    shutil.rmtree(stack_base)
-
-  if not os.path.exists(stack_base):
-    subprocess.check_call(['git', 'clone', '--recursive', repo],
-                          cwd=os.path.dirname(stack_base))
-
-  # git checkout.
-  jobset.message('START', 'git checkout %s from %s' % (release, stack_base),
-                 do_newline=True)
-  # We should NEVER do checkout on current tree !!!
-  assert not os.path.dirname(__file__).startswith(stack_base)
-  output = subprocess.check_output(
-      ['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT)
-  maybe_apply_patches_on_git_tag(stack_base, lang, release)
-  commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base)
-  jobset.message('SUCCESS', 'git checkout', 
-                 '%s: %s' % (str(output), commit_log), 
-                 do_newline=True)
-
-  # Write git log to commit_log so it can be packaged with the docker image.
-  with open(os.path.join(stack_base, 'commit_log'), 'w') as f:
-    f.write(commit_log)
-  return stack_base
+    """Invokes 'git check' for the lang/release and returns directory created."""
+    assert args.git_checkout and args.git_checkout_root
+
+    if not os.path.exists(args.git_checkout_root):
+        os.makedirs(args.git_checkout_root)
+
+    repo = client_matrix.get_github_repo(lang)
+    # Get the subdir name part of repo
+    # For example, 'git@github.com:grpc/grpc-go.git' should use 'grpc-go'.
+    repo_dir = os.path.splitext(os.path.basename(repo))[0]
+    stack_base = os.path.join(args.git_checkout_root, repo_dir)
+
+    # Clean up leftover repo dir if necessary.
+    if not args.reuse_git_root and os.path.exists(stack_base):
+        jobset.message('START', 'Removing git checkout root.', do_newline=True)
+        shutil.rmtree(stack_base)
+
+    if not os.path.exists(stack_base):
+        subprocess.check_call(
+            ['git', 'clone', '--recursive', repo],
+            cwd=os.path.dirname(stack_base))
+
+    # git checkout.
+    jobset.message(
+        'START',
+        'git checkout %s from %s' % (release, stack_base),
+        do_newline=True)
+    # We should NEVER do checkout on current tree !!!
+    assert not os.path.dirname(__file__).startswith(stack_base)
+    output = subprocess.check_output(
+        ['git', 'checkout', release], cwd=stack_base, stderr=subprocess.STDOUT)
+    maybe_apply_patches_on_git_tag(stack_base, lang, release)
+    commit_log = subprocess.check_output(['git', 'log', '-1'], cwd=stack_base)
+    jobset.message(
+        'SUCCESS',
+        'git checkout',
+        '%s: %s' % (str(output), commit_log),
+        do_newline=True)
+
+    # Write git log to commit_log so it can be packaged with the docker image.
+    with open(os.path.join(stack_base, 'commit_log'), 'w') as f:
+        f.write(commit_log)
+    return stack_base
+
 
 languages = args.language if args.language != ['all'] else _LANGUAGES
 for lang in languages:
-  docker_images = build_all_images_for_lang(lang)
-  for image in docker_images:
-    jobset.message('START', 'Uploading %s' % image, do_newline=True)
-    # docker image name must be in the format <gcr_path>/<image>:<gcr_tag>
-    assert image.startswith(args.gcr_path) and image.find(':') != -1
+    docker_images = build_all_images_for_lang(lang)
+    for image in docker_images:
+        jobset.message('START', 'Uploading %s' % image, do_newline=True)
+        # docker image name must be in the format <gcr_path>/<image>:<gcr_tag>
+        assert image.startswith(args.gcr_path) and image.find(':') != -1
 
-    subprocess.call(['gcloud', 'docker', '--', 'push', image])
+        subprocess.call(['gcloud', 'docker', '--', 'push', image])
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
index 1bc35d9e02..ff42bd77b0 100755
--- a/tools/interop_matrix/run_interop_matrix_tests.py
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Run tests using docker images in Google Container Registry per matrix."""
 
 from __future__ import print_function
@@ -30,8 +29,8 @@ import uuid
 # Langauage Runtime Matrix
 import client_matrix
 
-python_util_dir = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../run_tests/python_utils'))
+python_util_dir = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
 sys.path.append(python_util_dir)
 import dockerjob
 import jobset
@@ -40,46 +39,56 @@ import upload_test_results
 
 _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
 # All gRPC release tags, flattened, deduped and sorted.
-_RELEASES = sorted(list(set(
-    client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang)))
+_RELEASES = sorted(
+    list(
+        set(
+            client_matrix.get_release_tag_name(info)
+            for lang in client_matrix.LANG_RELEASE_MATRIX.values()
+            for info in lang)))
 _TEST_TIMEOUT = 30
 
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
-argp.add_argument('--gcr_path',
-                  default='gcr.io/grpc-testing',
-                  help='Path of docker images in Google Container Registry')
-argp.add_argument('--release',
-                  default='all',
-                  choices=['all', 'master'] + _RELEASES,
-                  help='Release tags to test.  When testing all '
-                  'releases defined in client_matrix.py, use "all".')
-
-argp.add_argument('-l', '--language',
-                  choices=['all'] + sorted(_LANGUAGES),
-                  nargs='+',
-                  default=['all'],
-                  help='Languages to test')
-
-argp.add_argument('--keep',
-                  action='store_true',
-                  help='keep the created local images after finishing the tests.')
-
-argp.add_argument('--report_file',
-                  default='report.xml',
-                  help='The result file to create.')
-
-argp.add_argument('--allow_flakes',
-                  default=False,
-                  action='store_const',
-                  const=True,
-                  help=('Allow flaky tests to show as passing (re-runs failed '
-                        'tests up to five times)'))
-argp.add_argument('--bq_result_table',
-                  default='',
-                  type=str,
-                  nargs='?',
-                  help='Upload test results to a specified BQ table.')
+argp.add_argument(
+    '--gcr_path',
+    default='gcr.io/grpc-testing',
+    help='Path of docker images in Google Container Registry')
+argp.add_argument(
+    '--release',
+    default='all',
+    choices=['all', 'master'] + _RELEASES,
+    help='Release tags to test.  When testing all '
+    'releases defined in client_matrix.py, use "all".')
+
+argp.add_argument(
+    '-l',
+    '--language',
+    choices=['all'] + sorted(_LANGUAGES),
+    nargs='+',
+    default=['all'],
+    help='Languages to test')
+
+argp.add_argument(
+    '--keep',
+    action='store_true',
+    help='keep the created local images after finishing the tests.')
+
+argp.add_argument(
+    '--report_file', default='report.xml', help='The result file to create.')
+
+argp.add_argument(
+    '--allow_flakes',
+    default=False,
+    action='store_const',
+    const=True,
+    help=('Allow flaky tests to show as passing (re-runs failed '
+          'tests up to five times)'))
+argp.add_argument(
+    '--bq_result_table',
+    default='',
+    type=str,
+    nargs='?',
+    help='Upload test results to a specified BQ table.')
 
 args = argp.parse_args()
 
@@ -87,138 +96,153 @@ print(str(args))
 
 
 def find_all_images_for_lang(lang):
-  """Find docker images for a language across releases and runtimes.
+    """Find docker images for a language across releases and runtimes.
 
   Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime.
   """
-  # Find all defined releases.
-  if args.release == 'all':
-    releases = ['master'] + client_matrix.get_release_tags(lang)
-  else:
-    # Look for a particular release.
-    if args.release not in ['master'] + client_matrix.get_release_tags(lang):
-      jobset.message('SKIPPED',
-                     '%s for %s is not defined' % (args.release, lang),
-                     do_newline=True)
-      return {}
-    releases = [args.release]
-
-  # Images tuples keyed by runtime.
-  images = {}
-  for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
-    image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
-    output = subprocess.check_output(['gcloud', 'beta', 'container', 'images',
-                                      'list-tags', '--format=json', image_path])
-    docker_image_list = json.loads(output)
-    # All images should have a single tag or no tag.
-    # TODO(adelez): Remove tagless images.
-    tags = [i['tags'][0] for i in docker_image_list if i['tags']]
-    jobset.message('START', 'Found images for %s: %s' % (image_path, tags),
-                   do_newline=True)
-    skipped = len(docker_image_list) - len(tags)
-    jobset.message('SKIPPED', 'Skipped images (no-tag/unknown-tag): %d' % skipped,
-                   do_newline=True)
-    # Filter tags based on the releases.
-    images[runtime] = [(tag,'%s:%s' % (image_path,tag)) for tag in tags if
-                       tag in releases]
-  return images
+    # Find all defined releases.
+    if args.release == 'all':
+        releases = ['master'] + client_matrix.get_release_tags(lang)
+    else:
+        # Look for a particular release.
+        if args.release not in ['master'] + client_matrix.get_release_tags(
+                lang):
+            jobset.message(
+                'SKIPPED',
+                '%s for %s is not defined' % (args.release, lang),
+                do_newline=True)
+            return {}
+        releases = [args.release]
+
+    # Images tuples keyed by runtime.
+    images = {}
+    for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
+        image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
+        output = subprocess.check_output([
+            'gcloud', 'beta', 'container', 'images', 'list-tags',
+            '--format=json', image_path
+        ])
+        docker_image_list = json.loads(output)
+        # All images should have a single tag or no tag.
+        # TODO(adelez): Remove tagless images.
+        tags = [i['tags'][0] for i in docker_image_list if i['tags']]
+        jobset.message(
+            'START',
+            'Found images for %s: %s' % (image_path, tags),
+            do_newline=True)
+        skipped = len(docker_image_list) - len(tags)
+        jobset.message(
+            'SKIPPED',
+            'Skipped images (no-tag/unknown-tag): %d' % skipped,
+            do_newline=True)
+        # Filter tags based on the releases.
+        images[runtime] = [(tag, '%s:%s' % (image_path, tag)) for tag in tags
+                           if tag in releases]
+    return images
+
 
 # caches test cases (list of JobSpec) loaded from file.  Keyed by lang and runtime.
 def find_test_cases(lang, runtime, release, suite_name):
-  """Returns the list of test cases from testcase files per lang/release."""
-  file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
-  testcase_release = release
-  filename_prefix = lang
-  if lang == 'csharp':
-    filename_prefix = runtime
-  if not os.path.exists(file_tmpl % (filename_prefix, release)):
-    testcase_release = 'master'
-  testcases = file_tmpl % (filename_prefix, testcase_release)
-
-  job_spec_list=[]
-  try:
-    with open(testcases) as f:
-      # Only line start with 'docker run' are test cases.
-      for line in f.readlines():
-        if line.startswith('docker run'):
-          m = re.search('--test_case=(.*)"', line)
-          shortname = m.group(1) if m else 'unknown_test'
-          m = re.search('--server_host_override=(.*).sandbox.googleapis.com', 
+    """Returns the list of test cases from testcase files per lang/release."""
+    file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
+    testcase_release = release
+    filename_prefix = lang
+    if lang == 'csharp':
+        filename_prefix = runtime
+    if not os.path.exists(file_tmpl % (filename_prefix, release)):
+        testcase_release = 'master'
+    testcases = file_tmpl % (filename_prefix, testcase_release)
+
+    job_spec_list = []
+    try:
+        with open(testcases) as f:
+            # Only line start with 'docker run' are test cases.
+            for line in f.readlines():
+                if line.startswith('docker run'):
+                    m = re.search('--test_case=(.*)"', line)
+                    shortname = m.group(1) if m else 'unknown_test'
+                    m = re.search(
+                        '--server_host_override=(.*).sandbox.googleapis.com',
                         line)
-          server = m.group(1) if m else 'unknown_server'
-          spec = jobset.JobSpec(cmdline=line,
-                                shortname='%s:%s:%s:%s' % (suite_name, lang, 
-                                                           server, shortname),
-                                timeout_seconds=_TEST_TIMEOUT,
-                                shell=True,
-                                flake_retries=5 if args.allow_flakes else 0)
-          job_spec_list.append(spec)
-      jobset.message('START',
-                     'Loaded %s tests from %s' % (len(job_spec_list), testcases),
-                     do_newline=True)
-  except IOError as err:
-    jobset.message('FAILED', err, do_newline=True)
-  return job_spec_list
+                    server = m.group(1) if m else 'unknown_server'
+                    spec = jobset.JobSpec(
+                        cmdline=line,
+                        shortname='%s:%s:%s:%s' % (suite_name, lang, server,
+                                                   shortname),
+                        timeout_seconds=_TEST_TIMEOUT,
+                        shell=True,
+                        flake_retries=5 if args.allow_flakes else 0)
+                    job_spec_list.append(spec)
+            jobset.message(
+                'START',
+                'Loaded %s tests from %s' % (len(job_spec_list), testcases),
+                do_newline=True)
+    except IOError as err:
+        jobset.message('FAILED', err, do_newline=True)
+    return job_spec_list
+
 
 _xml_report_tree = report_utils.new_junit_xml_tree()
+
+
 def run_tests_for_lang(lang, runtime, images):
-  """Find and run all test cases for a language.
+    """Find and run all test cases for a language.
 
   images is a list of (<release-tag>, <image-full-path>) tuple.
   """
-  total_num_failures = 0
-  for image_tuple in images:
-    release, image = image_tuple
-    jobset.message('START', 'Testing %s' % image, do_newline=True)
-    # Download the docker image before running each test case.
-    subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
-    suite_name = '%s__%s_%s' % (lang, runtime, release)
-    job_spec_list = find_test_cases(lang, runtime, release, suite_name)
-    
-    if not job_spec_list:  
-      jobset.message('FAILED', 'No test cases were found.', do_newline=True)
-      return 1
-
-    num_failures, resultset = jobset.run(job_spec_list,
-                                         newline_on_success=True,
-                                         add_env={'docker_image':image},
-                                         maxjobs=args.jobs)
-    if args.bq_result_table and resultset:
-      upload_test_results.upload_interop_results_to_bq(
-          resultset, args.bq_result_table, args)
-    if num_failures:
-      jobset.message('FAILED', 'Some tests failed', do_newline=True)
-      total_num_failures += num_failures
-    else:
-      jobset.message('SUCCESS', 'All tests passed', do_newline=True)
-
-    report_utils.append_junit_xml_results(
-        _xml_report_tree,
-        resultset,
-        'grpc_interop_matrix',
-        suite_name,
-        str(uuid.uuid4()))
-
-    if not args.keep:
-      cleanup(image)
-  
-  return total_num_failures
+    total_num_failures = 0
+    for image_tuple in images:
+        release, image = image_tuple
+        jobset.message('START', 'Testing %s' % image, do_newline=True)
+        # Download the docker image before running each test case.
+        subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
+        suite_name = '%s__%s_%s' % (lang, runtime, release)
+        job_spec_list = find_test_cases(lang, runtime, release, suite_name)
+
+        if not job_spec_list:
+            jobset.message(
+                'FAILED', 'No test cases were found.', do_newline=True)
+            return 1
+
+        num_failures, resultset = jobset.run(
+            job_spec_list,
+            newline_on_success=True,
+            add_env={'docker_image': image},
+            maxjobs=args.jobs)
+        if args.bq_result_table and resultset:
+            upload_test_results.upload_interop_results_to_bq(
+                resultset, args.bq_result_table, args)
+        if num_failures:
+            jobset.message('FAILED', 'Some tests failed', do_newline=True)
+            total_num_failures += num_failures
+        else:
+            jobset.message('SUCCESS', 'All tests passed', do_newline=True)
+
+        report_utils.append_junit_xml_results(_xml_report_tree, resultset,
+                                              'grpc_interop_matrix', suite_name,
+                                              str(uuid.uuid4()))
+
+        if not args.keep:
+            cleanup(image)
+
+    return total_num_failures
 
 
 def cleanup(image):
-  jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
-  dockerjob.remove_image(image, skip_nonexistent=True)
+    jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
+    dockerjob.remove_image(image, skip_nonexistent=True)
 
 
 languages = args.language if args.language != ['all'] else _LANGUAGES
 total_num_failures = 0
 for lang in languages:
-  docker_images = find_all_images_for_lang(lang)
-  for runtime in sorted(docker_images.keys()):
-    total_num_failures += run_tests_for_lang(lang, runtime, docker_images[runtime])
+    docker_images = find_all_images_for_lang(lang)
+    for runtime in sorted(docker_images.keys()):
+        total_num_failures += run_tests_for_lang(lang, runtime,
+                                                 docker_images[runtime])
 
 report_utils.create_xml_report_file(_xml_report_tree, args.report_file)
 
 if total_num_failures:
-  sys.exit(1)
+    sys.exit(1)
 sys.exit(0)