Merge pull request #5047 from adelez/cloud_to_canary

Added four different prod servers to run cloud_to_prod(_auth) tests against

3 files changed, 89 insertions, 45 deletions

diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop_html_report.template
index 114a508c31..46cce426b7 100644
--- a/tools/run_tests/interop_html_report.template
+++ b/tools/run_tests/interop_html_report.template
@@ -64,17 +64,8 @@
   % endif
 </%def>
 
-% if num_failures > 1:
-  <p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
-% elif num_failures:
-  <p><h2><font color="red">${num_failures} test failed!</font></h2></p>
-% else:
-  <p><h2><font color="green">All tests passed!</font></h2></p>
-% endif
-
-% if cloud_to_prod:
+<%def name="display_cloud_to_prod_result(prod_server)">
   ## Each column header is the client language.
-  <h2>Cloud to Prod</h2>
   <table style="width:100%" border="1">
   <tr bgcolor="#00BFFF">
   <th>Client languages &#9658;<br/>Test Cases &#9660;</th>
@@ -87,15 +78,32 @@
     % for client_lang in client_langs:
       <% 
         if test_case in auth_test_cases:
-          shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case)
+          shortname = 'cloud_to_prod_auth:%s:%s:%s' % (
+              prod_server, client_lang, test_case)
         else:
-          shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case)
+          shortname = 'cloud_to_prod:%s:%s:%s' % (
+              prod_server, client_lang, test_case)
       %>
       ${fill_one_test_result(shortname, resultset)}
     % endfor
     </tr> 
   % endfor
   </table>
+</%def>
+
+% if num_failures > 1:
+  <p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
+% elif num_failures:
+  <p><h2><font color="red">${num_failures} test failed!</font></h2></p>
+% else:
+  <p><h2><font color="green">All tests passed!</font></h2></p>
+% endif
+
+% if cloud_to_prod:
+  % for prod_server in prod_servers:
+    <h2>Cloud to ${prod_server}</h2>
+    ${display_cloud_to_prod_result(prod_server)}
+  % endfor
 % endif
 
 % if http2_interop:
@@ -108,7 +116,9 @@
     <th>${server_lang}</th>
   % endfor
   % if cloud_to_prod:
-    <th>prod</th>
+    % for prod_server in prod_servers:
+      <th>${prod_server}</th>
+    % endfor
   % endif
   </tr>
   % for test_case in http2_cases:
@@ -122,8 +132,10 @@
       ${fill_one_http2_test_result(shortname, resultset)}
     % endfor
     % if cloud_to_prod:
-      <% shortname = 'cloud_to_prod:http2:%s' % test_case %>
-      ${fill_one_http2_test_result(shortname, resultset)}
+      % for prod_server in prod_servers:
+        <% shortname = 'cloud_to_prod:%s:http2:%s' % (prod_server, test_case) %>
+        ${fill_one_http2_test_result(shortname, resultset)}
+      % endfor
     % endif
     </tr>
   % endfor
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 35f2069bee..437c4693a5 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -77,7 +77,7 @@ def render_junit_xml_report(resultset, xml_report):
 
 def render_interop_html_report(
   client_langs, server_langs, test_cases, auth_test_cases, http2_cases, 
-  resultset, num_failures, cloud_to_prod, http2_interop):
+  resultset, num_failures, cloud_to_prod, prod_servers, http2_interop):
   """Generate HTML report for interop tests."""
   template_file = 'tools/run_tests/interop_html_report.template'
   try:
@@ -94,6 +94,7 @@ def render_interop_html_report(
   sorted_http2_cases = sorted(http2_cases)
   sorted_client_langs = sorted(client_langs)
   sorted_server_langs = sorted(server_langs)
+  sorted_prod_servers = sorted(prod_servers)
 
   args = {'client_langs': sorted_client_langs, 
           'server_langs': sorted_server_langs,
@@ -103,6 +104,7 @@ def render_interop_html_report(
           'resultset': resultset,
           'num_failures': num_failures,
           'cloud_to_prod': cloud_to_prod,
+          'prod_servers': sorted_prod_servers,
           'http2_interop': http2_interop}
 
   html_report_out_dir = 'reports' 
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index fbc3b3cdf6..4a670e0c9d 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -422,12 +422,13 @@ def _job_kill_handler(job):
     time.sleep(2)
 
 
-def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
+def cloud_to_prod_jobspec(language, test_case, server_host_name, 
+                          server_host_detail, docker_image=None, auth=False):
   """Creates jobspec for cloud-to-prod interop test"""
   container_name = None
   cmdargs = [
-      '--server_host_override=grpc-test.sandbox.googleapis.com',
-      '--server_host=grpc-test.sandbox.googleapis.com',
+      '--server_host=%s' % server_host_detail[0],
+      '--server_host_override=%s' % server_host_detail[1],
       '--server_port=443',
       '--use_tls=true',
       '--test_case=%s' % test_case]
@@ -440,7 +441,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
   cwd = language.client_cwd
 
   if docker_image:
-    container_name = dockerjob.random_name('interop_client_%s' % language.safename)
+    container_name = dockerjob.random_name('interop_client_%s' % 
+                                           language.safename)
     cmdline = docker_run_cmdline(cmdline,
                                  image=docker_image,
                                  cwd=cwd,
@@ -455,7 +457,8 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
           cmdline=cmdline,
           cwd=cwd,
           environ=environ,
-          shortname='%s:%s:%s' % (suite_name, language, test_case),
+          shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language, 
+                                     test_case),
           timeout_seconds=90,
           flake_retries=5 if args.allow_flakes else 0,
           timeout_retries=2 if args.allow_flakes else 0,
@@ -491,7 +494,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
           cwd=cwd,
           environ=environ,
           shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name,
-                                                 test_case),
+                                                        test_case),
           timeout_seconds=90,
           flake_retries=5 if args.allow_flakes else 0,
           timeout_retries=2 if args.allow_flakes else 0,
@@ -572,6 +575,21 @@ def aggregate_http2_results(stdout):
     'percent': 1.0 * passed / (passed + failed)
   }
 
+# A dictionary of prod servers to test. 
+# Format: server_name: (server_host, server_host_override, errors_allowed)
+# TODO(adelez): implement logic for errors_allowed where if the indicated tests
+# fail, they don't impact the overall test result.
+prod_servers = {
+    'default': ('grpc-test.sandbox.googleapis.com', 
+                'grpc-test.sandbox.googleapis.com', False),
+    'gateway_v2': ('grpc-test2.sandbox.googleapis.com', 
+                   'grpc-test2.sandbox.googleapis.com', True),
+    'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com', 
+                      False),
+    'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com', 
+                         True)
+}
+
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(_LANGUAGES),
@@ -589,6 +607,12 @@ argp.add_argument('--cloud_to_prod_auth',
                   action='store_const',
                   const=True,
                   help='Run cloud_to_prod_auth tests.')
+argp.add_argument('--prod_servers',
+                  choices=prod_servers.keys(),
+                  default=['default'],
+                  nargs='+',
+                  help=('The servers to run cloud_to_prod and '
+                        'cloud_to_prod_auth tests against.'))
 argp.add_argument('-s', '--server',
                   choices=['all'] + sorted(_SERVERS),
                   action='append',
@@ -688,32 +712,37 @@ try:
     server_jobs[lang] = job
     server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
 
-
   jobs = []
   if args.cloud_to_prod:
-    for language in languages:
-      for test_case in _TEST_CASES:
-        if not test_case in language.unimplemented_test_cases():
-          if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
-            test_job = cloud_to_prod_jobspec(language, test_case,
-                                             docker_image=docker_images.get(str(language)))
-            jobs.append(test_job)
-
-    if args.http2_interop:
-      for test_case in _HTTP2_TEST_CASES:
-        test_job = cloud_to_prod_jobspec(http2Interop, test_case,
-                                         docker_image=docker_images.get(str(http2Interop)))
-        jobs.append(test_job)
-
+    for server_host_name in args.prod_servers:
+      for language in languages:
+        for test_case in _TEST_CASES:
+          if not test_case in language.unimplemented_test_cases():
+            if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
+              test_job = cloud_to_prod_jobspec(
+                  language, test_case, server_host_name, 
+                  prod_servers[server_host_name],
+                  docker_image=docker_images.get(str(language)))
+              jobs.append(test_job)
+
+      if args.http2_interop:
+        for test_case in _HTTP2_TEST_CASES:
+          test_job = cloud_to_prod_jobspec(
+              http2Interop, test_case, server_host_name, 
+              prod_servers[server_host_name],
+              docker_image=docker_images.get(str(http2Interop)))
+          jobs.append(test_job)
 
   if args.cloud_to_prod_auth:
-    for language in languages:
-      for test_case in _AUTH_TEST_CASES:
-        if not test_case in language.unimplemented_test_cases():
-          test_job = cloud_to_prod_jobspec(language, test_case,
-                                           docker_image=docker_images.get(str(language)),
-                                           auth=True)
-          jobs.append(test_job)
+    for server_host_name in args.prod_servers:
+      for language in languages:
+        for test_case in _AUTH_TEST_CASES:
+          if not test_case in language.unimplemented_test_cases():
+            test_job = cloud_to_prod_jobspec(
+                language, test_case, server_host_name, 
+                prod_servers[server_host_name],
+                docker_image=docker_images.get(str(language)), auth=True)
+            jobs.append(test_job)
 
   for server in args.override_server:
     server_name = server[0]
@@ -773,7 +802,8 @@ try:
   report_utils.render_interop_html_report(
       set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
       _HTTP2_TEST_CASES, resultset, num_failures,
-      args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop)
+      args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers, 
+      args.http2_interop)
 
 finally:
   # Check if servers are still running.