Merge github.com:grpc/grpc into census_suite

7 files changed, 158 insertions, 274 deletions

diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop_html_report.template
index 1ba2e6cfc2..c01bdf7a77 100644
--- a/tools/run_tests/interop_html_report.template
+++ b/tools/run_tests/interop_html_report.template
@@ -40,6 +40,26 @@
   % endif
 </%def>
 
+<%def name="fill_one_http2_test_result(shortname, resultset)">
+    ## keep this mostly in sync with the template above
+  % if shortname in resultset:
+    ## Because interop tests does not have runs_per_test flag, each test is 
+    ## run once. So there should only be one element for each result.
+    <% result = resultset[shortname][0] %>
+    <td bgcolor="white">
+      <div style="width:95%; border: 1px solid black; position: relative; padding: 3px;">
+        <span style="position: absolute; left: 45%;">${int(result.http2results['percent'] * 100)}&#37;</span>
+        <div style="height: 20px; 
+          background-color: hsl(${result.http2results['percent'] * 120}, 100%, 50%); 
+          width: ${result.http2results['percent'] * 100}%;"
+          title="${result.http2results['failed_cases'] | h}"></div>
+      </div>
+    </td>
+  % else:
+     <td bgcolor="magenta">Not implemented</td>
+  % endif
+</%def>
+
 % if num_failures > 1:
   <p><h2><font color="red">${num_failures} tests failed!</font></h2></p>
 % elif num_failures:
@@ -95,11 +115,11 @@
         shortname = 'cloud_to_cloud:http2:%s_server:%s' % (
             server_lang, test_case)
       %>
-      ${fill_one_test_result(shortname, resultset)}
+      ${fill_one_http2_test_result(shortname, resultset)}
     % endfor
     % if cloud_to_prod:
       <% shortname = 'cloud_to_prod:http2:%s' % test_case %>
-      ${fill_one_test_result(shortname, resultset)}
+      ${fill_one_http2_test_result(shortname, resultset)}
     % endif
     </tr>
   % endfor
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 01739be27c..fdbddf82ee 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -317,9 +317,13 @@ class Jobset(object):
     self._hashes = {}
     self._add_env = add_env
     self.resultset = {}
-    
+    self._remaining = None
+
+  def set_remaining(self, remaining):
+    self._remaining = remaining
+
   def get_num_failures(self):
-    return self._failures  
+    return self._failures
 
   def start(self, spec):
     """Start a job. Return True on success, False on failure."""
@@ -372,8 +376,9 @@ class Jobset(object):
         self._running.remove(job)
       if dead: return
       if (not self._travis):
-        message('WAITING', '%d jobs running, %d complete, %d failed' % (
-            len(self._running), self._completed, self._failures))
+        rstr = '' if self._remaining is None else '%d queued, ' % self._remaining
+        message('WAITING', '%s%d jobs running, %d complete, %d failed' % (
+            rstr, len(self._running), self._completed, self._failures))
       if platform_string() == 'windows':
         time.sleep(0.1)
       else:
@@ -412,6 +417,17 @@ class NoCache(object):
     pass
 
 
+def tag_remaining(xs):
+  staging = []
+  for x in xs:
+    staging.append(x)
+    if len(staging) > 1000:
+      yield (staging.pop(0), None)
+  n = len(staging)
+  for i, x in enumerate(staging):
+    yield (x, n - i - 1)
+
+
 def run(cmdlines,
         check_cancelled=_never_cancelled,
         maxjobs=None,
@@ -425,8 +441,11 @@ def run(cmdlines,
               maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
               newline_on_success, travis, stop_on_failure, add_env,
               cache if cache is not None else NoCache())
-  for cmdline in cmdlines:
+  for cmdline, remaining in tag_remaining(cmdlines):
     if not js.start(cmdline):
       break
-  js.finish()  
+    if remaining is not None:
+      js.set_remaining(remaining)
+  js.finish()
   return js.get_num_failures(), js.resultset
+
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 12b1972f1a..35f2069bee 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -32,6 +32,7 @@
 try:
   from mako.runtime import Context
   from mako.template import Template
+  from mako import exceptions
 except (ImportError):
   pass  # Mako not installed but it is ok. 
 import os
@@ -103,9 +104,15 @@ def render_interop_html_report(
           'num_failures': num_failures,
           'cloud_to_prod': cloud_to_prod,
           'http2_interop': http2_interop}
+
   html_report_out_dir = 'reports' 
   if not os.path.exists(html_report_out_dir):
     os.mkdir(html_report_out_dir) 
   html_file_path = os.path.join(html_report_out_dir, 'index.html')
-  with open(html_file_path, 'w') as output_file:
-    mytemplate.render_context(Context(output_file, **args))
+  try:
+    with open(html_file_path, 'w') as output_file:
+      mytemplate.render_context(Context(output_file, **args))
+  except:
+    print(exceptions.text_error_template().render())
+    raise
+
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 763ff5615c..7a09feb70d 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -31,17 +31,24 @@
 """Run interop (cross-language) tests in parallel."""
 
 import argparse
+import atexit
 import dockerjob
 import itertools
 import jobset
+import json
 import multiprocessing
 import os
+import re
 import report_utils
+import subprocess
 import sys
 import tempfile
 import time
 import uuid
 
+# Docker doesn't clean up after itself, so we do it on exit.
+atexit.register(lambda: subprocess.call(['stty', 'echo']))
+
 ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(ROOT)
 
@@ -549,6 +556,33 @@ def build_interop_image_jobspec(language, tag=None):
   return build_job
 
 
+def aggregate_http2_results(stdout):
+  match = re.search(r'\{"cases[^\]]*\]\}', stdout)
+  if not match:
+    return None
+    
+  results = json.loads(match.group(0))
+  skipped = 0
+  passed = 0
+  failed = 0
+  failed_cases = []
+  for case in results['cases']:
+    if case.get('skipped', False):
+      skipped += 1
+    else:
+      if case.get('passed', False):
+        passed += 1
+      else:
+        failed += 1
+        failed_cases.append(case.get('name', "NONAME"))
+  return {
+    'passed': passed,
+    'failed': failed,
+    'skipped': skipped,
+    'failed_cases': ', '.join(failed_cases),
+    'percent': 1.0 * passed / (passed + failed)
+  }
+
 argp = argparse.ArgumentParser(description='Run interop tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(_LANGUAGES),
@@ -676,9 +710,7 @@ try:
                                              docker_image=docker_images.get(str(language)))
             jobs.append(test_job)
 
-    # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable
-    # this once a better solution is in place.
-    if args.http2_interop and False:
+    if args.http2_interop:
       for test_case in _HTTP2_TEST_CASES:
         test_job = cloud_to_prod_jobspec(http2Interop, test_case,
                                          docker_image=docker_images.get(str(http2Interop)))
@@ -745,6 +777,10 @@ try:
 
   report_utils.render_junit_xml_report(resultset, 'report.xml')
 
+  for name, job in resultset.iteritems():
+    if "http2" in name:
+      job[0].http2results = aggregate_http2_results(job[0].message)
+
   report_utils.render_interop_html_report(
       set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, 
       _HTTP2_TEST_CASES, resultset, num_failures,
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index e3df912480..006f4bcdf1 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -152,7 +152,10 @@ class CLanguage(object):
       else:
         binary = 'bins/%s/%s' % (config.build_config, target['name'])
       if os.path.isfile(binary):
-        out.append(config.job_spec([binary], [binary]))
+        out.append(config.job_spec([binary], [binary],
+                                   environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
+                                            os.path.abspath(os.path.dirname(
+                                                sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
       elif args.regex == '.*' or platform_string() == 'windows':
         print '\nWARNING: binary not found, skipping', binary
     return sorted(out)
@@ -192,6 +195,7 @@ class CLanguage(object):
   def __str__(self):
     return self.make_target
 
+
 class NodeLanguage(object):
 
   def test_specs(self, config, args):
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 18113ac7b9..e6fce456a8 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -751,6 +751,20 @@
     ], 
     "headers": [], 
     "language": "c", 
+    "name": "httpscli_test", 
+    "src": [
+      "test/core/httpcli/httpscli_test.c"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
     "name": "init_test", 
     "src": [
       "test/core/surface/init_test.c"
@@ -939,6 +953,20 @@
       "gpr", 
       "gpr_test_util", 
       "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
+    "name": "server_chttp2_test", 
+    "src": [
+      "test/core/surface/server_chttp2_test.c"
+    ]
+  }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
       "grpc_test_util", 
       "test_tcp_server"
     ], 
@@ -8521,21 +8549,6 @@
     "deps": [
       "end2end_certs", 
       "end2end_fixture_h2_uchannel", 
-      "end2end_test_channel_connectivity", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc_test_util"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_channel_connectivity_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_certs", 
-      "end2end_fixture_h2_uchannel", 
       "end2end_test_compressed_payload", 
       "gpr", 
       "gpr_test_util", 
@@ -8551,36 +8564,6 @@
     "deps": [
       "end2end_certs", 
       "end2end_fixture_h2_uchannel", 
-      "end2end_test_default_host", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc_test_util"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_default_host_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_certs", 
-      "end2end_fixture_h2_uchannel", 
-      "end2end_test_disappearing_server", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc_test_util"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_disappearing_server_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_certs", 
-      "end2end_fixture_h2_uchannel", 
       "end2end_test_empty_batch", 
       "gpr", 
       "gpr_test_util", 
@@ -8866,21 +8849,6 @@
     "deps": [
       "end2end_certs", 
       "end2end_fixture_h2_uchannel", 
-      "end2end_test_simple_delayed_request", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc_test_util"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_simple_delayed_request_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_certs", 
-      "end2end_fixture_h2_uchannel", 
       "end2end_test_simple_request", 
       "gpr", 
       "gpr_test_util", 
@@ -13598,20 +13566,6 @@
   {
     "deps": [
       "end2end_nosec_fixture_h2_uchannel", 
-      "end2end_nosec_test_channel_connectivity", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc_test_util_unsecure", 
-      "grpc_unsecure"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_channel_connectivity_nosec_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_nosec_fixture_h2_uchannel", 
       "end2end_nosec_test_compressed_payload", 
       "gpr", 
       "gpr_test_util", 
@@ -13626,34 +13580,6 @@
   {
     "deps": [
       "end2end_nosec_fixture_h2_uchannel", 
-      "end2end_nosec_test_default_host", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc_test_util_unsecure", 
-      "grpc_unsecure"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_default_host_nosec_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_nosec_fixture_h2_uchannel", 
-      "end2end_nosec_test_disappearing_server", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc_test_util_unsecure", 
-      "grpc_unsecure"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_disappearing_server_nosec_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_nosec_fixture_h2_uchannel", 
       "end2end_nosec_test_empty_batch", 
       "gpr", 
       "gpr_test_util", 
@@ -13920,20 +13846,6 @@
   {
     "deps": [
       "end2end_nosec_fixture_h2_uchannel", 
-      "end2end_nosec_test_simple_delayed_request", 
-      "gpr", 
-      "gpr_test_util", 
-      "grpc_test_util_unsecure", 
-      "grpc_unsecure"
-    ], 
-    "headers": [], 
-    "language": "c", 
-    "name": "h2_uchannel_simple_delayed_request_nosec_test", 
-    "src": []
-  }, 
-  {
-    "deps": [
-      "end2end_nosec_fixture_h2_uchannel", 
       "end2end_nosec_test_simple_request", 
       "gpr", 
       "gpr_test_util", 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index c3eace76bc..2d043df6ff 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -853,6 +853,18 @@
   }, 
   {
     "ci_platforms": [
+      "linux"
+    ], 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
+    "name": "httpscli_test", 
+    "platforms": [
+      "linux"
+    ]
+  }, 
+  {
+    "ci_platforms": [
       "linux", 
       "mac", 
       "posix", 
@@ -1077,6 +1089,24 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
+    "name": "server_chttp2_test", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ]
+  }, 
+  {
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c", 
     "name": "set_initial_connect_string_test", 
     "platforms": [
       "linux", 
@@ -9129,24 +9159,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_channel_connectivity_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_compressed_payload_test", 
     "platforms": [
       "linux", 
@@ -9165,42 +9177,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_default_host_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_uchannel_disappearing_server_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_empty_batch_test", 
     "platforms": [
       "linux", 
@@ -9543,24 +9519,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_simple_delayed_request_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_simple_request_test", 
     "platforms": [
       "linux", 
@@ -14972,24 +14930,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_channel_connectivity_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_compressed_payload_nosec_test", 
     "platforms": [
       "linux", 
@@ -15008,42 +14948,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_default_host_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
-    "name": "h2_uchannel_disappearing_server_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_empty_batch_nosec_test", 
     "platforms": [
       "linux", 
@@ -15386,24 +15290,6 @@
     "exclude_configs": [], 
     "flaky": false, 
     "language": "c", 
-    "name": "h2_uchannel_simple_delayed_request_nosec_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ]
-  }, 
-  {
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix", 
-      "windows"
-    ], 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "language": "c", 
     "name": "h2_uchannel_simple_request_nosec_test", 
     "platforms": [
       "linux",