add health check for http2 server

4 files changed, 85 insertions, 6 deletions

diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py
new file mode 100644
index 0000000000..dd9402b855
--- /dev/null
+++ b/test/http2_test/http2_server_health_check.py
@@ -0,0 +1,49 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import hyper
+import sys
+
+# Utility to healthcheck the http2 server. Used when starting the server to
+# verify that the server is live before tests begin.
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--server_host', type=str, default='localhost')
+  parser.add_argument('--server_port', type=int, default=8080)
+  args = parser.parse_args()
+  server_host = args.server_host
+  server_port = args.server_port
+  conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port))
+  conn.request('POST', '/grpc.testing.TestService/UnaryCall')
+  resp = conn.get_response()
+  if resp.headers.get('grpc-encoding') is None:
+    sys.exit(1)
+  else:
+    sys.exit(0)
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
index 66d9b4f640..acc984acb0 100644
--- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
@@ -47,7 +47,7 @@ RUN pip install pip --upgrade
 RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
 
-RUN pip install twisted h2
+RUN pip install twisted h2 hyper
 
 # Define the default command.
 CMD ["bash"]
diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py
index 0869c5cee9..82466d3a93 100755
--- a/tools/run_tests/python_utils/dockerjob.py
+++ b/tools/run_tests/python_utils/dockerjob.py
@@ -70,6 +70,22 @@ def docker_mapped_port(cid, port, timeout_seconds=15):
                   (port, cid))
 
 
+def wait_for_healthy(cid, shortname, timeout_seconds):
+  """Wait timeout_seconds for the container to become healthy"""
+  started = time.time()
+  while time.time() - started < timeout_seconds:
+    try:
+      output = subprocess.check_output(
+          ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid],
+          stderr=_DEVNULL)
+      if output.strip('\n') == 'healthy':
+        return
+    except subprocess.CalledProcessError as e:
+      pass
+  raise Exception('Timed out waiting for %s (%s) to pass health check' %
+                  (shortname, cid))
+
+
 def finish_jobs(jobs):
   """Kills given docker containers and waits for corresponding jobs to finish"""
   for job in jobs:
@@ -113,6 +129,9 @@ class DockerJob:
   def mapped_port(self, port):
     return docker_mapped_port(self._container_name, port)
 
+  def wait_for_healthy(self, timeout_seconds=15):
+    wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds)
+
   def kill(self, suppress_failure=False):
     """Sends kill signal to the container."""
     if suppress_failure:
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index ac57e5a9c5..fb10a442d0 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -678,23 +678,32 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None):
       language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT,
                            '--use_tls=%s' % ('false' if insecure else 'true')]))
   environ = language.global_env()
+  docker_args = ['--name=%s' % container_name]
   if language.safename == 'http2':
     # we are running the http2 interop server. Open next N ports beginning
     # with the server port. These ports are used for http2 interop test
     # (one test case per port).
-    port_args = list(
+    docker_args += list(
         itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i))
                                       for i in range(
                                           len(_HTTP2_BADSERVER_TEST_CASES))))
+    docker_args += [
+        '--health-cmd=python test/http2_test/http2_server_health_check.py '
+        '--server_host=%s --server_port=%d'
+        % ('localhost', _DEFAULT_SERVER_PORT),
+        '--health-interval=1s',
+        '--health-retries=5',
+        '--health-timeout=1s',
+    ]
+
   else:
-    port_args = ['-p', str(_DEFAULT_SERVER_PORT)]
+    docker_args += ['-p', str(_DEFAULT_SERVER_PORT)]
 
   docker_cmdline = docker_run_cmdline(cmdline,
                                       image=docker_image,
                                       cwd=language.server_cwd,
                                       environ=environ,
-                                      docker_args=port_args +
-                                        ['--name=%s' % container_name])
+                                      docker_args=docker_args)
   if manual_cmd_log is not None:
       manual_cmd_log.append(manual_cmdline(docker_cmdline))
   server_job = jobset.JobSpec(
@@ -881,7 +890,8 @@ languages = set(_LANGUAGES[l]
 languages_http2_badserver_interop = set()
 if args.http2_badserver_interop:
   languages_http2_badserver_interop = set(
-      _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS)
+      _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS
+      if 'all' in args.language or l in args.language)
 
 http2Interop = Http2Client() if args.http2_interop else None
 http2InteropServer = Http2Server() if args.http2_badserver_interop else None
@@ -946,6 +956,7 @@ try:
                           manual_cmd_log=server_manual_cmd_log)
     if not args.manual_run:
       job = dockerjob.DockerJob(spec)
+      job.wait_for_healthy(timeout_seconds=15)
       server_jobs[lang] = job
       http2_badserver_ports = tuple([
           job.mapped_port(_DEFAULT_SERVER_PORT + i)