From 22afddf53c1de182082a022ee108f27db117d2d9 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 15 Mar 2017 15:43:43 -0700 Subject: add health check for http2 server --- test/http2_test/http2_server_health_check.py | 49 ++++++++++++++++++++++ .../interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/run_tests/python_utils/dockerjob.py | 19 +++++++++ tools/run_tests/run_interop_tests.py | 21 +++++++--- 4 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 test/http2_test/http2_server_health_check.py diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py new file mode 100644 index 0000000000..dd9402b855 --- /dev/null +++ b/test/http2_test/http2_server_health_check.py @@ -0,0 +1,49 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import hyper +import sys + +# Utility to healthcheck the http2 server. Used when starting the server to +# verify that the server is live before tests begin. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--server_host', type=str, default='localhost') + parser.add_argument('--server_port', type=int, default=8080) + args = parser.parse_args() + server_host = args.server_host + server_port = args.server_port + conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port)) + conn.request('POST', '/grpc.testing.TestService/UnaryCall') + resp = conn.get_response() + if resp.headers.get('grpc-encoding') is None: + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 66d9b4f640..acc984acb0 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 +RUN pip install twisted h2 hyper # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 0869c5cee9..82466d3a93 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -70,6 +70,22 @@ def docker_mapped_port(cid, port, timeout_seconds=15): (port, cid)) +def wait_for_healthy(cid, shortname, timeout_seconds): + """Wait timeout_seconds for the container to become healthy""" + started = time.time() + while time.time() - started < timeout_seconds: + try: + output = subprocess.check_output( + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], + stderr=_DEVNULL) + if output.strip('\n') == 'healthy': + return + except subprocess.CalledProcessError as e: + pass + raise Exception('Timed out waiting for %s (%s) to pass health check' % + (shortname, cid)) + + def finish_jobs(jobs): """Kills given docker containers and waits for corresponding jobs to finish""" for job in jobs: @@ -113,6 +129,9 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) + def wait_for_healthy(self, timeout_seconds=15): + wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) + def kill(self, suppress_failure=False): """Sends kill signal to the container.""" if suppress_failure: diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ac57e5a9c5..fb10a442d0 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -678,23 +678,32 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ('false' if insecure else 'true')])) environ = language.global_env() + docker_args = ['--name=%s' % container_name] if language.safename == 'http2': # we are running the http2 interop server. Open next N ports beginning # with the server port. These ports are used for http2 interop test # (one test case per port). - port_args = list( + docker_args += list( itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) for i in range( len(_HTTP2_BADSERVER_TEST_CASES)))) + docker_args += [ + '--health-cmd=python test/http2_test/http2_server_health_check.py ' + '--server_host=%s --server_port=%d' + % ('localhost', _DEFAULT_SERVER_PORT), + '--health-interval=1s', + '--health-retries=5', + '--health-timeout=1s', + ] + else: - port_args = ['-p', str(_DEFAULT_SERVER_PORT)] + docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, - docker_args=port_args + - ['--name=%s' % container_name]) + docker_args=docker_args) if manual_cmd_log is not None: manual_cmd_log.append(manual_cmdline(docker_cmdline)) server_job = jobset.JobSpec( @@ -881,7 +890,8 @@ languages = set(_LANGUAGES[l] languages_http2_badserver_interop = set() if args.http2_badserver_interop: languages_http2_badserver_interop = set( - _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS) + _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS + if 'all' in args.language or l in args.language) http2Interop = Http2Client() if args.http2_interop else None http2InteropServer = Http2Server() if args.http2_badserver_interop else None @@ -946,6 +956,7 @@ try: manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) + job.wait_for_healthy(timeout_seconds=15) server_jobs[lang] = job http2_badserver_ports = tuple([ job.mapped_port(_DEFAULT_SERVER_PORT + i) -- cgit v1.2.3