From 37268d0526036fcd29a319932f734a53ee35ae34 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Tue, 7 Mar 2017 09:22:04 -0800 Subject: Fix for using gem build/install When pre-compiled assets are missing for the current platform, and you run gem install, the compiled native extensions are placed in the first require paths folder. This differs from rake-compiler task, where they are always placed in lib. Setting the lib path as the first require path will ensure the compiled library is placed in the correct folder. --- templates/grpc.gemspec.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'templates') diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index 462ea52614..80ce643d80 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -26,7 +26,7 @@ s.files += Dir.glob('include/grpc/**/*') s.test_files = Dir.glob('src/ruby/spec/**/*') s.bindir = 'src/ruby/bin' - s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) + s.require_paths = %w( src/ruby/lib src/ruby/bin src/ruby/pb ) s.platform = Gem::Platform::RUBY s.add_dependency 'google-protobuf', '~> 3.1' -- cgit v1.2.3 From 846242dd5e7bce2a63b3edef96058fe371e6f508 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Fri, 24 Mar 2017 14:26:55 -0700 Subject: update go version in Dockerfile to 1.8 --- .../tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template | 2 +- .../tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template | 2 +- .../dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template | 2 +- tools/dockerfile/interoptest/grpc_interop_go/Dockerfile | 2 +- tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'templates') diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template index 38a5ca725d..bf025d8037 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 6204c3e2cb..0899e007d9 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template index e02254cd53..ad8ad71b5f 100644 --- a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index d3bf071c72..d7a0b1786b 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 66d9b4f640..42e9edf98a 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile index bbf7de7f91..c099f339ae 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -- cgit v1.2.3 From 358f1024ea1afaa12e923296cab7e74082f52b7a Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 5 Apr 2017 08:42:21 -0700 Subject: backport #10100 and #10377 --- .../grpc_interop_http2/Dockerfile.template | 2 +- test/http2_test/http2_server_health_check.py | 49 ++++++++++++++++ test/http2_test/http2_test_server.py | 21 ++++++- .../interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/run_tests/python_utils/dockerjob.py | 20 +++++++ tools/run_tests/run_interop_tests.py | 66 +++++++++++++--------- 6 files changed, 129 insertions(+), 31 deletions(-) create mode 100644 test/http2_test/http2_server_health_check.py (limited to 'templates') diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 0899e007d9..b517921f08 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -33,7 +33,7 @@ <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> - RUN pip install twisted h2 + RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py new file mode 100644 index 0000000000..dd9402b855 --- /dev/null +++ b/test/http2_test/http2_server_health_check.py @@ -0,0 +1,49 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import hyper +import sys + +# Utility to healthcheck the http2 server. Used when starting the server to +# verify that the server is live before tests begin. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--server_host', type=str, default='localhost') + parser.add_argument('--server_port', type=int, default=8080) + args = parser.parse_args() + server_host = args.server_host + server_port = args.server_port + conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port)) + conn.request('POST', '/grpc.testing.TestService/UnaryCall') + resp = conn.get_response() + if resp.headers.get('grpc-encoding') is None: + sys.exit(1) + else: + sys.exit(0) diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py index abde3433ad..46c3e00d18 100644 --- a/test/http2_test/http2_test_server.py +++ b/test/http2_test/http2_test_server.py @@ -31,6 +31,7 @@ import argparse import logging +import sys import twisted import twisted.internet import twisted.internet.endpoints @@ -53,9 +54,11 @@ _TEST_CASE_MAPPING = { 'max_streams': test_max_streams.TestcaseSettingsMaxStreams, } +_exit_code = 0 + class H2Factory(twisted.internet.protocol.Factory): def __init__(self, testcase): - logging.info('Creating H2Factory for new connection.') + logging.info('Creating H2Factory for new connection (%s)', testcase) self._num_streams = 0 self._testcase = testcase @@ -83,6 +86,17 @@ def parse_arguments(): ) return parser.parse_args() +def listen(endpoint, test_case): + deferred = endpoint.listen(H2Factory(test_case)) + def listen_error(reason): + # If listening fails, we stop the reactor and exit the program + # with exit code 1. + global _exit_code + _exit_code = 1 + logging.error('Listening failed: %s' % reason.value) + twisted.internet.reactor.stop() + deferred.addErrback(listen_error) + def start_test_servers(base_port): """ Start one server per test case on incrementing port numbers beginning with base_port """ @@ -92,7 +106,9 @@ def start_test_servers(base_port): logging.warning('serving on port %d : %s'%(portnum, test_case)) endpoint = twisted.internet.endpoints.TCP4ServerEndpoint( twisted.internet.reactor, portnum, backlog=128) - endpoint.listen(H2Factory(test_case)) + # Wait until the reactor is running before calling endpoint.listen(). + twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case) + index += 1 if __name__ == '__main__': @@ -102,3 +118,4 @@ if __name__ == '__main__': args = parse_arguments() start_test_servers(args.base_port) twisted.internet.reactor.run() + sys.exit(_exit_code) diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 42e9edf98a..094a4e096d 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 +RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 0869c5cee9..709fc121a9 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -70,6 +70,23 @@ def docker_mapped_port(cid, port, timeout_seconds=15): (port, cid)) +def wait_for_healthy(cid, shortname, timeout_seconds): + """Wait timeout_seconds for the container to become healthy""" + started = time.time() + while time.time() - started < timeout_seconds: + try: + output = subprocess.check_output( + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], + stderr=_DEVNULL) + if output.strip('\n') == 'healthy': + return + except subprocess.CalledProcessError as e: + pass + time.sleep(1) + raise Exception('Timed out waiting for %s (%s) to pass health check' % + (shortname, cid)) + + def finish_jobs(jobs): """Kills given docker containers and waits for corresponding jobs to finish""" for job in jobs: @@ -113,6 +130,9 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) + def wait_for_healthy(self, timeout_seconds): + wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) + def kill(self, suppress_failure=False): """Sends kill signal to the container.""" if suppress_failure: diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 0d5bec1d67..007e341ca9 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -607,17 +607,13 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, common_options = [ '--test_case=%s' % test_case, '--server_host=%s' % server_host, + '--server_port=%s' % server_port, ] if test_case in _HTTP2_BADSERVER_TEST_CASES: - # We are running the http2_badserver_interop test. Adjust command line accordingly. - offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) - client_options = common_options + ['--server_port=%s' % - (int(server_port)+offset)] - cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) + cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) cwd = language.http2_cwd else: - client_options = interop_only_options + common_options + ['--server_port=%s' % server_port] - cmdline = bash_cmdline(language.client_cmd(client_options)) + cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options)) cwd = language.client_cwd environ = language.global_env() @@ -653,30 +649,40 @@ def server_jobspec(language, docker_image, insecure=False): language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ('false' if insecure else 'true')])) environ = language.global_env() + docker_args = ['--name=%s' % container_name] if language.safename == 'http2': # we are running the http2 interop server. Open next N ports beginning # with the server port. These ports are used for http2 interop test - # (one test case per port). We also attach the docker container running - # the server to local network, so we don't have to mess with port mapping - port_args = [ - '-p', str(_DEFAULT_SERVER_PORT+0), - '-p', str(_DEFAULT_SERVER_PORT+1), - '-p', str(_DEFAULT_SERVER_PORT+2), - '-p', str(_DEFAULT_SERVER_PORT+3), - '-p', str(_DEFAULT_SERVER_PORT+4), - '-p', str(_DEFAULT_SERVER_PORT+5), - '-p', str(_DEFAULT_SERVER_PORT+6), - '--net=host', + # (one test case per port). + docker_args += list( + itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) + for i in range( + len(_HTTP2_BADSERVER_TEST_CASES)))) + # Enable docker's healthcheck mechanism. + # This runs a Python script inside the container every second. The script + # pings the http2 server to verify it is ready. The 'health-retries' flag + # specifies the number of consecutive failures before docker will report + # the container's status as 'unhealthy'. Prior to the first 'health_retries' + # failures or the first success, the status will be 'starting'. 'docker ps' + # or 'docker inspect' can be used to see the health of the container on the + # command line. + docker_args += [ + '--health-cmd=python test/http2_test/http2_server_health_check.py ' + '--server_host=%s --server_port=%d' + % ('localhost', _DEFAULT_SERVER_PORT), + '--health-interval=1s', + '--health-retries=5', + '--health-timeout=10s', ] + else: - port_args = ['-p', str(_DEFAULT_SERVER_PORT)] + docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, - docker_args=port_args + - ['--name', container_name]) + docker_args=docker_args) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, @@ -849,7 +855,8 @@ languages = set(_LANGUAGES[l] languages_http2_badserver_interop = set() if args.http2_badserver_interop: languages_http2_badserver_interop = set( - _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS) + _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS + if 'all' in args.language or l in args.language) http2Interop = Http2Client() if args.http2_interop else None http2InteropServer = Http2Server() if args.http2_badserver_interop else None @@ -888,8 +895,9 @@ if args.use_docker: sys.exit(1) # Start interop servers. -server_jobs={} -server_addresses={} +server_jobs = {} +server_addresses = {} +http2_badserver_ports = () try: for s in servers: lang = str(s) @@ -899,12 +907,13 @@ try: server_jobs[lang] = job server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) + http2_badserver_job = None if args.http2_badserver_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) spec = server_jobspec(http2InteropServer, docker_images.get(lang)) - job = dockerjob.DockerJob(spec) - server_jobs[lang] = job + http2_badserver_job = dockerjob.DockerJob(spec) + server_jobs[lang] = http2_badserver_job jobs = [] if args.cloud_to_prod: @@ -981,13 +990,16 @@ try: jobs.append(test_job) if args.http2_badserver_interop: + http2_badserver_job.wait_for_healthy(timeout_seconds=600) for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: + offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + server_port = http2_badserver_job.mapped_port(_DEFAULT_SERVER_PORT+offset) test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), 'localhost', - _DEFAULT_SERVER_PORT, + server_port, docker_image=docker_images.get(str(language))) jobs.append(test_job) -- cgit v1.2.3