diff options
Diffstat (limited to 'tools/run_tests/run_interop_tests.py')
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 189 |
1 files changed, 149 insertions, 40 deletions
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 83cfc429f9..b2dac7d1a8 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -34,20 +34,21 @@ from __future__ import print_function import argparse import atexit -import dockerjob import itertools -import jobset import json import multiprocessing import os import re -import report_utils import subprocess import sys import tempfile import time import uuid +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils + # Docker doesn't clean up after itself, so we do it on exit. atexit.register(lambda: subprocess.call(['stty', 'echo'])) @@ -85,7 +86,7 @@ class CXXLanguage: return {} def server_cmd(self, args): - return ['bins/opt/interop_server', '--use_tls=true'] + args + return ['bins/opt/interop_server'] + args def global_env(self): return {} @@ -114,7 +115,7 @@ class CSharpLanguage: return {} def server_cmd(self, args): - return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true'] + args + return ['mono', 'Grpc.IntegrationTesting.Server.exe'] + args def global_env(self): return {} @@ -143,7 +144,7 @@ class CSharpCoreCLRLanguage: return {} def server_cmd(self, args): - return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll', '--use_tls=true'] + args + return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll'] + args def global_env(self): return {} @@ -168,20 +169,23 @@ class JavaLanguage: def client_cmd(self, args): return ['./run-test-client.sh'] + args + def client_cmd_http2interop(self, args): + return ['./run-http2-client.sh'] + args + def cloud_to_prod_env(self): return {} def server_cmd(self, args): - return ['./run-test-server.sh', '--use_tls=true'] + args + return ['./run-test-server.sh'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return _SKIP_ADVANCED + _SKIP_COMPRESSION + return _SKIP_COMPRESSION def unimplemented_test_cases_server(self): - return _SKIP_ADVANCED + _SKIP_COMPRESSION + return _SKIP_COMPRESSION def __str__(self): return 'java' @@ -202,7 +206,7 @@ class GoLanguage: return {} def server_cmd(self, args): - return ['go', 'run', 'server.go', '--use_tls=true'] + args + return ['go', 'run', 'server.go'] + args def global_env(self): return {} @@ -216,6 +220,33 @@ class GoLanguage: def __str__(self): return 'go' +class Http2Server: + """Represents the HTTP/2 Interop Test server + + This pretends to be a language in order to be built and run, but really it + isn't. + """ + def __init__(self): + self.server_cwd = None + self.safename = str(self) + + def server_cmd(self, args): + return ['python test/http2_test/http2_test_server.py'] + + def cloud_to_prod_env(self): + return {} + + def global_env(self): + return {} + + def unimplemented_test_cases(self): + return _TEST_CASES + + def unimplemented_test_cases_server(self): + return _TEST_CASES + + def __str__(self): + return 'http2' class Http2Client: """Represents the HTTP/2 Interop Test @@ -261,8 +292,7 @@ class NodeLanguage: def server_cmd(self, args): return ['tools/run_tests/interop/with_nvm.sh', - 'node', 'src/node/interop/interop_server.js', - '--use_tls=true'] + args + 'node', 'src/node/interop/interop_server.js'] + args def global_env(self): return {} @@ -343,16 +373,16 @@ class RubyLanguage: def server_cmd(self, args): return ['tools/run_tests/interop/with_rvm.sh', - 'ruby', 'src/ruby/pb/test/server.rb', '--use_tls=true'] + args + 'ruby', 'src/ruby/pb/test/server.rb'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return _SKIP_ADVANCED + _SKIP_SERVER_COMPRESSION + return _SKIP_SERVER_COMPRESSION def unimplemented_test_cases_server(self): - return _SKIP_ADVANCED + _SKIP_COMPRESSION + return _SKIP_COMPRESSION def __str__(self): return 'ruby' @@ -374,6 +404,11 @@ class PythonLanguage: '--args="{}"'.format(' '.join(args)) ] + def client_cmd_http2interop(self, args): + return [ 'py27/bin/python', + 'src/python/grpcio_tests/tests/http2/_negative_http2_client.py', + ] + args + def cloud_to_prod_env(self): return {} @@ -383,7 +418,7 @@ class PythonLanguage: 'src/python/grpcio_tests/setup.py', 'run_interop', '--server', - '--args="{}"'.format(' '.join(args) + ' --use_tls=true') + '--args="{}"'.format(' '.join(args)) ] def global_env(self): @@ -428,7 +463,10 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] -_HTTP2_TEST_CASES = ["tls", "framing"] +_HTTP2_TEST_CASES = ['tls', 'framing'] + +_HTTP2_BADSERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', + 'goaway', 'ping', 'max_streams'] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -547,15 +585,27 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, - server_port, docker_image=None): + server_port, docker_image=None, insecure=False): """Creates jobspec for cloud-to-cloud interop test""" - cmdline = bash_cmdline(language.client_cmd([ + interop_only_options = [ '--server_host_override=foo.test.google.fr', - '--use_tls=true', + '--use_tls=%s' % ('false' if insecure else 'true'), '--use_test_ca=true', + ] + common_options = [ '--test_case=%s' % test_case, '--server_host=%s' % server_host, - '--server_port=%s' % server_port])) + ] + if test_case in _HTTP2_BADSERVER_TEST_CASES: + # We are running the http2_badserver_interop test. Adjust command line accordingly. + offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + client_options = common_options + ['--server_port=%s' % + (int(server_port)+offset)] + cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) + else: + client_options = interop_only_options + common_options + ['--server_port=%s' % server_port] + cmdline = bash_cmdline(language.client_cmd(client_options)) + cwd = language.client_cwd environ = language.global_env() if docker_image: @@ -583,19 +633,37 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, return test_job -def server_jobspec(language, docker_image): +def server_jobspec(language, docker_image, insecure=False): """Create jobspec for running a server""" container_name = dockerjob.random_name('interop_server_%s' % language.safename) cmdline = bash_cmdline( - language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT])) + language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, + '--use_tls=%s' % ('false' if insecure else 'true')])) environ = language.global_env() + if language.safename == 'http2': + # we are running the http2 interop server. Open next N ports beginning + # with the server port. These ports are used for http2 interop test + # (one test case per port). We also attach the docker container running + # the server to local network, so we don't have to mess with port mapping + port_args = [ + '-p', str(_DEFAULT_SERVER_PORT+0), + '-p', str(_DEFAULT_SERVER_PORT+1), + '-p', str(_DEFAULT_SERVER_PORT+2), + '-p', str(_DEFAULT_SERVER_PORT+3), + '-p', str(_DEFAULT_SERVER_PORT+4), + '-p', str(_DEFAULT_SERVER_PORT+5), + '-p', str(_DEFAULT_SERVER_PORT+6), + '--net=host', + ] + else: + port_args = ['-p', str(_DEFAULT_SERVER_PORT)] + docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, - docker_args=['-p', str(_DEFAULT_SERVER_PORT), - '--name', container_name]) - + docker_args=port_args + + ['--name', container_name]) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, @@ -729,7 +797,17 @@ argp.add_argument('--http2_interop', default=False, action='store_const', const=True, - help='Enable HTTP/2 interop tests') + help='Enable HTTP/2 client edge case testing. (Bad client, good server)') +argp.add_argument('--http2_badserver_interop', + default=False, + action='store_const', + const=True, + help='Enable HTTP/2 server edge case testing. (Good client, bad server)') +argp.add_argument('--insecure', + default=False, + action='store_const', + const=True, + help='Whether to use secure channel.') args = argp.parse_args() @@ -756,6 +834,7 @@ languages = set(_LANGUAGES[l] for x in args.language)) http2Interop = Http2Client() if args.http2_interop else None +http2InteropServer = Http2Server() if args.http2_badserver_interop else None docker_images={} if args.use_docker: @@ -765,6 +844,9 @@ if args.use_docker: if args.http2_interop: languages_to_build.add(http2Interop) + if args.http2_badserver_interop: + languages_to_build.add(http2InteropServer) + build_jobs = [] for l in languages_to_build: job = build_interop_image_jobspec(l) @@ -791,13 +873,24 @@ server_addresses={} try: for s in servers: lang = str(s) - spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang)) + spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), + args.insecure) job = dockerjob.DockerJob(spec) server_jobs[lang] = job server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) + if args.http2_badserver_interop: + # launch a HTTP2 server emulator that creates edge cases + lang = str(http2InteropServer) + spec = server_jobspec(http2InteropServer, docker_images.get(lang)) + job = dockerjob.DockerJob(spec) + server_jobs[lang] = job + server_addresses[lang] = ('localhost', _DEFAULT_SERVER_PORT) + jobs = [] if args.cloud_to_prod: + if args.insecure: + print('TLS is always enabled for cloud_to_prod scenarios.') for server_host_name in args.prod_servers: for language in languages: for test_case in _TEST_CASES: @@ -818,6 +911,8 @@ try: jobs.append(test_job) if args.cloud_to_prod_auth: + if args.insecure: + print('TLS is always enabled for cloud_to_prod scenarios.') for server_host_name in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: @@ -839,17 +934,19 @@ try: skip_server = [] # test cases unimplemented by server if server_language: skip_server = server_language.unimplemented_test_cases_server() - for language in languages: - for test_case in _TEST_CASES: - if not test_case in language.unimplemented_test_cases(): - if not test_case in skip_server: - test_job = cloud_to_cloud_jobspec(language, - test_case, - server_name, - server_host, - server_port, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) + if not args.http2_badserver_interop: + for language in languages: + for test_case in _TEST_CASES: + if not test_case in language.unimplemented_test_cases(): + if not test_case in skip_server: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port, + docker_image=docker_images.get(str(language)), + insecure=args.insecure) + jobs.append(test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -861,9 +958,21 @@ try: server_name, server_host, server_port, - docker_image=docker_images.get(str(http2Interop))) + docker_image=docker_images.get(str(http2Interop)), + insecure=args.insecure) jobs.append(test_job) + if args.http2_badserver_interop: + for language in languages: + for test_case in _HTTP2_BADSERVER_TEST_CASES: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) + if not jobs: print('No jobs to run.') for image in docker_images.itervalues(): |