diff options
Diffstat (limited to 'tools/run_tests/run_interop_tests.py')
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 279 |
1 files changed, 184 insertions, 95 deletions
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 5435a1d02f..e69e9877c5 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -31,27 +31,34 @@ """Run interop (cross-language) tests in parallel.""" import argparse +import atexit import dockerjob import itertools import jobset +import json import multiprocessing import os +import re import report_utils +import subprocess import sys import tempfile import time import uuid +# Docker doesn't clean up after itself, so we do it on exit. +atexit.register(lambda: subprocess.call(['stty', 'echo'])) + ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(ROOT) _DEFAULT_SERVER_PORT=8080 -# TOOD(jtattermusch) wrapped languages use this variable for location -# of roots.pem. We might want to use GRPC_DEFAULT_SSL_ROOTS_FILE_PATH -# supported by C core SslCredentials instead. -_SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' } +_SKIP_COMPRESSION = ['large_compressed_unary', + 'server_compressed_streaming'] +_SKIP_ADVANCED = ['custom_metadata', 'status_code_and_message', + 'unimplemented_method'] class CXXLanguage: @@ -60,20 +67,23 @@ class CXXLanguage: self.server_cwd = None self.safename = 'cxx' - def client_args(self): - return ['bins/opt/interop_client'] + def client_cmd(self, args): + return ['bins/opt/interop_client'] + args def cloud_to_prod_env(self): return {} - def server_args(self): - return ['bins/opt/interop_server', '--use_tls=true'] + def server_cmd(self, args): + return ['bins/opt/interop_server', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'c++' @@ -86,20 +96,24 @@ class CSharpLanguage: self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug' self.safename = str(self) - def client_args(self): - return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + def client_cmd(self, args): + return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args def cloud_to_prod_env(self): - return _SSL_CERT_ENV + return {} - def server_args(self): - return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true'] + def server_cmd(self, args): + return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + # TODO: status_code_and_message doesn't work against node_server + return _SKIP_COMPRESSION + ['status_code_and_message'] + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'csharp' @@ -112,20 +126,23 @@ class JavaLanguage: self.server_cwd = '../grpc-java' self.safename = str(self) - def client_args(self): - return ['./run-test-client.sh'] + def client_cmd(self, args): + return ['./run-test-client.sh'] + args def cloud_to_prod_env(self): return {} - def server_args(self): - return ['./run-test-server.sh', '--use_tls=true'] + def server_cmd(self, args): + return ['./run-test-server.sh', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'java' @@ -139,20 +156,23 @@ class GoLanguage: self.server_cwd = '/go/src/google.golang.org/grpc/interop/server' self.safename = str(self) - def client_args(self): - return ['go', 'run', 'client.go'] + def client_cmd(self, args): + return ['go', 'run', 'client.go'] + args def cloud_to_prod_env(self): return {} - def server_args(self): - return ['go', 'run', 'server.go', '--use_tls=true'] + def server_cmd(self, args): + return ['go', 'run', 'server.go', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'go' @@ -168,8 +188,8 @@ class Http2Client: self.client_cwd = None self.safename = str(self) - def client_args(self): - return ['tools/http2_interop/http2_interop.test', '-test.v'] + def client_cmd(self, args): + return ['tools/http2_interop/http2_interop.test', '-test.v'] + args def cloud_to_prod_env(self): return {} @@ -180,6 +200,9 @@ class Http2Client: def unimplemented_test_cases(self): return _TEST_CASES + def unimplemented_test_cases_server(self): + return [] + def __str__(self): return 'http2' @@ -190,20 +213,23 @@ class NodeLanguage: self.server_cwd = None self.safename = str(self) - def client_args(self): - return ['node', 'src/node/interop/interop_client.js'] + def client_cmd(self, args): + return ['node', 'src/node/interop/interop_client.js'] + args def cloud_to_prod_env(self): - return _SSL_CERT_ENV + return {} - def server_args(self): - return ['node', 'src/node/interop/interop_server.js', '--use_tls=true'] + def server_cmd(self, args): + return ['node', 'src/node/interop/interop_server.js', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + return _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'node' @@ -215,16 +241,19 @@ class PHPLanguage: self.client_cwd = None self.safename = str(self) - def client_args(self): - return ['src/php/bin/interop_client.sh'] + def client_cmd(self, args): + return ['src/php/bin/interop_client.sh'] + args def cloud_to_prod_env(self): - return _SSL_CERT_ENV + return {} def global_env(self): return {} def unimplemented_test_cases(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): return [] def __str__(self): @@ -238,20 +267,23 @@ class RubyLanguage: self.server_cwd = None self.safename = str(self) - def client_args(self): - return ['ruby', 'src/ruby/bin/interop/interop_client.rb'] + def client_cmd(self, args): + return ['ruby', 'src/ruby/bin/interop/interop_client.rb'] + args def cloud_to_prod_env(self): - return _SSL_CERT_ENV + return {} - def server_args(self): - return ['ruby', 'src/ruby/bin/interop/interop_server.rb', '--use_tls=true'] + def server_cmd(self, args): + return ['ruby', 'src/ruby/bin/interop/interop_server.rb', '--use_tls=true'] + args def global_env(self): return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'ruby' @@ -264,20 +296,36 @@ class PythonLanguage: self.server_cwd = None self.safename = str(self) - def client_args(self): - return ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.client'] + def client_cmd(self, args): + return [ + 'src/python/grpcio/.tox/py27/bin/python', + 'src/python/grpcio/setup.py', + 'run_interop', + '--client', + '--args=\'{}\''.format(' '.join(args)) + ] def cloud_to_prod_env(self): - return _SSL_CERT_ENV + return {} - def server_args(self): - return ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.server', '--use_tls=true'] + def server_cmd(self, args): + return [ + 'src/python/grpcio/.tox/py27/bin/python', + 'src/python/grpcio/setup.py', + 'run_interop', + '--server', + '--args=\'{}\''.format(' '.join(args) + ' --use_tls=true') + ] def global_env(self): - return {'LD_LIBRARY_PATH': 'libs/opt'} + return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)} def unimplemented_test_cases(self): - return ['jwt_token_creds', 'per_rpc_creds'] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + ['jwt_token_creds', + 'per_rpc_creds'] + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'python' @@ -300,13 +348,17 @@ _SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby', 'python'] _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 'client_streaming', 'server_streaming', 'cancel_after_begin', 'cancel_after_first_response', - 'timeout_on_sleeping_server'] + 'timeout_on_sleeping_server', 'custom_metadata', + 'status_code_and_message', 'unimplemented_method', + 'large_compressed_unary', 'server_compressed_streaming'] _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] _HTTP2_TEST_CASES = ["tls", "framing"] +DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' + def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): """Wraps given cmdline array to create 'docker run' cmdline from it.""" docker_cmdline = ['docker', 'run', '-i', '--rm=true'] @@ -317,7 +369,7 @@ def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): docker_cmdline += ['-e', '%s=%s' % (k,v)] # set working directory - workdir = '/var/local/git/grpc' + workdir = DOCKER_WORKDIR_ROOT if cwd: workdir = os.path.join(workdir, cwd) docker_cmdline += ['-w', workdir] @@ -334,12 +386,12 @@ def bash_login_cmdline(cmdline): return ['bash', '-l', '-c', ' '.join(cmdline)] -def add_auth_options(language, test_case, cmdline, env): +def auth_options(language, test_case): """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" language = str(language) - cmdline = list(cmdline) - env = env.copy() + cmdargs = [] + env = {} # TODO(jtattermusch): this file path only works inside docker key_filepath = '/root/service_account/stubbyCloudTestingTest-ee3fce360ac5.json' @@ -351,19 +403,19 @@ def add_auth_options(language, test_case, cmdline, env): if language in ['csharp', 'node', 'php', 'python', 'ruby']: env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath else: - cmdline += [key_file_arg] + cmdargs += [key_file_arg] if test_case in ['per_rpc_creds', 'oauth2_auth_token']: - cmdline += [oauth_scope_arg] + cmdargs += [oauth_scope_arg] if test_case == 'oauth2_auth_token' and language == 'c++': # C++ oauth2 test uses GCE creds and thus needs to know the default account - cmdline += [default_account_arg] + cmdargs += [default_account_arg] if test_case == 'compute_engine_creds': - cmdline += [oauth_scope_arg, default_account_arg] + cmdargs += [oauth_scope_arg, default_account_arg] - return (cmdline, env) + return (cmdargs, env) def _job_kill_handler(job): @@ -378,18 +430,20 @@ def _job_kill_handler(job): def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False): """Creates jobspec for cloud-to-prod interop test""" - cmdline = language.client_args() + [ + container_name = None + cmdargs = [ '--server_host_override=grpc-test.sandbox.google.com', '--server_host=grpc-test.sandbox.google.com', '--server_port=443', '--use_tls=true', '--test_case=%s' % test_case] - cwd = language.client_cwd environ = dict(language.cloud_to_prod_env(), **language.global_env()) - container_name = None if auth: - cmdline, environ = add_auth_options(language, test_case, cmdline, environ) - cmdline = bash_login_cmdline(cmdline) + auth_cmdargs, auth_env = auth_options(language, test_case) + cmdargs += auth_cmdargs + environ.update(auth_env) + cmdline = bash_login_cmdline(language.client_cmd(cmdargs)) + cwd = language.client_cwd if docker_image: container_name = dockerjob.random_name('interop_client_%s' % language.safename) @@ -419,13 +473,13 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False): def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, server_port, docker_image=None): """Creates jobspec for cloud-to-cloud interop test""" - cmdline = bash_login_cmdline(language.client_args() + - ['--server_host_override=foo.test.google.fr', - '--use_tls=true', - '--use_test_ca=true', - '--test_case=%s' % test_case, - '--server_host=%s' % server_host, - '--server_port=%s' % server_port]) + cmdline = bash_login_cmdline(language.client_cmd([ + '--server_host_override=foo.test.google.fr', + '--use_tls=true', + '--use_test_ca=true', + '--test_case=%s' % test_case, + '--server_host=%s' % server_host, + '--server_port=%s' % server_port])) cwd = language.client_cwd environ = language.global_env() if docker_image: @@ -455,8 +509,8 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, def server_jobspec(language, docker_image): """Create jobspec for running a server""" container_name = dockerjob.random_name('interop_server_%s' % language.safename) - cmdline = bash_login_cmdline(language.server_args() + - ['--port=%s' % _DEFAULT_SERVER_PORT]) + cmdline = bash_login_cmdline( + language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT])) environ = language.global_env() docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, @@ -497,6 +551,33 @@ def build_interop_image_jobspec(language, tag=None): return build_job +def aggregate_http2_results(stdout): + match = re.search(r'\{"cases[^\]]*\]\}', stdout) + if not match: + return None + + results = json.loads(match.group(0)) + skipped = 0 + passed = 0 + failed = 0 + failed_cases = [] + for case in results['cases']: + if case.get('skipped', False): + skipped += 1 + else: + if case.get('passed', False): + passed += 1 + else: + failed += 1 + failed_cases.append(case.get('name', "NONAME")) + return { + 'passed': passed, + 'failed': failed, + 'skipped': skipped, + 'failed_cases': ', '.join(failed_cases), + 'percent': 1.0 * passed / (passed + failed) + } + argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), @@ -547,7 +628,7 @@ argp.add_argument('--http2_interop', action='store_const', const=True, help='Enable HTTP/2 interop tests') - + args = argp.parse_args() servers = set(s for s in itertools.chain.from_iterable(_SERVERS @@ -571,7 +652,7 @@ languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( _LANGUAGES.iterkeys() if x == 'all' else [x] for x in args.language)) - + http2Interop = Http2Client() if args.http2_interop else None docker_images={} @@ -593,10 +674,10 @@ if args.use_docker: num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: - jobset.message('SUCCESS', 'All docker images built successfully.', + jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: - jobset.message('FAILED', 'Failed to build interop docker images.', + jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in docker_images.itervalues(): dockerjob.remove_image(image, skip_nonexistent=True) @@ -619,18 +700,17 @@ try: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec(language, test_case, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) - - # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable - # this once a better solution is in place. - if args.http2_interop and False: + if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: + test_job = cloud_to_prod_jobspec(language, test_case, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) + + if args.http2_interop: for test_case in _HTTP2_TEST_CASES: test_job = cloud_to_prod_jobspec(http2Interop, test_case, docker_image=docker_images.get(str(http2Interop))) jobs.append(test_job) - + if args.cloud_to_prod_auth: for language in languages: @@ -648,17 +728,22 @@ try: for server_name, server_address in server_addresses.iteritems(): (server_host, server_port) = server_address + server_language = _LANGUAGES.get(server_name, None) + skip_server = [] # test cases unimplemented by server + if server_language: + skip_server = server_language.unimplemented_test_cases_server() for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_cloud_jobspec(language, - test_case, - server_name, - server_host, - server_port, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) - + if not test_case in skip_server: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) + if args.http2_interop: for test_case in _HTTP2_TEST_CASES: if server_name == "go": @@ -678,7 +763,7 @@ try: dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) - num_failures, resultset = jobset.run(jobs, newline_on_success=True, + num_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) @@ -686,7 +771,11 @@ try: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.render_junit_xml_report(resultset, 'report.xml') - + + for name, job in resultset.iteritems(): + if "http2" in name: + job[0].http2results = aggregate_http2_results(job[0].message) + report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, |