diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/doxygen/Doxyfile.c++ | 2 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.c++.internal | 2 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core | 2 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 2 | ||||
| -rwxr-xr-x | tools/internal_ci/linux/grpc_interop_badserver_java.sh | 2 | ||||
| -rwxr-xr-x | tools/internal_ci/linux/grpc_interop_badserver_python.sh | 2 | ||||
| -rwxr-xr-x | tools/jenkins/run_interop.sh | 2 | ||||
| -rw-r--r-- | tools/run_tests/interop/interop_html_report.template | 10 | ||||
| -rw-r--r-- | tools/run_tests/python_utils/report_utils.py | 11 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 134 |
10 files changed, 106 insertions, 63 deletions
diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index b5021ede15..d21b48cabf 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -779,11 +779,11 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/http2-interop-test-descriptions.md \ doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ -doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 758ba2402a..8d6c2838bb 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -779,11 +779,11 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/http2-interop-test-descriptions.md \ doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ -doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index cd3f2af44c..b4fdcddbcf 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -778,11 +778,11 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/http2-interop-test-descriptions.md \ doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ -doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index b273350082..3e679e3a2b 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -778,11 +778,11 @@ doc/fail_fast.md \ doc/g_stands_for.md \ doc/health-checking.md \ doc/http-grpc-status-mapping.md \ +doc/http2-interop-test-descriptions.md \ doc/internationalization.md \ doc/interop-test-descriptions.md \ doc/load-balancing.md \ doc/naming.md \ -doc/negative-http2-interop-test-descriptions.md \ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ diff --git a/tools/internal_ci/linux/grpc_interop_badserver_java.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh index 0985e657c6..c309c623e0 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_java.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_java.sh @@ -37,5 +37,5 @@ cd $(dirname $0)/../../.. git submodule update --init -tools/run_tests/run_interop_tests.py -l java --use_docker --http2_badserver_interop $@ +tools/run_tests/run_interop_tests.py -l java --use_docker --http2_server_interop $@ diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh index 3fff537d2b..c3bb92f33d 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_python.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_python.sh @@ -37,5 +37,5 @@ cd $(dirname $0)/../../.. git submodule update --init -tools/run_tests/run_interop_tests.py -l python --use_docker --http2_badserver_interop $@ +tools/run_tests/run_interop_tests.py -l python --use_docker --http2_server_interop $@ diff --git a/tools/jenkins/run_interop.sh b/tools/jenkins/run_interop.sh index 2a9fc662a9..13c208e97c 100755 --- a/tools/jenkins/run_interop.sh +++ b/tools/jenkins/run_interop.sh @@ -36,4 +36,4 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../.. -tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop --http2_badserver_interop -t -j 12 $@ || true +tools/run_tests/run_interop_tests.py -l all -s all --cloud_to_prod --cloud_to_prod_auth --use_docker --http2_interop --http2_server_interop -t -j 12 $@ || true diff --git a/tools/run_tests/interop/interop_html_report.template b/tools/run_tests/interop/interop_html_report.template index 88ecd4e4db..6d9de5c62e 100644 --- a/tools/run_tests/interop/interop_html_report.template +++ b/tools/run_tests/interop/interop_html_report.template @@ -106,19 +106,19 @@ % endfor % endif -% if http2_badserver_cases: - <h2>HTTP/2 Bad Server Tests</h2> +% if http2_server_cases: + <h2>HTTP/2 Server Tests</h2> ## Each column header is the client language. <table style="width:100%" border="1"> <tr bgcolor="#00BFFF"> <th>Client languages ►<br/>Test Cases ▼</th> - % for client_lang in client_langs_http2_badserver_cases: + % for client_lang in client_langs: <th>${client_lang}</th> % endfor </tr> - % for test_case in http2_badserver_cases: + % for test_case in http2_server_cases: <tr><td><b>${test_case}</b></td> - % for client_lang in client_langs_http2_badserver_cases: + % for client_lang in client_langs: <% shortname = 'cloud_to_cloud:%s:http2_server:%s' % (client_lang, test_case) diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 131772f55f..3b2b4f8712 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -80,10 +80,9 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', tree = ET.ElementTree(root) tree.write(xml_report, encoding='UTF-8') - def render_interop_html_report( client_langs, server_langs, test_cases, auth_test_cases, http2_cases, - http2_badserver_cases, client_langs_http2_badserver_cases, resultset, + http2_server_cases, resultset, num_failures, cloud_to_prod, prod_servers, http2_interop): """Generate HTML report for interop tests.""" template_file = 'tools/run_tests/interop/interop_html_report.template' @@ -99,9 +98,7 @@ def render_interop_html_report( sorted_test_cases = sorted(test_cases) sorted_auth_test_cases = sorted(auth_test_cases) sorted_http2_cases = sorted(http2_cases) - sorted_http2_badserver_cases = sorted(http2_badserver_cases) - sorted_client_langs_http2_badserver_cases = sorted( - client_langs_http2_badserver_cases) + sorted_http2_server_cases = sorted(http2_server_cases) sorted_client_langs = sorted(client_langs) sorted_server_langs = sorted(server_langs) sorted_prod_servers = sorted(prod_servers) @@ -111,9 +108,7 @@ def render_interop_html_report( 'test_cases': sorted_test_cases, 'auth_test_cases': sorted_auth_test_cases, 'http2_cases': sorted_http2_cases, - 'http2_badserver_cases': sorted_http2_badserver_cases, - 'client_langs_http2_badserver_cases': ( - sorted_client_langs_http2_badserver_cases), + 'http2_server_cases': sorted_http2_server_cases, 'resultset': resultset, 'num_failures': num_failures, 'cloud_to_prod': cloud_to_prod, diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ce4dfb863e..2d7f4a625d 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -45,6 +45,7 @@ import tempfile import time import uuid import six +import traceback import python_utils.dockerjob as dockerjob import python_utils.jobset as jobset @@ -73,6 +74,10 @@ _SKIP_ADVANCED = ['status_code_and_message', _TEST_TIMEOUT = 3*60 +# disable this test on core-based languages, +# see https://github.com/grpc/grpc/issues/9779 +_SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] + class CXXLanguage: def __init__(self): @@ -97,7 +102,7 @@ class CXXLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return [] @@ -126,7 +131,7 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -155,7 +160,7 @@ class CSharpCoreCLRLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -250,7 +255,7 @@ class Http2Server: return {} def unimplemented_test_cases(self): - return _TEST_CASES + return _TEST_CASES + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _TEST_CASES @@ -281,7 +286,7 @@ class Http2Client: return _TEST_CASES def unimplemented_test_cases_server(self): - return [] + return _TEST_CASES def __str__(self): return 'http2' @@ -308,7 +313,7 @@ class NodeLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -333,7 +338,7 @@ class PHPLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return [] @@ -358,7 +363,7 @@ class PHP7Language: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return [] @@ -389,7 +394,7 @@ class RubyLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -437,7 +442,7 @@ class PythonLanguage: 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT)} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -476,10 +481,14 @@ _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', _HTTP2_TEST_CASES = ['tls', 'framing'] -_HTTP2_BADSERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', - 'goaway', 'ping', 'max_streams'] +_HTTP2_SERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data', + 'goaway', 'ping', 'max_streams', 'data_frame_padding', 'no_df_padding_sanity_test'] + +_GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { 'data_frame_padding': 'large_unary', 'no_df_padding_sanity_test': 'large_unary' } -_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java', 'go', 'python', 'c++'] +_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys() + +_LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = ['java', 'go', 'python', 'c++'] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -631,14 +640,28 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, '--use_tls=%s' % ('false' if insecure else 'true'), '--use_test_ca=true', ] + + client_test_case = test_case + if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: + client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[test_case] + if client_test_case in language.unimplemented_test_cases(): + print('asking client %s to run unimplemented test case %s' % (repr(language), client_test_case)) + sys.exit(1) + common_options = [ - '--test_case=%s' % test_case, + '--test_case=%s' % client_test_case, '--server_host=%s' % server_host, '--server_port=%s' % server_port, ] - if test_case in _HTTP2_BADSERVER_TEST_CASES: - cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) - cwd = language.http2_cwd + + if test_case in _HTTP2_SERVER_TEST_CASES: + if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: + client_options = interop_only_options + common_options + cmdline = bash_cmdline(language.client_cmd(client_options)) + cwd = language.client_cwd + else: + cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) + cwd = language.http2_cwd else: cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options)) cwd = language.client_cwd @@ -686,7 +709,7 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): docker_args += list( itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) for i in range( - len(_HTTP2_BADSERVER_TEST_CASES)))) + len(_HTTP2_SERVER_TEST_CASES)))) # Enable docker's healthcheck mechanism. # This runs a Python script inside the container every second. The script # pings the http2 server to verify it is ready. The 'health-retries' flag @@ -856,11 +879,11 @@ argp.add_argument('--http2_interop', action='store_const', const=True, help='Enable HTTP/2 client edge case testing. (Bad client, good server)') -argp.add_argument('--http2_badserver_interop', +argp.add_argument('--http2_server_interop', default=False, action='store_const', const=True, - help='Enable HTTP/2 server edge case testing. (Good client, bad server)') + help='Enable HTTP/2 server edge case testing. (Includes positive and negative tests') argp.add_argument('--insecure', default=False, action='store_const', @@ -895,26 +918,26 @@ languages = set(_LANGUAGES[l] six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) -languages_http2_badserver_interop = set() -if args.http2_badserver_interop: - languages_http2_badserver_interop = set( - _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS +languages_http2_clients_for_http2_server_interop = set() +if args.http2_server_interop: + languages_http2_clients_for_http2_server_interop = set( + _LANGUAGES[l] for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES if 'all' in args.language or l in args.language) http2Interop = Http2Client() if args.http2_interop else None -http2InteropServer = Http2Server() if args.http2_badserver_interop else None +http2InteropServer = Http2Server() if args.http2_server_interop else None docker_images={} if args.use_docker: # languages for which to build docker images languages_to_build = set( _LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers])) - languages_to_build = languages_to_build | languages_http2_badserver_interop + languages_to_build = languages_to_build | languages_http2_clients_for_http2_server_interop if args.http2_interop: languages_to_build.add(http2Interop) - if args.http2_badserver_interop: + if args.http2_server_interop: languages_to_build.add(http2InteropServer) build_jobs = [] @@ -943,7 +966,6 @@ client_manual_cmd_log = [] if args.manual_run else None # Start interop servers. server_jobs = {} server_addresses = {} -http2_badserver_ports = () try: for s in servers: lang = str(s) @@ -957,15 +979,15 @@ try: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') - http2_badserver_job = None - if args.http2_badserver_interop: + http2_server_job = None + if args.http2_server_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) spec = server_jobspec(http2InteropServer, docker_images.get(lang), manual_cmd_log=server_manual_cmd_log) if not args.manual_run: - http2_badserver_job = dockerjob.DockerJob(spec) - server_jobs[lang] = http2_badserver_job + http2_server_job = dockerjob.DockerJob(spec) + server_jobs[lang] = http2_server_job else: # don't run the server, set server port to a placeholder value server_addresses[lang] = ('localhost', '${SERVER_PORT}') @@ -1049,15 +1071,15 @@ try: manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) - if args.http2_badserver_interop: + if args.http2_server_interop: if not args.manual_run: - http2_badserver_job.wait_for_healthy(timeout_seconds=600) - for language in languages_http2_badserver_interop: - for test_case in _HTTP2_BADSERVER_TEST_CASES: - offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + http2_server_job.wait_for_healthy(timeout_seconds=600) + for language in languages_http2_clients_for_http2_server_interop: + for test_case in set(_HTTP2_SERVER_TEST_CASES) - set(_HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS): + offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) server_port = _DEFAULT_SERVER_PORT+offset if not args.manual_run: - server_port = http2_badserver_job.mapped_port(server_port) + server_port = http2_server_job.mapped_port(server_port) test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), @@ -1066,6 +1088,31 @@ try: docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) + for language in languages: + # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of + # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather + # than specialized http2 clients, reusing existing test implementations. + # For example, in the "data_frame_padding" test, use language's gRPC + # interop clients and make them think that theyre running "large_unary" + # test case. This avoids implementing a new test case in each language. + for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: + if test_case not in language.unimplemented_test_cases(): + offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) + server_port = _DEFAULT_SERVER_PORT+offset + if not args.manual_run: + server_port = http2_server_job.mapped_port(server_port) + if not args.insecure: + print(('Creating grpc cient to http2 server test case with insecure connection, even though' + ' args.insecure is False. Http2 test server only supports insecure connections.')) + test_job = cloud_to_cloud_jobspec(language, + test_case, + str(http2InteropServer), + 'localhost', + server_port, + docker_image=docker_images.get(str(language)), + insecure=True, + manual_cmd_log=client_manual_cmd_log) + jobs.append(test_job) if not jobs: print('No jobs to run.') @@ -1093,16 +1140,17 @@ try: if "http2" in name: job[0].http2results = aggregate_http2_results(job[0].message) - http2_badserver_test_cases = ( - _HTTP2_BADSERVER_TEST_CASES if args.http2_badserver_interop else []) + http2_server_test_cases = ( + _HTTP2_SERVER_TEST_CASES if args.http2_server_interop else []) report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, - _HTTP2_TEST_CASES, http2_badserver_test_cases, - _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS, resultset, num_failures, + _HTTP2_TEST_CASES, http2_server_test_cases, resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers, args.http2_interop) - +except Exception as e: + print('exception occurred:') + traceback.print_exc(file=sys.stdout) finally: # Check if servers are still running. for server, job in server_jobs.items(): |