From e7f8e8ecafdf36537a0b0ab032aeba0baf1f1cd3 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Tue, 8 Dec 2015 17:22:44 -0800 Subject: Make pretty interop tets output graphs --- tools/run_tests/run_interop_tests.py | 42 +++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 37b631bd0d..9a925bee4e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -31,17 +31,24 @@ """Run interop (cross-language) tests in parallel.""" import argparse +import atexit import dockerjob import itertools import jobset +import json import multiprocessing import os +import re import report_utils +import subprocess import sys import tempfile import time import uuid +# Docker doesn't clean up after itself, so we do it on exit. +atexit.register(lambda: subprocess.call(['stty', 'echo'])) + ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(ROOT) @@ -513,6 +520,33 @@ def build_interop_image_jobspec(language, tag=None): return build_job +def aggregate_http2_results(stdout): + match = re.search(r'\{"cases[^\]]*\]\}', stdout) + if not match: + return None + + results = json.loads(match.group(0)) + skipped = 0 + passed = 0 + failed = 0 + failed_cases = [] + for case in results['cases']: + if case.get('skipped', False): + skipped += 1 + else: + if case.get('passed', False): + passed += 1 + else: + failed += 1 + failed_cases.append(case.get('name', "NONAME")) + return { + 'passed': passed, + 'failed': failed, + 'skipped': skipped, + 'failed_cases': ', '.join(failed_cases), + 'percent': 1.0 * passed / (passed + failed) + } + argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), @@ -639,9 +673,7 @@ try: docker_image=docker_images.get(str(language))) jobs.append(test_job) - # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable - # this once a better solution is in place. - if args.http2_interop and False: + if args.http2_interop: for test_case in _HTTP2_TEST_CASES: test_job = cloud_to_prod_jobspec(http2Interop, test_case, docker_image=docker_images.get(str(http2Interop))) @@ -703,6 +735,10 @@ try: report_utils.render_junit_xml_report(resultset, 'report.xml') + for name, job in resultset.iteritems(): + if "http2" in name: + job[0].http2results = aggregate_http2_results(job[0].message) + report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, -- cgit v1.2.3 From 5c7a1c861f4b13f41bea2ffbff073b268329447c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 9 Dec 2015 19:56:51 -0800 Subject: teach interop script about new tests --- tools/run_tests/run_interop_tests.py | 77 +++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 18 deletions(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 37b631bd0d..8f9eedadba 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -52,6 +52,11 @@ _DEFAULT_SERVER_PORT=8080 # supported by C core SslCredentials instead. _SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' } +_SKIP_COMPRESSION = ['large_compressed_unary', + 'server_compressed_streaming'] + +_SKIP_ADVANCED = ['custom_metadata', 'status_code_and_message', + 'unimplemented_method'] class CXXLanguage: @@ -73,7 +78,10 @@ class CXXLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'c++' @@ -99,7 +107,10 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'csharp' @@ -125,7 +136,10 @@ class JavaLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'java' @@ -152,7 +166,10 @@ class GoLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'go' @@ -180,6 +197,9 @@ class Http2Client: def unimplemented_test_cases(self): return _TEST_CASES + def unimplemented_test_cases_server(self): + return [] + def __str__(self): return 'http2' @@ -203,7 +223,10 @@ class NodeLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'node' @@ -225,6 +248,9 @@ class PHPLanguage: return {} def unimplemented_test_cases(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): return [] def __str__(self): @@ -251,7 +277,10 @@ class RubyLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'ruby' @@ -289,7 +318,11 @@ class PythonLanguage: return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)} def unimplemented_test_cases(self): - return ['jwt_token_creds', 'per_rpc_creds'] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + ['jwt_token_creds', + 'per_rpc_creds'] + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'python' @@ -312,7 +345,9 @@ _SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby', 'python'] _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 'client_streaming', 'server_streaming', 'cancel_after_begin', 'cancel_after_first_response', - 'timeout_on_sleeping_server'] + 'timeout_on_sleeping_server', 'custom_metadata', + 'status_code_and_message', 'unimplemented_method', + 'large_compressed_unary', 'server_compressed_streaming'] _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] @@ -635,9 +670,10 @@ try: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec(language, test_case, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) + if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: + test_job = cloud_to_prod_jobspec(language, test_case, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable # this once a better solution is in place. @@ -664,16 +700,21 @@ try: for server_name, server_address in server_addresses.iteritems(): (server_host, server_port) = server_address + server_language = _LANGUAGES.get(server_name, None) + skip_server = [] # test cases unimplemented by server + if server_language: + skip_server = server_language.unimplemented_test_cases_server() for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_cloud_jobspec(language, - test_case, - server_name, - server_host, - server_port, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) + if not test_case in skip_server: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: -- cgit v1.2.3 From a86b41e7468c56cded1c4350bcd3d8f73d3af846 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 10 Dec 2015 08:00:28 -0800 Subject: Temporarily disable c# status_code_and_message client. --- tools/run_tests/run_interop_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 8f9eedadba..763ff5615c 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -107,7 +107,8 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + # TODO: status_code_and_message doesn't work against node_server + return _SKIP_COMPRESSION + ['status_code_and_message'] def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION -- cgit v1.2.3