diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/codegen/core/gen_legal_metadata_characters.c | 1 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 2 | ||||
| -rw-r--r-- | tools/run_tests/interop_html_report.template | 24 | ||||
| -rwxr-xr-x | tools/run_tests/jobset.py | 3 | ||||
| -rw-r--r-- | tools/run_tests/report_utils.py | 11 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 120 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 81 | ||||
| -rw-r--r-- | tools/run_tests/sources_and_headers.json | 48 | ||||
| -rw-r--r-- | tools/run_tests/tests.json | 48 |
9 files changed, 298 insertions, 40 deletions
diff --git a/tools/codegen/core/gen_legal_metadata_characters.c b/tools/codegen/core/gen_legal_metadata_characters.c index 677fa5c155..c6658f46c6 100644 --- a/tools/codegen/core/gen_legal_metadata_characters.c +++ b/tools/codegen/core/gen_legal_metadata_characters.c @@ -72,7 +72,6 @@ int main(void) { clear(); for (i = 32; i <= 126; i++) { - if (i == ',') continue; legal(i); } dump(); diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index e441d9e695..b998a38ebb 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -790,7 +790,6 @@ src/core/channel/connected_channel.h \ src/core/channel/context.h \ src/core/channel/http_client_filter.h \ src/core/channel/http_server_filter.h \ -src/core/channel/noop_filter.h \ src/core/channel/subchannel_call_holder.h \ src/core/client_config/client_config.h \ src/core/client_config/connector.h \ @@ -928,7 +927,6 @@ src/core/channel/compress_filter.c \ src/core/channel/connected_channel.c \ src/core/channel/http_client_filter.c \ src/core/channel/http_server_filter.c \ -src/core/channel/noop_filter.c \ src/core/channel/subchannel_call_holder.c \ src/core/client_config/client_config.c \ src/core/client_config/connector.c \ diff --git a/tools/run_tests/interop_html_report.template b/tools/run_tests/interop_html_report.template index 1ba2e6cfc2..c01bdf7a77 100644 --- a/tools/run_tests/interop_html_report.template +++ b/tools/run_tests/interop_html_report.template @@ -40,6 +40,26 @@ % endif </%def> +<%def name="fill_one_http2_test_result(shortname, resultset)"> + ## keep this mostly in sync with the template above + % if shortname in resultset: + ## Because interop tests does not have runs_per_test flag, each test is + ## run once. So there should only be one element for each result. + <% result = resultset[shortname][0] %> + <td bgcolor="white"> + <div style="width:95%; border: 1px solid black; position: relative; padding: 3px;"> + <span style="position: absolute; left: 45%;">${int(result.http2results['percent'] * 100)}%</span> + <div style="height: 20px; + background-color: hsl(${result.http2results['percent'] * 120}, 100%, 50%); + width: ${result.http2results['percent'] * 100}%;" + title="${result.http2results['failed_cases'] | h}"></div> + </div> + </td> + % else: + <td bgcolor="magenta">Not implemented</td> + % endif +</%def> + % if num_failures > 1: <p><h2><font color="red">${num_failures} tests failed!</font></h2></p> % elif num_failures: @@ -95,11 +115,11 @@ shortname = 'cloud_to_cloud:http2:%s_server:%s' % ( server_lang, test_case) %> - ${fill_one_test_result(shortname, resultset)} + ${fill_one_http2_test_result(shortname, resultset)} % endfor % if cloud_to_prod: <% shortname = 'cloud_to_prod:http2:%s' % test_case %> - ${fill_one_test_result(shortname, resultset)} + ${fill_one_http2_test_result(shortname, resultset)} % endif </tr> % endfor diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 5a26bff709..01739be27c 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -178,6 +178,9 @@ class JobSpec(object): def __cmp__(self, other): return self.identity() == other.identity() + + def __repr__(self): + return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline) class JobResult(object): diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index 12b1972f1a..35f2069bee 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -32,6 +32,7 @@ try: from mako.runtime import Context from mako.template import Template + from mako import exceptions except (ImportError): pass # Mako not installed but it is ok. import os @@ -103,9 +104,15 @@ def render_interop_html_report( 'num_failures': num_failures, 'cloud_to_prod': cloud_to_prod, 'http2_interop': http2_interop} + html_report_out_dir = 'reports' if not os.path.exists(html_report_out_dir): os.mkdir(html_report_out_dir) html_file_path = os.path.join(html_report_out_dir, 'index.html') - with open(html_file_path, 'w') as output_file: - mytemplate.render_context(Context(output_file, **args)) + try: + with open(html_file_path, 'w') as output_file: + mytemplate.render_context(Context(output_file, **args)) + except: + print(exceptions.text_error_template().render()) + raise + diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 37b631bd0d..7a09feb70d 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -31,17 +31,24 @@ """Run interop (cross-language) tests in parallel.""" import argparse +import atexit import dockerjob import itertools import jobset +import json import multiprocessing import os +import re import report_utils +import subprocess import sys import tempfile import time import uuid +# Docker doesn't clean up after itself, so we do it on exit. +atexit.register(lambda: subprocess.call(['stty', 'echo'])) + ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(ROOT) @@ -52,6 +59,11 @@ _DEFAULT_SERVER_PORT=8080 # supported by C core SslCredentials instead. _SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' } +_SKIP_COMPRESSION = ['large_compressed_unary', + 'server_compressed_streaming'] + +_SKIP_ADVANCED = ['custom_metadata', 'status_code_and_message', + 'unimplemented_method'] class CXXLanguage: @@ -73,7 +85,10 @@ class CXXLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'c++' @@ -99,7 +114,11 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - return [] + # TODO: status_code_and_message doesn't work against node_server + return _SKIP_COMPRESSION + ['status_code_and_message'] + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'csharp' @@ -125,7 +144,10 @@ class JavaLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'java' @@ -152,7 +174,10 @@ class GoLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'go' @@ -180,6 +205,9 @@ class Http2Client: def unimplemented_test_cases(self): return _TEST_CASES + def unimplemented_test_cases_server(self): + return [] + def __str__(self): return 'http2' @@ -203,7 +231,10 @@ class NodeLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION def __str__(self): return 'node' @@ -225,6 +256,9 @@ class PHPLanguage: return {} def unimplemented_test_cases(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): return [] def __str__(self): @@ -251,7 +285,10 @@ class RubyLanguage: return {} def unimplemented_test_cases(self): - return [] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'ruby' @@ -289,7 +326,11 @@ class PythonLanguage: return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT)} def unimplemented_test_cases(self): - return ['jwt_token_creds', 'per_rpc_creds'] + return _SKIP_ADVANCED + _SKIP_COMPRESSION + ['jwt_token_creds', + 'per_rpc_creds'] + + def unimplemented_test_cases_server(self): + return _SKIP_ADVANCED + _SKIP_COMPRESSION def __str__(self): return 'python' @@ -312,7 +353,9 @@ _SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby', 'python'] _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'empty_stream', 'client_streaming', 'server_streaming', 'cancel_after_begin', 'cancel_after_first_response', - 'timeout_on_sleeping_server'] + 'timeout_on_sleeping_server', 'custom_metadata', + 'status_code_and_message', 'unimplemented_method', + 'large_compressed_unary', 'server_compressed_streaming'] _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] @@ -513,6 +556,33 @@ def build_interop_image_jobspec(language, tag=None): return build_job +def aggregate_http2_results(stdout): + match = re.search(r'\{"cases[^\]]*\]\}', stdout) + if not match: + return None + + results = json.loads(match.group(0)) + skipped = 0 + passed = 0 + failed = 0 + failed_cases = [] + for case in results['cases']: + if case.get('skipped', False): + skipped += 1 + else: + if case.get('passed', False): + passed += 1 + else: + failed += 1 + failed_cases.append(case.get('name', "NONAME")) + return { + 'passed': passed, + 'failed': failed, + 'skipped': skipped, + 'failed_cases': ', '.join(failed_cases), + 'percent': 1.0 * passed / (passed + failed) + } + argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), @@ -635,13 +705,12 @@ try: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec(language, test_case, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) + if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: + test_job = cloud_to_prod_jobspec(language, test_case, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) - # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable - # this once a better solution is in place. - if args.http2_interop and False: + if args.http2_interop: for test_case in _HTTP2_TEST_CASES: test_job = cloud_to_prod_jobspec(http2Interop, test_case, docker_image=docker_images.get(str(http2Interop))) @@ -664,16 +733,21 @@ try: for server_name, server_address in server_addresses.iteritems(): (server_host, server_port) = server_address + server_language = _LANGUAGES.get(server_name, None) + skip_server = [] # test cases unimplemented by server + if server_language: + skip_server = server_language.unimplemented_test_cases_server() for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_cloud_jobspec(language, - test_case, - server_name, - server_host, - server_port, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) + if not test_case in skip_server: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port, + docker_image=docker_images.get(str(language))) + jobs.append(test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -703,6 +777,10 @@ try: report_utils.render_junit_xml_report(resultset, 'report.xml') + for name, job in resultset.iteritems(): + if "http2" in name: + job[0].http2results = aggregate_http2_results(job[0].message) + report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4c85f202f4..006f4bcdf1 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -152,7 +152,10 @@ class CLanguage(object): else: binary = 'bins/%s/%s' % (config.build_config, target['name']) if os.path.isfile(binary): - out.append(config.job_spec([binary], [binary])) + out.append(config.job_spec([binary], [binary], + environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + os.path.abspath(os.path.dirname( + sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) elif args.regex == '.*' or platform_string() == 'windows': print '\nWARNING: binary not found, skipping', binary return sorted(out) @@ -192,6 +195,7 @@ class CLanguage(object): def __str__(self): return self.make_target + class NodeLanguage(object): def test_specs(self, config, args): @@ -508,6 +512,43 @@ _WINDOWS_CONFIG = { } +def _windows_arch_option(arch): + """Returns msbuild cmdline option for selected architecture.""" + if arch == 'default' or arch == 'windows_x86': + return '/p:Platform=Win32' + elif arch == 'windows_x64': + return '/p:Platform=x64' + else: + print 'Architecture %s not supported on current platform.' % arch + sys.exit(1) + + +def _windows_build_bat(compiler): + """Returns name of build.bat for selected compiler.""" + if compiler == 'default' or compiler == 'vs2013': + return 'vsprojects\\build_vs2013.bat' + elif compiler == 'vs2015': + return 'vsprojects\\build_vs2015.bat' + elif compiler == 'vs2010': + return 'vsprojects\\build_vs2010.bat' + else: + print 'Compiler %s not supported.' % compiler + sys.exit(1) + + +def _windows_toolset_option(compiler): + """Returns msbuild PlatformToolset for selected compiler.""" + if compiler == 'default' or compiler == 'vs2013': + return '/p:PlatformToolset=v120' + elif compiler == 'vs2015': + return '/p:PlatformToolset=v140' + elif compiler == 'vs2010': + return '/p:PlatformToolset=v100' + else: + print 'Compiler %s not supported.' % compiler + sys.exit(1) + + def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -572,6 +613,19 @@ argp.add_argument('--allow_flakes', action='store_const', const=True, help='Allow flaky tests to show as passing (re-runs failed tests up to five times)') +argp.add_argument('--arch', + choices=['default', 'windows_x86', 'windows_x64'], + default='default', + help='Selects architecture to target. For some platforms "default" is the only supported choice.') +argp.add_argument('--compiler', + choices=['default', 'vs2010', 'vs2013', 'vs2015'], + default='default', + help='Selects compiler to use. For some platforms "default" is the only supported choice.') +argp.add_argument('--build_only', + default=False, + action='store_const', + const=True, + help='Perform all the build steps but dont run any tests.') argp.add_argument('-a', '--antagonists', default=0, type=int) argp.add_argument('-x', '--xml_report', default=None, type=str, help='Generates a JUnit-compatible XML report') @@ -633,6 +687,14 @@ if len(build_configs) > 1: print language, 'does not support multiple build configurations' sys.exit(1) +if platform_string() != 'windows': + if args.arch != 'default': + print 'Architecture %s not supported on current platform.' % args.arch + sys.exit(1) + if args.compiler != 'default': + print 'Compiler %s not supported on current platform.' % args.compiler + sys.exit(1) + if platform_string() == 'windows': def make_jobspec(cfg, targets, makefile='Makefile'): extra_args = [] @@ -643,9 +705,11 @@ if platform_string() == 'windows': # disable PDB generation: it's broken, and we don't need it during CI extra_args.extend(['/p:Jenkins=true']) return [ - jobset.JobSpec(['vsprojects\\build.bat', + jobset.JobSpec([_windows_build_bat(args.compiler), 'vsprojects\\%s.sln' % target, - '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg]] + + '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg], + _windows_toolset_option(args.compiler), + _windows_arch_option(args.arch)] + extra_args, shell=True, timeout_seconds=90*60) for target in targets] @@ -840,7 +904,7 @@ def _calculate_num_runs_failures(list_of_results): def _build_and_run( - check_cancelled, newline_on_success, cache, xml_report=None): + check_cancelled, newline_on_success, cache, xml_report=None, build_only=False): """Do one pass of building & running tests.""" # build latest sequentially num_failures, _ = jobset.run( @@ -848,6 +912,9 @@ def _build_and_run( newline_on_success=newline_on_success, travis=args.travis) if num_failures: return 1 + + if build_only: + return 0 # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) @@ -925,7 +992,8 @@ if forever: previous_success = success success = _build_and_run(check_cancelled=have_files_changed, newline_on_success=False, - cache=test_cache) == 0 + cache=test_cache, + build_only=args.build_only) == 0 if not previous_success and success: jobset.message('SUCCESS', 'All tests are now passing properly', @@ -937,7 +1005,8 @@ else: result = _build_and_run(check_cancelled=lambda: False, newline_on_success=args.newline_on_success, cache=test_cache, - xml_report=args.xml_report) + xml_report=args.xml_report, + build_only=args.build_only) if result == 0: jobset.message('SUCCESS', 'All tests passed', do_newline=True) else: diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 56f74a9c99..704131d5be 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -751,6 +751,20 @@ ], "headers": [], "language": "c", + "name": "httpscli_test", + "src": [ + "test/core/httpcli/httpscli_test.c" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "init_test", "src": [ "test/core/surface/init_test.c" @@ -939,6 +953,20 @@ "gpr", "gpr_test_util", "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", + "name": "server_chttp2_test", + "src": [ + "test/core/surface/server_chttp2_test.c" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", "grpc_test_util", "test_tcp_server" ], @@ -1084,6 +1112,20 @@ ], "headers": [], "language": "c", + "name": "transport_connectivity_state_test", + "src": [ + "test/core/transport/connectivity_state_test.c" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "transport_metadata_test", "src": [ "test/core/transport/metadata_test.c" @@ -14473,7 +14515,6 @@ "src/core/channel/context.h", "src/core/channel/http_client_filter.h", "src/core/channel/http_server_filter.h", - "src/core/channel/noop_filter.h", "src/core/channel/subchannel_call_holder.h", "src/core/client_config/client_config.h", "src/core/client_config/connector.h", @@ -14628,8 +14669,6 @@ "src/core/channel/http_client_filter.h", "src/core/channel/http_server_filter.c", "src/core/channel/http_server_filter.h", - "src/core/channel/noop_filter.c", - "src/core/channel/noop_filter.h", "src/core/channel/subchannel_call_holder.c", "src/core/channel/subchannel_call_holder.h", "src/core/client_config/client_config.c", @@ -14985,7 +15024,6 @@ "src/core/channel/context.h", "src/core/channel/http_client_filter.h", "src/core/channel/http_server_filter.h", - "src/core/channel/noop_filter.h", "src/core/channel/subchannel_call_holder.h", "src/core/client_config/client_config.h", "src/core/client_config/connector.h", @@ -15126,8 +15164,6 @@ "src/core/channel/http_client_filter.h", "src/core/channel/http_server_filter.c", "src/core/channel/http_server_filter.h", - "src/core/channel/noop_filter.c", - "src/core/channel/noop_filter.h", "src/core/channel/subchannel_call_holder.c", "src/core/channel/subchannel_call_holder.h", "src/core/client_config/client_config.c", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index f579a2d518..289a13d8c4 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -853,6 +853,18 @@ }, { "ci_platforms": [ + "linux" + ], + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "httpscli_test", + "platforms": [ + "linux" + ] + }, + { + "ci_platforms": [ "linux", "mac", "posix", @@ -1077,6 +1089,24 @@ "exclude_configs": [], "flaky": false, "language": "c", + "name": "server_chttp2_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "exclude_configs": [], + "flaky": false, + "language": "c", "name": "set_initial_connect_string_test", "platforms": [ "linux", @@ -1251,6 +1281,24 @@ "exclude_configs": [], "flaky": false, "language": "c", + "name": "transport_connectivity_state_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "exclude_configs": [], + "flaky": false, + "language": "c", "name": "transport_metadata_test", "platforms": [ "linux", |