Interop Test Result

From 3b2e1bd243395c7b471bda2dc4fbdfb8248fe96d Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Fri, 6 Nov 2015 14:31:55 -0800 Subject: Enable interop tests for http2 --- tools/run_tests/run_interop_tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 1686942b0a..2634164a21 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -624,7 +624,9 @@ try: docker_image=docker_images.get(str(language))) jobs.append(test_job) - if args.http2_interop: + # TODO(carl-mastrangelo): Currently prod TLS terminators aren't spec compliant. Reenable + # this once a better solution is in place. + if args.http2_interop and False: for test_case in _HTTP2_TEST_CASES: test_job = cloud_to_prod_jobspec(http2Interop, test_case, docker_image=docker_images.get(str(http2Interop))) @@ -660,6 +662,9 @@ try: if args.http2_interop: for test_case in _HTTP2_TEST_CASES: + if server_name == "go": + # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 + continue test_job = cloud_to_cloud_jobspec(http2Interop, test_case, server_name, -- cgit v1.2.3 From 3bc7ba4d950796fcb92ce19d3072e26bb62420f0 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Thu, 5 Nov 2015 10:21:58 -0800 Subject: Use mako template for HTML report. --- templates/interop_html_report.template | 141 +++++++++++++++++++++++++++ tools/run_tests/report_utils.py | 172 +++++++-------------------------- tools/run_tests/run_interop_tests.py | 4 +- tools/run_tests/run_tests.py | 2 +- 4 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 templates/interop_html_report.template (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/templates/interop_html_report.template b/templates/interop_html_report.template new file mode 100644 index 0000000000..1ba2e6cfc2 --- /dev/null +++ b/templates/interop_html_report.template @@ -0,0 +1,141 @@ + + +Interop Test Result + + +<%def name="fill_one_test_result(shortname, resultset)"> + % if shortname in resultset: + ## Because interop tests does not have runs_per_test flag, each test is + ## run once. So there should only be one element for each result. + <% result = resultset[shortname][0] %> + % if result.state == 'PASSED': + PASS + % else: + <% + tooltip = '' + if result.returncode > 0 or result.message: + if result.returncode > 0: + tooltip = 'returncode: %d ' % result.returncode + if result.message: + tooltip = '%smessage: %s' % (tooltip, result.message) + %> + % if result.state == 'FAILED': + + % if tooltip: + FAIL + % else: + FAIL + % endif + % elif result.state == 'TIMEOUT': + + % if tooltip: + TIMEOUT + % else: + TIMEOUT + % endif + % endif + % endif + % else: + Not implemented + % endif + + +% if num_failures > 1: +

${num_failures} tests failed!

+% elif num_failures: +

${num_failures} test failed!

+% else: +

All tests passed!

+% endif + +% if cloud_to_prod: + ## Each column header is the client language. +

Cloud to Prod

+ + + + % for client_lang in client_langs: + + % endfor + + % for test_case in test_cases + auth_test_cases: + + % for client_lang in client_langs: + <% + if test_case in auth_test_cases: + shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) + else: + shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + + % endfor +

Client languages ► Test Cases ▼	${client_lang}
${test_case}

+% endif + +% if http2_interop: + ## Each column header is the server language. +

HTTP/2 Interop

+ + + + % for server_lang in server_langs: + + % endfor + % if cloud_to_prod: + + % endif + + % for test_case in http2_cases: + + ## Fill up the cells with test result. + % for server_lang in server_langs: + <% + shortname = 'cloud_to_cloud:http2:%s_server:%s' % ( + server_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + % if cloud_to_prod: + <% shortname = 'cloud_to_prod:http2:%s' % test_case %> + ${fill_one_test_result(shortname, resultset)} + % endif + + % endfor +

Servers ► Test Cases ▼	${server_lang}	prod
${test_case}

+% endif + +% if server_langs: + % for test_case in test_cases: + ## Each column header is the client language. +

${test_case}

+ + + + % for client_lang in client_langs: + + % endfor + + ## Each row head is the server language. + % for server_lang in server_langs: + + + % for client_lang in client_langs: + <% + shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( + client_lang, server_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + + % endfor +

Client languages ► Server languages ▼	${client_lang}
${server_lang}

+ % endfor +% endif + + + + diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index bb9eca4254..dcc66f057d 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -29,6 +29,11 @@ """Generate XML and HTML test reports.""" +try: + from mako.runtime import Context + from mako.template import Template +except (ImportError): + pass # Mako not installed but it is ok. import os import string import xml.etree.cElementTree as ET @@ -49,7 +54,7 @@ def _filter_msg(msg, output_format): return msg -def render_xml_report(resultset, xml_report): +def render_junit_xml_report(resultset, xml_report): """Generate JUnit-like XML report.""" root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', @@ -69,147 +74,40 @@ def render_xml_report(resultset, xml_report): tree.write(xml_report, encoding='UTF-8') -# TODO(adelez): Use mako template. -def fill_one_test_result(shortname, resultset, html_str): - if shortname in resultset: - # Because interop tests does not have runs_per_test flag, each test is run - # once. So there should only be one element for each result. - result = resultset[shortname][0] - if result.state == 'PASSED': - html_str = '%sPASS\n' % html_str - else: - tooltip = '' - if result.returncode > 0 or result.message: - if result.returncode > 0: - tooltip = 'returncode: %d ' % result.returncode - if result.message: - escaped_msg = _filter_msg(result.message, 'HTML') - tooltip = '%smessage: %s' % (tooltip, escaped_msg) - if result.state == 'FAILED': - html_str = '%s' % html_str - if tooltip: - html_str = ('%sFAIL\n' % - (html_str, tooltip)) - else: - html_str = '%sFAIL\n' % html_str - elif result.state == 'TIMEOUT': - html_str = '%s' % html_str - if tooltip: - html_str = ('%sTIMEOUT\n' - % (html_str, tooltip)) - else: - html_str = '%sTIMEOUT\n' % html_str - else: - html_str = '%sNot implemented\n' % html_str - - return html_str +def render_interop_html_report( + client_langs, server_langs, test_cases, auth_test_cases, http2_cases, + resultset, num_failures, cloud_to_prod, http2_interop): + """Generate HTML report for interop tests.""" + template_file = 'templates/interop_html_report.template' + try: + mytemplate = Template(filename=template_file, format_exceptions=True) + except NameError: + print 'Mako template is not installed. Skipping HTML report generation.' + return + except IOError as e: + print 'Failed to find the template %s: %s' % (template_file, e) + return + # Write to reports/index.html as set up in Jenkins plugin. + html_report_dir = 'reports' + if not os.path.exists(html_report_dir): + os.mkdir(html_report_dir) + html_file_path = os.path.join(html_report_dir, 'index.html') -def render_html_report(client_langs, server_langs, test_cases, auth_test_cases, - http2_cases, resultset, num_failures, cloud_to_prod, - http2_interop): - """Generate html report.""" sorted_test_cases = sorted(test_cases) sorted_auth_test_cases = sorted(auth_test_cases) sorted_http2_cases = sorted(http2_cases) sorted_client_langs = sorted(client_langs) sorted_server_langs = sorted(server_langs) - html_str = ('\n' - '\n' - 'Interop Test Result\n' - '\n') - if num_failures > 1: - html_str = ( - '%s

%d tests failed!

\n' % - (html_str, num_failures)) - elif num_failures: - html_str = ( - '%s

%d test failed!

\n' % - (html_str, num_failures)) - else: - html_str = ( - '%s

All tests passed!

\n' % - html_str) - if cloud_to_prod: - # Each column header is the client language. - html_str = ('%s

Cloud to Prod

\n' - '\n' - '\n' - '\n') % html_str - for client_lang in sorted_client_langs: - html_str = '%s\n' % html_str - for test_case in sorted_test_cases + sorted_auth_test_cases: - html_str = '%s\n' % (html_str, test_case) - for client_lang in sorted_client_langs: - if not test_case in sorted_auth_test_cases: - shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) - else: - shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s\n' % html_str - html_str = '%s

Client languages ►	%s\n' % (html_str, client_lang) - html_str = '%s
%s

\n' % html_str - if http2_interop: - # Each column header is the server language. - html_str = ('%s

HTTP/2 Interop

\n' - '\n' - '\n' - '\n') % html_str - for server_lang in sorted_server_langs: - html_str = '%s\n' % html_str - for test_case in sorted_http2_cases: - html_str = '%s\n' % (html_str, test_case) - # Fill up the cells with test result. - for server_lang in sorted_server_langs: - shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( - "http2", server_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - if cloud_to_prod: - shortname = 'cloud_to_prod:%s:%s' % ("http2", test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s\n' % html_str - html_str = '%s

Servers ► ' - 'Test Cases ▼	%s\n' % (html_str, server_lang) - if cloud_to_prod: - html_str = '%s	%s\n' % (html_str, "prod") - html_str = '%s
%s

\n' % html_str - if server_langs: - for test_case in sorted_test_cases: - # Each column header is the client language. - html_str = ('%s

%s

\n' - '\n' - '\n' - '\n') % (html_str, test_case) - for client_lang in sorted_client_langs: - html_str = '%s\n' % html_str - # Each row head is the server language. - for server_lang in sorted_server_langs: - html_str = '%s\n' % (html_str, server_lang) - # Fill up the cells with test result. - for client_lang in sorted_client_langs: - shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( - client_lang, server_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s\n' % html_str - html_str = '%s

Client languages ► ' - 'Server languages ▼	%s\n' % (html_str, client_lang) - html_str = '%s
%s

\n' % html_str - html_str = ('%s\n' - '\n' - '\n' - '') % html_str - - # Write to reports/index.html as set up in Jenkins plugin. - html_report_dir = 'reports' - if not os.path.exists(html_report_dir): - os.mkdir(html_report_dir) - html_file_path = os.path.join(html_report_dir, 'index.html') - with open(html_file_path, 'w') as f: - f.write(html_str) + args = {'client_langs': sorted_client_langs, + 'server_langs': sorted_server_langs, + 'test_cases': sorted_test_cases, + 'auth_test_cases': sorted_auth_test_cases, + 'http2_cases': sorted_http2_cases, + 'resultset': resultset, + 'num_failures': num_failures, + 'cloud_to_prod': cloud_to_prod, + 'http2_interop': http2_interop} + with open(html_file_path, 'w') as output_file: + mytemplate.render_context(Context(output_file, **args)) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 2634164a21..ee3cddddd9 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -686,9 +686,9 @@ try: else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) - report_utils.render_xml_report(resultset, 'report.xml') + report_utils.render_junit_xml_report(resultset, 'report.xml') - report_utils.render_html_report( + report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ab2b71b80e..aa43337263 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -902,7 +902,7 @@ def _build_and_run( for antagonist in antagonists: antagonist.kill() if xml_report and resultset: - report_utils.render_xml_report(resultset, xml_report) + report_utils.render_junit_xml_report(resultset, xml_report) number_failures, _ = jobset.run( post_tests_steps, maxjobs=1, stop_on_failure=True, -- cgit v1.2.3 From 2dd55db4c10d32a5d7442e977fed878c3feb72c0 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Thu, 19 Nov 2015 10:51:48 -0800 Subject: Add framing http2 test case, enable verbose output, and properly skip tests --- tools/http2_interop/http2interop_test.go | 17 +++++++++++++---- tools/run_tests/run_interop_tests.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/http2_interop/http2interop_test.go b/tools/http2_interop/http2interop_test.go index 8fd838422b..e3d366f2f2 100644 --- a/tools/http2_interop/http2interop_test.go +++ b/tools/http2_interop/http2interop_test.go @@ -17,7 +17,7 @@ var ( serverHost = flag.String("server_host", "", "The host to test") serverPort = flag.Int("server_port", 443, "The port to test") useTls = flag.Bool("use_tls", true, "Should TLS tests be run") - testCase = flag.String("test_case", "", "What test cases to run") + testCase = flag.String("test_case", "", "What test cases to run (tls, framing)") // The rest of these are unused, but present to fulfill the client interface serverHostOverride = flag.String("server_host_override", "", "Unused") @@ -69,6 +69,9 @@ func (ctx *HTTP2InteropCtx) Close() error { } func TestShortPreface(t *testing.T) { + if *testCase != "framing" { + t.SkipNow() + } ctx := InteropCtx(t) for i := 0; i < len(Preface)-1; i++ { if err := testShortPreface(ctx, Preface[:i]+"X"); err != io.EOF { @@ -78,6 +81,9 @@ func TestShortPreface(t *testing.T) { } func TestUnknownFrameType(t *testing.T) { + if *testCase != "framing" { + t.SkipNow() + } ctx := InteropCtx(t) if err := testUnknownFrameType(ctx); err != nil { t.Fatal(err) @@ -86,7 +92,7 @@ func TestUnknownFrameType(t *testing.T) { func TestTLSApplicationProtocol(t *testing.T) { if *testCase != "tls" { - return + t.SkipNow() } ctx := InteropCtx(t) err := testTLSApplicationProtocol(ctx) @@ -95,7 +101,7 @@ func TestTLSApplicationProtocol(t *testing.T) { func TestTLSMaxVersion(t *testing.T) { if *testCase != "tls" { - return + t.SkipNow() } ctx := InteropCtx(t) err := testTLSMaxVersion(ctx, tls.VersionTLS11) @@ -106,7 +112,7 @@ func TestTLSMaxVersion(t *testing.T) { func TestTLSBadCipherSuites(t *testing.T) { if *testCase != "tls" { - return + t.SkipNow() } ctx := InteropCtx(t) err := testTLSBadCipherSuites(ctx) @@ -114,6 +120,9 @@ func TestTLSBadCipherSuites(t *testing.T) { } func TestClientPrefaceWithStreamId(t *testing.T) { + if *testCase != "framing" { + t.SkipNow() + } ctx := InteropCtx(t) err := testClientPrefaceWithStreamId(ctx) matchError(t, err, "EOF") diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ee3cddddd9..b1fb657733 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -170,7 +170,7 @@ class Http2Client: self.safename = str(self) def client_args(self): - return ['tools/http2_interop/http2_interop.test'] + return ['tools/http2_interop/http2_interop.test', '-v'] def cloud_to_prod_env(self): return {} @@ -306,7 +306,7 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds', 'oauth2_auth_token', 'per_rpc_creds'] -_HTTP2_TEST_CASES = ["tls"] +_HTTP2_TEST_CASES = ["tls", "framing"] def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): """Wraps given cmdline array to create 'docker run' cmdline from it.""" -- cgit v1.2.3 From 59096b404462742a769696e56510c7ea1bdc6946 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Thu, 19 Nov 2015 12:54:01 -0800 Subject: fix flag --- tools/run_tests/run_interop_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index b1fb657733..747ba4bd59 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -170,7 +170,7 @@ class Http2Client: self.safename = str(self) def client_args(self): - return ['tools/http2_interop/http2_interop.test', '-v'] + return ['tools/http2_interop/http2_interop.test', '-test.v'] def cloud_to_prod_env(self): return {} -- cgit v1.2.3 From 2d248a29f73be0cc9a6514afbe392ec0ed990485 Mon Sep 17 00:00:00 2001 From: Carl Mastrangelo Date: Thu, 19 Nov 2015 13:09:52 -0800 Subject: Fix stdin on run_interop_tests --- tools/run_tests/dockerjob.py | 3 +++ tools/run_tests/run_interop_tests.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/run_tests/run_interop_tests.py') diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py index 7d64222ba0..326c4faed9 100755 --- a/tools/run_tests/dockerjob.py +++ b/tools/run_tests/dockerjob.py @@ -47,6 +47,7 @@ def random_name(base_name): def docker_kill(cid): """Kills a docker container. Returns True if successful.""" return subprocess.call(['docker','kill', str(cid)], + stdin=subprocess.PIPE, stdout=_DEVNULL, stderr=subprocess.STDOUT) == 0 @@ -78,6 +79,7 @@ def finish_jobs(jobs): def image_exists(image): """Returns True if given docker image exists.""" return subprocess.call(['docker','inspect', image], + stdin=subprocess.PIPE, stdout=_DEVNULL, stderr=subprocess.STDOUT) == 0 @@ -88,6 +90,7 @@ def remove_image(image, skip_nonexistent=False, max_retries=10): return True for attempt in range(0, max_retries): if subprocess.call(['docker','rmi', '-f', image], + stdin=subprocess.PIPE, stdout=_DEVNULL, stderr=subprocess.STDOUT) == 0: return True diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index ee3cddddd9..37b8ab19f6 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -37,7 +37,6 @@ import jobset import multiprocessing import os import report_utils -import subprocess import sys import tempfile import time -- cgit v1.2.3