diff options
| -rw-r--r-- | reports/interop_html_report.template | 141 | ||||
| -rw-r--r-- | tools/run_tests/report_utils.py | 168 | ||||
| -rwxr-xr-x | tools/run_tests/run_interop_tests.py | 4 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 2 |
4 files changed, 175 insertions, 140 deletions
diff --git a/reports/interop_html_report.template b/reports/interop_html_report.template new file mode 100644 index 0000000000..1ba2e6cfc2 --- /dev/null +++ b/reports/interop_html_report.template @@ -0,0 +1,141 @@ +<!DOCTYPE html> +<html lang="en"> +<head><title>Interop Test Result</title></head> +<body> + +<%def name="fill_one_test_result(shortname, resultset)"> + % if shortname in resultset: + ## Because interop tests does not have runs_per_test flag, each test is + ## run once. So there should only be one element for each result. + <% result = resultset[shortname][0] %> + % if result.state == 'PASSED': + <td bgcolor="green">PASS</td> + % else: + <% + tooltip = '' + if result.returncode > 0 or result.message: + if result.returncode > 0: + tooltip = 'returncode: %d ' % result.returncode + if result.message: + tooltip = '%smessage: %s' % (tooltip, result.message) + %> + % if result.state == 'FAILED': + <td bgcolor="red"> + % if tooltip: + <a href="#" data-toggle="tooltip" data-placement="auto" title="${tooltip | h}">FAIL</a></td> + % else: + FAIL</td> + % endif + % elif result.state == 'TIMEOUT': + <td bgcolor="yellow"> + % if tooltip: + <a href="#" data-toggle="tooltip" data-placement="auto" title="${tooltip | h}">TIMEOUT</a></td> + % else: + TIMEOUT</td> + % endif + % endif + % endif + % else: + <td bgcolor="magenta">Not implemented</td> + % endif +</%def> + +% if num_failures > 1: + <p><h2><font color="red">${num_failures} tests failed!</font></h2></p> +% elif num_failures: + <p><h2><font color="red">${num_failures} test failed!</font></h2></p> +% else: + <p><h2><font color="green">All tests passed!</font></h2></p> +% endif + +% if cloud_to_prod: + ## Each column header is the client language. + <h2>Cloud to Prod</h2> + <table style="width:100%" border="1"> + <tr bgcolor="#00BFFF"> + <th>Client languages ►<br/>Test Cases ▼</th> + % for client_lang in client_langs: + <th>${client_lang}</th> + % endfor + </tr> + % for test_case in test_cases + auth_test_cases: + <tr><td><b>${test_case}</b></td> + % for client_lang in client_langs: + <% + if test_case in auth_test_cases: + shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) + else: + shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + </tr> + % endfor + </table> +% endif + +% if http2_interop: + ## Each column header is the server language. + <h2>HTTP/2 Interop</h2> + <table style="width:100%" border="1"> + <tr bgcolor="#00BFFF"> + <th>Servers ►<br/>Test Cases ▼</th> + % for server_lang in server_langs: + <th>${server_lang}</th> + % endfor + % if cloud_to_prod: + <th>prod</th> + % endif + </tr> + % for test_case in http2_cases: + <tr><td><b>${test_case}</b></td> + ## Fill up the cells with test result. + % for server_lang in server_langs: + <% + shortname = 'cloud_to_cloud:http2:%s_server:%s' % ( + server_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + % if cloud_to_prod: + <% shortname = 'cloud_to_prod:http2:%s' % test_case %> + ${fill_one_test_result(shortname, resultset)} + % endif + </tr> + % endfor + </table> +% endif + +% if server_langs: + % for test_case in test_cases: + ## Each column header is the client language. + <h2>${test_case}</h2> + <table style="width:100%" border="1"> + <tr bgcolor="#00BFFF"> + <th>Client languages ►<br/>Server languages ▼</th> + % for client_lang in client_langs: + <th>${client_lang}</th> + % endfor + </tr> + ## Each row head is the server language. + % for server_lang in server_langs: + <tr> + <td><b>${server_lang}</b></td> + % for client_lang in client_langs: + <% + shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( + client_lang, server_lang, test_case) + %> + ${fill_one_test_result(shortname, resultset)} + % endfor + </tr> + % endfor + </table> + % endfor +% endif + +<script> + $(document).ready(function(){$('[data-toggle="tooltip"]').tooltip();}); +</script> +</body> +</html> diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index bb9eca4254..adeb707a07 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -29,6 +29,11 @@ """Generate XML and HTML test reports.""" +try: + from mako.runtime import Context + from mako.template import Template +except (ImportError): + pass # Mako not installed but it is ok. import os import string import xml.etree.cElementTree as ET @@ -49,7 +54,7 @@ def _filter_msg(msg, output_format): return msg -def render_xml_report(resultset, xml_report): +def render_junit_xml_report(resultset, xml_report): """Generate JUnit-like XML report.""" root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', @@ -69,147 +74,36 @@ def render_xml_report(resultset, xml_report): tree.write(xml_report, encoding='UTF-8') -# TODO(adelez): Use mako template. -def fill_one_test_result(shortname, resultset, html_str): - if shortname in resultset: - # Because interop tests does not have runs_per_test flag, each test is run - # once. So there should only be one element for each result. - result = resultset[shortname][0] - if result.state == 'PASSED': - html_str = '%s<td bgcolor=\"green\">PASS</td>\n' % html_str - else: - tooltip = '' - if result.returncode > 0 or result.message: - if result.returncode > 0: - tooltip = 'returncode: %d ' % result.returncode - if result.message: - escaped_msg = _filter_msg(result.message, 'HTML') - tooltip = '%smessage: %s' % (tooltip, escaped_msg) - if result.state == 'FAILED': - html_str = '%s<td bgcolor=\"red\">' % html_str - if tooltip: - html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" ' - 'data-placement=\"auto\" title=\"%s\">FAIL</a></td>\n' % - (html_str, tooltip)) - else: - html_str = '%sFAIL</td>\n' % html_str - elif result.state == 'TIMEOUT': - html_str = '%s<td bgcolor=\"yellow\">' % html_str - if tooltip: - html_str = ('%s<a href=\"#\" data-toggle=\"tooltip\" ' - 'data-placement=\"auto\" title=\"%s\">TIMEOUT</a></td>\n' - % (html_str, tooltip)) - else: - html_str = '%sTIMEOUT</td>\n' % html_str - else: - html_str = '%s<td bgcolor=\"magenta\">Not implemented</td>\n' % html_str - - return html_str - +def render_interop_html_report( + client_langs, server_langs, test_cases, auth_test_cases, http2_cases, + resultset, num_failures, cloud_to_prod, http2_interop): + """Generate HTML report for interop tests.""" + html_report_dir = 'reports' + template_file = os.path.join(html_report_dir, 'interop_html_report.template') + try: + mytemplate = Template(filename=template_file, format_exceptions=True) + except NameError: + print 'Mako template is not installed. Skipping HTML report generation.' + return + except IOError as e: + print 'Failed to find the template %s: %s' % (template_file, e) + return -def render_html_report(client_langs, server_langs, test_cases, auth_test_cases, - http2_cases, resultset, num_failures, cloud_to_prod, - http2_interop): - """Generate html report.""" sorted_test_cases = sorted(test_cases) sorted_auth_test_cases = sorted(auth_test_cases) sorted_http2_cases = sorted(http2_cases) sorted_client_langs = sorted(client_langs) sorted_server_langs = sorted(server_langs) - html_str = ('<!DOCTYPE html>\n' - '<html lang=\"en\">\n' - '<head><title>Interop Test Result</title></head>\n' - '<body>\n') - if num_failures > 1: - html_str = ( - '%s<p><h2><font color=\"red\">%d tests failed!</font></h2></p>\n' % - (html_str, num_failures)) - elif num_failures: - html_str = ( - '%s<p><h2><font color=\"red\">%d test failed!</font></h2></p>\n' % - (html_str, num_failures)) - else: - html_str = ( - '%s<p><h2><font color=\"green\">All tests passed!</font></h2></p>\n' % - html_str) - if cloud_to_prod: - # Each column header is the client language. - html_str = ('%s<h2>Cloud to Prod</h2>\n' - '<table style=\"width:100%%\" border=\"1\">\n' - '<tr bgcolor=\"#00BFFF\">\n' - '<th>Client languages ►</th>\n') % html_str - for client_lang in sorted_client_langs: - html_str = '%s<th>%s\n' % (html_str, client_lang) - html_str = '%s</tr>\n' % html_str - for test_case in sorted_test_cases + sorted_auth_test_cases: - html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case) - for client_lang in sorted_client_langs: - if not test_case in sorted_auth_test_cases: - shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) - else: - shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s</tr>\n' % html_str - html_str = '%s</table>\n' % html_str - if http2_interop: - # Each column header is the server language. - html_str = ('%s<h2>HTTP/2 Interop</h2>\n' - '<table style=\"width:100%%\" border=\"1\">\n' - '<tr bgcolor=\"#00BFFF\">\n' - '<th>Servers ►<br/>' - 'Test Cases ▼</th>\n') % html_str - for server_lang in sorted_server_langs: - html_str = '%s<th>%s\n' % (html_str, server_lang) - if cloud_to_prod: - html_str = '%s<th>%s\n' % (html_str, "prod") - html_str = '%s</tr>\n' % html_str - for test_case in sorted_http2_cases: - html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case) - # Fill up the cells with test result. - for server_lang in sorted_server_langs: - shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( - "http2", server_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - if cloud_to_prod: - shortname = 'cloud_to_prod:%s:%s' % ("http2", test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s</tr>\n' % html_str - html_str = '%s</table>\n' % html_str - if server_langs: - for test_case in sorted_test_cases: - # Each column header is the client language. - html_str = ('%s<h2>%s</h2>\n' - '<table style=\"width:100%%\" border=\"1\">\n' - '<tr bgcolor=\"#00BFFF\">\n' - '<th>Client languages ►<br/>' - 'Server languages ▼</th>\n') % (html_str, test_case) - for client_lang in sorted_client_langs: - html_str = '%s<th>%s\n' % (html_str, client_lang) - html_str = '%s</tr>\n' % html_str - # Each row head is the server language. - for server_lang in sorted_server_langs: - html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, server_lang) - # Fill up the cells with test result. - for client_lang in sorted_client_langs: - shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( - client_lang, server_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s</tr>\n' % html_str - html_str = '%s</table>\n' % html_str - html_str = ('%s\n' - '<script>\n' - '$(document).ready(function(){' - '$(\'[data-toggle=\"tooltip\"]\').tooltip();\n' - '});\n' - '</script>\n' - '</body>\n' - '</html>') % html_str - - # Write to reports/index.html as set up in Jenkins plugin. - html_report_dir = 'reports' - if not os.path.exists(html_report_dir): - os.mkdir(html_report_dir) + args = {'client_langs': sorted_client_langs, + 'server_langs': sorted_server_langs, + 'test_cases': sorted_test_cases, + 'auth_test_cases': sorted_auth_test_cases, + 'http2_cases': sorted_http2_cases, + 'resultset': resultset, + 'num_failures': num_failures, + 'cloud_to_prod': cloud_to_prod, + 'http2_interop': http2_interop} html_file_path = os.path.join(html_report_dir, 'index.html') - with open(html_file_path, 'w') as f: - f.write(html_str) + with open(html_file_path, 'w') as output_file: + mytemplate.render_context(Context(output_file, **args)) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 2634164a21..ee3cddddd9 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -686,9 +686,9 @@ try: else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) - report_utils.render_xml_report(resultset, 'report.xml') + report_utils.render_junit_xml_report(resultset, 'report.xml') - report_utils.render_html_report( + report_utils.render_interop_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, _HTTP2_TEST_CASES, resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ab2b71b80e..aa43337263 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -902,7 +902,7 @@ def _build_and_run( for antagonist in antagonists: antagonist.kill() if xml_report and resultset: - report_utils.render_xml_report(resultset, xml_report) + report_utils.render_junit_xml_report(resultset, xml_report) number_failures, _ = jobset.run( post_tests_steps, maxjobs=1, stop_on_failure=True, |