Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 1 | # Copyright 2015 gRPC authors. |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 2 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 6 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 8 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 14 | """Generate XML and HTML test reports.""" |
| 15 | |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 16 | from __future__ import print_function |
| 17 | |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 18 | try: |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 19 | from mako.runtime import Context |
| 20 | from mako.template import Template |
| 21 | from mako import exceptions |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 22 | except (ImportError): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 23 | pass # Mako not installed but it is ok. |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 24 | import datetime |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 25 | import os |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 26 | import string |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 27 | import xml.etree.cElementTree as ET |
Siddharth Shukla | d194f59 | 2017-03-11 19:12:43 +0100 | [diff] [blame] | 28 | import six |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 29 | |
| 30 | |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 31 | def _filter_msg(msg, output_format): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 32 | """Filters out nonprintable and illegal characters from the message.""" |
| 33 | if output_format in ['XML', 'HTML']: |
| 34 | # keep whitespaces but remove formfeed and vertical tab characters |
| 35 | # that make XML report unparseable. |
| 36 | filtered_msg = filter( |
| 37 | lambda x: x in string.printable and x != '\f' and x != '\v', |
| 38 | msg.decode('UTF-8', 'ignore')) |
| 39 | if output_format == 'HTML': |
| 40 | filtered_msg = filtered_msg.replace('"', '"') |
| 41 | return filtered_msg |
| 42 | else: |
| 43 | return msg |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 44 | |
| 45 | |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 46 | def new_junit_xml_tree(): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 47 | return ET.ElementTree(ET.Element('testsuites')) |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 48 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 49 | |
| 50 | def render_junit_xml_report(resultset, |
| 51 | report_file, |
| 52 | suite_package='grpc', |
Jan Tattermusch | cfcc075 | 2016-10-09 17:02:34 +0200 | [diff] [blame] | 53 | suite_name='tests'): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 54 | """Generate JUnit-like XML report.""" |
| 55 | tree = new_junit_xml_tree() |
| 56 | append_junit_xml_results(tree, resultset, suite_package, suite_name, '1') |
| 57 | create_xml_report_file(tree, report_file) |
| 58 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 59 | |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 60 | def create_xml_report_file(tree, report_file): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 61 | """Generate JUnit-like report file from xml tree .""" |
| 62 | # ensure the report directory exists |
| 63 | report_dir = os.path.dirname(os.path.abspath(report_file)) |
| 64 | if not os.path.exists(report_dir): |
| 65 | os.makedirs(report_dir) |
| 66 | tree.write(report_file, encoding='UTF-8') |
| 67 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 68 | |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 69 | def append_junit_xml_results(tree, resultset, suite_package, suite_name, id): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 70 | """Append a JUnit-like XML report tree with test results as a new suite.""" |
| 71 | testsuite = ET.SubElement( |
| 72 | tree.getroot(), |
| 73 | 'testsuite', |
| 74 | id=id, |
| 75 | package=suite_package, |
| 76 | name=suite_name, |
| 77 | timestamp=datetime.datetime.now().isoformat()) |
| 78 | failure_count = 0 |
| 79 | error_count = 0 |
| 80 | for shortname, results in six.iteritems(resultset): |
| 81 | for result in results: |
| 82 | xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) |
| 83 | if result.elapsed_time: |
| 84 | xml_test.set('time', str(result.elapsed_time)) |
| 85 | filtered_msg = _filter_msg(result.message, 'XML') |
| 86 | if result.state == 'FAILED': |
| 87 | ET.SubElement( |
| 88 | xml_test, 'failure', message='Failure').text = filtered_msg |
| 89 | failure_count += 1 |
| 90 | elif result.state == 'TIMEOUT': |
| 91 | ET.SubElement( |
| 92 | xml_test, 'error', message='Timeout').text = filtered_msg |
| 93 | error_count += 1 |
| 94 | elif result.state == 'SKIPPED': |
| 95 | ET.SubElement(xml_test, 'skipped', message='Skipped') |
| 96 | testsuite.set('failures', str(failure_count)) |
| 97 | testsuite.set('errors', str(error_count)) |
Matt Kwong | 52ff986 | 2017-04-17 13:56:51 -0700 | [diff] [blame] | 98 | |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 99 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 100 | def render_interop_html_report(client_langs, server_langs, test_cases, |
| 101 | auth_test_cases, http2_cases, http2_server_cases, |
| 102 | resultset, num_failures, cloud_to_prod, |
| 103 | prod_servers, http2_interop): |
| 104 | """Generate HTML report for interop tests.""" |
| 105 | template_file = 'tools/run_tests/interop/interop_html_report.template' |
| 106 | try: |
| 107 | mytemplate = Template(filename=template_file, format_exceptions=True) |
| 108 | except NameError: |
| 109 | print( |
| 110 | 'Mako template is not installed. Skipping HTML report generation.') |
| 111 | return |
| 112 | except IOError as e: |
| 113 | print('Failed to find the template %s: %s' % (template_file, e)) |
| 114 | return |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 115 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 116 | sorted_test_cases = sorted(test_cases) |
| 117 | sorted_auth_test_cases = sorted(auth_test_cases) |
| 118 | sorted_http2_cases = sorted(http2_cases) |
| 119 | sorted_http2_server_cases = sorted(http2_server_cases) |
| 120 | sorted_client_langs = sorted(client_langs) |
| 121 | sorted_server_langs = sorted(server_langs) |
| 122 | sorted_prod_servers = sorted(prod_servers) |
Carl Mastrangelo | e7f8e8e | 2015-12-08 17:22:44 -0800 | [diff] [blame] | 123 | |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 124 | args = { |
| 125 | 'client_langs': sorted_client_langs, |
| 126 | 'server_langs': sorted_server_langs, |
| 127 | 'test_cases': sorted_test_cases, |
| 128 | 'auth_test_cases': sorted_auth_test_cases, |
| 129 | 'http2_cases': sorted_http2_cases, |
| 130 | 'http2_server_cases': sorted_http2_server_cases, |
| 131 | 'resultset': resultset, |
| 132 | 'num_failures': num_failures, |
| 133 | 'cloud_to_prod': cloud_to_prod, |
| 134 | 'prod_servers': sorted_prod_servers, |
| 135 | 'http2_interop': http2_interop |
| 136 | } |
| 137 | |
| 138 | html_report_out_dir = 'reports' |
| 139 | if not os.path.exists(html_report_out_dir): |
| 140 | os.mkdir(html_report_out_dir) |
| 141 | html_file_path = os.path.join(html_report_out_dir, 'index.html') |
| 142 | try: |
| 143 | with open(html_file_path, 'w') as output_file: |
| 144 | mytemplate.render_context(Context(output_file, **args)) |
| 145 | except: |
| 146 | print(exceptions.text_error_template().render()) |
| 147 | raise |
| 148 | |
Alexander Polcyn | 9f08d11 | 2016-10-24 12:25:02 -0700 | [diff] [blame] | 149 | |
| 150 | def render_perf_profiling_results(output_filepath, profile_names): |
ncteisen | 05687c3 | 2017-12-11 16:54:47 -0800 | [diff] [blame] | 151 | with open(output_filepath, 'w') as output_file: |
| 152 | output_file.write('<ul>\n') |
| 153 | for name in profile_names: |
| 154 | output_file.write('<li><a href=%s>%s</a></li>\n' % (name, name)) |
| 155 | output_file.write('</ul>\n') |