Craig Tiller | 6169d5f | 2016-03-31 07:46:18 -0700 | [diff] [blame] | 1 | # Copyright 2015, Google Inc. |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 2 | # All rights reserved. |
| 3 | # |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following disclaimer |
| 12 | # in the documentation and/or other materials provided with the |
| 13 | # distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived from |
| 16 | # this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | """Generate XML and HTML test reports.""" |
| 31 | |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 32 | from __future__ import print_function |
| 33 | |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 34 | try: |
| 35 | from mako.runtime import Context |
| 36 | from mako.template import Template |
Carl Mastrangelo | e7f8e8e | 2015-12-08 17:22:44 -0800 | [diff] [blame] | 37 | from mako import exceptions |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 38 | except (ImportError): |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 39 | pass # Mako not installed but it is ok. |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 40 | import os |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 41 | import string |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 42 | import xml.etree.cElementTree as ET |
| 43 | |
| 44 | |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 45 | def _filter_msg(msg, output_format): |
| 46 | """Filters out nonprintable and illegal characters from the message.""" |
| 47 | if output_format in ['XML', 'HTML']: |
| 48 | # keep whitespaces but remove formfeed and vertical tab characters |
| 49 | # that make XML report unparseable. |
| 50 | filtered_msg = filter( |
| 51 | lambda x: x in string.printable and x != '\f' and x != '\v', |
Jan Tattermusch | 60ab05a | 2016-02-25 17:42:44 -0800 | [diff] [blame] | 52 | msg.decode('UTF-8', 'ignore')) |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 53 | if output_format == 'HTML': |
| 54 | filtered_msg = filtered_msg.replace('"', '"') |
| 55 | return filtered_msg |
| 56 | else: |
| 57 | return msg |
| 58 | |
| 59 | |
Jan Tattermusch | cfcc075 | 2016-10-09 17:02:34 +0200 | [diff] [blame] | 60 | def render_junit_xml_report(resultset, xml_report, suite_package='grpc', |
| 61 | suite_name='tests'): |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 62 | """Generate JUnit-like XML report.""" |
| 63 | root = ET.Element('testsuites') |
Jan Tattermusch | cfcc075 | 2016-10-09 17:02:34 +0200 | [diff] [blame] | 64 | testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package, |
| 65 | name=suite_name) |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 66 | for shortname, results in resultset.iteritems(): |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 67 | for result in results: |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 68 | xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 69 | if result.elapsed_time: |
| 70 | xml_test.set('time', str(result.elapsed_time)) |
Adele Zhou | d01cbe3 | 2015-11-02 14:20:43 -0800 | [diff] [blame] | 71 | ET.SubElement(xml_test, 'system-out').text = _filter_msg(result.message, |
| 72 | 'XML') |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 73 | if result.state == 'FAILED': |
| 74 | ET.SubElement(xml_test, 'failure', message='Failure') |
| 75 | elif result.state == 'TIMEOUT': |
| 76 | ET.SubElement(xml_test, 'error', message='Timeout') |
| 77 | tree = ET.ElementTree(root) |
| 78 | tree.write(xml_report, encoding='UTF-8') |
| 79 | |
| 80 | |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 81 | def render_interop_html_report( |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 82 | client_langs, server_langs, test_cases, auth_test_cases, http2_cases, |
Adele Zhou | b9e66cc | 2016-02-03 13:12:23 -0800 | [diff] [blame] | 83 | resultset, num_failures, cloud_to_prod, prod_servers, http2_interop): |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 84 | """Generate HTML report for interop tests.""" |
Adele Zhou | 12877c9 | 2015-12-09 11:22:25 -0800 | [diff] [blame] | 85 | template_file = 'tools/run_tests/interop_html_report.template' |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 86 | try: |
| 87 | mytemplate = Template(filename=template_file, format_exceptions=True) |
| 88 | except NameError: |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 89 | print('Mako template is not installed. Skipping HTML report generation.') |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 90 | return |
| 91 | except IOError as e: |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 92 | print('Failed to find the template %s: %s' % (template_file, e)) |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 93 | return |
Adele Zhou | 2271ab5 | 2015-10-28 13:59:14 -0700 | [diff] [blame] | 94 | |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 95 | sorted_test_cases = sorted(test_cases) |
| 96 | sorted_auth_test_cases = sorted(auth_test_cases) |
| 97 | sorted_http2_cases = sorted(http2_cases) |
| 98 | sorted_client_langs = sorted(client_langs) |
| 99 | sorted_server_langs = sorted(server_langs) |
Adele Zhou | b9e66cc | 2016-02-03 13:12:23 -0800 | [diff] [blame] | 100 | sorted_prod_servers = sorted(prod_servers) |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 101 | |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 102 | args = {'client_langs': sorted_client_langs, |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 103 | 'server_langs': sorted_server_langs, |
| 104 | 'test_cases': sorted_test_cases, |
| 105 | 'auth_test_cases': sorted_auth_test_cases, |
| 106 | 'http2_cases': sorted_http2_cases, |
| 107 | 'resultset': resultset, |
| 108 | 'num_failures': num_failures, |
| 109 | 'cloud_to_prod': cloud_to_prod, |
Adele Zhou | b9e66cc | 2016-02-03 13:12:23 -0800 | [diff] [blame] | 110 | 'prod_servers': sorted_prod_servers, |
Adele Zhou | 3bc7ba4 | 2015-11-05 10:21:58 -0800 | [diff] [blame] | 111 | 'http2_interop': http2_interop} |
Carl Mastrangelo | e7f8e8e | 2015-12-08 17:22:44 -0800 | [diff] [blame] | 112 | |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 113 | html_report_out_dir = 'reports' |
Adele Zhou | 12877c9 | 2015-12-09 11:22:25 -0800 | [diff] [blame] | 114 | if not os.path.exists(html_report_out_dir): |
murgatroid99 | 5d6ae93 | 2016-10-28 11:48:22 -0700 | [diff] [blame^] | 115 | os.mkdir(html_report_out_dir) |
Adele Zhou | 12877c9 | 2015-12-09 11:22:25 -0800 | [diff] [blame] | 116 | html_file_path = os.path.join(html_report_out_dir, 'index.html') |
Carl Mastrangelo | e7f8e8e | 2015-12-08 17:22:44 -0800 | [diff] [blame] | 117 | try: |
| 118 | with open(html_file_path, 'w') as output_file: |
| 119 | mytemplate.render_context(Context(output_file, **args)) |
| 120 | except: |
| 121 | print(exceptions.text_error_template().render()) |
| 122 | raise |