Craig Tiller | 6169d5f | 2016-03-31 07:46:18 -0700 | [diff] [blame] | 1 | # Copyright 2015, Google Inc. |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 2 | # All rights reserved. |
| 3 | # |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following disclaimer |
| 12 | # in the documentation and/or other materials provided with the |
| 13 | # distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived from |
| 16 | # this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
Leifur Halldor Asgeirsson | 554c7dd | 2016-02-18 17:24:05 -0500 | [diff] [blame] | 30 | from __future__ import absolute_import |
| 31 | |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 32 | import collections |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 33 | import multiprocessing |
| 34 | import os |
| 35 | import select |
| 36 | import signal |
| 37 | import sys |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 38 | import tempfile |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 39 | import threading |
| 40 | import time |
| 41 | import unittest |
| 42 | import uuid |
| 43 | |
Leifur Halldor Asgeirsson | ae21956 | 2016-03-01 10:35:05 -0500 | [diff] [blame] | 44 | import six |
Leifur Halldor Asgeirsson | 554c7dd | 2016-02-18 17:24:05 -0500 | [diff] [blame] | 45 | from six import moves |
| 46 | |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 47 | from tests import _loader |
| 48 | from tests import _result |
| 49 | |
Masood Malekghassemi | cbd1bce | 2016-03-15 13:24:10 -0700 | [diff] [blame] | 50 | |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 51 | class CaptureFile(object): |
| 52 | """A context-managed file to redirect output to a byte array. |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 53 | |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 54 | Use by invoking `start` (`__enter__`) and at some point invoking `stop` |
| 55 | (`__exit__`). At any point after the initial call to `start` call `output` to |
| 56 | get the current redirected output. Note that we don't currently use file |
| 57 | locking, so calling `output` between calls to `start` and `stop` may muddle |
| 58 | the result (you should only be doing this during a Python-handled interrupt as |
| 59 | a last ditch effort to provide output to the user). |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 60 | |
| 61 | Attributes: |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 62 | _redirected_fd (int): File descriptor of file to redirect writes from. |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 63 | _saved_fd (int): A copy of the original value of the redirected file |
| 64 | descriptor. |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 65 | _into_file (TemporaryFile or None): File to which writes are redirected. |
| 66 | Only non-None when self is started. |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 67 | """ |
| 68 | |
| 69 | def __init__(self, fd): |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 70 | self._redirected_fd = fd |
| 71 | self._saved_fd = os.dup(self._redirected_fd) |
| 72 | self._into_file = None |
| 73 | |
| 74 | def output(self): |
| 75 | """Get all output from the redirected-to file if it exists.""" |
| 76 | if self._into_file: |
| 77 | self._into_file.seek(0) |
| 78 | return bytes(self._into_file.read()) |
| 79 | else: |
| 80 | return bytes() |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 81 | |
| 82 | def start(self): |
| 83 | """Start redirection of writes to the file descriptor.""" |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 84 | self._into_file = tempfile.TemporaryFile() |
| 85 | os.dup2(self._into_file.fileno(), self._redirected_fd) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 86 | |
| 87 | def stop(self): |
| 88 | """Stop redirection of writes to the file descriptor.""" |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 89 | # n.b. this dup2 call auto-closes self._redirected_fd |
| 90 | os.dup2(self._saved_fd, self._redirected_fd) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 91 | |
| 92 | def write_bypass(self, value): |
| 93 | """Bypass the redirection and write directly to the original file. |
| 94 | |
| 95 | Arguments: |
| 96 | value (str): What to write to the original file. |
| 97 | """ |
Leifur Halldor Asgeirsson | ae21956 | 2016-03-01 10:35:05 -0500 | [diff] [blame] | 98 | if six.PY3 and not isinstance(value, six.binary_type): |
| 99 | value = bytes(value, 'ascii') |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 100 | if self._saved_fd is None: |
| 101 | os.write(self._redirect_fd, value) |
| 102 | else: |
| 103 | os.write(self._saved_fd, value) |
| 104 | |
| 105 | def __enter__(self): |
| 106 | self.start() |
| 107 | return self |
| 108 | |
| 109 | def __exit__(self, type, value, traceback): |
| 110 | self.stop() |
| 111 | |
| 112 | def close(self): |
| 113 | """Close any resources used by self not closed by stop().""" |
| 114 | os.close(self._saved_fd) |
| 115 | |
| 116 | |
| 117 | class AugmentedCase(collections.namedtuple('AugmentedCase', [ |
| 118 | 'case', 'id'])): |
| 119 | """A test case with a guaranteed unique externally specified identifier. |
| 120 | |
| 121 | Attributes: |
| 122 | case (unittest.TestCase): TestCase we're decorating with an additional |
| 123 | identifier. |
| 124 | id (object): Any identifier that may be considered 'unique' for testing |
| 125 | purposes. |
| 126 | """ |
| 127 | |
| 128 | def __new__(cls, case, id=None): |
| 129 | if id is None: |
| 130 | id = uuid.uuid4() |
| 131 | return super(cls, AugmentedCase).__new__(cls, case, id) |
| 132 | |
| 133 | |
| 134 | class Runner(object): |
| 135 | |
| 136 | def run(self, suite): |
| 137 | """See setuptools' test_runner setup argument for information.""" |
Jan Tattermusch | 072ebaa | 2016-03-01 18:33:12 -0800 | [diff] [blame] | 138 | # only run test cases with id starting with given prefix |
Masood Malekghassemi | ec49e15 | 2016-03-09 11:51:23 -0800 | [diff] [blame] | 139 | testcase_filter = os.getenv('GRPC_PYTHON_TESTRUNNER_FILTER') |
Jan Tattermusch | 072ebaa | 2016-03-01 18:33:12 -0800 | [diff] [blame] | 140 | filtered_cases = [] |
| 141 | for case in _loader.iterate_suite_cases(suite): |
| 142 | if not testcase_filter or case.id().startswith(testcase_filter): |
| 143 | filtered_cases.append(case) |
| 144 | |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 145 | # Ensure that every test case has no collision with any other test case in |
| 146 | # the augmented results. |
| 147 | augmented_cases = [AugmentedCase(case, uuid.uuid4()) |
Jan Tattermusch | 072ebaa | 2016-03-01 18:33:12 -0800 | [diff] [blame] | 148 | for case in filtered_cases] |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 149 | case_id_by_case = dict((augmented_case.case, augmented_case.id) |
| 150 | for augmented_case in augmented_cases) |
Leifur Halldor Asgeirsson | 554c7dd | 2016-02-18 17:24:05 -0500 | [diff] [blame] | 151 | result_out = moves.cStringIO() |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 152 | result = _result.TerminalResult( |
| 153 | result_out, id_map=lambda case: case_id_by_case[case]) |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 154 | stdout_pipe = CaptureFile(sys.stdout.fileno()) |
| 155 | stderr_pipe = CaptureFile(sys.stderr.fileno()) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 156 | kill_flag = [False] |
| 157 | |
| 158 | def sigint_handler(signal_number, frame): |
| 159 | if signal_number == signal.SIGINT: |
| 160 | kill_flag[0] = True # Python 2.7 not having 'local'... :-( |
| 161 | signal.signal(signal_number, signal.SIG_DFL) |
| 162 | |
| 163 | def fault_handler(signal_number, frame): |
| 164 | stdout_pipe.write_bypass( |
| 165 | 'Received fault signal {}\nstdout:\n{}\n\nstderr:{}\n' |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 166 | .format(signal_number, stdout_pipe.output(), |
| 167 | stderr_pipe.output())) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 168 | os._exit(1) |
| 169 | |
| 170 | def check_kill_self(): |
| 171 | if kill_flag[0]: |
| 172 | stdout_pipe.write_bypass('Stopping tests short...') |
| 173 | result.stopTestRun() |
| 174 | stdout_pipe.write_bypass(result_out.getvalue()) |
| 175 | stdout_pipe.write_bypass( |
Leifur Halldor Asgeirsson | ae21956 | 2016-03-01 10:35:05 -0500 | [diff] [blame] | 176 | '\ninterrupted stdout:\n{}\n'.format(stdout_pipe.output().decode())) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 177 | stderr_pipe.write_bypass( |
Leifur Halldor Asgeirsson | ae21956 | 2016-03-01 10:35:05 -0500 | [diff] [blame] | 178 | '\ninterrupted stderr:\n{}\n'.format(stderr_pipe.output().decode())) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 179 | os._exit(1) |
Masood Malekghassemi | cab9d4f | 2016-06-28 09:09:31 -0700 | [diff] [blame^] | 180 | def try_set_handler(name, handler): |
| 181 | try: |
| 182 | signal.signal(getattr(signal, name), handler) |
| 183 | except AttributeError: |
| 184 | pass |
| 185 | try_set_handler('SIGINT', sigint_handler) |
| 186 | try_set_handler('SIGSEGV', fault_handler) |
| 187 | try_set_handler('SIGBUS', fault_handler) |
| 188 | try_set_handler('SIGABRT', fault_handler) |
| 189 | try_set_handler('SIGFPE', fault_handler) |
| 190 | try_set_handler('SIGILL', fault_handler) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 191 | # Sometimes output will lag after a test has successfully finished; we |
| 192 | # ignore such writes to our pipes. |
Masood Malekghassemi | cab9d4f | 2016-06-28 09:09:31 -0700 | [diff] [blame^] | 193 | try_set_handler('SIGPIPE', signal.SIG_IGN) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 194 | |
| 195 | # Run the tests |
| 196 | result.startTestRun() |
| 197 | for augmented_case in augmented_cases: |
| 198 | sys.stdout.write('Running {}\n'.format(augmented_case.case.id())) |
| 199 | sys.stdout.flush() |
| 200 | case_thread = threading.Thread( |
| 201 | target=augmented_case.case.run, args=(result,)) |
| 202 | try: |
| 203 | with stdout_pipe, stderr_pipe: |
| 204 | case_thread.start() |
| 205 | while case_thread.is_alive(): |
| 206 | check_kill_self() |
| 207 | time.sleep(0) |
| 208 | case_thread.join() |
| 209 | except: |
| 210 | # re-raise the exception after forcing the with-block to end |
| 211 | raise |
| 212 | result.set_output( |
Masood Malekghassemi | fd5a3ef | 2016-03-16 18:49:54 -0700 | [diff] [blame] | 213 | augmented_case.case, stdout_pipe.output(), stderr_pipe.output()) |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 214 | sys.stdout.write(result_out.getvalue()) |
| 215 | sys.stdout.flush() |
| 216 | result_out.truncate(0) |
| 217 | check_kill_self() |
| 218 | result.stopTestRun() |
| 219 | stdout_pipe.close() |
| 220 | stderr_pipe.close() |
| 221 | |
| 222 | # Report results |
| 223 | sys.stdout.write(result_out.getvalue()) |
| 224 | sys.stdout.flush() |
| 225 | signal.signal(signal.SIGINT, signal.SIG_DFL) |
Leifur Halldor Asgeirsson | ae21956 | 2016-03-01 10:35:05 -0500 | [diff] [blame] | 226 | with open('report.xml', 'wb') as report_xml_file: |
Masood Malekghassemi | 7566c9a | 2015-10-21 20:29:23 -0700 | [diff] [blame] | 227 | _result.jenkins_junit_xml(result).write(report_xml_file) |
| 228 | return result |
| 229 | |