Zachary Turner | b4733e6 | 2015-12-08 01:15:44 +0000 | [diff] [blame] | 1 | """ |
| 2 | The LLVM Compiler Infrastructure |
| 3 | |
| 4 | This file is distributed under the University of Illinois Open Source |
| 5 | License. See LICENSE.TXT for details. |
| 6 | |
| 7 | Provides the LLDBTestResult class, which holds information about progress |
| 8 | and results of a single test run. |
| 9 | """ |
| 10 | |
| 11 | from __future__ import absolute_import |
| 12 | from __future__ import print_function |
| 13 | |
| 14 | # System modules |
| 15 | import inspect |
| 16 | import os |
| 17 | import platform |
| 18 | import subprocess |
| 19 | |
| 20 | |
| 21 | # Third-party modules |
| 22 | import unittest2 |
| 23 | |
| 24 | # LLDB Modules |
| 25 | import lldbsuite |
| 26 | from . import configuration |
| 27 | from .result_formatter import EventBuilder |
| 28 | |
| 29 | |
| 30 | class LLDBTestResult(unittest2.TextTestResult): |
| 31 | """ |
| 32 | Enforce a singleton pattern to allow introspection of test progress. |
| 33 | |
| 34 | Overwrite addError(), addFailure(), and addExpectedFailure() methods |
| 35 | to enable each test instance to track its failure/error status. It |
| 36 | is used in the LLDB test framework to emit detailed trace messages |
| 37 | to a log file for easier human inspection of test failures/errors. |
| 38 | """ |
| 39 | __singleton__ = None |
| 40 | __ignore_singleton__ = False |
| 41 | |
| 42 | @staticmethod |
| 43 | def getTerminalSize(): |
| 44 | import os |
| 45 | env = os.environ |
| 46 | def ioctl_GWINSZ(fd): |
| 47 | try: |
| 48 | import fcntl, termios, struct, os |
| 49 | cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, |
| 50 | '1234')) |
| 51 | except: |
| 52 | return |
| 53 | return cr |
| 54 | cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) |
| 55 | if not cr: |
| 56 | try: |
| 57 | fd = os.open(os.ctermid(), os.O_RDONLY) |
| 58 | cr = ioctl_GWINSZ(fd) |
| 59 | os.close(fd) |
| 60 | except: |
| 61 | pass |
| 62 | if not cr: |
| 63 | cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) |
| 64 | return int(cr[1]), int(cr[0]) |
| 65 | |
| 66 | def __init__(self, *args): |
| 67 | if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: |
| 68 | raise Exception("LLDBTestResult instantiated more than once") |
| 69 | super(LLDBTestResult, self).__init__(*args) |
| 70 | LLDBTestResult.__singleton__ = self |
| 71 | # Now put this singleton into the lldb module namespace. |
| 72 | configuration.test_result = self |
| 73 | # Computes the format string for displaying the counter. |
| 74 | counterWidth = len(str(configuration.suite.countTestCases())) |
| 75 | self.fmt = "%" + str(counterWidth) + "d: " |
| 76 | self.indentation = ' ' * (counterWidth + 2) |
| 77 | # This counts from 1 .. suite.countTestCases(). |
| 78 | self.counter = 0 |
| 79 | (width, height) = LLDBTestResult.getTerminalSize() |
Zachary Turner | b4733e6 | 2015-12-08 01:15:44 +0000 | [diff] [blame] | 80 | self.results_formatter = configuration.results_formatter_object |
| 81 | |
| 82 | def _config_string(self, test): |
| 83 | compiler = getattr(test, "getCompiler", None) |
| 84 | arch = getattr(test, "getArchitecture", None) |
| 85 | return "%s-%s" % (compiler() if compiler else "", arch() if arch else "") |
| 86 | |
| 87 | def _exc_info_to_string(self, err, test): |
| 88 | """Overrides superclass TestResult's method in order to append |
| 89 | our test config info string to the exception info string.""" |
| 90 | if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): |
| 91 | return '%sConfig=%s-%s' % (super(LLDBTestResult, self)._exc_info_to_string(err, test), |
| 92 | test.getArchitecture(), |
| 93 | test.getCompiler()) |
| 94 | else: |
| 95 | return super(LLDBTestResult, self)._exc_info_to_string(err, test) |
| 96 | |
| 97 | def getDescription(self, test): |
| 98 | doc_first_line = test.shortDescription() |
| 99 | if self.descriptions and doc_first_line: |
| 100 | return '\n'.join((str(test), self.indentation + doc_first_line)) |
| 101 | else: |
| 102 | return str(test) |
| 103 | |
| 104 | def getCategoriesForTest(self,test): |
| 105 | if hasattr(test,"_testMethodName"): |
| 106 | test_method = getattr(test,"_testMethodName") |
| 107 | test_method = getattr(test,test_method) |
| 108 | else: |
| 109 | test_method = None |
| 110 | if test_method != None and hasattr(test_method,"getCategories"): |
| 111 | test_categories = test_method.getCategories(test) |
| 112 | elif hasattr(test,"getCategories"): |
| 113 | test_categories = test.getCategories() |
| 114 | elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"): |
| 115 | test_categories = test.__self__.getCategories() |
| 116 | else: |
| 117 | test_categories = [] |
| 118 | if test_categories == None: |
| 119 | test_categories = [] |
| 120 | return test_categories |
| 121 | |
| 122 | def hardMarkAsSkipped(self,test): |
| 123 | getattr(test, test._testMethodName).__func__.__unittest_skip__ = True |
| 124 | getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" |
| 125 | test.__class__.__unittest_skip__ = True |
| 126 | test.__class__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" |
| 127 | |
| 128 | def startTest(self, test): |
| 129 | if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)): |
| 130 | self.hardMarkAsSkipped(test) |
| 131 | configuration.setCrashInfoHook("%s at %s" % (str(test),inspect.getfile(test.__class__))) |
| 132 | self.counter += 1 |
| 133 | #if self.counter == 4: |
| 134 | # import crashinfo |
| 135 | # crashinfo.testCrashReporterDescription(None) |
| 136 | test.test_number = self.counter |
| 137 | if self.showAll: |
| 138 | self.stream.write(self.fmt % self.counter) |
| 139 | super(LLDBTestResult, self).startTest(test) |
| 140 | if self.results_formatter: |
| 141 | self.results_formatter.handle_event( |
| 142 | EventBuilder.event_for_start(test)) |
| 143 | |
| 144 | def addSuccess(self, test): |
| 145 | super(LLDBTestResult, self).addSuccess(test) |
| 146 | if configuration.parsable: |
| 147 | self.stream.write("PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 148 | if self.results_formatter: |
| 149 | self.results_formatter.handle_event( |
| 150 | EventBuilder.event_for_success(test)) |
| 151 | |
| 152 | def addError(self, test, err): |
| 153 | configuration.sdir_has_content = True |
| 154 | super(LLDBTestResult, self).addError(test, err) |
| 155 | method = getattr(test, "markError", None) |
| 156 | if method: |
| 157 | method() |
| 158 | if configuration.parsable: |
| 159 | self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 160 | if self.results_formatter: |
| 161 | self.results_formatter.handle_event( |
| 162 | EventBuilder.event_for_error(test, err)) |
| 163 | |
| 164 | def addCleanupError(self, test, err): |
| 165 | configuration.sdir_has_content = True |
| 166 | super(LLDBTestResult, self).addCleanupError(test, err) |
| 167 | method = getattr(test, "markCleanupError", None) |
| 168 | if method: |
| 169 | method() |
| 170 | if configuration.parsable: |
| 171 | self.stream.write("CLEANUP ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 172 | if self.results_formatter: |
| 173 | self.results_formatter.handle_event( |
| 174 | EventBuilder.event_for_cleanup_error( |
| 175 | test, err)) |
| 176 | |
| 177 | def addFailure(self, test, err): |
| 178 | configuration.sdir_has_content = True |
| 179 | super(LLDBTestResult, self).addFailure(test, err) |
| 180 | method = getattr(test, "markFailure", None) |
| 181 | if method: |
| 182 | method() |
| 183 | if configuration.parsable: |
| 184 | self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 185 | if configuration.useCategories: |
| 186 | test_categories = self.getCategoriesForTest(test) |
| 187 | for category in test_categories: |
| 188 | if category in configuration.failuresPerCategory: |
| 189 | configuration.failuresPerCategory[category] = configuration.failuresPerCategory[category] + 1 |
| 190 | else: |
| 191 | configuration.failuresPerCategory[category] = 1 |
| 192 | if self.results_formatter: |
| 193 | self.results_formatter.handle_event( |
| 194 | EventBuilder.event_for_failure(test, err)) |
| 195 | |
| 196 | |
| 197 | def addExpectedFailure(self, test, err, bugnumber): |
| 198 | configuration.sdir_has_content = True |
| 199 | super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) |
| 200 | method = getattr(test, "markExpectedFailure", None) |
| 201 | if method: |
| 202 | method(err, bugnumber) |
| 203 | if configuration.parsable: |
| 204 | self.stream.write("XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 205 | if self.results_formatter: |
| 206 | self.results_formatter.handle_event( |
| 207 | EventBuilder.event_for_expected_failure( |
| 208 | test, err, bugnumber)) |
| 209 | |
| 210 | def addSkip(self, test, reason): |
| 211 | configuration.sdir_has_content = True |
| 212 | super(LLDBTestResult, self).addSkip(test, reason) |
| 213 | method = getattr(test, "markSkippedTest", None) |
| 214 | if method: |
| 215 | method() |
| 216 | if configuration.parsable: |
| 217 | self.stream.write("UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason)) |
| 218 | if self.results_formatter: |
| 219 | self.results_formatter.handle_event( |
| 220 | EventBuilder.event_for_skip(test, reason)) |
| 221 | |
| 222 | def addUnexpectedSuccess(self, test, bugnumber): |
| 223 | configuration.sdir_has_content = True |
| 224 | super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) |
| 225 | method = getattr(test, "markUnexpectedSuccess", None) |
| 226 | if method: |
| 227 | method(bugnumber) |
| 228 | if configuration.parsable: |
| 229 | self.stream.write("XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) |
| 230 | if self.results_formatter: |
| 231 | self.results_formatter.handle_event( |
| 232 | EventBuilder.event_for_unexpected_success( |
| 233 | test, bugnumber)) |