blob: ae968e33eef58e1b06b2c55eba536210f0501957 [file] [log] [blame]
Zachary Turnerb4733e62015-12-08 01:15:44 +00001"""
2 The LLVM Compiler Infrastructure
3
4This file is distributed under the University of Illinois Open Source
5License. See LICENSE.TXT for details.
6
7Provides the LLDBTestResult class, which holds information about progress
8and results of a single test run.
9"""
10
11from __future__ import absolute_import
12from __future__ import print_function
13
14# System modules
15import inspect
16import os
17import platform
18import subprocess
19
20
21# Third-party modules
22import unittest2
23
24# LLDB Modules
25import lldbsuite
26from . import configuration
27from .result_formatter import EventBuilder
28
29
30class LLDBTestResult(unittest2.TextTestResult):
31 """
32 Enforce a singleton pattern to allow introspection of test progress.
33
34 Overwrite addError(), addFailure(), and addExpectedFailure() methods
35 to enable each test instance to track its failure/error status. It
36 is used in the LLDB test framework to emit detailed trace messages
37 to a log file for easier human inspection of test failures/errors.
38 """
39 __singleton__ = None
40 __ignore_singleton__ = False
41
42 @staticmethod
43 def getTerminalSize():
44 import os
45 env = os.environ
46 def ioctl_GWINSZ(fd):
47 try:
48 import fcntl, termios, struct, os
49 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
50 '1234'))
51 except:
52 return
53 return cr
54 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
55 if not cr:
56 try:
57 fd = os.open(os.ctermid(), os.O_RDONLY)
58 cr = ioctl_GWINSZ(fd)
59 os.close(fd)
60 except:
61 pass
62 if not cr:
63 cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
64 return int(cr[1]), int(cr[0])
65
66 def __init__(self, *args):
67 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
68 raise Exception("LLDBTestResult instantiated more than once")
69 super(LLDBTestResult, self).__init__(*args)
70 LLDBTestResult.__singleton__ = self
71 # Now put this singleton into the lldb module namespace.
72 configuration.test_result = self
73 # Computes the format string for displaying the counter.
74 counterWidth = len(str(configuration.suite.countTestCases()))
75 self.fmt = "%" + str(counterWidth) + "d: "
76 self.indentation = ' ' * (counterWidth + 2)
77 # This counts from 1 .. suite.countTestCases().
78 self.counter = 0
79 (width, height) = LLDBTestResult.getTerminalSize()
Zachary Turnerb4733e62015-12-08 01:15:44 +000080 self.results_formatter = configuration.results_formatter_object
81
82 def _config_string(self, test):
83 compiler = getattr(test, "getCompiler", None)
84 arch = getattr(test, "getArchitecture", None)
85 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")
86
87 def _exc_info_to_string(self, err, test):
88 """Overrides superclass TestResult's method in order to append
89 our test config info string to the exception info string."""
90 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
91 return '%sConfig=%s-%s' % (super(LLDBTestResult, self)._exc_info_to_string(err, test),
92 test.getArchitecture(),
93 test.getCompiler())
94 else:
95 return super(LLDBTestResult, self)._exc_info_to_string(err, test)
96
97 def getDescription(self, test):
98 doc_first_line = test.shortDescription()
99 if self.descriptions and doc_first_line:
100 return '\n'.join((str(test), self.indentation + doc_first_line))
101 else:
102 return str(test)
103
104 def getCategoriesForTest(self,test):
105 if hasattr(test,"_testMethodName"):
106 test_method = getattr(test,"_testMethodName")
107 test_method = getattr(test,test_method)
108 else:
109 test_method = None
110 if test_method != None and hasattr(test_method,"getCategories"):
111 test_categories = test_method.getCategories(test)
112 elif hasattr(test,"getCategories"):
113 test_categories = test.getCategories()
114 elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"):
115 test_categories = test.__self__.getCategories()
116 else:
117 test_categories = []
118 if test_categories == None:
119 test_categories = []
120 return test_categories
121
122 def hardMarkAsSkipped(self,test):
123 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
124 getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
125 test.__class__.__unittest_skip__ = True
126 test.__class__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
127
128 def startTest(self, test):
129 if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
130 self.hardMarkAsSkipped(test)
131 configuration.setCrashInfoHook("%s at %s" % (str(test),inspect.getfile(test.__class__)))
132 self.counter += 1
133 #if self.counter == 4:
134 # import crashinfo
135 # crashinfo.testCrashReporterDescription(None)
136 test.test_number = self.counter
137 if self.showAll:
138 self.stream.write(self.fmt % self.counter)
139 super(LLDBTestResult, self).startTest(test)
140 if self.results_formatter:
141 self.results_formatter.handle_event(
142 EventBuilder.event_for_start(test))
143
144 def addSuccess(self, test):
145 super(LLDBTestResult, self).addSuccess(test)
146 if configuration.parsable:
147 self.stream.write("PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
148 if self.results_formatter:
149 self.results_formatter.handle_event(
150 EventBuilder.event_for_success(test))
151
152 def addError(self, test, err):
153 configuration.sdir_has_content = True
154 super(LLDBTestResult, self).addError(test, err)
155 method = getattr(test, "markError", None)
156 if method:
157 method()
158 if configuration.parsable:
159 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
160 if self.results_formatter:
161 self.results_formatter.handle_event(
162 EventBuilder.event_for_error(test, err))
163
164 def addCleanupError(self, test, err):
165 configuration.sdir_has_content = True
166 super(LLDBTestResult, self).addCleanupError(test, err)
167 method = getattr(test, "markCleanupError", None)
168 if method:
169 method()
170 if configuration.parsable:
171 self.stream.write("CLEANUP ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
172 if self.results_formatter:
173 self.results_formatter.handle_event(
174 EventBuilder.event_for_cleanup_error(
175 test, err))
176
177 def addFailure(self, test, err):
178 configuration.sdir_has_content = True
179 super(LLDBTestResult, self).addFailure(test, err)
180 method = getattr(test, "markFailure", None)
181 if method:
182 method()
183 if configuration.parsable:
184 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
185 if configuration.useCategories:
186 test_categories = self.getCategoriesForTest(test)
187 for category in test_categories:
188 if category in configuration.failuresPerCategory:
189 configuration.failuresPerCategory[category] = configuration.failuresPerCategory[category] + 1
190 else:
191 configuration.failuresPerCategory[category] = 1
192 if self.results_formatter:
193 self.results_formatter.handle_event(
194 EventBuilder.event_for_failure(test, err))
195
196
197 def addExpectedFailure(self, test, err, bugnumber):
198 configuration.sdir_has_content = True
199 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
200 method = getattr(test, "markExpectedFailure", None)
201 if method:
202 method(err, bugnumber)
203 if configuration.parsable:
204 self.stream.write("XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
205 if self.results_formatter:
206 self.results_formatter.handle_event(
207 EventBuilder.event_for_expected_failure(
208 test, err, bugnumber))
209
210 def addSkip(self, test, reason):
211 configuration.sdir_has_content = True
212 super(LLDBTestResult, self).addSkip(test, reason)
213 method = getattr(test, "markSkippedTest", None)
214 if method:
215 method()
216 if configuration.parsable:
217 self.stream.write("UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason))
218 if self.results_formatter:
219 self.results_formatter.handle_event(
220 EventBuilder.event_for_skip(test, reason))
221
222 def addUnexpectedSuccess(self, test, bugnumber):
223 configuration.sdir_has_content = True
224 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
225 method = getattr(test, "markUnexpectedSuccess", None)
226 if method:
227 method(bugnumber)
228 if configuration.parsable:
229 self.stream.write("XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)))
230 if self.results_formatter:
231 self.results_formatter.handle_event(
232 EventBuilder.event_for_unexpected_success(
233 test, bugnumber))