Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame^] | 1 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | import logging |
| 6 | import os |
| 7 | import re |
| 8 | import tempfile |
| 9 | |
| 10 | from devil.android import apk_helper |
| 11 | from pylib import constants |
| 12 | from pylib.constants import host_paths |
| 13 | from pylib.base import base_test_result |
| 14 | from pylib.base import test_instance |
| 15 | from pylib.utils import isolator |
| 16 | |
| 17 | with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): |
| 18 | import unittest_util # pylint: disable=import-error |
| 19 | |
| 20 | |
| 21 | BROWSER_TEST_SUITES = [ |
| 22 | 'components_browsertests', |
| 23 | 'content_browsertests', |
| 24 | ] |
| 25 | |
| 26 | RUN_IN_SUB_THREAD_TEST_SUITES = ['net_unittests'] |
| 27 | |
| 28 | |
| 29 | _DEFAULT_ISOLATE_FILE_PATHS = { |
| 30 | 'base_unittests': 'base/base_unittests.isolate', |
| 31 | 'blink_heap_unittests': |
| 32 | 'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate', |
| 33 | 'blink_platform_unittests': |
| 34 | 'third_party/WebKit/Source/platform/blink_platform_unittests.isolate', |
| 35 | 'cc_perftests': 'cc/cc_perftests.isolate', |
| 36 | 'components_browsertests': 'components/components_browsertests.isolate', |
| 37 | 'components_unittests': 'components/components_unittests.isolate', |
| 38 | 'content_browsertests': 'content/content_browsertests.isolate', |
| 39 | 'content_unittests': 'content/content_unittests.isolate', |
| 40 | 'media_perftests': 'media/media_perftests.isolate', |
| 41 | 'media_unittests': 'media/media_unittests.isolate', |
| 42 | 'midi_unittests': 'media/midi/midi_unittests.isolate', |
| 43 | 'net_unittests': 'net/net_unittests.isolate', |
| 44 | 'sql_unittests': 'sql/sql_unittests.isolate', |
| 45 | 'sync_unit_tests': 'sync/sync_unit_tests.isolate', |
| 46 | 'ui_base_unittests': 'ui/base/ui_base_tests.isolate', |
| 47 | 'unit_tests': 'chrome/unit_tests.isolate', |
| 48 | 'webkit_unit_tests': |
| 49 | 'third_party/WebKit/Source/web/WebKitUnitTests.isolate', |
| 50 | } |
| 51 | |
| 52 | |
| 53 | # Used for filtering large data deps at a finer grain than what's allowed in |
| 54 | # isolate files since pushing deps to devices is expensive. |
| 55 | # Wildcards are allowed. |
| 56 | _DEPS_EXCLUSION_LIST = [ |
| 57 | 'chrome/test/data/extensions/api_test', |
| 58 | 'chrome/test/data/extensions/secure_shell', |
| 59 | 'chrome/test/data/firefox*', |
| 60 | 'chrome/test/data/gpu', |
| 61 | 'chrome/test/data/image_decoding', |
| 62 | 'chrome/test/data/import', |
| 63 | 'chrome/test/data/page_cycler', |
| 64 | 'chrome/test/data/perf', |
| 65 | 'chrome/test/data/pyauto_private', |
| 66 | 'chrome/test/data/safari_import', |
| 67 | 'chrome/test/data/scroll', |
| 68 | 'chrome/test/data/third_party', |
| 69 | 'third_party/hunspell_dictionaries/*.dic', |
| 70 | # crbug.com/258690 |
| 71 | 'webkit/data/bmp_decoder', |
| 72 | 'webkit/data/ico_decoder', |
| 73 | ] |
| 74 | |
| 75 | |
| 76 | _EXTRA_NATIVE_TEST_ACTIVITY = ( |
| 77 | 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' |
| 78 | 'NativeTestActivity') |
| 79 | _EXTRA_RUN_IN_SUB_THREAD = ( |
| 80 | 'org.chromium.native_test.NativeTestActivity.RunInSubThread') |
| 81 | EXTRA_SHARD_NANO_TIMEOUT = ( |
| 82 | 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' |
| 83 | 'ShardNanoTimeout') |
| 84 | _EXTRA_SHARD_SIZE_LIMIT = ( |
| 85 | 'org.chromium.native_test.NativeTestInstrumentationTestRunner.' |
| 86 | 'ShardSizeLimit') |
| 87 | |
| 88 | # TODO(jbudorick): Remove these once we're no longer parsing stdout to generate |
| 89 | # results. |
| 90 | _RE_TEST_STATUS = re.compile( |
| 91 | r'\[ +((?:RUN)|(?:FAILED)|(?:OK)|(?:CRASHED)) +\]' |
| 92 | r' ?([^ ]+)?(?: \((\d+) ms\))?$') |
| 93 | _RE_TEST_RUN_STATUS = re.compile( |
| 94 | r'\[ +(PASSED|RUNNER_FAILED|CRASHED) \] ?[^ ]+') |
| 95 | # Crash detection constants. |
| 96 | _RE_TEST_ERROR = re.compile(r'FAILURES!!! Tests run: \d+,' |
| 97 | r' Failures: \d+, Errors: 1') |
| 98 | _RE_TEST_CURRENTLY_RUNNING = re.compile(r'\[ERROR:.*?\]' |
| 99 | r' Currently running: (.*)') |
| 100 | |
| 101 | # TODO(jbudorick): Make this a class method of GtestTestInstance once |
| 102 | # test_package_apk and test_package_exe are gone. |
| 103 | def ParseGTestListTests(raw_list): |
| 104 | """Parses a raw test list as provided by --gtest_list_tests. |
| 105 | |
| 106 | Args: |
| 107 | raw_list: The raw test listing with the following format: |
| 108 | |
| 109 | IPCChannelTest. |
| 110 | SendMessageInChannelConnected |
| 111 | IPCSyncChannelTest. |
| 112 | Simple |
| 113 | DISABLED_SendWithTimeoutMixedOKAndTimeout |
| 114 | |
| 115 | Returns: |
| 116 | A list of all tests. For the above raw listing: |
| 117 | |
| 118 | [IPCChannelTest.SendMessageInChannelConnected, IPCSyncChannelTest.Simple, |
| 119 | IPCSyncChannelTest.DISABLED_SendWithTimeoutMixedOKAndTimeout] |
| 120 | """ |
| 121 | ret = [] |
| 122 | current = '' |
| 123 | for test in raw_list: |
| 124 | if not test: |
| 125 | continue |
| 126 | if test[0] != ' ': |
| 127 | test_case = test.split()[0] |
| 128 | if test_case.endswith('.'): |
| 129 | current = test_case |
| 130 | elif not 'YOU HAVE' in test: |
| 131 | test_name = test.split()[0] |
| 132 | ret += [current + test_name] |
| 133 | return ret |
| 134 | |
| 135 | |
| 136 | class GtestTestInstance(test_instance.TestInstance): |
| 137 | |
| 138 | def __init__(self, args, isolate_delegate, error_func): |
| 139 | super(GtestTestInstance, self).__init__() |
| 140 | # TODO(jbudorick): Support multiple test suites. |
| 141 | if len(args.suite_name) > 1: |
| 142 | raise ValueError('Platform mode currently supports only 1 gtest suite') |
| 143 | self._extract_test_list_from_filter = args.extract_test_list_from_filter |
| 144 | self._shard_timeout = args.shard_timeout |
| 145 | self._suite = args.suite_name[0] |
| 146 | self._exe_dist_dir = None |
| 147 | |
| 148 | # GYP: |
| 149 | if args.executable_dist_dir: |
| 150 | self._exe_dist_dir = os.path.abspath(args.executable_dist_dir) |
| 151 | else: |
| 152 | # TODO(agrieve): Remove auto-detection once recipes pass flag explicitly. |
| 153 | exe_dist_dir = os.path.join(constants.GetOutDirectory(), |
| 154 | '%s__dist' % self._suite) |
| 155 | |
| 156 | if os.path.exists(exe_dist_dir): |
| 157 | self._exe_dist_dir = exe_dist_dir |
| 158 | |
| 159 | incremental_part = '' |
| 160 | if args.test_apk_incremental_install_script: |
| 161 | incremental_part = '_incremental' |
| 162 | |
| 163 | apk_path = os.path.join( |
| 164 | constants.GetOutDirectory(), '%s_apk' % self._suite, |
| 165 | '%s-debug%s.apk' % (self._suite, incremental_part)) |
| 166 | self._test_apk_incremental_install_script = ( |
| 167 | args.test_apk_incremental_install_script) |
| 168 | if not os.path.exists(apk_path): |
| 169 | self._apk_helper = None |
| 170 | else: |
| 171 | self._apk_helper = apk_helper.ApkHelper(apk_path) |
| 172 | self._extras = { |
| 173 | _EXTRA_NATIVE_TEST_ACTIVITY: self._apk_helper.GetActivityName(), |
| 174 | } |
| 175 | if self._suite in RUN_IN_SUB_THREAD_TEST_SUITES: |
| 176 | self._extras[_EXTRA_RUN_IN_SUB_THREAD] = 1 |
| 177 | if self._suite in BROWSER_TEST_SUITES: |
| 178 | self._extras[_EXTRA_SHARD_SIZE_LIMIT] = 1 |
| 179 | self._extras[EXTRA_SHARD_NANO_TIMEOUT] = int(1e9 * self._shard_timeout) |
| 180 | self._shard_timeout = 900 |
| 181 | |
| 182 | if not self._apk_helper and not self._exe_dist_dir: |
| 183 | error_func('Could not find apk or executable for %s' % self._suite) |
| 184 | |
| 185 | self._data_deps = [] |
| 186 | if args.test_filter: |
| 187 | self._gtest_filter = args.test_filter |
| 188 | elif args.test_filter_file: |
| 189 | with open(args.test_filter_file, 'r') as f: |
| 190 | self._gtest_filter = ':'.join(l.strip() for l in f) |
| 191 | else: |
| 192 | self._gtest_filter = None |
| 193 | |
| 194 | if not args.isolate_file_path: |
| 195 | default_isolate_file_path = _DEFAULT_ISOLATE_FILE_PATHS.get(self._suite) |
| 196 | if default_isolate_file_path: |
| 197 | args.isolate_file_path = os.path.join( |
| 198 | host_paths.DIR_SOURCE_ROOT, default_isolate_file_path) |
| 199 | |
| 200 | if (args.isolate_file_path and |
| 201 | not isolator.IsIsolateEmpty(args.isolate_file_path)): |
| 202 | self._isolate_abs_path = os.path.abspath(args.isolate_file_path) |
| 203 | self._isolate_delegate = isolate_delegate |
| 204 | self._isolated_abs_path = os.path.join( |
| 205 | constants.GetOutDirectory(), '%s.isolated' % self._suite) |
| 206 | else: |
| 207 | logging.warning('No isolate file provided. No data deps will be pushed.') |
| 208 | self._isolate_delegate = None |
| 209 | |
| 210 | if args.app_data_files: |
| 211 | self._app_data_files = args.app_data_files |
| 212 | if args.app_data_file_dir: |
| 213 | self._app_data_file_dir = args.app_data_file_dir |
| 214 | else: |
| 215 | self._app_data_file_dir = tempfile.mkdtemp() |
| 216 | logging.critical('Saving app files to %s', self._app_data_file_dir) |
| 217 | else: |
| 218 | self._app_data_files = None |
| 219 | self._app_data_file_dir = None |
| 220 | |
| 221 | self._test_arguments = args.test_arguments |
| 222 | |
| 223 | @property |
| 224 | def activity(self): |
| 225 | return self._apk_helper and self._apk_helper.GetActivityName() |
| 226 | |
| 227 | @property |
| 228 | def apk(self): |
| 229 | return self._apk_helper and self._apk_helper.path |
| 230 | |
| 231 | @property |
| 232 | def apk_helper(self): |
| 233 | return self._apk_helper |
| 234 | |
| 235 | @property |
| 236 | def app_file_dir(self): |
| 237 | return self._app_data_file_dir |
| 238 | |
| 239 | @property |
| 240 | def app_files(self): |
| 241 | return self._app_data_files |
| 242 | |
| 243 | @property |
| 244 | def exe_dist_dir(self): |
| 245 | return self._exe_dist_dir |
| 246 | |
| 247 | @property |
| 248 | def extras(self): |
| 249 | return self._extras |
| 250 | |
| 251 | @property |
| 252 | def gtest_filter(self): |
| 253 | return self._gtest_filter |
| 254 | |
| 255 | @property |
| 256 | def package(self): |
| 257 | return self._apk_helper and self._apk_helper.GetPackageName() |
| 258 | |
| 259 | @property |
| 260 | def permissions(self): |
| 261 | return self._apk_helper and self._apk_helper.GetPermissions() |
| 262 | |
| 263 | @property |
| 264 | def runner(self): |
| 265 | return self._apk_helper and self._apk_helper.GetInstrumentationName() |
| 266 | |
| 267 | @property |
| 268 | def shard_timeout(self): |
| 269 | return self._shard_timeout |
| 270 | |
| 271 | @property |
| 272 | def suite(self): |
| 273 | return self._suite |
| 274 | |
| 275 | @property |
| 276 | def test_apk_incremental_install_script(self): |
| 277 | return self._test_apk_incremental_install_script |
| 278 | |
| 279 | @property |
| 280 | def test_arguments(self): |
| 281 | return self._test_arguments |
| 282 | |
| 283 | @property |
| 284 | def extract_test_list_from_filter(self): |
| 285 | return self._extract_test_list_from_filter |
| 286 | |
| 287 | #override |
| 288 | def TestType(self): |
| 289 | return 'gtest' |
| 290 | |
| 291 | #override |
| 292 | def SetUp(self): |
| 293 | """Map data dependencies via isolate.""" |
| 294 | if self._isolate_delegate: |
| 295 | self._isolate_delegate.Remap( |
| 296 | self._isolate_abs_path, self._isolated_abs_path) |
| 297 | self._isolate_delegate.PurgeExcluded(_DEPS_EXCLUSION_LIST) |
| 298 | self._isolate_delegate.MoveOutputDeps() |
| 299 | dest_dir = None |
| 300 | self._data_deps.extend([ |
| 301 | (self._isolate_delegate.isolate_deps_dir, dest_dir)]) |
| 302 | |
| 303 | |
| 304 | def GetDataDependencies(self): |
| 305 | """Returns the test suite's data dependencies. |
| 306 | |
| 307 | Returns: |
| 308 | A list of (host_path, device_path) tuples to push. If device_path is |
| 309 | None, the client is responsible for determining where to push the file. |
| 310 | """ |
| 311 | return self._data_deps |
| 312 | |
| 313 | def FilterTests(self, test_list, disabled_prefixes=None): |
| 314 | """Filters |test_list| based on prefixes and, if present, a filter string. |
| 315 | |
| 316 | Args: |
| 317 | test_list: The list of tests to filter. |
| 318 | disabled_prefixes: A list of test prefixes to filter. Defaults to |
| 319 | DISABLED_, FLAKY_, FAILS_, PRE_, and MANUAL_ |
| 320 | Returns: |
| 321 | A filtered list of tests to run. |
| 322 | """ |
| 323 | gtest_filter_strings = [ |
| 324 | self._GenerateDisabledFilterString(disabled_prefixes)] |
| 325 | if self._gtest_filter: |
| 326 | gtest_filter_strings.append(self._gtest_filter) |
| 327 | |
| 328 | filtered_test_list = test_list |
| 329 | for gtest_filter_string in gtest_filter_strings: |
| 330 | logging.debug('Filtering tests using: %s', gtest_filter_string) |
| 331 | filtered_test_list = unittest_util.FilterTestNames( |
| 332 | filtered_test_list, gtest_filter_string) |
| 333 | return filtered_test_list |
| 334 | |
| 335 | def _GenerateDisabledFilterString(self, disabled_prefixes): |
| 336 | disabled_filter_items = [] |
| 337 | |
| 338 | if disabled_prefixes is None: |
| 339 | disabled_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_', 'PRE_', 'MANUAL_'] |
| 340 | disabled_filter_items += ['%s*' % dp for dp in disabled_prefixes] |
| 341 | disabled_filter_items += ['*.%s*' % dp for dp in disabled_prefixes] |
| 342 | |
| 343 | disabled_tests_file_path = os.path.join( |
| 344 | host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'gtest', |
| 345 | 'filter', '%s_disabled' % self._suite) |
| 346 | if disabled_tests_file_path and os.path.exists(disabled_tests_file_path): |
| 347 | with open(disabled_tests_file_path) as disabled_tests_file: |
| 348 | disabled_filter_items += [ |
| 349 | '%s' % l for l in (line.strip() for line in disabled_tests_file) |
| 350 | if l and not l.startswith('#')] |
| 351 | |
| 352 | return '*-%s' % ':'.join(disabled_filter_items) |
| 353 | |
| 354 | # pylint: disable=no-self-use |
| 355 | def ParseGTestOutput(self, output): |
| 356 | """Parses raw gtest output and returns a list of results. |
| 357 | |
| 358 | Args: |
| 359 | output: A list of output lines. |
| 360 | Returns: |
| 361 | A list of base_test_result.BaseTestResults. |
| 362 | """ |
| 363 | log = [] |
| 364 | result_type = None |
| 365 | results = [] |
| 366 | test_name = None |
| 367 | for l in output: |
| 368 | logging.info(l) |
| 369 | matcher = _RE_TEST_STATUS.match(l) |
| 370 | if matcher: |
| 371 | # Be aware that test name and status might not appear on same line. |
| 372 | test_name = matcher.group(2) if matcher.group(2) else test_name |
| 373 | duration = int(matcher.group(3)) if matcher.group(3) else 0 |
| 374 | if matcher.group(1) == 'RUN': |
| 375 | log = [] |
| 376 | elif matcher.group(1) == 'OK': |
| 377 | result_type = base_test_result.ResultType.PASS |
| 378 | elif matcher.group(1) == 'FAILED': |
| 379 | result_type = base_test_result.ResultType.FAIL |
| 380 | elif matcher.group(1) == 'CRASHED': |
| 381 | result_type = base_test_result.ResultType.CRASH |
| 382 | |
| 383 | # Needs another matcher here to match crashes, like those of DCHECK. |
| 384 | matcher = _RE_TEST_CURRENTLY_RUNNING.match(l) |
| 385 | if matcher: |
| 386 | test_name = matcher.group(1) |
| 387 | result_type = base_test_result.ResultType.CRASH |
| 388 | duration = 0 # Don't know. |
| 389 | |
| 390 | if log is not None: |
| 391 | log.append(l) |
| 392 | |
| 393 | if result_type: |
| 394 | results.append(base_test_result.BaseTestResult( |
| 395 | test_name, result_type, duration, |
| 396 | log=('\n'.join(log) if log else ''))) |
| 397 | log = None |
| 398 | result_type = None |
| 399 | |
| 400 | return results |
| 401 | |
| 402 | #override |
| 403 | def TearDown(self): |
| 404 | """Clear the mappings created by SetUp.""" |
| 405 | if self._isolate_delegate: |
| 406 | self._isolate_delegate.Clear() |
| 407 | |