Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame^] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | import collections |
| 7 | import glob |
| 8 | import hashlib |
| 9 | import json |
| 10 | import os |
| 11 | import random |
| 12 | import re |
| 13 | import shutil |
| 14 | import sys |
| 15 | |
| 16 | import bb_utils |
| 17 | import bb_annotations |
| 18 | |
| 19 | sys.path.append(os.path.join(os.path.dirname(__file__), '..')) |
| 20 | import devil_chromium |
| 21 | import provision_devices |
| 22 | from devil.android import device_utils |
| 23 | from pylib import constants |
| 24 | from pylib.gtest import gtest_config |
| 25 | |
| 26 | CHROME_SRC_DIR = bb_utils.CHROME_SRC |
| 27 | DIR_BUILD_ROOT = os.path.dirname(CHROME_SRC_DIR) |
| 28 | CHROME_OUT_DIR = bb_utils.CHROME_OUT_DIR |
| 29 | BLINK_SCRIPTS_DIR = 'third_party/WebKit/Tools/Scripts' |
| 30 | |
| 31 | SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave') |
| 32 | LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat') |
| 33 | GS_URL = 'https://storage.googleapis.com' |
| 34 | GS_AUTH_URL = 'https://storage.cloud.google.com' |
| 35 | |
| 36 | # Describes an instrumation test suite: |
| 37 | # test: Name of test we're running. |
| 38 | # apk: apk to be installed. |
| 39 | # apk_package: package for the apk to be installed. |
| 40 | # test_apk: apk to run tests on. |
| 41 | # test_data: data folder in format destination:source. |
| 42 | # host_driven_root: The host-driven test root directory. |
| 43 | # annotation: Annotation of the tests to include. |
| 44 | # exclude_annotation: The annotation of the tests to exclude. |
| 45 | I_TEST = collections.namedtuple('InstrumentationTest', [ |
| 46 | 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'isolate_file_path', |
| 47 | 'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags']) |
| 48 | |
| 49 | |
| 50 | def SrcPath(*path): |
| 51 | return os.path.join(CHROME_SRC_DIR, *path) |
| 52 | |
| 53 | |
| 54 | def I(name, apk, apk_package, test_apk, test_data, isolate_file_path=None, |
| 55 | host_driven_root=None, annotation=None, exclude_annotation=None, |
| 56 | extra_flags=None): |
| 57 | return I_TEST(name, apk, apk_package, test_apk, test_data, isolate_file_path, |
| 58 | host_driven_root, annotation, exclude_annotation, extra_flags) |
| 59 | |
| 60 | INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ |
| 61 | I('ContentShell', |
| 62 | 'ContentShell.apk', |
| 63 | 'org.chromium.content_shell_apk', |
| 64 | 'ContentShellTest', |
| 65 | 'content:content/test/data/android/device_files', |
| 66 | isolate_file_path='content/content_shell_test_data.isolate'), |
| 67 | I('ChromePublic', |
| 68 | 'ChromePublic.apk', |
| 69 | 'org.chromium.chrome', |
| 70 | 'ChromePublicTest', |
| 71 | 'chrome:chrome/test/data/android/device_files', |
| 72 | isolate_file_path='chrome/chrome_public_test_apk.isolate'), |
| 73 | I('AndroidWebView', |
| 74 | 'AndroidWebView.apk', |
| 75 | 'org.chromium.android_webview.shell', |
| 76 | 'AndroidWebViewTest', |
| 77 | 'webview:android_webview/test/data/device_files', |
| 78 | isolate_file_path='android_webview/android_webview_test_data.isolate'), |
| 79 | I('ChromeSyncShell', |
| 80 | 'ChromeSyncShell.apk', |
| 81 | 'org.chromium.chrome.browser.sync', |
| 82 | 'ChromeSyncShellTest', |
| 83 | None), |
| 84 | ]) |
| 85 | |
| 86 | InstallablePackage = collections.namedtuple('InstallablePackage', [ |
| 87 | 'name', 'apk', 'apk_package']) |
| 88 | |
| 89 | INSTALLABLE_PACKAGES = dict((package.name, package) for package in ( |
| 90 | [InstallablePackage(i.name, i.apk, i.apk_package) |
| 91 | for i in INSTRUMENTATION_TESTS.itervalues()] + |
| 92 | [InstallablePackage('ChromeDriverWebViewShell', |
| 93 | 'ChromeDriverWebViewShell.apk', |
| 94 | 'org.chromium.chromedriver_webview_shell')])) |
| 95 | |
| 96 | VALID_TESTS = set([ |
| 97 | 'base_junit_tests', |
| 98 | 'chromedriver', |
| 99 | 'components_browsertests', |
| 100 | 'gfx_unittests', |
| 101 | 'gl_unittests', |
| 102 | 'gpu', |
| 103 | 'python_unittests', |
| 104 | 'ui', |
| 105 | 'unit', |
| 106 | 'webkit', |
| 107 | 'webkit_layout' |
| 108 | ]) |
| 109 | |
| 110 | RunCmd = bb_utils.RunCmd |
| 111 | |
| 112 | |
| 113 | def _GetRevision(options): |
| 114 | """Get the SVN revision number. |
| 115 | |
| 116 | Args: |
| 117 | options: options object. |
| 118 | |
| 119 | Returns: |
| 120 | The revision number. |
| 121 | """ |
| 122 | revision = options.build_properties.get('got_revision') |
| 123 | if not revision: |
| 124 | revision = options.build_properties.get('revision', 'testing') |
| 125 | return revision |
| 126 | |
| 127 | |
| 128 | def _RunTest(options, cmd, suite): |
| 129 | """Run test command with runtest.py. |
| 130 | |
| 131 | Args: |
| 132 | options: options object. |
| 133 | cmd: the command to run. |
| 134 | suite: test name. |
| 135 | """ |
| 136 | property_args = bb_utils.EncodeProperties(options) |
| 137 | args = [os.path.join(SLAVE_SCRIPTS_DIR, 'runtest.py')] + property_args |
| 138 | args += ['--test-platform', 'android'] |
| 139 | if options.factory_properties.get('generate_gtest_json'): |
| 140 | args.append('--generate-json-file') |
| 141 | args += ['-o', 'gtest-results/%s' % suite, |
| 142 | '--annotate', 'gtest', |
| 143 | '--build-number', str(options.build_properties.get('buildnumber', |
| 144 | '')), |
| 145 | '--builder-name', options.build_properties.get('buildername', '')] |
| 146 | if options.target == 'Release': |
| 147 | args += ['--target', 'Release'] |
| 148 | else: |
| 149 | args += ['--target', 'Debug'] |
| 150 | if options.flakiness_server: |
| 151 | args += ['--flakiness-dashboard-server=%s' % |
| 152 | options.flakiness_server] |
| 153 | args += cmd |
| 154 | RunCmd(args, cwd=DIR_BUILD_ROOT) |
| 155 | |
| 156 | |
| 157 | def RunTestSuites(options, suites, suites_options=None): |
| 158 | """Manages an invocation of test_runner.py for gtests. |
| 159 | |
| 160 | Args: |
| 161 | options: options object. |
| 162 | suites: List of suite names to run. |
| 163 | suites_options: Command line options dictionary for particular suites. |
| 164 | For example, |
| 165 | {'content_browsertests', ['--num_retries=1', '--release']} |
| 166 | will add the options only to content_browsertests. |
| 167 | """ |
| 168 | |
| 169 | if not suites_options: |
| 170 | suites_options = {} |
| 171 | |
| 172 | args = ['--verbose', '--blacklist-file', 'out/bad_devices.json'] |
| 173 | if options.target == 'Release': |
| 174 | args.append('--release') |
| 175 | if options.asan: |
| 176 | args.append('--tool=asan') |
| 177 | if options.gtest_filter: |
| 178 | args.append('--gtest-filter=%s' % options.gtest_filter) |
| 179 | |
| 180 | for suite in suites: |
| 181 | bb_annotations.PrintNamedStep(suite) |
| 182 | cmd = [suite] + args |
| 183 | cmd += suites_options.get(suite, []) |
| 184 | if suite == 'content_browsertests' or suite == 'components_browsertests': |
| 185 | cmd.append('--num_retries=1') |
| 186 | _RunTest(options, cmd, suite) |
| 187 | |
| 188 | |
| 189 | def RunJunitSuite(suite): |
| 190 | bb_annotations.PrintNamedStep(suite) |
| 191 | RunCmd(['build/android/test_runner.py', 'junit', '-s', suite]) |
| 192 | |
| 193 | |
| 194 | def RunChromeDriverTests(options): |
| 195 | """Run all the steps for running chromedriver tests.""" |
| 196 | bb_annotations.PrintNamedStep('chromedriver_annotation') |
| 197 | RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', |
| 198 | '--android-packages=%s,%s,%s,%s' % |
| 199 | ('chromium', |
| 200 | 'chrome_stable', |
| 201 | 'chrome_beta', |
| 202 | 'chromedriver_webview_shell'), |
| 203 | '--revision=%s' % _GetRevision(options), |
| 204 | '--update-log']) |
| 205 | |
| 206 | |
| 207 | def InstallApk(options, test, print_step=False): |
| 208 | """Install an apk to all phones. |
| 209 | |
| 210 | Args: |
| 211 | options: options object |
| 212 | test: An I_TEST namedtuple |
| 213 | print_step: Print a buildbot step |
| 214 | """ |
| 215 | if print_step: |
| 216 | bb_annotations.PrintNamedStep('install_%s' % test.name.lower()) |
| 217 | |
| 218 | args = [ |
| 219 | '--apk_package', test.apk_package, |
| 220 | '--blacklist-file', 'out/bad_devices.json', |
| 221 | ] |
| 222 | if options.target == 'Release': |
| 223 | args.append('--release') |
| 224 | args.append(test.apk) |
| 225 | |
| 226 | RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True) |
| 227 | |
| 228 | |
| 229 | def RunInstrumentationSuite(options, test, flunk_on_failure=True, |
| 230 | python_only=False, official_build=False): |
| 231 | """Manages an invocation of test_runner.py for instrumentation tests. |
| 232 | |
| 233 | Args: |
| 234 | options: options object |
| 235 | test: An I_TEST namedtuple |
| 236 | flunk_on_failure: Flunk the step if tests fail. |
| 237 | Python: Run only host driven Python tests. |
| 238 | official_build: Run official-build tests. |
| 239 | """ |
| 240 | bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower()) |
| 241 | |
| 242 | if test.apk: |
| 243 | InstallApk(options, test) |
| 244 | args = [ |
| 245 | '--test-apk', test.test_apk, '--verbose', |
| 246 | '--blacklist-file', 'out/bad_devices.json' |
| 247 | ] |
| 248 | if test.test_data: |
| 249 | args.extend(['--test_data', test.test_data]) |
| 250 | if options.target == 'Release': |
| 251 | args.append('--release') |
| 252 | if options.asan: |
| 253 | args.append('--tool=asan') |
| 254 | if options.flakiness_server: |
| 255 | args.append('--flakiness-dashboard-server=%s' % |
| 256 | options.flakiness_server) |
| 257 | if options.coverage_bucket: |
| 258 | args.append('--coverage-dir=%s' % options.coverage_dir) |
| 259 | if test.isolate_file_path: |
| 260 | args.append('--isolate-file-path=%s' % test.isolate_file_path) |
| 261 | if test.host_driven_root: |
| 262 | args.append('--host-driven-root=%s' % test.host_driven_root) |
| 263 | if test.annotation: |
| 264 | args.extend(['-A', test.annotation]) |
| 265 | if test.exclude_annotation: |
| 266 | args.extend(['-E', test.exclude_annotation]) |
| 267 | if test.extra_flags: |
| 268 | args.extend(test.extra_flags) |
| 269 | if python_only: |
| 270 | args.append('-p') |
| 271 | if official_build: |
| 272 | # The option needs to be assigned 'True' as it does not have an action |
| 273 | # associated with it. |
| 274 | args.append('--official-build') |
| 275 | |
| 276 | RunCmd(['build/android/test_runner.py', 'instrumentation'] + args, |
| 277 | flunk_on_failure=flunk_on_failure) |
| 278 | |
| 279 | |
| 280 | def RunWebkitLint(): |
| 281 | """Lint WebKit's TestExpectation files.""" |
| 282 | bb_annotations.PrintNamedStep('webkit_lint') |
| 283 | RunCmd([SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'lint-test-expectations'))]) |
| 284 | |
| 285 | |
| 286 | def RunWebkitLayoutTests(options): |
| 287 | """Run layout tests on an actual device.""" |
| 288 | bb_annotations.PrintNamedStep('webkit_tests') |
| 289 | cmd_args = [ |
| 290 | '--no-show-results', |
| 291 | '--no-new-test-results', |
| 292 | '--full-results-html', |
| 293 | '--clobber-old-results', |
| 294 | '--exit-after-n-failures', '5000', |
| 295 | '--exit-after-n-crashes-or-timeouts', '100', |
| 296 | '--debug-rwt-logging', |
| 297 | '--results-directory', '../layout-test-results', |
| 298 | '--target', options.target, |
| 299 | '--builder-name', options.build_properties.get('buildername', ''), |
| 300 | '--build-number', str(options.build_properties.get('buildnumber', '')), |
| 301 | '--master-name', 'ChromiumWebkit', # TODO: Get this from the cfg. |
| 302 | '--build-name', options.build_properties.get('buildername', ''), |
| 303 | '--platform=android'] |
| 304 | |
| 305 | for flag in 'test_results_server', 'driver_name', 'additional_driver_flag': |
| 306 | if flag in options.factory_properties: |
| 307 | cmd_args.extend(['--%s' % flag.replace('_', '-'), |
| 308 | options.factory_properties.get(flag)]) |
| 309 | |
| 310 | for f in options.factory_properties.get('additional_expectations', []): |
| 311 | cmd_args.extend( |
| 312 | ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) |
| 313 | |
| 314 | # TODO(dpranke): Remove this block after |
| 315 | # https://codereview.chromium.org/12927002/ lands. |
| 316 | for f in options.factory_properties.get('additional_expectations_files', []): |
| 317 | cmd_args.extend( |
| 318 | ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) |
| 319 | |
| 320 | exit_code = RunCmd( |
| 321 | [SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] + cmd_args) |
| 322 | if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS |
| 323 | bb_annotations.PrintMsg('?? (crashed or hung)') |
| 324 | elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS |
| 325 | bb_annotations.PrintMsg('?? (no devices found)') |
| 326 | elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS |
| 327 | bb_annotations.PrintMsg('?? (no tests found)') |
| 328 | else: |
| 329 | full_results_path = os.path.join('..', 'layout-test-results', |
| 330 | 'full_results.json') |
| 331 | if os.path.exists(full_results_path): |
| 332 | full_results = json.load(open(full_results_path)) |
| 333 | unexpected_passes, unexpected_failures, unexpected_flakes = ( |
| 334 | _ParseLayoutTestResults(full_results)) |
| 335 | if unexpected_failures: |
| 336 | _PrintDashboardLink('failed', unexpected_failures.keys(), |
| 337 | max_tests=25) |
| 338 | elif unexpected_passes: |
| 339 | _PrintDashboardLink('unexpected passes', unexpected_passes.keys(), |
| 340 | max_tests=10) |
| 341 | if unexpected_flakes: |
| 342 | _PrintDashboardLink('unexpected flakes', unexpected_flakes.keys(), |
| 343 | max_tests=10) |
| 344 | |
| 345 | if exit_code == 0 and (unexpected_passes or unexpected_flakes): |
| 346 | # If exit_code != 0, RunCmd() will have already printed an error. |
| 347 | bb_annotations.PrintWarning() |
| 348 | else: |
| 349 | bb_annotations.PrintError() |
| 350 | bb_annotations.PrintMsg('?? (results missing)') |
| 351 | |
| 352 | if options.factory_properties.get('archive_webkit_results', False): |
| 353 | bb_annotations.PrintNamedStep('archive_webkit_results') |
| 354 | base = 'https://storage.googleapis.com/chromium-layout-test-archives' |
| 355 | builder_name = options.build_properties.get('buildername', '') |
| 356 | build_number = str(options.build_properties.get('buildnumber', '')) |
| 357 | results_link = '%s/%s/%s/layout-test-results/results.html' % ( |
| 358 | base, EscapeBuilderName(builder_name), build_number) |
| 359 | bb_annotations.PrintLink('results', results_link) |
| 360 | bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % ( |
| 361 | base, EscapeBuilderName(builder_name), build_number)) |
| 362 | gs_bucket = 'gs://chromium-layout-test-archives' |
| 363 | RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium', |
| 364 | 'archive_layout_test_results.py'), |
| 365 | '--results-dir', '../../layout-test-results', |
| 366 | '--build-number', build_number, |
| 367 | '--builder-name', builder_name, |
| 368 | '--gs-bucket', gs_bucket], |
| 369 | cwd=DIR_BUILD_ROOT) |
| 370 | |
| 371 | |
| 372 | def _ParseLayoutTestResults(results): |
| 373 | """Extract the failures from the test run.""" |
| 374 | # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results |
| 375 | tests = _ConvertTrieToFlatPaths(results['tests']) |
| 376 | failures = {} |
| 377 | flakes = {} |
| 378 | passes = {} |
| 379 | for (test, result) in tests.iteritems(): |
| 380 | if result.get('is_unexpected'): |
| 381 | actual_results = result['actual'].split() |
| 382 | expected_results = result['expected'].split() |
| 383 | if len(actual_results) > 1: |
| 384 | # We report the first failure type back, even if the second |
| 385 | # was more severe. |
| 386 | if actual_results[1] in expected_results: |
| 387 | flakes[test] = actual_results[0] |
| 388 | else: |
| 389 | failures[test] = actual_results[0] |
| 390 | elif actual_results[0] == 'PASS': |
| 391 | passes[test] = result |
| 392 | else: |
| 393 | failures[test] = actual_results[0] |
| 394 | |
| 395 | return (passes, failures, flakes) |
| 396 | |
| 397 | |
| 398 | def _ConvertTrieToFlatPaths(trie, prefix=None): |
| 399 | """Flatten the trie of failures into a list.""" |
| 400 | # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results |
| 401 | result = {} |
| 402 | for name, data in trie.iteritems(): |
| 403 | if prefix: |
| 404 | name = prefix + '/' + name |
| 405 | |
| 406 | if len(data) and 'actual' not in data and 'expected' not in data: |
| 407 | result.update(_ConvertTrieToFlatPaths(data, name)) |
| 408 | else: |
| 409 | result[name] = data |
| 410 | |
| 411 | return result |
| 412 | |
| 413 | |
| 414 | def _PrintDashboardLink(link_text, tests, max_tests): |
| 415 | """Add a link to the flakiness dashboard in the step annotations.""" |
| 416 | if len(tests) > max_tests: |
| 417 | test_list_text = ' '.join(tests[:max_tests]) + ' and more' |
| 418 | else: |
| 419 | test_list_text = ' '.join(tests) |
| 420 | |
| 421 | dashboard_base = ('http://test-results.appspot.com' |
| 422 | '/dashboards/flakiness_dashboard.html#' |
| 423 | 'master=ChromiumWebkit&tests=') |
| 424 | |
| 425 | bb_annotations.PrintLink('%d %s: %s' % |
| 426 | (len(tests), link_text, test_list_text), |
| 427 | dashboard_base + ','.join(tests)) |
| 428 | |
| 429 | |
| 430 | def EscapeBuilderName(builder_name): |
| 431 | return re.sub('[ ()]', '_', builder_name) |
| 432 | |
| 433 | |
| 434 | def SpawnLogcatMonitor(): |
| 435 | shutil.rmtree(LOGCAT_DIR, ignore_errors=True) |
| 436 | bb_utils.SpawnCmd([ |
| 437 | os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'), |
| 438 | LOGCAT_DIR]) |
| 439 | |
| 440 | # Wait for logcat_monitor to pull existing logcat |
| 441 | RunCmd(['sleep', '5']) |
| 442 | |
| 443 | |
| 444 | def ProvisionDevices(options): |
| 445 | bb_annotations.PrintNamedStep('provision_devices') |
| 446 | |
| 447 | if not bb_utils.TESTING: |
| 448 | # Restart adb to work around bugs, sleep to wait for usb discovery. |
| 449 | device_utils.RestartServer() |
| 450 | RunCmd(['sleep', '1']) |
| 451 | provision_cmd = [ |
| 452 | 'build/android/provision_devices.py', '-t', options.target, |
| 453 | '--blacklist-file', 'out/bad_devices.json' |
| 454 | ] |
| 455 | if options.auto_reconnect: |
| 456 | provision_cmd.append('--auto-reconnect') |
| 457 | if options.skip_wipe: |
| 458 | provision_cmd.append('--skip-wipe') |
| 459 | if options.disable_location: |
| 460 | provision_cmd.append('--disable-location') |
| 461 | RunCmd(provision_cmd, halt_on_failure=True) |
| 462 | |
| 463 | |
| 464 | def DeviceStatusCheck(options): |
| 465 | bb_annotations.PrintNamedStep('device_status_check') |
| 466 | cmd = [ |
| 467 | 'build/android/buildbot/bb_device_status_check.py', |
| 468 | '--blacklist-file', 'out/bad_devices.json', |
| 469 | ] |
| 470 | if options.restart_usb: |
| 471 | cmd.append('--restart-usb') |
| 472 | RunCmd(cmd, halt_on_failure=True) |
| 473 | |
| 474 | |
| 475 | def GetDeviceSetupStepCmds(): |
| 476 | return [ |
| 477 | ('device_status_check', DeviceStatusCheck), |
| 478 | ('provision_devices', ProvisionDevices), |
| 479 | ] |
| 480 | |
| 481 | |
| 482 | def RunUnitTests(options): |
| 483 | suites = gtest_config.STABLE_TEST_SUITES |
| 484 | if options.asan: |
| 485 | suites = [s for s in suites |
| 486 | if s not in gtest_config.ASAN_EXCLUDED_TEST_SUITES] |
| 487 | RunTestSuites(options, suites) |
| 488 | |
| 489 | |
| 490 | def RunInstrumentationTests(options): |
| 491 | for test in INSTRUMENTATION_TESTS.itervalues(): |
| 492 | RunInstrumentationSuite(options, test) |
| 493 | |
| 494 | |
| 495 | def RunWebkitTests(options): |
| 496 | RunTestSuites(options, ['webkit_unit_tests', 'blink_heap_unittests']) |
| 497 | RunWebkitLint() |
| 498 | |
| 499 | |
| 500 | def RunGPUTests(options): |
| 501 | exit_code = 0 |
| 502 | revision = _GetRevision(options) |
| 503 | builder_name = options.build_properties.get('buildername', 'noname') |
| 504 | |
| 505 | bb_annotations.PrintNamedStep('pixel_tests') |
| 506 | exit_code = RunCmd(['content/test/gpu/run_gpu_test.py', |
| 507 | 'pixel', '-v', |
| 508 | '--browser', |
| 509 | 'android-content-shell', |
| 510 | '--build-revision', |
| 511 | str(revision), |
| 512 | '--upload-refimg-to-cloud-storage', |
| 513 | '--refimg-cloud-storage-bucket', |
| 514 | 'chromium-gpu-archive/reference-images', |
| 515 | '--os-type', |
| 516 | 'android', |
| 517 | '--test-machine-name', |
| 518 | EscapeBuilderName(builder_name), |
| 519 | '--android-blacklist-file', |
| 520 | 'out/bad_devices.json']) or exit_code |
| 521 | |
| 522 | bb_annotations.PrintNamedStep('webgl_conformance_tests') |
| 523 | exit_code = RunCmd(['content/test/gpu/run_gpu_test.py', '-v', |
| 524 | '--browser=android-content-shell', 'webgl_conformance', |
| 525 | '--webgl-conformance-version=1.0.1', |
| 526 | '--android-blacklist-file', |
| 527 | 'out/bad_devices.json']) or exit_code |
| 528 | |
| 529 | bb_annotations.PrintNamedStep('android_webview_webgl_conformance_tests') |
| 530 | exit_code = RunCmd(['content/test/gpu/run_gpu_test.py', '-v', |
| 531 | '--browser=android-webview-shell', 'webgl_conformance', |
| 532 | '--webgl-conformance-version=1.0.1', |
| 533 | '--android-blacklist-file', |
| 534 | 'out/bad_devices.json']) or exit_code |
| 535 | |
| 536 | bb_annotations.PrintNamedStep('gpu_rasterization_tests') |
| 537 | exit_code = RunCmd(['content/test/gpu/run_gpu_test.py', |
| 538 | 'gpu_rasterization', '-v', |
| 539 | '--browser', |
| 540 | 'android-content-shell', |
| 541 | '--build-revision', |
| 542 | str(revision), |
| 543 | '--test-machine-name', |
| 544 | EscapeBuilderName(builder_name), |
| 545 | '--android-blacklist-file', |
| 546 | 'out/bad_devices.json']) or exit_code |
| 547 | |
| 548 | return exit_code |
| 549 | |
| 550 | |
| 551 | def RunPythonUnitTests(_options): |
| 552 | for suite in constants.PYTHON_UNIT_TEST_SUITES: |
| 553 | bb_annotations.PrintNamedStep(suite) |
| 554 | RunCmd(['build/android/test_runner.py', 'python', '-s', suite]) |
| 555 | |
| 556 | |
| 557 | def GetTestStepCmds(): |
| 558 | return [ |
| 559 | ('base_junit_tests', |
| 560 | lambda _options: RunJunitSuite('base_junit_tests')), |
| 561 | ('chromedriver', RunChromeDriverTests), |
| 562 | ('components_browsertests', |
| 563 | lambda options: RunTestSuites(options, ['components_browsertests'])), |
| 564 | ('gfx_unittests', |
| 565 | lambda options: RunTestSuites(options, ['gfx_unittests'])), |
| 566 | ('gl_unittests', |
| 567 | lambda options: RunTestSuites(options, ['gl_unittests'])), |
| 568 | ('gpu', RunGPUTests), |
| 569 | ('python_unittests', RunPythonUnitTests), |
| 570 | ('ui', RunInstrumentationTests), |
| 571 | ('unit', RunUnitTests), |
| 572 | ('webkit', RunWebkitTests), |
| 573 | ('webkit_layout', RunWebkitLayoutTests), |
| 574 | ] |
| 575 | |
| 576 | |
| 577 | def MakeGSPath(options, gs_base_dir): |
| 578 | revision = _GetRevision(options) |
| 579 | bot_id = options.build_properties.get('buildername', 'testing') |
| 580 | randhash = hashlib.sha1(str(random.random())).hexdigest() |
| 581 | gs_path = '%s/%s/%s/%s' % (gs_base_dir, bot_id, revision, randhash) |
| 582 | # remove double slashes, happens with blank revisions and confuses gsutil |
| 583 | gs_path = re.sub('/+', '/', gs_path) |
| 584 | return gs_path |
| 585 | |
| 586 | def UploadHTML(options, gs_base_dir, dir_to_upload, link_text, |
| 587 | link_rel_path='index.html', gs_url=GS_URL): |
| 588 | """Uploads directory at |dir_to_upload| to Google Storage and output a link. |
| 589 | |
| 590 | Args: |
| 591 | options: Command line options. |
| 592 | gs_base_dir: The Google Storage base directory (e.g. |
| 593 | 'chromium-code-coverage/java') |
| 594 | dir_to_upload: Absolute path to the directory to be uploaded. |
| 595 | link_text: Link text to be displayed on the step. |
| 596 | link_rel_path: Link path relative to |dir_to_upload|. |
| 597 | gs_url: Google storage URL. |
| 598 | """ |
| 599 | gs_path = MakeGSPath(options, gs_base_dir) |
| 600 | RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', dir_to_upload, 'gs://%s' % gs_path]) |
| 601 | bb_annotations.PrintLink(link_text, |
| 602 | '%s/%s/%s' % (gs_url, gs_path, link_rel_path)) |
| 603 | |
| 604 | |
| 605 | def GenerateJavaCoverageReport(options): |
| 606 | """Generates an HTML coverage report using EMMA and uploads it.""" |
| 607 | bb_annotations.PrintNamedStep('java_coverage_report') |
| 608 | |
| 609 | coverage_html = os.path.join(options.coverage_dir, 'coverage_html') |
| 610 | RunCmd(['build/android/generate_emma_html.py', |
| 611 | '--coverage-dir', options.coverage_dir, |
| 612 | '--metadata-dir', os.path.join(CHROME_OUT_DIR, options.target), |
| 613 | '--cleanup', |
| 614 | '--output', os.path.join(coverage_html, 'index.html')]) |
| 615 | return coverage_html |
| 616 | |
| 617 | |
| 618 | def LogcatDump(options): |
| 619 | # Print logcat, kill logcat monitor |
| 620 | bb_annotations.PrintNamedStep('logcat_dump') |
| 621 | logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt') |
| 622 | RunCmd([SrcPath('build', 'android', 'adb_logcat_printer.py'), |
| 623 | '--output-path', logcat_file, LOGCAT_DIR]) |
| 624 | gs_path = MakeGSPath(options, 'chromium-android/logcat_dumps') |
| 625 | RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-z', 'txt', logcat_file, |
| 626 | 'gs://%s' % gs_path]) |
| 627 | bb_annotations.PrintLink('logcat dump', '%s/%s' % (GS_AUTH_URL, gs_path)) |
| 628 | |
| 629 | |
| 630 | def RunStackToolSteps(options): |
| 631 | """Run stack tool steps. |
| 632 | |
| 633 | Stack tool is run for logcat dump, optionally for ASAN. |
| 634 | """ |
| 635 | bb_annotations.PrintNamedStep('Run stack tool with logcat dump') |
| 636 | build_dir = os.path.join(CHROME_OUT_DIR, options.target) |
| 637 | logcat_file = os.path.join(build_dir, 'full_log.txt') |
| 638 | RunCmd([os.path.join(CHROME_SRC_DIR, 'third_party', 'android_platform', |
| 639 | 'development', 'scripts', 'stack'), |
| 640 | '--output-directory', build_dir, |
| 641 | '--more-info', logcat_file]) |
| 642 | if options.asan_symbolize: |
| 643 | bb_annotations.PrintNamedStep('Run stack tool for ASAN') |
| 644 | RunCmd([ |
| 645 | os.path.join(CHROME_SRC_DIR, 'build', 'android', 'asan_symbolize.py'), |
| 646 | '--output-directory', build_dir, |
| 647 | '-l', logcat_file]) |
| 648 | |
| 649 | |
| 650 | def GenerateTestReport(options): |
| 651 | bb_annotations.PrintNamedStep('test_report') |
| 652 | for report in glob.glob( |
| 653 | os.path.join(CHROME_OUT_DIR, options.target, 'test_logs', '*.log')): |
| 654 | RunCmd(['cat', report]) |
| 655 | os.remove(report) |
| 656 | |
| 657 | |
| 658 | def MainTestWrapper(options): |
| 659 | exit_code = 0 |
| 660 | try: |
| 661 | # Spawn logcat monitor |
| 662 | SpawnLogcatMonitor() |
| 663 | |
| 664 | # Run all device setup steps |
| 665 | for _, cmd in GetDeviceSetupStepCmds(): |
| 666 | cmd(options) |
| 667 | |
| 668 | if options.install: |
| 669 | for i in options.install: |
| 670 | install_obj = INSTALLABLE_PACKAGES[i] |
| 671 | InstallApk(options, install_obj, print_step=True) |
| 672 | |
| 673 | if options.test_filter: |
| 674 | exit_code = bb_utils.RunSteps( |
| 675 | options.test_filter, GetTestStepCmds(), options) or exit_code |
| 676 | |
| 677 | if options.coverage_bucket: |
| 678 | coverage_html = GenerateJavaCoverageReport(options) |
| 679 | UploadHTML(options, '%s/java' % options.coverage_bucket, coverage_html, |
| 680 | 'Coverage Report') |
| 681 | shutil.rmtree(coverage_html, ignore_errors=True) |
| 682 | |
| 683 | if options.experimental: |
| 684 | exit_code = RunTestSuites( |
| 685 | options, gtest_config.EXPERIMENTAL_TEST_SUITES) or exit_code |
| 686 | |
| 687 | return exit_code |
| 688 | |
| 689 | finally: |
| 690 | # Run all post test steps |
| 691 | LogcatDump(options) |
| 692 | if not options.disable_stack_tool: |
| 693 | RunStackToolSteps(options) |
| 694 | GenerateTestReport(options) |
| 695 | # KillHostHeartbeat() has logic to check if heartbeat process is running, |
| 696 | # and kills only if it finds the process is running on the host. |
| 697 | provision_devices.KillHostHeartbeat() |
| 698 | if options.cleanup: |
| 699 | shutil.rmtree(os.path.join(CHROME_OUT_DIR, options.target), |
| 700 | ignore_errors=True) |
| 701 | |
| 702 | |
| 703 | def GetDeviceStepsOptParser(): |
| 704 | parser = bb_utils.GetParser() |
| 705 | parser.add_option('--experimental', action='store_true', |
| 706 | help='Run experiemental tests') |
| 707 | parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], |
| 708 | action='append', |
| 709 | help=('Run a test suite. Test suites: "%s"' % |
| 710 | '", "'.join(VALID_TESTS))) |
| 711 | parser.add_option('--gtest-filter', |
| 712 | help='Filter for running a subset of tests of a gtest test') |
| 713 | parser.add_option('--asan', action='store_true', help='Run tests with asan.') |
| 714 | parser.add_option('--install', metavar='<apk name>', action="append", |
| 715 | help='Install an apk by name') |
| 716 | parser.add_option('--no-reboot', action='store_true', |
| 717 | help='Do not reboot devices during provisioning.') |
| 718 | parser.add_option('--coverage-bucket', |
| 719 | help=('Bucket name to store coverage results. Coverage is ' |
| 720 | 'only run if this is set.')) |
| 721 | parser.add_option('--restart-usb', action='store_true', |
| 722 | help='Restart usb ports before device status check.') |
| 723 | parser.add_option( |
| 724 | '--flakiness-server', |
| 725 | help=('The flakiness dashboard server to which the results should be ' |
| 726 | 'uploaded.')) |
| 727 | parser.add_option( |
| 728 | '--auto-reconnect', action='store_true', |
| 729 | help='Push script to device which restarts adbd on disconnections.') |
| 730 | parser.add_option('--skip-wipe', action='store_true', |
| 731 | help='Do not wipe devices during provisioning.') |
| 732 | parser.add_option('--disable-location', action='store_true', |
| 733 | help='Disable location settings.') |
| 734 | parser.add_option( |
| 735 | '--logcat-dump-output', |
| 736 | help='The logcat dump output will be "tee"-ed into this file') |
| 737 | # During processing perf bisects, a seperate working directory created under |
| 738 | # which builds are produced. Therefore we should look for relevent output |
| 739 | # file under this directory.(/b/build/slave/<slave_name>/build/bisect/src/out) |
| 740 | parser.add_option( |
| 741 | '--chrome-output-dir', |
| 742 | help='Chrome output directory to be used while bisecting.') |
| 743 | |
| 744 | parser.add_option('--disable-stack-tool', action='store_true', |
| 745 | help='Do not run stack tool.') |
| 746 | parser.add_option('--asan-symbolize', action='store_true', |
| 747 | help='Run stack tool for ASAN') |
| 748 | parser.add_option('--cleanup', action='store_true', |
| 749 | help='Delete out/<target> directory at the end of the run.') |
| 750 | return parser |
| 751 | |
| 752 | |
| 753 | def main(argv): |
| 754 | parser = GetDeviceStepsOptParser() |
| 755 | options, args = parser.parse_args(argv[1:]) |
| 756 | |
| 757 | devil_chromium.Initialize() |
| 758 | |
| 759 | if args: |
| 760 | return sys.exit('Unused args %s' % args) |
| 761 | |
| 762 | unknown_tests = set(options.test_filter) - VALID_TESTS |
| 763 | if unknown_tests: |
| 764 | return sys.exit('Unknown tests %s' % list(unknown_tests)) |
| 765 | |
| 766 | setattr(options, 'target', options.factory_properties.get('target', 'Debug')) |
| 767 | |
| 768 | # pylint: disable=global-statement |
| 769 | if options.chrome_output_dir: |
| 770 | global CHROME_OUT_DIR |
| 771 | global LOGCAT_DIR |
| 772 | CHROME_OUT_DIR = options.chrome_output_dir |
| 773 | LOGCAT_DIR = os.path.join(CHROME_OUT_DIR, 'logcat') |
| 774 | |
| 775 | if options.coverage_bucket: |
| 776 | setattr(options, 'coverage_dir', |
| 777 | os.path.join(CHROME_OUT_DIR, options.target, 'coverage')) |
| 778 | |
| 779 | return MainTestWrapper(options) |
| 780 | |
| 781 | |
| 782 | if __name__ == '__main__': |
| 783 | sys.exit(main(sys.argv)) |