Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 1 | # Copyright 2018 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Run the skylab staging test. |
| 6 | |
| 7 | This script runs a suite of autotest tests and some other sanity checks against |
| 8 | a given Skylab instance. If all sanity checks and tests have the expected |
| 9 | results, the script exits with success. |
| 10 | |
| 11 | This script is intended to be used for the Autotest staging lab's test_push. |
| 12 | This script does not update any software before running the tests (i.e. caller |
| 13 | is responsible for setting up the staging lab with the correct software |
| 14 | beforehand), nor does it update any software refs on success (i.e. caller is |
| 15 | responsible for blessing the newer version of software as needed). |
| 16 | """ |
| 17 | |
| 18 | from __future__ import absolute_import |
| 19 | from __future__ import division |
| 20 | from __future__ import print_function |
| 21 | |
| 22 | import argparse |
Alex Zamorzaev | f0573b5 | 2019-04-05 12:07:59 -0700 | [diff] [blame] | 23 | import collections |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 24 | import json |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 25 | import logging |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 26 | import os |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 27 | import sys |
| 28 | import time |
| 29 | |
| 30 | from lucifer import autotest |
| 31 | from lucifer import loglib |
| 32 | from skylab_staging import errors |
| 33 | from skylab_staging import swarming |
| 34 | |
Aviv Keshet | 7ee12c5 | 2019-05-21 14:53:15 -0700 | [diff] [blame] | 35 | |
| 36 | cros_build_lib = autotest.deferred_chromite_load('cros_build_lib') |
| 37 | metrics = autotest.deferred_chromite_load('metrics') |
| 38 | ts_mon_config = autotest.deferred_chromite_load('ts_mon_config') |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 39 | |
| 40 | |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 41 | _METRICS_PREFIX = 'chromeos/autotest/test_push/skylab' |
| 42 | _POLLING_INTERVAL_S = 10 |
| 43 | _WAIT_FOR_DUTS_TIMEOUT_S = 20 * 60 |
| 44 | |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 45 | # Dictionary of test results expected in suite:skylab_staging_test. |
Alex Zamorzaev | f0573b5 | 2019-04-05 12:07:59 -0700 | [diff] [blame] | 46 | _EXPECTED_TEST_RESULTS = {'login_LoginSuccess.*': ['GOOD'], |
| 47 | 'provision_AutoUpdate.double': ['GOOD'], |
Alex Zamorzaev | 9033aca | 2019-04-18 14:08:19 -0700 | [diff] [blame] | 48 | 'dummy_Pass$': ['GOOD'], |
| 49 | 'dummy_Pass.actionable$': ['GOOD'], |
Alex Zamorzaev | 0891e4e | 2019-04-23 16:42:19 -0700 | [diff] [blame] | 50 | 'dummy_Pass.bluetooth$': ['GOOD'], |
Alex Zamorzaev | 9033aca | 2019-04-18 14:08:19 -0700 | [diff] [blame] | 51 | # ssp and nossp. |
| 52 | 'dummy_PassServer$': ['GOOD', 'GOOD'], |
Aviv Keshet | 52fcf66 | 2019-05-24 00:12:19 +0000 | [diff] [blame] | 53 | 'dummy_Fail.Fail$': ['FAIL'], |
| 54 | 'dummy_Fail.Error$': ['ERROR'], |
| 55 | 'dummy_Fail.Warn$': ['WARN'], |
| 56 | 'dummy_Fail.NAError$': ['TEST_NA'], |
| 57 | 'dummy_Fail.Crash$': ['GOOD'], |
Alex Zamorzaev | 9033aca | 2019-04-18 14:08:19 -0700 | [diff] [blame] | 58 | 'tast.*': ['GOOD'], |
Alex Zamorzaev | f0573b5 | 2019-04-05 12:07:59 -0700 | [diff] [blame] | 59 | } |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 60 | |
| 61 | # Some test could be missing from the test results for various reasons. Add |
| 62 | # such test in this list and explain the reason. |
| 63 | _IGNORED_TESTS = [ |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 64 | ] |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 65 | |
| 66 | _logger = logging.getLogger(__name__) |
| 67 | |
| 68 | |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 69 | def main(): |
| 70 | """Entry point of test_push.""" |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 71 | parser = _get_parser() |
| 72 | loglib.add_logging_options(parser) |
| 73 | args = parser.parse_args() |
| 74 | loglib.configure_logging_with_args(parser, args) |
| 75 | |
| 76 | with ts_mon_config.SetupTsMonGlobalState(service_name='skylab_test_push', |
| 77 | indirect=True): |
| 78 | success = False |
| 79 | try: |
| 80 | with metrics.SecondsTimer(_METRICS_PREFIX + '/durations/total', |
| 81 | add_exception_field=True): |
| 82 | _run_test_push(args) |
| 83 | success = True |
| 84 | finally: |
| 85 | metrics.Counter(_METRICS_PREFIX + '/tick').increment( |
| 86 | fields={'success': success}) |
| 87 | |
| 88 | def _get_parser(): |
| 89 | parser = argparse.ArgumentParser( |
| 90 | description='Run test_push against Skylab instance.') |
| 91 | parser.add_argument( |
| 92 | '--swarming-url', |
| 93 | required=True, |
| 94 | help='Full URL to the Swarming instance to use', |
| 95 | ) |
| 96 | parser.add_argument( |
| 97 | '--swarming-cli', |
| 98 | required=True, |
| 99 | help='Path to the Swarming cli tool.', |
| 100 | ) |
| 101 | # TODO(crbug.com/867969) Use model instead of board once skylab inventory has |
| 102 | # model information. |
| 103 | parser.add_argument( |
| 104 | '--dut-board', |
| 105 | required=True, |
| 106 | help='Label board of the DUTs to use for testing', |
| 107 | ) |
| 108 | parser.add_argument( |
| 109 | '--dut-pool', |
| 110 | required=True, |
| 111 | choices=('DUT_POOL_CQ', 'DUT_POOL_BVT', 'DUT_POOL_SUITES'), |
| 112 | help='Label pool of the DUTs to use for testing', |
| 113 | ) |
| 114 | parser.add_argument( |
| 115 | '--build', |
| 116 | required=True, |
| 117 | help='ChromeOS build to use for provisioning' |
| 118 | ' (e.g.: gandolf-release/R54-8743.25.0).', |
| 119 | ) |
| 120 | parser.add_argument( |
| 121 | '--timeout-mins', |
| 122 | type=int, |
| 123 | required=True, |
| 124 | help='(Optional) Overall timeout for the test_push. On timeout, test_push' |
| 125 | ' attempts to abort any in-flight test suites before quitting.', |
| 126 | ) |
| 127 | parser.add_argument( |
| 128 | '--num-min-duts', |
| 129 | type=int, |
| 130 | help='Minimum number of Ready DUTs required for test suite.', |
| 131 | ) |
| 132 | parser.add_argument( |
| 133 | '--service-account-json', |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 134 | required=True, |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 135 | help='(Optional) Path to the service account credentials file to' |
| 136 | ' authenticate with Swarming service.', |
| 137 | ) |
| 138 | return parser |
| 139 | |
| 140 | |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 141 | def _skylab_tool(): |
| 142 | """Return path to skylab tool.""" |
| 143 | return os.environ.get('SKYLAB_TOOL', '/opt/infra-tools/skylab') |
| 144 | |
| 145 | |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 146 | def _run_test_push(args): |
| 147 | """Meat of the test_push flow.""" |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 148 | deadline = time.time() + (args.timeout_mins * 60) |
| 149 | swclient = swarming.Client(args.swarming_cli, args.swarming_url, |
| 150 | args.service_account_json) |
| 151 | if args.num_min_duts: |
| 152 | _ensure_duts_ready( |
| 153 | swclient, |
| 154 | args.dut_board, |
| 155 | args.dut_pool, |
| 156 | args.num_min_duts, |
| 157 | min(deadline - time.time(), _WAIT_FOR_DUTS_TIMEOUT_S), |
| 158 | ) |
| 159 | |
| 160 | # Just like the builders, first run a provision suite to provision required |
| 161 | # DUTs, then run the actual suite. |
| 162 | with metrics.SecondsTimer(_METRICS_PREFIX + '/durations/provision_suite', |
| 163 | add_exception_field=True): |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 164 | _create_suite_and_wait( |
| 165 | args.dut_board, args.dut_pool, args.build, deadline, |
| 166 | args.service_account_json, 'provision') |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 167 | |
| 168 | with metrics.SecondsTimer(_METRICS_PREFIX + '/durations/push_to_prod_suite', |
| 169 | add_exception_field=True): |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 170 | task_id = _create_suite_and_wait( |
| 171 | args.dut_board, args.dut_pool, args.build, deadline, |
| 172 | args.service_account_json, 'skylab_staging_test', |
| 173 | require_success=False) |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 174 | |
| 175 | _verify_test_results(task_id, _EXPECTED_TEST_RESULTS) |
| 176 | |
| 177 | |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 178 | def _create_suite_and_wait(dut_board, dut_pool, build, deadline, |
| 179 | service_account_json, suite, require_success=True): |
| 180 | """Create and wait for a skylab suite (in staging). |
| 181 | |
Aviv Keshet | 3202b14 | 2019-05-23 14:57:45 -0700 | [diff] [blame] | 182 | Returns: string task run id of the completed suite. |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 183 | |
Aviv Keshet | 3202b14 | 2019-05-23 14:57:45 -0700 | [diff] [blame] | 184 | Raises: errors.TestPushError if the suite failed and require_success is True. |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 185 | """ |
| 186 | mins_remaining = int((deadline - time.time())/60) |
| 187 | cmd = [ |
Aviv Keshet | af01647 | 2019-08-01 13:36:36 -0700 | [diff] [blame^] | 188 | _skylab_tool(), 'create-suite', '-bb=False', |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 189 | # test_push always runs in dev instance of skylab |
| 190 | '-dev', |
| 191 | '-board', dut_board, |
| 192 | '-pool', dut_pool, |
| 193 | '-image', build, |
| 194 | '-timeout-mins', str(mins_remaining), |
| 195 | '-service-account-json', service_account_json, |
| 196 | '-json', |
| 197 | suite, |
| 198 | ] |
| 199 | |
| 200 | cmd_result = cros_build_lib.RunCommand(cmd, redirect_stdout=True) |
| 201 | task_id = json.loads(cmd_result.output)['task_id'] |
| 202 | _logger.info('Triggered suite %s. Task id: %s', suite, task_id) |
| 203 | |
| 204 | cmd = [ |
Aviv Keshet | af01647 | 2019-08-01 13:36:36 -0700 | [diff] [blame^] | 205 | _skylab_tool(), 'wait-task', '-bb=False', |
Aviv Keshet | e10d3e5 | 2019-05-17 13:42:37 -0700 | [diff] [blame] | 206 | '-dev', |
| 207 | '-service-account-json', service_account_json, |
| 208 | task_id |
| 209 | ] |
| 210 | cmd_result = cros_build_lib.RunCommand(cmd, redirect_stdout=True) |
| 211 | |
| 212 | _logger.info( |
| 213 | 'Finished suite %s with output: \n%s', suite, |
| 214 | json.loads(cmd_result.output)['stdout'] |
| 215 | ) |
| 216 | if (require_success and |
| 217 | not json.loads(cmd_result.output)['task-result']['success']): |
| 218 | raise errors.TestPushError('Suite %s did not succeed.' % suite) |
Xixuan Wu | 5dcca50 | 2019-01-07 12:24:37 -0800 | [diff] [blame] | 219 | |
Aviv Keshet | 3202b14 | 2019-05-23 14:57:45 -0700 | [diff] [blame] | 220 | return json.loads(cmd_result.output)['task-result']['task-run-id'] |
Aviv Keshet | 300f69f | 2019-05-23 14:28:33 -0700 | [diff] [blame] | 221 | |
Xixuan Wu | 5dcca50 | 2019-01-07 12:24:37 -0800 | [diff] [blame] | 222 | |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 223 | def _verify_test_results(task_id, expected_results): |
| 224 | """Verify if test results are expected.""" |
Aviv Keshet | 4c22d92 | 2019-05-23 13:44:12 -0700 | [diff] [blame] | 225 | _logger.info('Comparing test results for suite task %s...', task_id) |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 226 | test_views = _get_test_views(task_id) |
| 227 | available_views = [v for v in test_views if _view_is_preserved(v)] |
| 228 | logging.debug('Test results:') |
| 229 | for v in available_views: |
| 230 | logging.debug('%s%s', v['test_name'].ljust(30), v['status']) |
| 231 | |
| 232 | summary = _verify_and_summarize(available_views, expected_results) |
| 233 | if summary: |
| 234 | logging.error('\n'.join(summary)) |
| 235 | raise errors.TestPushError('Test results are not consistent with ' |
| 236 | 'expected results') |
| 237 | |
| 238 | |
| 239 | def _get_test_views(task_id): |
| 240 | """Retrieve test views from TKO for skylab task id.""" |
| 241 | tko_db = autotest.load('tko.db') |
| 242 | db = tko_db.db() |
| 243 | return db.get_child_tests_by_parent_task_id(task_id) |
| 244 | |
| 245 | |
| 246 | def _view_is_preserved(view): |
| 247 | """Detect whether to keep the test view for further comparison.""" |
| 248 | job_status = autotest.load('server.cros.dynamic_suite.job_status') |
| 249 | return (job_status.view_is_relevant(view) and |
| 250 | (not job_status.view_is_for_suite_job(view))) |
| 251 | |
| 252 | |
| 253 | def _verify_and_summarize(available_views, expected_results): |
| 254 | """Verify and generate summaries for test_push results.""" |
| 255 | test_push_common = autotest.load('site_utils.test_push_common') |
Alex Zamorzaev | f0573b5 | 2019-04-05 12:07:59 -0700 | [diff] [blame] | 256 | views = collections.defaultdict(list) |
| 257 | for view in available_views: |
| 258 | views[view['test_name']].append(view['status']) |
Xixuan Wu | 51f41d8 | 2018-08-10 15:19:07 -0700 | [diff] [blame] | 259 | return test_push_common.summarize_push(views, expected_results, |
| 260 | _IGNORED_TESTS) |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 261 | |
| 262 | |
| 263 | def _ensure_duts_ready(swclient, board, pool, min_duts, timeout_s): |
| 264 | """Ensure that at least num_duts are in the ready dut_state.""" |
| 265 | start_time = time.time() |
| 266 | while True: |
| 267 | _logger.debug('Checking whether %d DUTs are available', min_duts) |
| 268 | num_duts = swclient.num_ready_duts(board, pool) |
| 269 | if num_duts >= min_duts: |
| 270 | _logger.info( |
| 271 | '%d available DUTs satisfy the minimum requirement of %d DUTs', |
| 272 | num_duts, min_duts, |
| 273 | ) |
| 274 | return |
| 275 | if time.time() - start_time > timeout_s: |
| 276 | raise errors.TestPushError( |
| 277 | 'Could not find %d ready DUTs with (board:%s, pool:%s) within %d' |
| 278 | ' seconds' % (min_duts, board, pool, timeout_s) |
| 279 | ) |
| 280 | time.sleep(_POLLING_INTERVAL_S) |
| 281 | |
| 282 | |
| 283 | if __name__ == '__main__': |
Aviv Keshet | 7ee12c5 | 2019-05-21 14:53:15 -0700 | [diff] [blame] | 284 | autotest.monkeypatch() |
Prathmesh Prabhu | dd9071f | 2018-07-25 12:36:07 -0700 | [diff] [blame] | 285 | sys.exit(main()) |