Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 1 | # Copyright 2018 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Definition of a CrOS suite in skylab. |
| 6 | |
| 7 | This file is a simplicication of dynamic_suite.suite without any useless |
| 8 | features for skylab suite. |
| 9 | |
| 10 | Suite class in this file mainly has 2 features: |
| 11 | 1. Integrate parameters from control file & passed in arguments. |
| 12 | 2. Find proper child tests for a given suite. |
| 13 | |
| 14 | Use case: |
| 15 | See _run_suite() in skylab_suite.run_suite_skylab. |
| 16 | """ |
| 17 | |
| 18 | from __future__ import absolute_import |
| 19 | from __future__ import division |
| 20 | from __future__ import print_function |
| 21 | |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 22 | import collections |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 23 | import logging |
Xixuan Wu | 8873a0e | 2018-07-20 17:51:33 -0700 | [diff] [blame] | 24 | import os |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 25 | |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 26 | from lucifer import autotest |
Xixuan Wu | 8980338 | 2018-07-27 12:33:33 -0700 | [diff] [blame] | 27 | from skylab_suite import errors |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 28 | from skylab_suite import swarming_lib |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 29 | |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 30 | |
Xixuan Wu | 9287fda | 2018-07-12 16:22:06 -0700 | [diff] [blame] | 31 | SuiteSpec = collections.namedtuple( |
| 32 | 'SuiteSpec', |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 33 | [ |
| 34 | 'builds', |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 35 | 'suite_name', |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 36 | 'suite_file_name', |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 37 | 'test_source_build', |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 38 | 'suite_args', |
Xixuan Wu | 70217a9 | 2018-06-04 16:43:42 -0700 | [diff] [blame] | 39 | 'priority', |
Xixuan Wu | 77d4a59 | 2018-06-08 10:40:57 -0700 | [diff] [blame] | 40 | 'board', |
| 41 | 'pool', |
Xixuan Wu | b279566 | 2018-06-28 16:02:53 -0700 | [diff] [blame] | 42 | 'job_keyvals', |
Xixuan Wu | 0956b63 | 2018-08-06 14:40:23 -0700 | [diff] [blame] | 43 | 'minimum_duts', |
Xixuan Wu | ba0276c | 2018-08-10 09:36:23 -0700 | [diff] [blame] | 44 | 'timeout_mins', |
Aviv Keshet | be570b3 | 2018-12-06 14:46:19 -0800 | [diff] [blame] | 45 | 'quota_account', |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 46 | ]) |
| 47 | |
Xixuan Wu | 834cb4b | 2018-07-12 16:33:49 -0700 | [diff] [blame] | 48 | SuiteHandlerSpec = collections.namedtuple( |
| 49 | 'SuiteHandlerSpec', |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 50 | [ |
Xixuan Wu | 08354a0 | 2018-08-01 09:15:26 -0700 | [diff] [blame] | 51 | 'suite_name', |
Xixuan Wu | f2da195 | 2018-07-10 10:19:42 -0700 | [diff] [blame] | 52 | 'wait', |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 53 | 'suite_id', |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 54 | 'timeout_mins', |
Xixuan Wu | c7dce5ef | 2018-08-19 21:09:53 -0700 | [diff] [blame] | 55 | 'passed_mins', |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 56 | 'test_retry', |
| 57 | 'max_retries', |
| 58 | 'provision_num_required', |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 59 | ]) |
| 60 | |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 61 | TestHandlerSpec = collections.namedtuple( |
| 62 | 'TestHandlerSpec', |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 63 | [ |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 64 | 'test_spec', |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 65 | 'remaining_retries', |
| 66 | 'previous_retried_ids', |
| 67 | ]) |
| 68 | |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 69 | TestSpec = collections.namedtuple( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 70 | 'TestSpec', |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 71 | [ |
| 72 | 'test', |
Xixuan Wu | 70217a9 | 2018-06-04 16:43:42 -0700 | [diff] [blame] | 73 | 'priority', |
Xixuan Wu | 77d4a59 | 2018-06-08 10:40:57 -0700 | [diff] [blame] | 74 | 'board', |
| 75 | 'pool', |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 76 | 'build', |
Xixuan Wu | b279566 | 2018-06-28 16:02:53 -0700 | [diff] [blame] | 77 | 'keyvals', |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 78 | 'bot_id', |
Xixuan Wu | ff19abe | 2018-06-20 10:44:45 -0700 | [diff] [blame] | 79 | 'dut_name', |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 80 | 'expiration_secs', |
| 81 | 'grace_period_secs', |
| 82 | 'execution_timeout_secs', |
| 83 | 'io_timeout_secs', |
Aviv Keshet | be570b3 | 2018-12-06 14:46:19 -0800 | [diff] [blame] | 84 | 'quota_account', |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 85 | ]) |
| 86 | |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 87 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 88 | class SuiteHandler(object): |
| 89 | """The class for handling a CrOS suite run. |
| 90 | |
| 91 | Its responsibility includes handling retries for child tests. |
| 92 | """ |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 93 | |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 94 | def __init__(self, specs): |
Xixuan Wu | 08354a0 | 2018-08-01 09:15:26 -0700 | [diff] [blame] | 95 | self._suite_name = specs.suite_name |
Xixuan Wu | f2da195 | 2018-07-10 10:19:42 -0700 | [diff] [blame] | 96 | self._wait = specs.wait |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 97 | self._timeout_mins = specs.timeout_mins |
| 98 | self._provision_num_required = specs.provision_num_required |
| 99 | self._test_retry = specs.test_retry |
| 100 | self._max_retries = specs.max_retries |
Xixuan Wu | c7dce5ef | 2018-08-19 21:09:53 -0700 | [diff] [blame] | 101 | self.passed_mins = specs.passed_mins |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 102 | |
Xixuan Wu | 8873a0e | 2018-07-20 17:51:33 -0700 | [diff] [blame] | 103 | # The swarming task id of the suite that this suite_handler is handling. |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 104 | self._suite_id = specs.suite_id |
Xixuan Wu | 8873a0e | 2018-07-20 17:51:33 -0700 | [diff] [blame] | 105 | # The swarming task id of current run_suite_skylab process. It could be |
| 106 | # different from self._suite_id if a suite_id is passed in. |
| 107 | self._task_id = os.environ.get('SWARMING_TASK_ID') |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 108 | self._task_to_test_maps = {} |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 109 | self.successfully_provisioned_duts = set() |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 110 | |
| 111 | # It only maintains the swarming task of the final run of each |
| 112 | # child task, i.e. it doesn't include failed swarming tasks of |
| 113 | # each child task which will get retried later. |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 114 | self._active_child_tasks = [] |
| 115 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 116 | def should_wait(self): |
| 117 | """Return whether to wait for a suite's result.""" |
| 118 | return self._wait |
| 119 | |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 120 | def is_provision(self): |
| 121 | """Return whether the suite handler is for provision suite.""" |
Xixuan Wu | 08354a0 | 2018-08-01 09:15:26 -0700 | [diff] [blame] | 122 | return self._suite_name == 'provision' |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 123 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 124 | def set_suite_id(self, suite_id): |
| 125 | """Set swarming task id for a suite. |
| 126 | |
| 127 | @param suite_id: The swarming task id of this suite. |
| 128 | """ |
| 129 | self._suite_id = suite_id |
| 130 | |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 131 | def add_test_by_task_id(self, task_id, test_handler_spec): |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 132 | """Record a child test and its swarming task id. |
| 133 | |
| 134 | @param task_id: the swarming task id of a child test. |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 135 | @param test_handler_spec: a TestHandlerSpec object. |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 136 | """ |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 137 | self._task_to_test_maps[task_id] = test_handler_spec |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 138 | |
| 139 | def get_test_by_task_id(self, task_id): |
| 140 | """Get a child test by its swarming task id. |
| 141 | |
| 142 | @param task_id: the swarming task id of a child test. |
| 143 | """ |
| 144 | return self._task_to_test_maps[task_id] |
| 145 | |
| 146 | def remove_test_by_task_id(self, task_id): |
| 147 | """Delete a child test by its swarming task id. |
| 148 | |
| 149 | @param task_id: the swarming task id of a child test. |
| 150 | """ |
| 151 | self._task_to_test_maps.pop(task_id, None) |
| 152 | |
| 153 | def set_max_retries(self, max_retries): |
| 154 | """Set the max retries for a suite. |
| 155 | |
| 156 | @param max_retries: The current maximum retries to set. |
| 157 | """ |
| 158 | self._max_retries = max_retries |
| 159 | |
| 160 | @property |
Xixuan Wu | 61d6283 | 2018-07-20 17:10:19 -0700 | [diff] [blame] | 161 | def task_to_test_maps(self): |
| 162 | """Get the task_to_test_maps of a suite.""" |
| 163 | return self._task_to_test_maps |
| 164 | |
| 165 | @property |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 166 | def timeout_mins(self): |
| 167 | """Get the timeout minutes of a suite.""" |
| 168 | return self._timeout_mins |
| 169 | |
| 170 | @property |
| 171 | def suite_id(self): |
| 172 | """Get the swarming task id of a suite.""" |
| 173 | return self._suite_id |
| 174 | |
| 175 | @property |
Xixuan Wu | 8873a0e | 2018-07-20 17:51:33 -0700 | [diff] [blame] | 176 | def task_id(self): |
| 177 | """Get swarming task id of current process.""" |
| 178 | return self._task_id |
| 179 | |
| 180 | @property |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 181 | def max_retries(self): |
| 182 | """Get the max num of retries of a suite.""" |
| 183 | return self._max_retries |
| 184 | |
Xixuan Wu | c6e28d3 | 2018-08-27 14:48:14 -0700 | [diff] [blame] | 185 | def get_active_child_tasks(self, suite_id): |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 186 | """Get the child tasks which is actively monitored by a suite. |
| 187 | |
| 188 | The active child tasks list includes tasks which are currently running |
| 189 | or finished without following retries. E.g. |
| 190 | Suite task X: |
| 191 | child task 1: x1 (first try x1_1, second try x1_2) |
| 192 | child task 2: x2 (first try: x2_1) |
| 193 | The final active child task list will include task x1_2 and x2_1, won't |
| 194 | include x1_1 since it's a task which is finished but get retried later. |
| 195 | """ |
Xixuan Wu | c6e28d3 | 2018-08-27 14:48:14 -0700 | [diff] [blame] | 196 | all_tasks = swarming_lib.get_child_tasks(suite_id) |
| 197 | return [t for t in all_tasks if t['task_id'] in self._task_to_test_maps] |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 198 | |
Xixuan Wu | c6e28d3 | 2018-08-27 14:48:14 -0700 | [diff] [blame] | 199 | def handle_results(self, suite_id): |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 200 | """Handle child tasks' results.""" |
Xixuan Wu | c6e28d3 | 2018-08-27 14:48:14 -0700 | [diff] [blame] | 201 | self._active_child_tasks = self.get_active_child_tasks(suite_id) |
| 202 | self.retried_tasks = [t for t in self._active_child_tasks |
| 203 | if self._should_retry(t)] |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 204 | logging.info('Found %d tests to be retried.', len(self.retried_tasks)) |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 205 | |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 206 | def _check_all_tasks_finished(self): |
| 207 | """Check whether all tasks are finished, including retried tasks.""" |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 208 | finished_tasks = [t for t in self._active_child_tasks if |
| 209 | t['state'] in swarming_lib.TASK_FINISHED_STATUS] |
| 210 | logging.info('%d/%d child tasks finished, %d got retried.', |
| 211 | len(finished_tasks), len(self._active_child_tasks), |
| 212 | len(self.retried_tasks)) |
| 213 | return (len(finished_tasks) == len(self._active_child_tasks) |
| 214 | and not self.retried_tasks) |
| 215 | |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 216 | def _set_successful_provisioned_duts(self): |
| 217 | """Set successfully provisioned duts.""" |
| 218 | for t in self._active_child_tasks: |
| 219 | if (swarming_lib.get_task_final_state(t) == |
| 220 | swarming_lib.TASK_COMPLETED_SUCCESS): |
Xixuan Wu | ff19abe | 2018-06-20 10:44:45 -0700 | [diff] [blame] | 221 | dut_name = self.get_test_by_task_id( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 222 | t['task_id']).test_spec.dut_name |
Xixuan Wu | ff19abe | 2018-06-20 10:44:45 -0700 | [diff] [blame] | 223 | if dut_name: |
Xixuan Wu | 415e821 | 2018-06-04 17:01:12 -0700 | [diff] [blame] | 224 | self.successfully_provisioned_duts.add(dut_name) |
| 225 | |
| 226 | def is_provision_successfully_finished(self): |
| 227 | """Check whether provision succeeds.""" |
| 228 | logging.info('Found %d successfully provisioned duts, ' |
| 229 | 'the minimum requirement is %d', |
| 230 | len(self.successfully_provisioned_duts), |
| 231 | self._provision_num_required) |
| 232 | return (len(self.successfully_provisioned_duts) >= |
| 233 | self._provision_num_required) |
| 234 | |
| 235 | def is_finished_waiting(self): |
| 236 | """Check whether the suite should finish its waiting.""" |
| 237 | if self.is_provision(): |
| 238 | self._set_successful_provisioned_duts() |
| 239 | return (self.is_provision_successfully_finished() or |
| 240 | self._check_all_tasks_finished()) |
| 241 | |
| 242 | return self._check_all_tasks_finished() |
| 243 | |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 244 | def _should_retry(self, test_result): |
| 245 | """Check whether a test should be retried. |
| 246 | |
| 247 | We will retry a test if: |
| 248 | 1. The test-level retry is enabled for this suite. |
| 249 | 2. The test fails. |
| 250 | 3. The test is currently monitored by the suite, i.e. |
| 251 | it's not a previous retried test. |
| 252 | 4. The test has remaining retries based on JOB_RETRIES in |
| 253 | its control file. |
| 254 | 5. The suite-level max retries isn't hit. |
| 255 | |
| 256 | @param test_result: A json test result from swarming API. |
| 257 | |
| 258 | @return True if we should retry the test. |
| 259 | """ |
| 260 | task_id = test_result['task_id'] |
| 261 | state = test_result['state'] |
| 262 | is_failure = test_result['failure'] |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 263 | return (self._test_retry and |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 264 | ((state == swarming_lib.TASK_COMPLETED and is_failure) |
Xixuan Wu | aff23c7 | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 265 | or (state in swarming_lib.TASK_STATUS_TO_RETRY)) |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 266 | and (task_id in self._task_to_test_maps) |
| 267 | and (self._task_to_test_maps[task_id].remaining_retries > 0) |
| 268 | and (self._max_retries > 0)) |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 269 | |
| 270 | |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 271 | class Suite(object): |
| 272 | """The class for a CrOS suite.""" |
Xixuan Wu | 606e218 | 2018-06-14 11:30:35 -0700 | [diff] [blame] | 273 | EXPIRATION_SECS = swarming_lib.DEFAULT_EXPIRATION_SECS |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 274 | |
Xixuan Wu | 9287fda | 2018-07-12 16:22:06 -0700 | [diff] [blame] | 275 | def __init__(self, spec): |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 276 | """Initialize a suite. |
| 277 | |
Xixuan Wu | 9287fda | 2018-07-12 16:22:06 -0700 | [diff] [blame] | 278 | @param spec: A SuiteSpec object. |
Xixuan Wu | 7cc10e5 | 2018-04-25 17:04:51 -0700 | [diff] [blame] | 279 | """ |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 280 | self._ds = None |
| 281 | |
| 282 | self.control_file = '' |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 283 | self.test_specs = [] |
Xixuan Wu | 9287fda | 2018-07-12 16:22:06 -0700 | [diff] [blame] | 284 | self.builds = spec.builds |
| 285 | self.test_source_build = spec.test_source_build |
| 286 | self.suite_name = spec.suite_name |
| 287 | self.suite_file_name = spec.suite_file_name |
| 288 | self.priority = spec.priority |
| 289 | self.board = spec.board |
| 290 | self.pool = spec.pool |
| 291 | self.job_keyvals = spec.job_keyvals |
Xixuan Wu | 0956b63 | 2018-08-06 14:40:23 -0700 | [diff] [blame] | 292 | self.minimum_duts = spec.minimum_duts |
Xixuan Wu | ba0276c | 2018-08-10 09:36:23 -0700 | [diff] [blame] | 293 | self.timeout_mins = spec.timeout_mins |
Aviv Keshet | be570b3 | 2018-12-06 14:46:19 -0800 | [diff] [blame] | 294 | self.quota_account = spec.quota_account |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 295 | |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 296 | @property |
| 297 | def ds(self): |
| 298 | """Getter for private |self._ds| property. |
| 299 | |
| 300 | This ensures that once self.ds is called, there's a devserver ready |
| 301 | for it. |
| 302 | """ |
| 303 | if self._ds is None: |
Xixuan Wu | 8980338 | 2018-07-27 12:33:33 -0700 | [diff] [blame] | 304 | raise errors.InValidPropertyError( |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 305 | 'Property self.ds is None. Please call stage_suite_artifacts() ' |
| 306 | 'before calling it.') |
| 307 | |
| 308 | return self._ds |
| 309 | |
Xixuan Wu | b279566 | 2018-06-28 16:02:53 -0700 | [diff] [blame] | 310 | def _get_cros_build(self): |
| 311 | provision = autotest.load('server.cros.provision') |
| 312 | return self.builds.get(provision.CROS_VERSION_PREFIX, |
| 313 | self.builds.values()[0]) |
| 314 | |
| 315 | def _create_suite_keyvals(self): |
| 316 | constants = autotest.load('server.cros.dynamic_suite.constants') |
| 317 | provision = autotest.load('server.cros.provision') |
| 318 | cros_build = self._get_cros_build() |
| 319 | keyvals = { |
| 320 | constants.JOB_BUILD_KEY: cros_build, |
| 321 | constants.JOB_SUITE_KEY: self.suite_name, |
| 322 | constants.JOB_BUILDS_KEY: self.builds |
| 323 | } |
| 324 | if (cros_build != self.test_source_build or |
| 325 | len(self.builds) > 1): |
| 326 | keyvals[constants.JOB_TEST_SOURCE_BUILD_KEY] = ( |
| 327 | self.test_source_build) |
| 328 | for prefix, build in self.builds.iteritems(): |
| 329 | if prefix == provision.FW_RW_VERSION_PREFIX: |
| 330 | keyvals[constants.FWRW_BUILD]= build |
| 331 | elif prefix == provision.FW_RO_VERSION_PREFIX: |
| 332 | keyvals[constants.FWRO_BUILD] = build |
| 333 | |
| 334 | for key in self.job_keyvals: |
| 335 | if key in constants.INHERITED_KEYVALS: |
| 336 | keyvals[key] = self.job_keyvals[key] |
| 337 | |
| 338 | return keyvals |
| 339 | |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 340 | def prepare(self): |
| 341 | """Prepare a suite job for execution.""" |
| 342 | self._stage_suite_artifacts() |
| 343 | self._parse_suite_args() |
Xixuan Wu | b279566 | 2018-06-28 16:02:53 -0700 | [diff] [blame] | 344 | keyvals = self._create_suite_keyvals() |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 345 | available_bots = self._get_available_bots() |
Xixuan Wu | 0956b63 | 2018-08-06 14:40:23 -0700 | [diff] [blame] | 346 | if len(available_bots) < self.minimum_duts: |
| 347 | raise errors.NoAvailableDUTsError( |
Xixuan Wu | efa9452 | 2018-08-07 17:51:34 -0700 | [diff] [blame] | 348 | self.board, self.pool, len(available_bots), |
| 349 | self.minimum_duts) |
| 350 | |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 351 | tests = self._find_tests(available_bots_num=len(available_bots)) |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 352 | self.test_specs = self._get_test_specs(tests, available_bots, keyvals) |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 353 | |
Xixuan Wu | 19df5f3 | 2018-08-07 16:19:44 -0700 | [diff] [blame] | 354 | def _create_test_spec(self, test, keyvals, bot_id='', dut_name=''): |
| 355 | return TestSpec( |
| 356 | test=test, |
| 357 | priority=self.priority, |
| 358 | board=self.board, |
| 359 | pool=self.pool, |
| 360 | build=self.test_source_build, |
| 361 | bot_id=bot_id, |
| 362 | dut_name=dut_name, |
| 363 | keyvals=keyvals, |
Xixuan Wu | ba0276c | 2018-08-10 09:36:23 -0700 | [diff] [blame] | 364 | expiration_secs=self.timeout_mins * 60, |
Xixuan Wu | 19df5f3 | 2018-08-07 16:19:44 -0700 | [diff] [blame] | 365 | grace_period_secs=swarming_lib.DEFAULT_TIMEOUT_SECS, |
| 366 | execution_timeout_secs=swarming_lib.DEFAULT_TIMEOUT_SECS, |
| 367 | io_timeout_secs=swarming_lib.DEFAULT_TIMEOUT_SECS, |
Aviv Keshet | be570b3 | 2018-12-06 14:46:19 -0800 | [diff] [blame] | 368 | quota_account=self.quota_account, |
Xixuan Wu | 19df5f3 | 2018-08-07 16:19:44 -0700 | [diff] [blame] | 369 | ) |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 370 | |
Xixuan Wu | 19df5f3 | 2018-08-07 16:19:44 -0700 | [diff] [blame] | 371 | def _get_test_specs(self, tests, available_bots, keyvals): |
| 372 | return [self._create_test_spec(test, keyvals) for test in tests] |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 373 | |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 374 | def _stage_suite_artifacts(self): |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 375 | """Stage suite control files and suite-to-tests mapping file. |
| 376 | |
| 377 | @param build: The build to stage artifacts. |
| 378 | """ |
| 379 | suite_common = autotest.load('server.cros.dynamic_suite.suite_common') |
| 380 | ds, _ = suite_common.stage_build_artifacts(self.test_source_build) |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 381 | self._ds = ds |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 382 | |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 383 | def _parse_suite_args(self): |
Xixuan Wu | 48c45b9 | 2018-04-26 11:09:35 -0700 | [diff] [blame] | 384 | """Get the suite args. |
| 385 | |
| 386 | The suite args includes: |
| 387 | a. suite args in suite control file. |
| 388 | b. passed-in suite args by user. |
| 389 | """ |
Xixuan Wu | e3e362f | 2018-04-26 16:34:28 -0700 | [diff] [blame] | 390 | suite_common = autotest.load('server.cros.dynamic_suite.suite_common') |
| 391 | self.control_file = suite_common.get_control_file_by_build( |
| 392 | self.test_source_build, self.ds, self.suite_file_name) |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 393 | |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 394 | def _find_tests(self, available_bots_num=0): |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 395 | """Fetch the child tests.""" |
| 396 | control_file_getter = autotest.load( |
| 397 | 'server.cros.dynamic_suite.control_file_getter') |
| 398 | suite_common = autotest.load('server.cros.dynamic_suite.suite_common') |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 399 | cf_getter = control_file_getter.DevServerGetter( |
| 400 | self.test_source_build, self.ds) |
| 401 | tests = suite_common.retrieve_for_suite( |
| 402 | cf_getter, self.suite_name) |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 403 | return suite_common.filter_tests(tests) |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 404 | |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 405 | def _get_available_bots(self): |
Xixuan Wu | 0956b63 | 2018-08-06 14:40:23 -0700 | [diff] [blame] | 406 | """Get available bots for suites.""" |
| 407 | bots = swarming_lib.query_bots_list({ |
| 408 | 'pool': swarming_lib.SKYLAB_DRONE_POOL, |
Xixuan Wu | 6a00456 | 2018-12-10 11:56:17 -0800 | [diff] [blame^] | 409 | 'label-pool': swarming_lib.to_swarming_pool_label(self.pool), |
Xixuan Wu | 0956b63 | 2018-08-06 14:40:23 -0700 | [diff] [blame] | 410 | 'label-board': self.board}) |
| 411 | return [bot for bot in bots if swarming_lib.bot_available(bot)] |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 412 | |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 413 | |
| 414 | class ProvisionSuite(Suite): |
| 415 | """The class for a CrOS provision suite.""" |
Xixuan Wu | 6bd67ea | 2018-08-01 09:24:59 -0700 | [diff] [blame] | 416 | EXPIRATION_SECS = swarming_lib.DEFAULT_EXPIRATION_SECS |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 417 | |
Xixuan Wu | 9287fda | 2018-07-12 16:22:06 -0700 | [diff] [blame] | 418 | def __init__(self, spec): |
| 419 | super(ProvisionSuite, self).__init__(spec) |
| 420 | self._num_required = spec.suite_args['num_required'] |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 421 | |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 422 | def _find_tests(self, available_bots_num=0): |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 423 | """Fetch the child tests for provision suite.""" |
| 424 | control_file_getter = autotest.load( |
| 425 | 'server.cros.dynamic_suite.control_file_getter') |
| 426 | suite_common = autotest.load('server.cros.dynamic_suite.suite_common') |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 427 | cf_getter = control_file_getter.DevServerGetter( |
| 428 | self.test_source_build, self.ds) |
| 429 | dummy_test = suite_common.retrieve_control_data_for_test( |
| 430 | cf_getter, 'dummy_Pass') |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 431 | logging.info('Get %d available DUTs for provision.', available_bots_num) |
Xixuan Wu | 8980338 | 2018-07-27 12:33:33 -0700 | [diff] [blame] | 432 | if available_bots_num < self._num_required: |
| 433 | logging.warning('Not enough available DUTs for provision.') |
| 434 | raise errors.NoAvailableDUTsError( |
Xixuan Wu | efa9452 | 2018-08-07 17:51:34 -0700 | [diff] [blame] | 435 | self.board, self.pool, available_bots_num, |
| 436 | self._num_required) |
Xixuan Wu | 8980338 | 2018-07-27 12:33:33 -0700 | [diff] [blame] | 437 | |
Xixuan Wu | 0c01b09 | 2018-06-13 14:12:55 -0700 | [diff] [blame] | 438 | return [dummy_test] * max(self._num_required, available_bots_num) |
Xixuan Wu | cb46951 | 2018-06-08 15:17:23 -0700 | [diff] [blame] | 439 | |
Xixuan Wu | 19df5f3 | 2018-08-07 16:19:44 -0700 | [diff] [blame] | 440 | def _get_test_specs(self, tests, available_bots, keyvals): |
| 441 | test_specs = [] |
| 442 | for idx, test in enumerate(tests): |
| 443 | if idx < len(available_bots): |
| 444 | bot = available_bots[idx] |
| 445 | test_specs.append(self._create_test_spec( |
| 446 | test, keyvals, bot_id=bot['bot_id'], |
| 447 | dut_name=swarming_lib.get_task_dut_name( |
| 448 | bot['dimensions']))) |
| 449 | else: |
| 450 | test_specs.append(self._create_test_spec(test, keyvals)) |
| 451 | |
| 452 | return test_specs |