Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 1 | # Copyright 2018 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Module for CrOS dynamic test suite generation and execution.""" |
| 6 | |
| 7 | from __future__ import absolute_import |
| 8 | from __future__ import division |
| 9 | from __future__ import print_function |
| 10 | |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 11 | import contextlib |
| 12 | import itertools |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 13 | import json |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 14 | import logging |
Xixuan Wu | e71c893 | 2018-05-07 17:18:34 -0700 | [diff] [blame] | 15 | import os |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 16 | import re |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 17 | import time |
Xixuan Wu | 6c04133 | 2018-05-07 16:04:36 -0700 | [diff] [blame] | 18 | |
Xixuan Wu | e71c893 | 2018-05-07 17:18:34 -0700 | [diff] [blame] | 19 | from lucifer import autotest |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 20 | from skylab_suite import cros_suite |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 21 | from skylab_suite import swarming_lib |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 22 | |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 23 | |
Allen Li | 1ccca8f | 2018-08-29 12:11:06 -0700 | [diff] [blame] | 24 | SKYLAB_DRONE_SWARMING_WORKER = '/opt/infra-tools/skylab_swarming_worker' |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 25 | SKYLAB_SUITE_USER = 'skylab_suite_runner' |
| 26 | SKYLAB_TOOL = '/opt/infra-tools/skylab' |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 27 | |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 28 | SUITE_WAIT_SLEEP_INTERVAL_SECONDS = 30 |
| 29 | |
Xixuan Wu | 79d1466 | 2018-08-20 11:15:41 -0700 | [diff] [blame] | 30 | # See #5 in crbug.com/873886 for more details. |
| 31 | _NOT_SUPPORTED_DEPENDENCIES = ['skip_provision', 'cleanup-reboot', 'rpm', |
| 32 | 'modem_repair'] |
| 33 | |
Xixuan Wu | e71c893 | 2018-05-07 17:18:34 -0700 | [diff] [blame] | 34 | |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 35 | def run(client, test_specs, suite_handler, dry_run=False): |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 36 | """Run a CrOS dynamic test suite. |
| 37 | |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 38 | @param client: A swarming_lib.Client instance. |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 39 | @param test_specs: A list of cros_suite.TestSpec objects. |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 40 | @param suite_handler: A cros_suite.SuiteHandler object. |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 41 | @param dry_run: Whether to kick off dry runs of the tests. |
| 42 | """ |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 43 | assert isinstance(client, swarming_lib.Client) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 44 | if suite_handler.suite_id: |
| 45 | # Resume an existing suite. |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 46 | _resume_suite(client, test_specs, suite_handler, dry_run) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 47 | else: |
| 48 | # Make a new suite. |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 49 | _run_suite(test_specs, suite_handler, dry_run) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 50 | |
| 51 | |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 52 | def _resume_suite(client, test_specs, suite_handler, dry_run=False): |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 53 | """Resume a suite and its child tasks by given suite id.""" |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 54 | assert isinstance(client, swarming_lib.Client) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 55 | suite_id = suite_handler.suite_id |
Aviv Keshet | d3adbfa | 2019-03-19 11:43:24 -0700 | [diff] [blame] | 56 | all_tasks = client.get_child_tasks(suite_id) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 57 | not_yet_scheduled = _get_unscheduled_test_specs( |
Xixuan Wu | 6c1866b | 2018-07-12 17:04:39 -0700 | [diff] [blame] | 58 | test_specs, suite_handler, all_tasks) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 59 | |
| 60 | logging.info('Not yet scheduled test_specs: %r', not_yet_scheduled) |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 61 | _create_test_tasks(not_yet_scheduled, suite_handler, suite_id, dry_run) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 62 | |
| 63 | if suite_id is not None and suite_handler.should_wait(): |
| 64 | _wait_for_results(suite_handler, dry_run=dry_run) |
| 65 | |
| 66 | |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 67 | def _get_unscheduled_test_specs(test_specs, suite_handler, all_tasks): |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 68 | not_yet_scheduled = [] |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 69 | for test_spec in test_specs: |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 70 | if suite_handler.is_provision(): |
Xixuan Wu | 58bbb64 | 2018-07-12 14:12:14 -0700 | [diff] [blame] | 71 | # We cannot check bot_id because pending tasks do not have it yet. |
| 72 | bot_id_tag = 'id:%s' % test_spec.bot_id |
| 73 | tasks = [t for t in all_tasks if bot_id_tag in t['tags']] |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 74 | else: |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 75 | tasks = [t for t in all_tasks if t['name']==test_spec.test.name] |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 76 | |
| 77 | if not tasks: |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 78 | not_yet_scheduled.append(test_spec) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 79 | continue |
| 80 | |
| 81 | current_task = _get_current_task(tasks) |
| 82 | test_task_id = (current_task['task_id'] if current_task |
| 83 | else tasks[0]['task_id']) |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 84 | remaining_retries = test_spec.test.job_retries - len(tasks) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 85 | previous_retried_ids = [t['task_id'] for t in tasks |
| 86 | if t['task_id'] != test_task_id] |
| 87 | suite_handler.add_test_by_task_id( |
| 88 | test_task_id, |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 89 | cros_suite.TestHandlerSpec( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 90 | test_spec=test_spec, |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 91 | remaining_retries=remaining_retries, |
| 92 | previous_retried_ids=previous_retried_ids)) |
| 93 | |
| 94 | return not_yet_scheduled |
| 95 | |
| 96 | |
| 97 | def _get_current_task(tasks): |
| 98 | """Get current running task. |
| 99 | |
| 100 | @param tasks: A list of task dicts including task_id, state, etc. |
| 101 | |
| 102 | @return a dict representing the current running task. |
| 103 | """ |
| 104 | current_task = None |
| 105 | for t in tasks: |
| 106 | if t['state'] not in swarming_lib.TASK_FINISHED_STATUS: |
| 107 | if current_task: |
| 108 | raise ValueError( |
| 109 | 'Parent task has 2 same running child tasks: %s, %s' |
| 110 | % (current_task['task_id'], t['task_id'])) |
| 111 | |
| 112 | current_task = t |
| 113 | |
| 114 | return current_task |
| 115 | |
| 116 | |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 117 | def _run_suite(test_specs, suite_handler, dry_run=False): |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 118 | """Make a new suite.""" |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 119 | suite_id = os.environ.get('SWARMING_TASK_ID') |
Aviv Keshet | 73b9066 | 2019-03-28 14:01:58 -0700 | [diff] [blame] | 120 | if not suite_id: |
| 121 | raise ValueError("Unable to determine suite's task id from env var " |
| 122 | "SWARMING_TASK_ID.") |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 123 | _create_test_tasks(test_specs, suite_handler, suite_id, dry_run) |
Xixuan Wu | a79b5f7 | 2018-12-26 12:29:39 -0800 | [diff] [blame] | 124 | suite_handler.set_suite_id(suite_id) |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 125 | |
Aviv Keshet | 73b9066 | 2019-03-28 14:01:58 -0700 | [diff] [blame] | 126 | if suite_handler.should_wait(): |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 127 | _wait_for_results(suite_handler, dry_run=dry_run) |
| 128 | |
| 129 | |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 130 | def _create_test_tasks(test_specs, suite_handler, suite_id, dry_run=False): |
| 131 | """Create test tasks for a list of tests (TestSpecs). |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 132 | |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 133 | Given a list of TestSpec object, this function will schedule them on |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 134 | swarming one by one, and add them to the swarming_task_id-to-test map |
| 135 | of suite_handler to keep monitoring them. |
| 136 | |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 137 | @param test_specs: A list of cros_suite.TestSpec objects to schedule. |
Xixuan Wu | c743071 | 2018-07-10 12:04:34 -0700 | [diff] [blame] | 138 | @param suite_handler: A cros_suite.SuiteHandler object to monitor the |
| 139 | test_specs' progress. |
| 140 | @param suite_id: A string ID for a suite task, it's the parent task id for |
| 141 | these to-be-scheduled test_specs. |
| 142 | @param dry_run: Whether to kick off dry runs of the tests. |
| 143 | """ |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 144 | for test_spec in test_specs: |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 145 | test_task_id = _create_test_task( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 146 | test_spec, |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 147 | suite_id=suite_id, |
Xixuan Wu | 814ceb6 | 2018-08-27 15:47:34 -0700 | [diff] [blame] | 148 | is_provision=suite_handler.is_provision(), |
Xixuan Wu | 5cb5a40 | 2018-06-04 16:37:23 -0700 | [diff] [blame] | 149 | dry_run=dry_run) |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 150 | suite_handler.add_test_by_task_id( |
| 151 | test_task_id, |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 152 | cros_suite.TestHandlerSpec( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 153 | test_spec=test_spec, |
| 154 | remaining_retries=test_spec.test.job_retries - 1, |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 155 | previous_retried_ids=[])) |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 156 | |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 157 | |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 158 | def _create_test_task(test_spec, suite_id=None, |
| 159 | is_provision=False, dry_run=False): |
| 160 | """Create a test task for a given test spec. |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 161 | |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 162 | @param test_spec: A cros_suite.TestSpec object. |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 163 | @param suite_id: the suite task id of the test. |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 164 | @param dry_run: If true, don't actually create task. |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 165 | |
| 166 | @return the swarming task id of this task. |
Xixuan Wu | c7bf77c | 2018-04-24 12:05:40 -0700 | [diff] [blame] | 167 | """ |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 168 | logging.info('Creating task for test %s', test_spec.test.name) |
Aviv Keshet | 2c25d06 | 2019-03-19 13:18:23 -0700 | [diff] [blame] | 169 | skylab_tool_path = os.environ.get('SKYLAB_TOOL', SKYLAB_TOOL) |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 170 | |
| 171 | cmd = [ |
Aviv Keshet | 2c25d06 | 2019-03-19 13:18:23 -0700 | [diff] [blame] | 172 | skylab_tool_path, 'create-test', |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 173 | '-board', test_spec.board, |
| 174 | '-image', test_spec.build, |
Aviv Keshet | 399692f | 2019-04-02 12:17:51 -0700 | [diff] [blame] | 175 | '-service-account-json', os.environ['SWARMING_CREDS'], |
Aviv Keshet | f0f1b49 | 2019-04-03 15:30:48 -0700 | [diff] [blame^] | 176 | '-timeout-mins', str(test_spec.execution_timeout_mins), |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 177 | ] |
Aviv Keshet | b70c709 | 2019-04-02 12:05:56 -0700 | [diff] [blame] | 178 | if _is_dev(): |
| 179 | cmd += ['-dev'] |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 180 | if test_spec.pool: |
| 181 | # TODO(akeshet): Clean up this hack around pool name translation. |
| 182 | autotest_pool_label = 'pool:%s' % test_spec.pool |
| 183 | pool_dependency_value = swarming_lib.task_dependencies_from_labels( |
| 184 | [autotest_pool_label])['label-pool'] |
| 185 | cmd += ['-pool', pool_dependency_value] |
| 186 | |
| 187 | if test_spec.model: |
| 188 | cmd += ['-model', test_spec.model] |
| 189 | if test_spec.quota_account: |
| 190 | cmd += ['-qs-account', test_spec.quota_account] |
| 191 | if test_spec.test.test_type.lower() == 'client': |
| 192 | cmd += ['-client-test'] |
| 193 | |
Aviv Keshet | f0f1b49 | 2019-04-03 15:30:48 -0700 | [diff] [blame^] | 194 | |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 195 | tags = _compute_tags(test_spec.build, suite_id) |
| 196 | dimensions = _compute_dimensions( |
| 197 | test_spec.bot_id, test_spec.test.dependencies) |
| 198 | keyvals_flat = _compute_job_keyvals_flat(test_spec.keyvals, suite_id) |
| 199 | |
| 200 | for tag in tags: |
| 201 | cmd += ['-tag', tag] |
| 202 | for keyval in keyvals_flat: |
| 203 | cmd += ['-keyval', keyval] |
| 204 | cmd += [test_spec.test.name] |
| 205 | cmd += dimensions |
| 206 | |
Xixuan Wu | e71c893 | 2018-05-07 17:18:34 -0700 | [diff] [blame] | 207 | if dry_run: |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 208 | logging.info('Would have created task with command %s', cmd) |
| 209 | return |
Xixuan Wu | e71c893 | 2018-05-07 17:18:34 -0700 | [diff] [blame] | 210 | |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 211 | # TODO(akeshet): Avoid this late chromite import. |
| 212 | cros_build_lib = autotest.chromite_load('cros_build_lib') |
| 213 | result = cros_build_lib.RunCommand(cmd, capture_output=True) |
| 214 | # TODO(akeshet): Use -json flag and json-parse output of the command instead |
| 215 | # of regex matching to determine task_id. |
| 216 | m = re.match('.*id=(.*)$', result.output) |
| 217 | task_id = m.group(1) |
| 218 | logging.info('Created task with id %s', task_id) |
| 219 | return task_id |
Xixuan Wu | 3dea7cf | 2018-12-10 17:50:45 -0800 | [diff] [blame] | 220 | |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 221 | |
Aviv Keshet | b70c709 | 2019-04-02 12:05:56 -0700 | [diff] [blame] | 222 | # TODO(akeshet): Eliminate the need for this, by either adding an explicit |
| 223 | # swarming_server argument to skylab tool, or having the tool respect the |
| 224 | # SWARMING_SERVER environment variable. See crbug.com/948774 |
| 225 | def _is_dev(): |
| 226 | """Detect whether skylab tool should be invoked with -dev flag.""" |
Aviv Keshet | aa3839d | 2019-04-02 16:13:42 -0700 | [diff] [blame] | 227 | return 'chromium-swarm-dev' in os.environ['SWARMING_SERVER'] |
Aviv Keshet | b70c709 | 2019-04-02 12:05:56 -0700 | [diff] [blame] | 228 | |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 229 | def _compute_tags(build, suite_id): |
| 230 | tags = [ |
| 231 | 'build:%s' % build, |
| 232 | ] |
| 233 | if suite_id is not None: |
| 234 | tags += ['parent_task_id:%s' % suite_id] |
| 235 | return tags |
| 236 | |
| 237 | |
| 238 | def _compute_dimensions(bot_id, dependencies): |
| 239 | dimensions = [] |
| 240 | if bot_id: |
| 241 | dimensions += ['id:%s' % bot_id] |
| 242 | deps = _filter_unsupported_dependencies(dependencies) |
| 243 | flattened_swarming_deps = sorted([ |
| 244 | '%s:%s' % (k, v) for |
| 245 | k, v in swarming_lib.task_dependencies_from_labels(deps).items() |
| 246 | ]) |
| 247 | dimensions += flattened_swarming_deps |
| 248 | return dimensions |
| 249 | |
| 250 | |
| 251 | def _compute_job_keyvals_flat(keyvals, suite_id): |
| 252 | # Job keyvals calculation. |
| 253 | job_keyvals = keyvals.copy() |
| 254 | if suite_id is not None: |
| 255 | # TODO(akeshet): Avoid this late autotest constants import. |
| 256 | constants = autotest.load('server.cros.dynamic_suite.constants') |
| 257 | job_keyvals[constants.PARENT_JOB_ID] = suite_id |
| 258 | keyvals_flat = sorted( |
| 259 | ['%s:%s' % (k, v) for k, v in job_keyvals.items()]) |
| 260 | return keyvals_flat |
| 261 | |
| 262 | |
| 263 | def _filter_unsupported_dependencies(dependencies): |
| 264 | """Filter out Skylab-unsupported test dependencies, with a warning.""" |
Xixuan Wu | db053c8 | 2019-01-31 20:07:06 -0800 | [diff] [blame] | 265 | deps = [] |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 266 | for dep in dependencies: |
Xixuan Wu | 79d1466 | 2018-08-20 11:15:41 -0700 | [diff] [blame] | 267 | if dep in _NOT_SUPPORTED_DEPENDENCIES: |
| 268 | logging.warning('Dependency %s is not supported in skylab', dep) |
Aviv Keshet | f095121 | 2019-03-18 14:54:32 -0700 | [diff] [blame] | 269 | else: |
| 270 | deps.append(dep) |
| 271 | return deps |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 272 | |
| 273 | |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 274 | @contextlib.contextmanager |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 275 | def disable_logging(logging_level): |
| 276 | """Context manager for disabling logging of a given logging level.""" |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 277 | try: |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 278 | logging.disable(logging_level) |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 279 | yield |
| 280 | finally: |
| 281 | logging.disable(logging.NOTSET) |
| 282 | |
| 283 | |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 284 | def _loop_and_wait_forever(suite_handler, dry_run): |
| 285 | """Wait for child tasks to finish or break.""" |
| 286 | for iterations in itertools.count(0): |
| 287 | # Log progress every 300 seconds. |
| 288 | no_logging = bool(iterations * SUITE_WAIT_SLEEP_INTERVAL_SECONDS % 300) |
| 289 | with disable_logging(logging.INFO if no_logging else logging.NOTSET): |
Xixuan Wu | c6e28d3 | 2018-08-27 14:48:14 -0700 | [diff] [blame] | 290 | suite_handler.handle_results(suite_handler.suite_id) |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 291 | if suite_handler.is_finished_waiting(): |
| 292 | break |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 293 | |
Xixuan Wu | 4d5d014 | 2018-08-27 15:26:58 -0700 | [diff] [blame] | 294 | for t in suite_handler.retried_tasks: |
| 295 | _retry_test(suite_handler, t['task_id'], dry_run=dry_run) |
| 296 | |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 297 | time.sleep(SUITE_WAIT_SLEEP_INTERVAL_SECONDS) |
Xixuan Wu | 80795c8 | 2018-06-12 11:56:17 -0700 | [diff] [blame] | 298 | |
| 299 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 300 | def _wait_for_results(suite_handler, dry_run=False): |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 301 | """Wait for child tasks to finish and return their results. |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 302 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 303 | @param suite_handler: a cros_suite.SuiteHandler object. |
Xixuan Wu | 2406be3 | 2018-05-14 13:51:30 -0700 | [diff] [blame] | 304 | """ |
| 305 | timeout_util = autotest.chromite_load('timeout_util') |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 306 | try: |
Xixuan Wu | b098363 | 2018-08-17 17:54:42 -0700 | [diff] [blame] | 307 | with timeout_util.Timeout(suite_handler.timeout_mins * 60 - |
| 308 | suite_handler.passed_mins * 60): |
Xixuan Wu | f52e40d | 2018-06-14 12:10:44 -0700 | [diff] [blame] | 309 | _loop_and_wait_forever(suite_handler, dry_run) |
| 310 | except timeout_util.TimeoutError: |
| 311 | logging.error('Timeout in waiting for child tasks.') |
| 312 | return |
Xixuan Wu | 0bea952 | 2018-05-08 17:49:19 -0700 | [diff] [blame] | 313 | |
| 314 | logging.info('Finished to wait for child tasks.') |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 315 | |
| 316 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 317 | def _retry_test(suite_handler, task_id, dry_run=False): |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 318 | """Retry test for a suite. |
| 319 | |
| 320 | We will execute the following actions for retrying a test: |
| 321 | 1. Schedule the test. |
| 322 | 2. Add the test with the new swarming task id to the suite's |
| 323 | retry handler, but reduce its remaining retries by 1. |
| 324 | 3. Reduce the suite-level max retries by 1. |
| 325 | 4. Remove prevous failed test from retry handler since it's not |
| 326 | actively monitored by the suite. |
| 327 | |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 328 | @param suite_handler: a cros_suite.SuiteHandler object. |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 329 | @param task_id: The swarming task id for the retried test. |
| 330 | @param dry_run: Whether to retry a dry run of the test. |
| 331 | """ |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 332 | last_retry_spec = suite_handler.get_test_by_task_id(task_id) |
Xixuan Wu | 56424bc | 2018-05-15 11:03:27 -0700 | [diff] [blame] | 333 | logging.info('Retrying test %s, remaining %d retries.', |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 334 | last_retry_spec.test_spec.test.name, |
| 335 | last_retry_spec.remaining_retries - 1) |
Aviv Keshet | d993510 | 2019-03-18 14:44:28 -0700 | [diff] [blame] | 336 | retried_task_id = _create_test_task( |
Xixuan Wu | b60c143 | 2019-03-07 17:15:39 +0000 | [diff] [blame] | 337 | last_retry_spec.test_spec, |
| 338 | suite_id=suite_handler.suite_id, |
| 339 | is_provision=suite_handler.is_provision(), |
| 340 | dry_run=dry_run) |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 341 | previous_retried_ids = last_retry_spec.previous_retried_ids + [task_id] |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 342 | suite_handler.add_test_by_task_id( |
| 343 | retried_task_id, |
Xixuan Wu | 9d5d703 | 2018-07-12 16:44:02 -0700 | [diff] [blame] | 344 | cros_suite.TestHandlerSpec( |
Xixuan Wu | 5811e83 | 2018-07-12 11:56:24 -0700 | [diff] [blame] | 345 | test_spec=last_retry_spec.test_spec, |
| 346 | remaining_retries=last_retry_spec.remaining_retries - 1, |
Xixuan Wu | 9af95a2 | 2018-05-18 10:46:42 -0700 | [diff] [blame] | 347 | previous_retried_ids=previous_retried_ids)) |
| 348 | suite_handler.set_max_retries(suite_handler.max_retries - 1) |
| 349 | suite_handler.remove_test_by_task_id(task_id) |
Xixuan Wu | 7a450c5 | 2018-07-20 15:07:51 -0700 | [diff] [blame] | 350 | |
| 351 | |
| 352 | def _convert_dict_to_string(input_dict): |
| 353 | """Convert dictionary to a string. |
| 354 | |
| 355 | @param input_dict: A dictionary. |
| 356 | """ |
| 357 | for k, v in input_dict.iteritems(): |
| 358 | if isinstance(v, dict): |
| 359 | input_dict[k] = _convert_dict_to_string(v) |
| 360 | else: |
| 361 | input_dict[k] = str(v) |
| 362 | |
| 363 | return json.dumps(input_dict) |