Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 5 | import hashlib, logging, os, re, traceback |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 6 | |
| 7 | import common |
| 8 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 9 | from autotest_lib.client.common_lib import control_data |
beeps | c4fb147 | 2013-05-08 21:49:48 -0700 | [diff] [blame] | 10 | from autotest_lib.client.common_lib import site_utils, utils |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 11 | from autotest_lib.server.cros.dynamic_suite import constants |
| 12 | from autotest_lib.server.cros.dynamic_suite import control_file_getter |
| 13 | from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 14 | from autotest_lib.server.cros.dynamic_suite import job_status |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 15 | from autotest_lib.server.cros.dynamic_suite.job_status import Status |
beeps | 1ccbbb8 | 2013-02-21 20:35:01 -0800 | [diff] [blame] | 16 | from autotest_lib.server.cros.dynamic_suite import reporting |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 17 | |
| 18 | class Suite(object): |
| 19 | """ |
| 20 | A suite of tests, defined by some predicate over control file variables. |
| 21 | |
| 22 | Given a place to search for control files a predicate to match the desired |
| 23 | tests, can gather tests and fire off jobs to run them, and then wait for |
| 24 | results. |
| 25 | |
| 26 | @var _predicate: a function that should return True when run over a |
| 27 | ControlData representation of a control file that should be in |
| 28 | this Suite. |
| 29 | @var _tag: a string with which to tag jobs run in this suite. |
| 30 | @var _build: the build on which we're running this suite. |
| 31 | @var _afe: an instance of AFE as defined in server/frontend.py. |
| 32 | @var _tko: an instance of TKO as defined in server/frontend.py. |
| 33 | @var _jobs: currently scheduled jobs, if any. |
| 34 | @var _cf_getter: a control_file_getter.ControlFileGetter |
| 35 | """ |
| 36 | |
| 37 | |
| 38 | @staticmethod |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 39 | def create_ds_getter(build, devserver): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 40 | """ |
| 41 | @param build: the build on which we're running this suite. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 42 | @param devserver: the devserver which contains the build. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 43 | @return a FileSystemGetter instance that looks under |autotest_dir|. |
| 44 | """ |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 45 | return control_file_getter.DevServerGetter(build, devserver) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 46 | |
| 47 | |
| 48 | @staticmethod |
| 49 | def create_fs_getter(autotest_dir): |
| 50 | """ |
| 51 | @param autotest_dir: the place to find autotests. |
| 52 | @return a FileSystemGetter instance that looks under |autotest_dir|. |
| 53 | """ |
| 54 | # currently hard-coded places to look for tests. |
| 55 | subpaths = ['server/site_tests', 'client/site_tests', |
| 56 | 'server/tests', 'client/tests'] |
| 57 | directories = [os.path.join(autotest_dir, p) for p in subpaths] |
| 58 | return control_file_getter.FileSystemGetter(directories) |
| 59 | |
| 60 | |
| 61 | @staticmethod |
| 62 | def parse_tag(tag): |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 63 | """Splits a string on ',' optionally surrounded by whitespace. |
| 64 | @param tag: string to split. |
| 65 | """ |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 66 | return map(lambda x: x.strip(), tag.split(',')) |
| 67 | |
| 68 | |
| 69 | @staticmethod |
| 70 | def name_in_tag_predicate(name): |
| 71 | """Returns predicate that takes a control file and looks for |name|. |
| 72 | |
| 73 | Builds a predicate that takes in a parsed control file (a ControlData) |
| 74 | and returns True if the SUITE tag is present and contains |name|. |
| 75 | |
| 76 | @param name: the suite name to base the predicate on. |
| 77 | @return a callable that takes a ControlData and looks for |name| in that |
| 78 | ControlData object's suite member. |
| 79 | """ |
| 80 | return lambda t: hasattr(t, 'suite') and \ |
| 81 | name in Suite.parse_tag(t.suite) |
| 82 | |
| 83 | |
| 84 | @staticmethod |
Aviv Keshet | 40222a4 | 2013-06-04 16:25:49 -0700 | [diff] [blame] | 85 | def not_in_blacklist_predicate(blacklist): |
| 86 | """Returns predicate that takes a control file and looks for its |
| 87 | path to not be in given blacklist. |
| 88 | |
| 89 | @param blacklist: A list of strings both paths on control_files that |
| 90 | should be blacklisted. |
| 91 | |
| 92 | @return a callable that takes a ControlData and looks for it to be |
| 93 | absent from blacklist. |
| 94 | """ |
| 95 | return lambda t: hasattr(t, 'path') and \ |
| 96 | not any(b.endswith(t.path) for b in blacklist) |
| 97 | |
| 98 | |
| 99 | @staticmethod |
| 100 | def test_name_equals_predicate(test_name): |
| 101 | """Returns predicate that matched based on a test's name. |
| 102 | |
| 103 | Builds a predicate that takes in a parsed control file (a ControlData) |
| 104 | and returns True if the test name is equal to |test_name|. |
| 105 | |
| 106 | @param test_name: the test name to base the predicate on. |
| 107 | @return a callable that takes a ControlData and looks for |test_name| |
| 108 | in that ControlData's name. |
| 109 | """ |
| 110 | return lambda t: hasattr(t, 'name') and test_name == t.name |
| 111 | |
| 112 | |
| 113 | @staticmethod |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 114 | def list_all_suites(build, devserver, cf_getter=None): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 115 | """ |
| 116 | Parses all ControlData objects with a SUITE tag and extracts all |
| 117 | defined suite names. |
| 118 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 119 | @param build: the build on which we're running this suite. |
| 120 | @param devserver: the devserver which contains the build. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 121 | @param cf_getter: control_file_getter.ControlFileGetter. Defaults to |
| 122 | using DevServerGetter. |
| 123 | |
| 124 | @return list of suites |
| 125 | """ |
| 126 | if cf_getter is None: |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 127 | cf_getter = Suite.create_ds_getter(build, devserver) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 128 | |
| 129 | suites = set() |
| 130 | predicate = lambda t: hasattr(t, 'suite') |
| 131 | for test in Suite.find_and_parse_tests(cf_getter, predicate, |
| 132 | add_experimental=True): |
| 133 | suites.update(Suite.parse_tag(test.suite)) |
| 134 | return list(suites) |
| 135 | |
| 136 | |
| 137 | @staticmethod |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 138 | def create_from_predicates(predicates, build, board, devserver, |
| 139 | cf_getter=None, name='ad_hoc_suite', **dargs): |
Aviv Keshet | 69ebb6c | 2013-06-11 13:58:44 -0700 | [diff] [blame] | 140 | """ |
| 141 | Create a Suite using a given predicate test filters. |
| 142 | |
| 143 | Uses supplied predicate(s) to instantiate a Suite. Looks for tests in |
| 144 | |autotest_dir| and will schedule them using |afe|. Pulls control files |
| 145 | from the default dev server. Results will be pulled from |tko| upon |
| 146 | completion. |
| 147 | |
| 148 | @param predicates: A list of callables that accept ControlData |
| 149 | representations of control files. A test will be |
Aviv Keshet | 938a677 | 2013-07-25 14:05:45 -0700 | [diff] [blame] | 150 | included in suite if all callables in this list |
Aviv Keshet | 69ebb6c | 2013-06-11 13:58:44 -0700 | [diff] [blame] | 151 | return True on the given control file. |
| 152 | @param build: the build on which we're running this suite. |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 153 | @param board: the board on which we're running this suite. |
Aviv Keshet | 69ebb6c | 2013-06-11 13:58:44 -0700 | [diff] [blame] | 154 | @param devserver: the devserver which contains the build. |
| 155 | @param cf_getter: control_file_getter.ControlFileGetter. Defaults to |
| 156 | using DevServerGetter. |
| 157 | @param name: name of suite. Defaults to 'ad_hoc_suite' |
| 158 | @param **dargs: Any other Suite constructor parameters, as described |
| 159 | in Suite.__init__ docstring. |
| 160 | @return a Suite instance. |
| 161 | """ |
| 162 | if cf_getter is None: |
| 163 | cf_getter = Suite.create_ds_getter(build, devserver) |
| 164 | |
| 165 | return Suite(predicates, |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 166 | name, build, board, cf_getter, **dargs) |
Aviv Keshet | 69ebb6c | 2013-06-11 13:58:44 -0700 | [diff] [blame] | 167 | |
| 168 | |
| 169 | @staticmethod |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 170 | def create_from_name(name, build, board, devserver, cf_getter=None, |
| 171 | **dargs): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 172 | """ |
| 173 | Create a Suite using a predicate based on the SUITE control file var. |
| 174 | |
| 175 | Makes a predicate based on |name| and uses it to instantiate a Suite |
| 176 | that looks for tests in |autotest_dir| and will schedule them using |
| 177 | |afe|. Pulls control files from the default dev server. |
| 178 | Results will be pulled from |tko| upon completion. |
| 179 | |
| 180 | @param name: a value of the SUITE control file variable to search for. |
| 181 | @param build: the build on which we're running this suite. |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 182 | @param board: the board on which we're running this suite. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 183 | @param devserver: the devserver which contains the build. |
Aviv Keshet | 813d678 | 2013-06-04 17:11:03 -0700 | [diff] [blame] | 184 | @param cf_getter: control_file_getter.ControlFileGetter. Defaults to |
| 185 | using DevServerGetter. |
| 186 | @param **dargs: Any other Suite constructor parameters, as described |
| 187 | in Suite.__init__ docstring. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 188 | @return a Suite instance. |
| 189 | """ |
| 190 | if cf_getter is None: |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 191 | cf_getter = Suite.create_ds_getter(build, devserver) |
| 192 | |
Aviv Keshet | 40222a4 | 2013-06-04 16:25:49 -0700 | [diff] [blame] | 193 | return Suite([Suite.name_in_tag_predicate(name)], |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 194 | name, build, board, cf_getter, **dargs) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 195 | |
| 196 | |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 197 | @staticmethod |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 198 | def create_from_name_and_blacklist(name, blacklist, build, board, devserver, |
Aviv Keshet | 813d678 | 2013-06-04 17:11:03 -0700 | [diff] [blame] | 199 | cf_getter=None, **dargs): |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 200 | """ |
| 201 | Create a Suite using a predicate based on the SUITE control file var. |
| 202 | |
| 203 | Makes a predicate based on |name| and uses it to instantiate a Suite |
| 204 | that looks for tests in |autotest_dir| and will schedule them using |
| 205 | |afe|. Pulls control files from the default dev server. |
| 206 | Results will be pulled from |tko| upon completion. |
| 207 | |
| 208 | @param name: a value of the SUITE control file variable to search for. |
| 209 | @param blacklist: iterable of control file paths to skip. |
| 210 | @param build: the build on which we're running this suite. |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 211 | @param board: the board on which we're running this suite. |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 212 | @param devserver: the devserver which contains the build. |
Aviv Keshet | 813d678 | 2013-06-04 17:11:03 -0700 | [diff] [blame] | 213 | @param cf_getter: control_file_getter.ControlFileGetter. Defaults to |
| 214 | using DevServerGetter. |
| 215 | @param **dargs: Any other Suite constructor parameters, as described |
| 216 | in Suite.__init__ docstring. |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 217 | @return a Suite instance. |
| 218 | """ |
| 219 | if cf_getter is None: |
| 220 | cf_getter = Suite.create_ds_getter(build, devserver) |
| 221 | |
Aviv Keshet | 40222a4 | 2013-06-04 16:25:49 -0700 | [diff] [blame] | 222 | predicates = [Suite.name_in_tag_predicate(name), |
| 223 | Suite.not_in_blacklist_predicate(blacklist)] |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 224 | |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 225 | return Suite(predicates, name, build, board, cf_getter, **dargs) |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 226 | |
| 227 | |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 228 | def __init__(self, predicates, tag, build, board, cf_getter, afe=None, |
| 229 | tko=None, pool=None, results_dir=None, max_runtime_mins=24*60, |
Alex Miller | a3a4fe7 | 2013-01-22 09:57:47 -0800 | [diff] [blame] | 230 | version_prefix=constants.VERSION_PREFIX, |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 231 | file_bugs=False, file_experimental_bugs=False, |
| 232 | suite_job_id=None, ignore_deps=False): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 233 | """ |
| 234 | Constructor |
| 235 | |
Aviv Keshet | 40222a4 | 2013-06-04 16:25:49 -0700 | [diff] [blame] | 236 | @param predicates: A list of callables that accept ControlData |
| 237 | representations of control files. A test will be |
| 238 | included in suite is all callables in this list |
| 239 | return True on the given control file. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 240 | @param tag: a string with which to tag jobs run in this suite. |
| 241 | @param build: the build on which we're running this suite. |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 242 | @param board: the board on which we're running this suite. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 243 | @param cf_getter: a control_file_getter.ControlFileGetter |
| 244 | @param afe: an instance of AFE as defined in server/frontend.py. |
| 245 | @param tko: an instance of TKO as defined in server/frontend.py. |
| 246 | @param pool: Specify the pool of machines to use for scheduling |
| 247 | purposes. |
| 248 | @param results_dir: The directory where the job can write results to. |
| 249 | This must be set if you want job_id of sub-jobs |
| 250 | list in the job keyvals. |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 251 | @param max_runtime_mins: Maximum suite runtime, in minutes. |
Vadim Bendebury | ab14bf1 | 2012-12-28 13:51:46 -0800 | [diff] [blame] | 252 | @param version_prefix: a string, prefix for the database label |
| 253 | associated with the build |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 254 | @param suite_job_id: Job id that will act as parent id to all sub jobs. |
| 255 | Default: None |
Aviv Keshet | d7959f3 | 2013-05-17 15:58:43 -0700 | [diff] [blame] | 256 | @param ignore_deps: True if jobs should ignore the DEPENDENCIES |
| 257 | attribute and skip applying of dependency labels. |
| 258 | (Default:False) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 259 | """ |
Aviv Keshet | 40222a4 | 2013-06-04 16:25:49 -0700 | [diff] [blame] | 260 | def combined_predicate(test): |
| 261 | #pylint: disable-msg=C0111 |
| 262 | return all((f(test) for f in predicates)) |
| 263 | self._predicate = combined_predicate |
| 264 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 265 | self._tag = tag |
| 266 | self._build = build |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 267 | self._board = board |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 268 | self._cf_getter = cf_getter |
| 269 | self._results_dir = results_dir |
| 270 | self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30, |
| 271 | delay_sec=10, |
| 272 | debug=False) |
| 273 | self._tko = tko or frontend_wrappers.RetryingTKO(timeout_min=30, |
| 274 | delay_sec=10, |
| 275 | debug=False) |
| 276 | self._pool = pool |
| 277 | self._jobs = [] |
| 278 | self._tests = Suite.find_and_parse_tests(self._cf_getter, |
| 279 | self._predicate, |
beeps | c594c1c | 2013-07-09 22:33:18 -0700 | [diff] [blame] | 280 | self._tag, |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 281 | add_experimental=True) |
Simran Basi | c68cda4 | 2012-11-19 17:03:18 -0800 | [diff] [blame] | 282 | self._max_runtime_mins = max_runtime_mins |
Vadim Bendebury | ab14bf1 | 2012-12-28 13:51:46 -0800 | [diff] [blame] | 283 | self._version_prefix = version_prefix |
Alex Miller | a3a4fe7 | 2013-01-22 09:57:47 -0800 | [diff] [blame] | 284 | self._file_bugs = file_bugs |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 285 | self._file_experimental_bugs = file_experimental_bugs |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 286 | self._suite_job_id = suite_job_id |
Aviv Keshet | d7959f3 | 2013-05-17 15:58:43 -0700 | [diff] [blame] | 287 | self._ignore_deps = ignore_deps |
Alex Miller | a3a4fe7 | 2013-01-22 09:57:47 -0800 | [diff] [blame] | 288 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 289 | |
| 290 | @property |
| 291 | def tests(self): |
| 292 | """ |
| 293 | A list of ControlData objects in the suite, with added |text| attr. |
| 294 | """ |
| 295 | return self._tests |
| 296 | |
| 297 | |
| 298 | def stable_tests(self): |
| 299 | """ |
| 300 | |self.tests|, filtered for non-experimental tests. |
| 301 | """ |
| 302 | return filter(lambda t: not t.experimental, self.tests) |
| 303 | |
| 304 | |
| 305 | def unstable_tests(self): |
| 306 | """ |
| 307 | |self.tests|, filtered for experimental tests. |
| 308 | """ |
| 309 | return filter(lambda t: t.experimental, self.tests) |
| 310 | |
| 311 | |
| 312 | def _create_job(self, test): |
| 313 | """ |
| 314 | Thin wrapper around frontend.AFE.create_job(). |
| 315 | |
| 316 | @param test: ControlData object for a test to run. |
| 317 | @return a frontend.Job object with an added test_name member. |
| 318 | test_name is used to preserve the higher level TEST_NAME |
| 319 | name of the job. |
| 320 | """ |
Aviv Keshet | d7959f3 | 2013-05-17 15:58:43 -0700 | [diff] [blame] | 321 | if self._ignore_deps: |
| 322 | job_deps = [] |
| 323 | else: |
| 324 | job_deps = list(test.dependencies) |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 325 | |
| 326 | cros_label = self._version_prefix + self._build |
| 327 | job_deps.append(cros_label) |
| 328 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 329 | if self._pool: |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 330 | job_deps.append(self._pool) |
| 331 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 332 | test_obj = self._afe.create_job( |
| 333 | control_file=test.text, |
| 334 | name='/'.join([self._build, self._tag, test.name]), |
| 335 | control_type=test.test_type.capitalize(), |
Alex Miller | a091307 | 2013-06-12 10:01:51 -0700 | [diff] [blame] | 336 | meta_hosts=[self._board], |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 337 | dependencies=job_deps, |
| 338 | keyvals={constants.JOB_BUILD_KEY: self._build, |
Simran Basi | c68cda4 | 2012-11-19 17:03:18 -0800 | [diff] [blame] | 339 | constants.JOB_SUITE_KEY: self._tag}, |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 340 | max_runtime_mins=self._max_runtime_mins, |
Aviv Keshet | 6f45526 | 2013-03-01 16:02:29 -0800 | [diff] [blame] | 341 | parent_job_id=self._suite_job_id, |
| 342 | test_retry=test.retries) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 343 | |
| 344 | setattr(test_obj, 'test_name', test.name) |
| 345 | |
| 346 | return test_obj |
| 347 | |
| 348 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 349 | def schedule_and_wait(self, record, add_experimental=True): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 350 | """ |
| 351 | Synchronously run tests in |self.tests|. |
| 352 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 353 | See |schedule| and |wait| for more information. |
| 354 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 355 | Schedules tests against a device running image |self._build|, and |
| 356 | then polls for status, using |record| to print status when each |
| 357 | completes. |
| 358 | |
| 359 | Tests returned by self.stable_tests() will always be run, while tests |
| 360 | in self.unstable_tests() will only be run if |add_experimental| is true. |
| 361 | |
| 362 | @param record: callable that records job status. |
| 363 | prototype: |
| 364 | record(base_job.status_log_entry) |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 365 | @param manager: a populated HostLockManager instance to handle |
| 366 | unlocking DUTs that we already reimaged. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 367 | @param add_experimental: schedule experimental tests as well, or not. |
| 368 | """ |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 369 | # This method still exists for unittesting convenience. |
| 370 | self.schedule(record, add_experimental) |
| 371 | self.wait(record) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 372 | |
| 373 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 374 | def schedule(self, record, add_experimental=True): |
Aviv Keshet | 1830892 | 2013-02-19 17:49:49 -0800 | [diff] [blame] | 375 | #pylint: disable-msg=C0111 |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 376 | """ |
| 377 | Schedule jobs using |self._afe|. |
| 378 | |
| 379 | frontend.Job objects representing each scheduled job will be put in |
| 380 | |self._jobs|. |
| 381 | |
| 382 | @param add_experimental: schedule experimental tests as well, or not. |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 383 | @returns: The number of tests that were scheduled. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 384 | """ |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 385 | logging.debug('Discovered %d stable tests.', len(self.stable_tests())) |
| 386 | logging.debug('Discovered %d unstable tests.', |
| 387 | len(self.unstable_tests())) |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 388 | n_scheduled = 0 |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 389 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 390 | Status('INFO', 'Start %s' % self._tag).record_result(record) |
| 391 | try: |
| 392 | for test in self.stable_tests(): |
| 393 | logging.debug('Scheduling %s', test.name) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 394 | self._jobs.append(self._create_job(test)) |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 395 | n_scheduled += 1 |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 396 | |
| 397 | if add_experimental: |
| 398 | for test in self.unstable_tests(): |
| 399 | logging.debug('Scheduling experimental %s', test.name) |
| 400 | test.name = constants.EXPERIMENTAL_PREFIX + test.name |
| 401 | self._jobs.append(self._create_job(test)) |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 402 | n_scheduled += 1 |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 403 | |
| 404 | if self._results_dir: |
| 405 | self._remember_scheduled_job_ids() |
| 406 | except Exception: # pylint: disable=W0703 |
| 407 | logging.error(traceback.format_exc()) |
| 408 | Status('FAIL', self._tag, |
| 409 | 'Exception while scheduling suite').record_result(record) |
| 410 | |
Aviv Keshet | e9170d9 | 2013-07-19 11:20:45 -0700 | [diff] [blame] | 411 | return n_scheduled |
| 412 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 413 | |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 414 | def should_file_bug(self, result): |
| 415 | """ |
| 416 | Returns True if this failure requires a bug. |
| 417 | |
| 418 | @param result: A result, encapsulating the status of the failed job. |
| 419 | @return: True if we should file bugs for this failure. |
| 420 | """ |
beeps | beefc06 | 2013-08-02 11:17:09 -0700 | [diff] [blame^] | 421 | is_not_experimental = ( |
| 422 | constants.EXPERIMENTAL_PREFIX not in result._test_name and |
| 423 | constants.EXPERIMENTAL_PREFIX not in result._job_name) |
| 424 | |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 425 | return (self._file_bugs and |
beeps | beefc06 | 2013-08-02 11:17:09 -0700 | [diff] [blame^] | 426 | (is_not_experimental or self._file_experimental_bugs) and |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 427 | result.is_worse_than(job_status.Status('WARN', '', 'reason'))) |
| 428 | |
| 429 | |
beeps | c8a875b | 2013-03-25 10:20:38 -0700 | [diff] [blame] | 430 | def wait(self, record, bug_template={}): |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 431 | """ |
| 432 | Polls for the job statuses, using |record| to print status when each |
| 433 | completes. |
| 434 | |
| 435 | @param record: callable that records job status. |
| 436 | prototype: |
| 437 | record(base_job.status_log_entry) |
beeps | c8a875b | 2013-03-25 10:20:38 -0700 | [diff] [blame] | 438 | @param bug_template: A template dictionary specifying the default bug |
| 439 | filing options for failures in this suite. |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 440 | """ |
Alex Miller | a3a4fe7 | 2013-01-22 09:57:47 -0800 | [diff] [blame] | 441 | if self._file_bugs: |
| 442 | bug_reporter = reporting.Reporter() |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 443 | try: |
| 444 | for result in job_status.wait_for_results(self._afe, |
| 445 | self._tko, |
| 446 | self._jobs): |
| 447 | result.record_all(record) |
| 448 | if (self._results_dir and |
| 449 | job_status.is_for_infrastructure_fail(result)): |
| 450 | self._remember_provided_job_id(result) |
Dan Shi | d152180 | 2013-05-24 13:08:37 -0700 | [diff] [blame] | 451 | elif (self._results_dir and isinstance(result, Status)): |
| 452 | self._remember_test_status_job_id(result) |
Alex Miller | a3a4fe7 | 2013-01-22 09:57:47 -0800 | [diff] [blame] | 453 | |
beeps | da5b711 | 2013-05-30 11:34:14 -0700 | [diff] [blame] | 454 | if self.should_file_bug(result): |
beeps | c4fb147 | 2013-05-08 21:49:48 -0700 | [diff] [blame] | 455 | job_views = self._tko.run('get_detailed_test_views', |
| 456 | afe_job_id=result.id) |
| 457 | |
beeps | c208bf3 | 2013-04-16 14:36:35 -0700 | [diff] [blame] | 458 | failure = reporting.TestFailure(self._build, |
beeps | c4fb147 | 2013-05-08 21:49:48 -0700 | [diff] [blame] | 459 | site_utils.get_chrome_version(job_views), |
| 460 | self._tag, |
| 461 | result) |
beeps | 1ccbbb8 | 2013-02-21 20:35:01 -0800 | [diff] [blame] | 462 | |
beeps | 8ead53c | 2013-04-26 19:12:46 -0700 | [diff] [blame] | 463 | bug_id = bug_reporter.report(failure, bug_template) |
| 464 | try: |
| 465 | # Attempting to use the name of a job with special |
| 466 | # characters as a keyval will throw a ValueError. One |
| 467 | # such case is with aborted jobs. Luckily, we don't |
| 468 | # really care about the name, since the same name we |
| 469 | # have here is inserted into the results database we can |
| 470 | # use it as a key to retrieve the bug id. An example key |
| 471 | # for an aborted job after replacing the '/' with '_': |
| 472 | # lumpy-release_R28-3947.0.0_dummy_experimental_dummy_\ |
| 473 | # Pass-Bug_Id=xxxx, where xxxx is the id of the bug. |
| 474 | utils.write_keyval(self._results_dir, { |
| 475 | (result.test_name.replace('/', '_')+ |
| 476 | constants.BUG_KEYVAL): bug_id}) |
| 477 | except ValueError: |
| 478 | logging.error('Unable to log keyval for test:%s ' |
| 479 | 'bugid: %s', result.test_name, bug_id) |
| 480 | |
Alex Miller | 3a69adc | 2012-12-19 13:38:31 -0800 | [diff] [blame] | 481 | except Exception: # pylint: disable=W0703 |
| 482 | logging.error(traceback.format_exc()) |
| 483 | Status('FAIL', self._tag, |
| 484 | 'Exception waiting for results').record_result(record) |
| 485 | |
| 486 | |
| 487 | def abort(self): |
| 488 | """ |
| 489 | Abort all scheduled test jobs. |
| 490 | """ |
| 491 | if self._jobs: |
| 492 | job_ids = [job.id for job in self._jobs] |
| 493 | self._afe.run('abort_host_queue_entries', job__id__in=job_ids) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 494 | |
| 495 | |
Chris Masone | d9f13c5 | 2012-08-29 10:37:08 -0700 | [diff] [blame] | 496 | def _remember_scheduled_job_ids(self): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 497 | """ |
| 498 | Record scheduled job ids as keyvals, so they can be referenced later. |
| 499 | """ |
| 500 | for job in self._jobs: |
Chris Masone | d9f13c5 | 2012-08-29 10:37:08 -0700 | [diff] [blame] | 501 | self._remember_provided_job_id(job) |
| 502 | |
| 503 | |
| 504 | def _remember_provided_job_id(self, job): |
| 505 | """ |
| 506 | Record provided job as a suite job keyval, for later referencing. |
| 507 | |
| 508 | @param job: some representation of a job, including id, test_name |
| 509 | and owner |
| 510 | """ |
| 511 | if job.id and job.owner and job.test_name: |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 512 | job_id_owner = '%s-%s' % (job.id, job.owner) |
Chris Masone | d9f13c5 | 2012-08-29 10:37:08 -0700 | [diff] [blame] | 513 | logging.debug('Adding job keyval for %s=%s', |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 514 | job.test_name, job_id_owner) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 515 | utils.write_keyval( |
| 516 | self._results_dir, |
| 517 | {hashlib.md5(job.test_name).hexdigest(): job_id_owner}) |
| 518 | |
| 519 | |
Dan Shi | d152180 | 2013-05-24 13:08:37 -0700 | [diff] [blame] | 520 | def _remember_test_status_job_id(self, status): |
| 521 | """ |
| 522 | Record provided status as a test status keyval, for later referencing. |
| 523 | |
| 524 | @param status: Test status, including properties such as id, test_name |
| 525 | and owner. |
| 526 | """ |
| 527 | if status.id and status.owner and status.test_name: |
| 528 | test_id_owner = '%s-%s' % (status.id, status.owner) |
| 529 | logging.debug('Adding status keyval for %s=%s', |
| 530 | status.test_name, test_id_owner) |
| 531 | utils.write_keyval( |
| 532 | self._results_dir, |
| 533 | {hashlib.md5(status.test_name).hexdigest(): test_id_owner}) |
| 534 | |
| 535 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 536 | @staticmethod |
beeps | c594c1c | 2013-07-09 22:33:18 -0700 | [diff] [blame] | 537 | def find_and_parse_tests(cf_getter, predicate, suite_name='', |
| 538 | add_experimental=False): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 539 | """ |
| 540 | Function to scan through all tests and find eligible tests. |
| 541 | |
| 542 | Looks at control files returned by _cf_getter.get_control_file_list() |
beeps | c594c1c | 2013-07-09 22:33:18 -0700 | [diff] [blame] | 543 | for tests that pass self._predicate(). When this method is called |
| 544 | with a file system ControlFileGetter, it performs a full parse of the |
| 545 | root directory associated with the getter. This is the case when it's |
| 546 | invoked from suite_preprocessor. When it's invoked with a devserver |
| 547 | getter it looks up the suite_name in a suite to control file map |
| 548 | generated at build time, and parses the relevant control files alone. |
| 549 | This lookup happens on the devserver, so as far as this method is |
| 550 | concerned, both cases are equivalent. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 551 | |
| 552 | @param cf_getter: a control_file_getter.ControlFileGetter used to list |
| 553 | and fetch the content of control files |
| 554 | @param predicate: a function that should return True when run over a |
| 555 | ControlData representation of a control file that should be in |
| 556 | this Suite. |
beeps | c594c1c | 2013-07-09 22:33:18 -0700 | [diff] [blame] | 557 | @param suite_name: If specified, this method will attempt to restrain |
| 558 | the search space to just this suite's control files. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 559 | @param add_experimental: add tests with experimental attribute set. |
| 560 | |
| 561 | @return list of ControlData objects that should be run, with control |
Dan Shi | ef5b53f | 2013-01-22 10:22:01 -0800 | [diff] [blame] | 562 | file text added in |text| attribute. Results are sorted based |
| 563 | on the TIME setting in control file, slowest test comes first. |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 564 | """ |
| 565 | tests = {} |
beeps | c594c1c | 2013-07-09 22:33:18 -0700 | [diff] [blame] | 566 | files = cf_getter.get_control_file_list(suite_name=suite_name) |
| 567 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 568 | matcher = re.compile(r'[^/]+/(deps|profilers)/.+') |
Aviv Keshet | e8765c0 | 2013-06-04 14:33:43 -0700 | [diff] [blame] | 569 | parsed_count = 0 |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 570 | for file in filter(lambda f: not matcher.match(f), files): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 571 | text = cf_getter.get_control_file_contents(file) |
| 572 | try: |
| 573 | found_test = control_data.parse_control_string( |
| 574 | text, raise_warnings=True) |
Aviv Keshet | e8765c0 | 2013-06-04 14:33:43 -0700 | [diff] [blame] | 575 | parsed_count += 1 |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 576 | if not add_experimental and found_test.experimental: |
| 577 | continue |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 578 | found_test.text = text |
| 579 | found_test.path = file |
| 580 | tests[file] = found_test |
| 581 | except control_data.ControlVariableException, e: |
| 582 | logging.warn("Skipping %s\n%s", file, e) |
| 583 | except Exception, e: |
| 584 | logging.error("Bad %s\n%s", file, e) |
Aviv Keshet | e8765c0 | 2013-06-04 14:33:43 -0700 | [diff] [blame] | 585 | logging.debug('Parsed %s control files.', parsed_count) |
Dan Shi | ef5b53f | 2013-01-22 10:22:01 -0800 | [diff] [blame] | 586 | tests = [test for test in tests.itervalues() if predicate(test)] |
| 587 | tests.sort(key=lambda t: |
| 588 | control_data.ControlData.get_test_time_index(t.time), |
| 589 | reverse=True) |
| 590 | return tests |