J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 1 | import abc |
| 2 | import datetime |
| 3 | import glob |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 4 | import json |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 5 | import os |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 6 | import re |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 7 | import shutil |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 8 | import time |
| 9 | |
| 10 | import common |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 11 | from autotest_lib.client.common_lib import time_utils |
Dan Shi | 8180063 | 2015-09-29 12:16:48 -0700 | [diff] [blame] | 12 | from autotest_lib.client.common_lib import utils |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 13 | from autotest_lib.server.cros.dynamic_suite import constants |
J. Richard Barnette | acdb013 | 2014-09-03 16:44:12 -0700 | [diff] [blame] | 14 | from autotest_lib.server.cros.dynamic_suite import frontend_wrappers |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 15 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 16 | |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 17 | _AFE = frontend_wrappers.RetryingAFE() |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 18 | |
Dan Shi | 1b4c7c3 | 2015-10-05 10:38:57 -0700 | [diff] [blame] | 19 | SPECIAL_TASK_PATTERN = '.*/hosts/[^/]+/(\d+)-[^/]+' |
| 20 | JOB_PATTERN = '.*/(\d+)-[^/]+' |
Dan Shi | afa6387 | 2016-02-23 15:32:31 -0800 | [diff] [blame] | 21 | # Pattern of a job folder, e.g., 123-debug_user, where 123 is job id and |
| 22 | # debug_user is the name of user starts the job. |
| 23 | JOB_FOLDER_PATTERN = '.*/(\d+-[^/]+)' |
Dan Shi | 1b4c7c3 | 2015-10-05 10:38:57 -0700 | [diff] [blame] | 24 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 25 | def _is_job_expired(age_limit, timestamp): |
| 26 | """Check whether a job timestamp is older than an age limit. |
| 27 | |
| 28 | @param age_limit: Minimum age, measured in days. If the value is |
| 29 | not positive, the job is always expired. |
| 30 | @param timestamp: Timestamp of the job whose age we are checking. |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 31 | The format must match time_utils.TIME_FMT. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 32 | |
| 33 | @returns True iff the job is old enough to be expired. |
| 34 | """ |
| 35 | if age_limit <= 0: |
| 36 | return True |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 37 | job_time = time_utils.time_string_to_datetime(timestamp) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 38 | expiration = job_time + datetime.timedelta(days=age_limit) |
| 39 | return datetime.datetime.now() >= expiration |
| 40 | |
| 41 | |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 42 | def get_job_id_or_task_id(result_dir): |
| 43 | """Extract job id or special task id from result_dir |
| 44 | |
| 45 | @param result_dir: path to the result dir. |
| 46 | For test job: |
| 47 | /usr/local/autotest/results/2032-chromeos-test/chromeos1-rack5-host6 |
| 48 | The hostname at the end is optional. |
| 49 | For special task: |
| 50 | /usr/local/autotest/results/hosts/chromeos1-rack5-host6/1343-cleanup |
| 51 | |
Dan Shi | 8180063 | 2015-09-29 12:16:48 -0700 | [diff] [blame] | 52 | @returns: integer representing the job id or task id. Returns None if fail |
| 53 | to parse job or task id from the result_dir. |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 54 | """ |
| 55 | if not result_dir: |
| 56 | return |
| 57 | result_dir = os.path.abspath(result_dir) |
Dan Shi | 8180063 | 2015-09-29 12:16:48 -0700 | [diff] [blame] | 58 | # Result folder for job running inside container has only job id. |
| 59 | ssp_job_pattern = '.*/(\d+)$' |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 60 | # Try to get the job ID from the last pattern of number-text. This avoids |
| 61 | # issue with path like 123-results/456-debug_user, in which 456 is the real |
| 62 | # job ID. |
Dan Shi | 1b4c7c3 | 2015-10-05 10:38:57 -0700 | [diff] [blame] | 63 | m_job = re.findall(JOB_PATTERN, result_dir) |
Dan Shi | 8180063 | 2015-09-29 12:16:48 -0700 | [diff] [blame] | 64 | if m_job: |
| 65 | return int(m_job[-1]) |
Dan Shi | 1b4c7c3 | 2015-10-05 10:38:57 -0700 | [diff] [blame] | 66 | m_special_task = re.match(SPECIAL_TASK_PATTERN, result_dir) |
Dan Shi | 8180063 | 2015-09-29 12:16:48 -0700 | [diff] [blame] | 67 | if m_special_task: |
| 68 | return int(m_special_task.group(1)) |
| 69 | m_ssp_job_pattern = re.match(ssp_job_pattern, result_dir) |
| 70 | if m_ssp_job_pattern and utils.is_in_container(): |
| 71 | return int(m_ssp_job_pattern.group(1)) |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 72 | |
| 73 | |
Dan Shi | afa6387 | 2016-02-23 15:32:31 -0800 | [diff] [blame] | 74 | def get_job_folder_name(result_dir): |
| 75 | """Extract folder name of a job from result_dir. |
| 76 | |
| 77 | @param result_dir: path to the result dir. |
| 78 | For test job: |
| 79 | /usr/local/autotest/results/2032-chromeos-test/chromeos1-rack5-host6 |
| 80 | The hostname at the end is optional. |
| 81 | For special task: |
| 82 | /usr/local/autotest/results/hosts/chromeos1-rack5-host6/1343-cleanup |
| 83 | |
| 84 | @returns: The name of the folder of a job. Returns None if fail to parse |
| 85 | the name matching pattern JOB_FOLDER_PATTERN from the result_dir. |
| 86 | """ |
| 87 | if not result_dir: |
| 88 | return |
| 89 | m_job = re.findall(JOB_FOLDER_PATTERN, result_dir) |
| 90 | if m_job: |
| 91 | return m_job[-1] |
| 92 | |
| 93 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 94 | class _JobDirectory(object): |
| 95 | """State associated with a job to be offloaded. |
| 96 | |
| 97 | The full life-cycle of a job (including failure events that |
| 98 | normally don't occur) looks like this: |
| 99 | 1. The job's results directory is discovered by |
| 100 | `get_job_directories()`, and a job instance is created for it. |
| 101 | 2. Calls to `offload()` have no effect so long as the job |
| 102 | isn't complete in the database and the job isn't expired |
| 103 | according to the `age_limit` parameter. |
| 104 | 3. Eventually, the job is both finished and expired. The next |
| 105 | call to `offload()` makes the first attempt to offload the |
| 106 | directory to GS. Offload is attempted, but fails to complete |
| 107 | (e.g. because of a GS problem). |
| 108 | 4. After the first failed offload `is_offloaded()` is false, |
| 109 | but `is_reportable()` is also false, so the failure is not |
| 110 | reported. |
| 111 | 5. Another call to `offload()` again tries to offload the |
| 112 | directory, and again fails. |
| 113 | 6. After a second failure, `is_offloaded()` is false and |
| 114 | `is_reportable()` is true, so the failure generates an e-mail |
| 115 | notification. |
| 116 | 7. Finally, a call to `offload()` succeeds, and the directory no |
| 117 | longer exists. Now `is_offloaded()` is true, so the job |
| 118 | instance is deleted, and future failures will not mention this |
| 119 | directory any more. |
| 120 | |
| 121 | Only steps 1. and 7. are guaranteed to occur. The others depend |
| 122 | on the timing of calls to `offload()`, and on the reliability of |
| 123 | the actual offload process. |
| 124 | |
| 125 | """ |
| 126 | |
| 127 | __metaclass__ = abc.ABCMeta |
| 128 | |
| 129 | GLOB_PATTERN = None # must be redefined in subclass |
| 130 | |
| 131 | def __init__(self, resultsdir): |
| 132 | self._dirname = resultsdir |
Dan Shi | cf4d203 | 2015-03-12 15:04:21 -0700 | [diff] [blame] | 133 | self._id = get_job_id_or_task_id(resultsdir) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 134 | self._offload_count = 0 |
| 135 | self._first_offload_start = 0 |
| 136 | |
| 137 | @classmethod |
| 138 | def get_job_directories(cls): |
| 139 | """Return a list of directories of jobs that need offloading.""" |
| 140 | return [d for d in glob.glob(cls.GLOB_PATTERN) if os.path.isdir(d)] |
| 141 | |
| 142 | @abc.abstractmethod |
| 143 | def get_timestamp_if_finished(self): |
| 144 | """Return this job's timestamp from the database. |
| 145 | |
| 146 | If the database has not marked the job as finished, return |
| 147 | `None`. Otherwise, return a timestamp for the job. The |
| 148 | timestamp is to be used to determine expiration in |
| 149 | `_is_job_expired()`. |
| 150 | |
| 151 | @return Return `None` if the job is still running; otherwise |
| 152 | return a string with a timestamp in the appropriate |
| 153 | format. |
| 154 | """ |
| 155 | raise NotImplementedError("_JobDirectory.get_timestamp_if_finished") |
| 156 | |
| 157 | def enqueue_offload(self, queue, age_limit): |
| 158 | """Enqueue the job for offload, if it's eligible. |
| 159 | |
| 160 | The job is eligible for offloading if the database has marked |
| 161 | it finished, and the job is older than the `age_limit` |
| 162 | parameter. |
| 163 | |
| 164 | If the job is eligible, offload processing is requested by |
| 165 | passing the `queue` parameter's `put()` method a sequence with |
J. Richard Barnette | 2c72ddd | 2014-05-20 12:17:37 -0700 | [diff] [blame] | 166 | the job's `_dirname` attribute and its directory name. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 167 | |
| 168 | @param queue If the job should be offloaded, put the offload |
| 169 | parameters into this queue for processing. |
| 170 | @param age_limit Minimum age for a job to be offloaded. A value |
| 171 | of 0 means that the job will be offloaded as |
| 172 | soon as it is finished. |
| 173 | |
| 174 | """ |
| 175 | if not self._offload_count: |
| 176 | timestamp = self.get_timestamp_if_finished() |
| 177 | if not timestamp: |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 178 | return |
| 179 | if not _is_job_expired(age_limit, timestamp): |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 180 | return |
| 181 | self._first_offload_start = time.time() |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 182 | self._offload_count += 1 |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 183 | if self.process_gs_instructions(): |
| 184 | queue.put([self._dirname, os.path.dirname(self._dirname)]) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 185 | |
| 186 | def is_offloaded(self): |
| 187 | """Return whether this job has been successfully offloaded.""" |
| 188 | return not os.path.exists(self._dirname) |
| 189 | |
| 190 | def is_reportable(self): |
| 191 | """Return whether this job has a reportable failure.""" |
| 192 | return self._offload_count > 1 |
| 193 | |
| 194 | def get_failure_time(self): |
| 195 | """Return the time of the first offload failure.""" |
| 196 | return self._first_offload_start |
| 197 | |
| 198 | def get_failure_count(self): |
| 199 | """Return the number of times this job has failed to offload.""" |
| 200 | return self._offload_count |
| 201 | |
| 202 | def get_job_directory(self): |
| 203 | """Return the name of this job's results directory.""" |
| 204 | return self._dirname |
| 205 | |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 206 | def process_gs_instructions(self): |
| 207 | """Process any gs_offloader instructions for this special task. |
| 208 | |
| 209 | @returns True/False if there is anything left to offload. |
| 210 | """ |
| 211 | # Default support is to still offload the directory. |
| 212 | return True |
| 213 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 214 | |
| 215 | class RegularJobDirectory(_JobDirectory): |
| 216 | """Subclass of _JobDirectory for regular test jobs.""" |
| 217 | |
| 218 | GLOB_PATTERN = '[0-9]*-*' |
| 219 | |
Simran Basi | 1e10e92 | 2015-04-16 15:09:56 -0700 | [diff] [blame] | 220 | def process_gs_instructions(self): |
| 221 | """Process any gs_offloader instructions for this job. |
| 222 | |
| 223 | @returns True/False if there is anything left to offload. |
| 224 | """ |
| 225 | # Go through the gs_offloader instructions file for each test in this job. |
| 226 | for path in glob.glob(os.path.join(self._dirname, '*', |
| 227 | constants.GS_OFFLOADER_INSTRUCTIONS)): |
| 228 | with open(path, 'r') as f: |
| 229 | gs_off_instructions = json.load(f) |
| 230 | if gs_off_instructions.get(constants.GS_OFFLOADER_NO_OFFLOAD): |
| 231 | shutil.rmtree(os.path.dirname(path)) |
| 232 | |
| 233 | # Finally check if there's anything left to offload. |
| 234 | if not os.listdir(self._dirname): |
| 235 | shutil.rmtree(self._dirname) |
| 236 | return False |
| 237 | return True |
| 238 | |
| 239 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 240 | def get_timestamp_if_finished(self): |
Simran Basi | fb98e46 | 2014-08-18 12:35:44 -0700 | [diff] [blame] | 241 | """Get the timestamp to use for finished jobs. |
| 242 | |
| 243 | @returns the latest hqe finished_on time. If the finished_on times are null |
| 244 | returns the job's created_on time. |
| 245 | """ |
J. Richard Barnette | dd0227d | 2015-04-10 15:18:48 -0700 | [diff] [blame] | 246 | entry = _AFE.get_jobs(id=self._id, finished=True) |
Simran Basi | fb98e46 | 2014-08-18 12:35:44 -0700 | [diff] [blame] | 247 | if not entry: |
| 248 | return None |
J. Richard Barnette | dd0227d | 2015-04-10 15:18:48 -0700 | [diff] [blame] | 249 | hqes = _AFE.get_host_queue_entries(finished_on__isnull=False, |
| 250 | job_id=self._id) |
Simran Basi | fb98e46 | 2014-08-18 12:35:44 -0700 | [diff] [blame] | 251 | if not hqes: |
J. Richard Barnette | dd0227d | 2015-04-10 15:18:48 -0700 | [diff] [blame] | 252 | return entry[0].created_on |
Simran Basi | fb98e46 | 2014-08-18 12:35:44 -0700 | [diff] [blame] | 253 | # While most Jobs have 1 HQE, some can have multiple, so check them all. |
J. Richard Barnette | dd0227d | 2015-04-10 15:18:48 -0700 | [diff] [blame] | 254 | return max([hqe.finished_on for hqe in hqes]) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 255 | |
| 256 | |
| 257 | class SpecialJobDirectory(_JobDirectory): |
| 258 | """Subclass of _JobDirectory for special (per-host) jobs.""" |
| 259 | |
| 260 | GLOB_PATTERN = 'hosts/*/[0-9]*-*' |
| 261 | |
| 262 | def __init__(self, resultsdir): |
| 263 | super(SpecialJobDirectory, self).__init__(resultsdir) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 264 | |
| 265 | def get_timestamp_if_finished(self): |
J. Richard Barnette | dd0227d | 2015-04-10 15:18:48 -0700 | [diff] [blame] | 266 | entry = _AFE.get_special_tasks(id=self._id, is_complete=True) |
| 267 | return entry[0].time_finished if entry else None |