Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | # |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 3 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 7 | """Script to archive old Autotest results to Google Storage. |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 8 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 9 | Uses gsutil to archive files to the configured Google Storage bucket. |
| 10 | Upon successful copy, the local results directory is deleted. |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 11 | """ |
| 12 | |
Simran Basi | bd9ded0 | 2013-11-04 15:49:11 -0800 | [diff] [blame] | 13 | import datetime |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 14 | import logging |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 15 | import os |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 16 | import shutil |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 17 | import signal |
Simran Basi | 981a927 | 2012-11-14 10:46:03 -0800 | [diff] [blame] | 18 | import socket |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 19 | import subprocess |
| 20 | import sys |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 21 | import tempfile |
| 22 | import time |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 23 | |
Simran Basi | 7d9a149 | 2012-10-25 13:51:54 -0700 | [diff] [blame] | 24 | from optparse import OptionParser |
| 25 | |
Simran Basi | 981a927 | 2012-11-14 10:46:03 -0800 | [diff] [blame] | 26 | import common |
| 27 | |
Alex Miller | c900b34 | 2014-06-09 16:52:07 -0700 | [diff] [blame] | 28 | try: |
| 29 | # Does not exist, nor is needed, on moblab. |
| 30 | import psutil |
| 31 | except ImportError: |
| 32 | psutil = None |
| 33 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 34 | import job_directories |
Simran Basi | 981a927 | 2012-11-14 10:46:03 -0800 | [diff] [blame] | 35 | from autotest_lib.client.common_lib import global_config |
| 36 | from autotest_lib.scheduler import email_manager |
Fang Deng | 970b6a7 | 2013-04-09 11:59:16 -0700 | [diff] [blame] | 37 | from chromite.lib import parallel |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 38 | |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 39 | # Google Storage bucket URI to store results in. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 40 | GS_URI = global_config.global_config.get_config_value( |
| 41 | 'CROS', 'results_storage_server') |
| 42 | GS_URI_PATTERN = GS_URI + '%s' |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 43 | |
| 44 | # Nice setting for process, the higher the number the lower the priority. |
| 45 | NICENESS = 10 |
| 46 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 47 | # Maximum number of seconds to allow for offloading a single |
| 48 | # directory. |
J. Richard Barnette | 7e0f859 | 2014-09-03 17:00:55 -0700 | [diff] [blame^] | 49 | OFFLOAD_TIMEOUT_SECS = 60 * 60 |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 50 | |
Simran Basi | 392d4a5 | 2012-12-14 10:29:44 -0800 | [diff] [blame] | 51 | # Sleep time per loop. |
| 52 | SLEEP_TIME_SECS = 5 |
| 53 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 54 | # Minimum number of seconds between e-mail reports. |
| 55 | REPORT_INTERVAL_SECS = 60 * 60 |
| 56 | |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 57 | # Location of Autotest results on disk. |
| 58 | RESULTS_DIR = '/usr/local/autotest/results' |
| 59 | |
Simran Basi | 31d561d | 2012-07-31 13:44:40 -0700 | [diff] [blame] | 60 | # Hosts sub-directory that contains cleanup, verify and repair jobs. |
| 61 | HOSTS_SUB_DIR = 'hosts' |
| 62 | |
Alex Miller | 0c8db6d | 2013-02-15 15:41:00 -0800 | [diff] [blame] | 63 | LOG_LOCATION = '/usr/local/autotest/logs/' |
| 64 | LOG_FILENAME_FORMAT = 'gs_offloader_%s_log_%s.txt' |
| 65 | LOG_TIMESTAMP_FORMAT = '%Y%m%d_%H%M%S' |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 66 | LOGGING_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' |
| 67 | |
Alex Miller | 0c8db6d | 2013-02-15 15:41:00 -0800 | [diff] [blame] | 68 | # pylint: disable=E1120 |
Simran Basi | 981a927 | 2012-11-14 10:46:03 -0800 | [diff] [blame] | 69 | NOTIFY_ADDRESS = global_config.global_config.get_config_value( |
| 70 | 'SCHEDULER', 'notify_email', default='') |
| 71 | |
| 72 | ERROR_EMAIL_SUBJECT_FORMAT = 'GS Offloader notifications from %s' |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 73 | ERROR_EMAIL_REPORT_FORMAT = '''\ |
| 74 | gs_offloader is failing to offload results directories. |
Simran Basi | 981a927 | 2012-11-14 10:46:03 -0800 | [diff] [blame] | 75 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 76 | First failure Count Directory name |
| 77 | =================== ====== ============================== |
| 78 | ''' |
| 79 | # --+----1----+---- ----+ ----+----1----+----2----+----3 |
| 80 | |
| 81 | ERROR_EMAIL_DIRECTORY_FORMAT = '%19s %5d %-1s\n' |
| 82 | ERROR_EMAIL_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 83 | |
| 84 | class TimeoutException(Exception): |
beeps | 8c30db1 | 2013-03-30 18:20:27 -0700 | [diff] [blame] | 85 | """Exception raised by the timeout_handler.""" |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 86 | pass |
| 87 | |
| 88 | |
| 89 | def timeout_handler(_signum, _frame): |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 90 | """Handler for SIGALRM when the offloading process times out. |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 91 | |
beeps | 8c30db1 | 2013-03-30 18:20:27 -0700 | [diff] [blame] | 92 | @param _signum: Signal number of the signal that was just caught. |
| 93 | 14 for SIGALRM. |
| 94 | @param _frame: Current stack frame. |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 95 | @raise TimeoutException: Automatically raises so that the time out is caught |
| 96 | by the try/except surrounding the Popen call. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 97 | |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 98 | """ |
| 99 | raise TimeoutException('Process Timed Out') |
| 100 | |
| 101 | |
Simran Basi | 31d561d | 2012-07-31 13:44:40 -0700 | [diff] [blame] | 102 | def get_cmd_list(dir_entry, relative_path): |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 103 | """Return the command to offload a specified directory. |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 104 | |
| 105 | @param dir_entry: Directory entry/path that which we need a cmd_list to |
| 106 | offload. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 107 | @param relative_path: Location in google storage where we will |
| 108 | offload the directory. |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 109 | |
| 110 | @return: A command list to be executed by Popen. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 111 | |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 112 | """ |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 113 | return ['gsutil', '-m', 'cp', '-eR', '-a', 'project-private', |
| 114 | dir_entry, GS_URI_PATTERN % relative_path] |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 115 | |
| 116 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 117 | def offload_dir(dir_entry, dest_path): |
| 118 | """Offload the specified directory entry to Google storage. |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 119 | |
| 120 | @param dir_entry: Directory entry to offload. |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 121 | @param dest_path: Location in google storage where we will offload |
| 122 | the directory. |
| 123 | |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 124 | """ |
| 125 | try: |
| 126 | error = False |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 127 | stdout_file = tempfile.TemporaryFile('w+') |
| 128 | stderr_file = tempfile.TemporaryFile('w+') |
J. Richard Barnette | 2e443ef | 2014-05-20 12:31:35 -0700 | [diff] [blame] | 129 | process = None |
| 130 | signal.alarm(OFFLOAD_TIMEOUT_SECS) |
Simran Basi | 31d561d | 2012-07-31 13:44:40 -0700 | [diff] [blame] | 131 | process = subprocess.Popen(get_cmd_list(dir_entry, dest_path), |
| 132 | stdout=stdout_file, stderr=stderr_file) |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 133 | process.wait() |
| 134 | signal.alarm(0) |
| 135 | if process.returncode == 0: |
| 136 | shutil.rmtree(dir_entry) |
| 137 | else: |
| 138 | error = True |
| 139 | except TimeoutException: |
J. Richard Barnette | 2e443ef | 2014-05-20 12:31:35 -0700 | [diff] [blame] | 140 | # If we finished the call to Popen(), we may need to terminate |
| 141 | # the child process. We don't bother calling process.poll(); |
| 142 | # that inherently races because the child can die any time it |
| 143 | # wants. |
| 144 | if process: |
| 145 | try: |
| 146 | process.terminate() |
| 147 | except OSError: |
| 148 | # We don't expect any error other than "No such |
| 149 | # process". |
| 150 | pass |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 151 | logging.error('Offloading %s timed out after waiting %d seconds.', |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 152 | dir_entry, OFFLOAD_TIMEOUT_SECS) |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 153 | error = True |
| 154 | finally: |
| 155 | signal.alarm(0) |
| 156 | if error: |
| 157 | # Rewind the log files for stdout and stderr and log their contents. |
| 158 | stdout_file.seek(0) |
| 159 | stderr_file.seek(0) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 160 | logging.error('Error occurred when offloading %s:', dir_entry) |
| 161 | logging.error('Stdout:\n%s \nStderr:\n%s', |
| 162 | stdout_file.read(), stderr_file.read()) |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 163 | stdout_file.close() |
| 164 | stderr_file.close() |
| 165 | |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 166 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 167 | def delete_files(dir_entry, dest_path): |
Simran Basi | bd9ded0 | 2013-11-04 15:49:11 -0800 | [diff] [blame] | 168 | """Simply deletes the dir_entry from the filesystem. |
| 169 | |
| 170 | Uses same arguments as offload_dir so that it can be used in replace of it on |
| 171 | systems that only want to delete files instead of offloading them. |
| 172 | |
| 173 | @param dir_entry: Directory entry to offload. |
| 174 | @param dest_path: NOT USED. |
| 175 | """ |
| 176 | shutil.rmtree(dir_entry) |
| 177 | |
| 178 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 179 | def report_offload_failures(joblist): |
| 180 | """Generate e-mail notification for failed offloads. |
| 181 | |
| 182 | The e-mail report will include data from all jobs in `joblist`. |
| 183 | |
| 184 | @param joblist List of jobs to be reported in the message. |
| 185 | |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 186 | """ |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 187 | def _format_job(job): |
| 188 | d = datetime.datetime.fromtimestamp(job.get_failure_time()) |
| 189 | data = (d.strftime(ERROR_EMAIL_TIME_FORMAT), |
| 190 | job.get_failure_count(), |
| 191 | job.get_job_directory()) |
| 192 | return ERROR_EMAIL_DIRECTORY_FORMAT % data |
| 193 | joblines = [_format_job(job) for job in joblist] |
| 194 | joblines.sort() |
| 195 | email_subject = ERROR_EMAIL_SUBJECT_FORMAT % socket.gethostname() |
| 196 | email_message = ERROR_EMAIL_REPORT_FORMAT + ''.join(joblines) |
| 197 | email_manager.manager.send_email(NOTIFY_ADDRESS, email_subject, |
| 198 | email_message) |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 199 | |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 200 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 201 | class Offloader(object): |
| 202 | """State of the offload process. |
| 203 | |
| 204 | Contains the following member fields: |
| 205 | * _offload_func: Function to call for each attempt to offload |
| 206 | a job directory. |
| 207 | * _jobdir_classes: List of classes of job directory to be |
| 208 | offloaded. |
| 209 | * _processes: Maximum number of outstanding offload processes |
| 210 | to allow during an offload cycle. |
| 211 | * _age_limit: Minimum age in days at which a job may be |
| 212 | offloaded. |
| 213 | * _open_jobs: a dictionary mapping directory paths to Job |
| 214 | objects. |
| 215 | * _next_report_time: Earliest time that we should send e-mail |
| 216 | if there are failures to be reported. |
| 217 | |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 218 | """ |
Alex Miller | 95c3a4e | 2012-11-30 19:14:39 -0800 | [diff] [blame] | 219 | |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 220 | def __init__(self, options): |
| 221 | if options.delete_only: |
| 222 | self._offload_func = delete_files |
| 223 | else: |
| 224 | self._offload_func = offload_dir |
| 225 | classlist = [] |
| 226 | if options.process_hosts_only or options.process_all: |
| 227 | classlist.append(job_directories.SpecialJobDirectory) |
| 228 | if not options.process_hosts_only: |
| 229 | classlist.append(job_directories.RegularJobDirectory) |
| 230 | self._jobdir_classes = classlist |
| 231 | assert self._jobdir_classes |
| 232 | self._processes = options.parallelism |
| 233 | self._age_limit = options.days_old |
| 234 | self._open_jobs = {} |
| 235 | self._next_report_time = time.time() |
| 236 | |
| 237 | def _add_new_jobs(self): |
| 238 | """Find new job directories that need offloading. |
| 239 | |
| 240 | Go through the file system looking for valid job directories |
| 241 | that are currently not in `self._open_jobs`, and add them in. |
| 242 | |
| 243 | """ |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 244 | new_job_count = 0 |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 245 | for cls in self._jobdir_classes: |
| 246 | for resultsdir in cls.get_job_directories(): |
| 247 | if resultsdir in self._open_jobs: |
| 248 | continue |
| 249 | self._open_jobs[resultsdir] = cls(resultsdir) |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 250 | new_job_count += 1 |
| 251 | logging.debug("Start of offload cycle - found %d new jobs", |
| 252 | new_job_count) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 253 | |
| 254 | def _remove_offloaded_jobs(self): |
| 255 | """Removed offloaded jobs from `self._open_jobs`.""" |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 256 | removed_job_count = 0 |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 257 | for jobkey, job in self._open_jobs.items(): |
| 258 | if job.is_offloaded(): |
| 259 | del self._open_jobs[jobkey] |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 260 | removed_job_count += 1 |
| 261 | logging.debug("End of offload cycle - cleared %d new jobs, " |
| 262 | "carrying %d open jobs", |
| 263 | removed_job_count, len(self._open_jobs)) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 264 | |
| 265 | def _have_reportable_errors(self): |
| 266 | """Return whether any jobs need reporting via e-mail. |
| 267 | |
| 268 | @returns True if there are reportable jobs in `self._open_jobs`, |
| 269 | or False otherwise. |
| 270 | """ |
| 271 | for job in self._open_jobs.values(): |
| 272 | if job.is_reportable(): |
| 273 | return True |
| 274 | return False |
| 275 | |
| 276 | def _update_offload_results(self): |
| 277 | """Check and report status after attempting offload. |
| 278 | |
| 279 | This function processes all jobs in `self._open_jobs`, assuming |
| 280 | an attempt has just been made to offload all of them. |
| 281 | |
| 282 | Any jobs that have been successfully offloaded are removed. |
| 283 | |
| 284 | If any jobs have reportable errors, and we haven't generated |
| 285 | an e-mail report in the last `REPORT_INTERVAL_SECS` seconds, |
| 286 | send new e-mail describing the failures. |
| 287 | |
| 288 | """ |
| 289 | self._remove_offloaded_jobs() |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 290 | if self._have_reportable_errors(): |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 291 | # N.B. We include all jobs that have failed at least once, |
| 292 | # which may include jobs that aren't otherwise reportable. |
J. Richard Barnette | 22dd748 | 2014-06-23 12:25:02 -0700 | [diff] [blame] | 293 | failed_jobs = [j for j in self._open_jobs.values() |
| 294 | if j.get_failure_time()] |
| 295 | logging.debug("Currently there are %d jobs with offload failures", |
| 296 | len(failed_jobs)) |
| 297 | if time.time() >= self._next_report_time: |
| 298 | logging.debug("Reporting failures by e-mail") |
| 299 | report_offload_failures(failed_jobs) |
| 300 | self._next_report_time = time.time() + REPORT_INTERVAL_SECS |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 301 | |
| 302 | def offload_once(self): |
| 303 | """Perform one offload cycle. |
| 304 | |
| 305 | Find all job directories for new jobs that we haven't seen |
| 306 | before. Then, attempt to offload the directories for any |
| 307 | jobs that have finished running. Offload of multiple jobs |
| 308 | is done in parallel, up to `self._processes` at a time. |
| 309 | |
| 310 | After we've tried uploading all directories, go through the list |
| 311 | checking the status of all uploaded directories. If necessary, |
| 312 | report failures via e-mail. |
| 313 | |
| 314 | """ |
| 315 | self._add_new_jobs() |
Fang Deng | 970b6a7 | 2013-04-09 11:59:16 -0700 | [diff] [blame] | 316 | with parallel.BackgroundTaskRunner( |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 317 | self._offload_func, processes=self._processes) as queue: |
| 318 | for job in self._open_jobs.values(): |
| 319 | job.enqueue_offload(queue, self._age_limit) |
| 320 | self._update_offload_results() |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 321 | |
| 322 | |
Simran Basi | 7d9a149 | 2012-10-25 13:51:54 -0700 | [diff] [blame] | 323 | def parse_options(): |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 324 | """Parse the args passed into gs_offloader.""" |
Simran Basi | 7d9a149 | 2012-10-25 13:51:54 -0700 | [diff] [blame] | 325 | defaults = 'Defaults:\n Destination: %s\n Results Path: %s' % (GS_URI, |
| 326 | RESULTS_DIR) |
| 327 | usage = 'usage: %prog [options]\n' + defaults |
| 328 | parser = OptionParser(usage) |
| 329 | parser.add_option('-a', '--all', dest='process_all', action='store_true', |
| 330 | help='Offload all files in the results directory.') |
| 331 | parser.add_option('-s', '--hosts', dest='process_hosts_only', |
| 332 | action='store_true', |
| 333 | help='Offload only the special tasks result files located' |
| 334 | 'in the results/hosts subdirectory') |
Fang Deng | 970b6a7 | 2013-04-09 11:59:16 -0700 | [diff] [blame] | 335 | parser.add_option('-p', '--parallelism', dest='parallelism', type='int', |
| 336 | default=1, help='Number of parallel workers to use.') |
Simran Basi | 9244c33 | 2013-11-12 15:40:03 -0800 | [diff] [blame] | 337 | parser.add_option('-o', '--delete_only', dest='delete_only', |
Simran Basi | bd9ded0 | 2013-11-04 15:49:11 -0800 | [diff] [blame] | 338 | action='store_true', |
| 339 | help='GS Offloader will only the delete the directories ' |
| 340 | 'and will not offload them to google storage.', |
| 341 | default=False) |
Simran Basi | 9244c33 | 2013-11-12 15:40:03 -0800 | [diff] [blame] | 342 | parser.add_option('-d', '--days_old', dest='days_old', |
Simran Basi | bd9ded0 | 2013-11-04 15:49:11 -0800 | [diff] [blame] | 343 | help='Minimum job age in days before a result can be ' |
| 344 | 'offloaded.', type='int', default=0) |
Simran Basi | 7d9a149 | 2012-10-25 13:51:54 -0700 | [diff] [blame] | 345 | options = parser.parse_args()[0] |
| 346 | if options.process_all and options.process_hosts_only: |
| 347 | parser.print_help() |
| 348 | print ('Cannot process all files and only the hosts subdirectory. ' |
| 349 | 'Please remove an argument.') |
| 350 | sys.exit(1) |
| 351 | return options |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 352 | |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 353 | |
| 354 | def main(): |
beeps | 8c30db1 | 2013-03-30 18:20:27 -0700 | [diff] [blame] | 355 | """Main method of gs_offloader.""" |
Simran Basi | 7d9a149 | 2012-10-25 13:51:54 -0700 | [diff] [blame] | 356 | options = parse_options() |
Alex Miller | 0c8db6d | 2013-02-15 15:41:00 -0800 | [diff] [blame] | 357 | |
| 358 | if options.process_all: |
| 359 | offloader_type = 'all' |
| 360 | elif options.process_hosts_only: |
| 361 | offloader_type = 'hosts' |
| 362 | else: |
| 363 | offloader_type = 'jobs' |
| 364 | |
| 365 | log_timestamp = time.strftime(LOG_TIMESTAMP_FORMAT) |
| 366 | log_filename = os.path.join(LOG_LOCATION, |
| 367 | LOG_FILENAME_FORMAT % (offloader_type, log_timestamp)) |
Simran Basi | 9523eaa | 2012-06-28 17:18:45 -0700 | [diff] [blame] | 368 | logging.basicConfig(filename=log_filename, level=logging.DEBUG, |
| 369 | format=LOGGING_FORMAT) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 370 | |
| 371 | # Nice our process (carried to subprocesses) so we don't overload |
| 372 | # the system. |
| 373 | logging.debug('Set process to nice value: %d', NICENESS) |
| 374 | os.nice(NICENESS) |
Alex Miller | c900b34 | 2014-06-09 16:52:07 -0700 | [diff] [blame] | 375 | if psutil: |
| 376 | proc = psutil.Process() |
| 377 | logging.debug('Set process to ionice IDLE') |
| 378 | proc.ionice(psutil.IOPRIO_CLASS_IDLE) |
J. Richard Barnette | ea78536 | 2014-03-17 16:00:53 -0700 | [diff] [blame] | 379 | |
| 380 | # os.listdir returns relative paths, so change to where we need to be to avoid |
| 381 | # an os.path.join on each loop. |
| 382 | logging.debug('Offloading Autotest results in %s', RESULTS_DIR) |
| 383 | os.chdir(RESULTS_DIR) |
| 384 | |
| 385 | signal.signal(signal.SIGALRM, timeout_handler) |
| 386 | |
| 387 | offloader = Offloader(options) |
| 388 | while True: |
| 389 | offloader.offload_once() |
| 390 | time.sleep(SLEEP_TIME_SECS) |
Scott Zawalski | cb2e2b7 | 2012-04-17 12:10:05 -0400 | [diff] [blame] | 391 | |
| 392 | |
Scott Zawalski | 20a9b58 | 2011-11-21 11:49:40 -0800 | [diff] [blame] | 393 | if __name__ == '__main__': |
| 394 | main() |