mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1 | #!/usr/bin/python -u |
| 2 | |
| 3 | """ |
| 4 | Autotest scheduler |
| 5 | """ |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 6 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 7 | |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 8 | import datetime, errno, MySQLdb, optparse, os, pwd, Queue, re, shutil, signal |
| 9 | import smtplib, socket, stat, subprocess, sys, tempfile, time, traceback |
mbligh | 70feeee | 2008-06-11 16:20:49 +0000 | [diff] [blame] | 10 | import common |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 11 | from autotest_lib.client.common_lib import global_config |
| 12 | from autotest_lib.client.common_lib import host_protections, utils |
mbligh | 70feeee | 2008-06-11 16:20:49 +0000 | [diff] [blame] | 13 | |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 14 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 15 | RESULTS_DIR = '.' |
| 16 | AUTOSERV_NICE_LEVEL = 10 |
| 17 | |
| 18 | AUTOTEST_PATH = os.path.join(os.path.dirname(__file__), '..') |
| 19 | |
| 20 | if os.environ.has_key('AUTOTEST_DIR'): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 21 | AUTOTEST_PATH = os.environ['AUTOTEST_DIR'] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 22 | AUTOTEST_SERVER_DIR = os.path.join(AUTOTEST_PATH, 'server') |
| 23 | AUTOTEST_TKO_DIR = os.path.join(AUTOTEST_PATH, 'tko') |
| 24 | |
| 25 | if AUTOTEST_SERVER_DIR not in sys.path: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 26 | sys.path.insert(0, AUTOTEST_SERVER_DIR) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 27 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 28 | AUTOSERV_PID_FILE = '.autoserv_execute' |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 29 | # how long to wait for autoserv to write a pidfile |
| 30 | PIDFILE_TIMEOUT = 5 * 60 # 5 min |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 31 | |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 32 | _db = None |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 33 | _shutdown = False |
| 34 | _notify_email = None |
mbligh | 4314a71 | 2008-02-29 22:44:30 +0000 | [diff] [blame] | 35 | _autoserv_path = 'autoserv' |
| 36 | _testing_mode = False |
showard | ec11316 | 2008-05-08 00:52:49 +0000 | [diff] [blame] | 37 | _global_config_section = 'SCHEDULER' |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 38 | _base_url = None |
| 39 | # see os.getlogin() online docs |
| 40 | _email_from = pwd.getpwuid(os.getuid())[0] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 41 | |
| 42 | |
| 43 | def main(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 44 | usage = 'usage: %prog [options] results_dir' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 45 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 46 | parser = optparse.OptionParser(usage) |
| 47 | parser.add_option('--recover-hosts', help='Try to recover dead hosts', |
| 48 | action='store_true') |
| 49 | parser.add_option('--logfile', help='Set a log file that all stdout ' + |
| 50 | 'should be redirected to. Stderr will go to this ' + |
| 51 | 'file + ".err"') |
| 52 | parser.add_option('--test', help='Indicate that scheduler is under ' + |
| 53 | 'test and should use dummy autoserv and no parsing', |
| 54 | action='store_true') |
| 55 | (options, args) = parser.parse_args() |
| 56 | if len(args) != 1: |
| 57 | parser.print_usage() |
| 58 | return |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 59 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 60 | global RESULTS_DIR |
| 61 | RESULTS_DIR = args[0] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 62 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 63 | # read in notify_email from global_config |
| 64 | c = global_config.global_config |
| 65 | global _notify_email |
| 66 | val = c.get_config_value(_global_config_section, "notify_email") |
| 67 | if val != "": |
| 68 | _notify_email = val |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 69 | |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 70 | tick_pause = c.get_config_value( |
| 71 | _global_config_section, 'tick_pause_sec', type=int) |
| 72 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 73 | if options.test: |
| 74 | global _autoserv_path |
| 75 | _autoserv_path = 'autoserv_dummy' |
| 76 | global _testing_mode |
| 77 | _testing_mode = True |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 78 | |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 79 | # read in base url |
| 80 | global _base_url |
| 81 | val = c.get_config_value("AUTOTEST_WEB", "base_url") |
| 82 | if val: |
| 83 | _base_url = val |
| 84 | else: |
| 85 | _base_url = "http://your_autotest_server/afe/" |
| 86 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 87 | init(options.logfile) |
| 88 | dispatcher = Dispatcher() |
| 89 | dispatcher.do_initial_recovery(recover_hosts=options.recover_hosts) |
| 90 | |
| 91 | try: |
| 92 | while not _shutdown: |
| 93 | dispatcher.tick() |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 94 | time.sleep(tick_pause) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 95 | except: |
| 96 | log_stacktrace("Uncaught exception; terminating monitor_db") |
| 97 | |
| 98 | email_manager.send_queued_emails() |
| 99 | _db.disconnect() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 100 | |
| 101 | |
| 102 | def handle_sigint(signum, frame): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 103 | global _shutdown |
| 104 | _shutdown = True |
| 105 | print "Shutdown request received." |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 106 | |
| 107 | |
| 108 | def init(logfile): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 109 | if logfile: |
| 110 | enable_logging(logfile) |
| 111 | print "%s> dispatcher starting" % time.strftime("%X %x") |
| 112 | print "My PID is %d" % os.getpid() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 113 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 114 | os.environ['PATH'] = AUTOTEST_SERVER_DIR + ':' + os.environ['PATH'] |
| 115 | global _db |
| 116 | _db = DatabaseConn() |
| 117 | _db.connect() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 118 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 119 | print "Setting signal handler" |
| 120 | signal.signal(signal.SIGINT, handle_sigint) |
| 121 | |
| 122 | print "Connected! Running..." |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 123 | |
| 124 | |
| 125 | def enable_logging(logfile): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 126 | out_file = logfile |
| 127 | err_file = "%s.err" % logfile |
| 128 | print "Enabling logging to %s (%s)" % (out_file, err_file) |
| 129 | out_fd = open(out_file, "a", buffering=0) |
| 130 | err_fd = open(err_file, "a", buffering=0) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 131 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 132 | os.dup2(out_fd.fileno(), sys.stdout.fileno()) |
| 133 | os.dup2(err_fd.fileno(), sys.stderr.fileno()) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 134 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 135 | sys.stdout = out_fd |
| 136 | sys.stderr = err_fd |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 137 | |
| 138 | |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 139 | def queue_entries_to_abort(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 140 | rows = _db.execute(""" |
| 141 | SELECT * FROM host_queue_entries WHERE status='Abort'; |
| 142 | """) |
| 143 | qe = [HostQueueEntry(row=i) for i in rows] |
| 144 | return qe |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 145 | |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 146 | def remove_file_or_dir(path): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 147 | if stat.S_ISDIR(os.stat(path).st_mode): |
| 148 | # directory |
| 149 | shutil.rmtree(path) |
| 150 | else: |
| 151 | # file |
| 152 | os.remove(path) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 153 | |
| 154 | |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 155 | class DatabaseConn: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 156 | def __init__(self): |
| 157 | self.reconnect_wait = 20 |
| 158 | self.conn = None |
| 159 | self.cur = None |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 160 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 161 | import MySQLdb.converters |
| 162 | self.convert_dict = MySQLdb.converters.conversions |
| 163 | self.convert_dict.setdefault(bool, self.convert_boolean) |
mbligh | 4eb2df2 | 2008-03-13 15:39:29 +0000 | [diff] [blame] | 164 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 165 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 166 | @staticmethod |
| 167 | def convert_boolean(boolean, conversion_dict): |
| 168 | 'Convert booleans to integer strings' |
| 169 | return str(int(boolean)) |
mbligh | 4eb2df2 | 2008-03-13 15:39:29 +0000 | [diff] [blame] | 170 | |
| 171 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 172 | def connect(self, db_name=None): |
| 173 | self.disconnect() |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 174 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 175 | # get global config and parse for info |
| 176 | c = global_config.global_config |
| 177 | dbase = "AUTOTEST_WEB" |
| 178 | db_host = c.get_config_value(dbase, "host") |
| 179 | if db_name is None: |
| 180 | db_name = c.get_config_value(dbase, "database") |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 181 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 182 | if _testing_mode: |
| 183 | db_name = 'stresstest_autotest_web' |
showard | 30eed1f | 2008-05-27 22:31:58 +0000 | [diff] [blame] | 184 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 185 | db_user = c.get_config_value(dbase, "user") |
| 186 | db_pass = c.get_config_value(dbase, "password") |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 187 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 188 | while not self.conn: |
| 189 | try: |
| 190 | self.conn = MySQLdb.connect( |
| 191 | host=db_host, user=db_user, passwd=db_pass, |
| 192 | db=db_name, conv=self.convert_dict) |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 193 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 194 | self.conn.autocommit(True) |
| 195 | self.cur = self.conn.cursor() |
| 196 | except MySQLdb.OperationalError: |
| 197 | traceback.print_exc() |
| 198 | print "Can't connect to MYSQL; reconnecting" |
| 199 | time.sleep(self.reconnect_wait) |
| 200 | self.disconnect() |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 201 | |
| 202 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 203 | def disconnect(self): |
| 204 | if self.conn: |
| 205 | self.conn.close() |
| 206 | self.conn = None |
| 207 | self.cur = None |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 208 | |
| 209 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 210 | def execute(self, *args, **dargs): |
| 211 | while (True): |
| 212 | try: |
| 213 | self.cur.execute(*args, **dargs) |
| 214 | return self.cur.fetchall() |
| 215 | except MySQLdb.OperationalError: |
| 216 | traceback.print_exc() |
| 217 | print "MYSQL connection died; reconnecting" |
| 218 | time.sleep(self.reconnect_wait) |
| 219 | self.connect() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 220 | |
| 221 | |
mbligh | dbdac6c | 2008-03-05 15:49:58 +0000 | [diff] [blame] | 222 | def generate_parse_command(results_dir, flags=""): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 223 | parse = os.path.abspath(os.path.join(AUTOTEST_TKO_DIR, 'parse')) |
| 224 | output = os.path.abspath(os.path.join(results_dir, '.parse.log')) |
| 225 | cmd = "%s %s -r -o %s > %s 2>&1 &" |
| 226 | return cmd % (parse, flags, results_dir, output) |
mbligh | dbdac6c | 2008-03-05 15:49:58 +0000 | [diff] [blame] | 227 | |
| 228 | |
showard | 970a6db | 2008-09-03 20:02:39 +0000 | [diff] [blame] | 229 | _parse_command_queue = [] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 230 | def parse_results(results_dir, flags=""): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 231 | if _testing_mode: |
| 232 | return |
showard | 970a6db | 2008-09-03 20:02:39 +0000 | [diff] [blame] | 233 | _parse_command_queue.append(generate_parse_command(results_dir, flags)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 234 | |
| 235 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 236 | |
| 237 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 238 | def log_stacktrace(reason): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 239 | (type, value, tb) = sys.exc_info() |
| 240 | str = "EXCEPTION: %s\n" % reason |
| 241 | str += ''.join(traceback.format_exception(type, value, tb)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 242 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 243 | sys.stderr.write("\n%s\n" % str) |
| 244 | email_manager.enqueue_notify_email("monitor_db exception", str) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 245 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 246 | |
| 247 | def get_proc_poll_fn(pid): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 248 | proc_path = os.path.join('/proc', str(pid)) |
| 249 | def poll_fn(): |
| 250 | if os.path.exists(proc_path): |
| 251 | return None |
| 252 | return 0 # we can't get a real exit code |
| 253 | return poll_fn |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 254 | |
| 255 | |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 256 | def send_email(from_addr, to_string, subject, body): |
| 257 | """Mails out emails to the addresses listed in to_string. |
| 258 | |
| 259 | to_string is split into a list which can be delimited by any of: |
| 260 | ';', ',', ':' or any whitespace |
| 261 | """ |
| 262 | |
| 263 | # Create list from string removing empty strings from the list. |
| 264 | to_list = [x for x in re.split('\s|,|;|:', to_string) if x] |
showard | 7d182aa | 2008-09-22 16:17:24 +0000 | [diff] [blame] | 265 | if not to_list: |
| 266 | return |
| 267 | |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 268 | msg = "From: %s\nTo: %s\nSubject: %s\n\n%s" % ( |
| 269 | from_addr, ', '.join(to_list), subject, body) |
showard | 7d182aa | 2008-09-22 16:17:24 +0000 | [diff] [blame] | 270 | try: |
| 271 | mailer = smtplib.SMTP('localhost') |
| 272 | try: |
| 273 | mailer.sendmail(from_addr, to_list, msg) |
| 274 | finally: |
| 275 | mailer.quit() |
| 276 | except Exception, e: |
| 277 | print "Sending email failed. Reason: %s" % repr(e) |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 278 | |
| 279 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 280 | def kill_autoserv(pid, poll_fn=None): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 281 | print 'killing', pid |
| 282 | if poll_fn is None: |
| 283 | poll_fn = get_proc_poll_fn(pid) |
| 284 | if poll_fn() == None: |
| 285 | os.kill(pid, signal.SIGCONT) |
| 286 | os.kill(pid, signal.SIGTERM) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 287 | |
| 288 | |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 289 | class EmailNotificationManager(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 290 | def __init__(self): |
| 291 | self._emails = [] |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 292 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 293 | def enqueue_notify_email(self, subject, message): |
| 294 | if not _notify_email: |
| 295 | return |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 296 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 297 | body = 'Subject: ' + subject + '\n' |
| 298 | body += "%s / %s / %s\n%s" % (socket.gethostname(), |
| 299 | os.getpid(), |
| 300 | time.strftime("%X %x"), message) |
| 301 | self._emails.append(body) |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 302 | |
| 303 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 304 | def send_queued_emails(self): |
| 305 | if not self._emails: |
| 306 | return |
| 307 | subject = 'Scheduler notifications from ' + socket.gethostname() |
| 308 | separator = '\n' + '-' * 40 + '\n' |
| 309 | body = separator.join(self._emails) |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 310 | |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 311 | send_email(_email_from, _notify_email, subject, body) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 312 | self._emails = [] |
showard | 7cf9a9b | 2008-05-15 21:15:52 +0000 | [diff] [blame] | 313 | |
| 314 | email_manager = EmailNotificationManager() |
| 315 | |
| 316 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 317 | class HostScheduler(object): |
| 318 | def _get_ready_hosts(self): |
| 319 | # avoid any host with a currently active queue entry against it |
| 320 | hosts = Host.fetch( |
| 321 | joins='LEFT JOIN host_queue_entries AS active_hqe ' |
| 322 | 'ON (hosts.id = active_hqe.host_id AND ' |
| 323 | 'active_hqe.active = TRUE)', |
| 324 | where="active_hqe.host_id IS NULL " |
| 325 | "AND hosts.locked = FALSE " |
| 326 | "AND (hosts.status IS NULL OR hosts.status = 'Ready')") |
| 327 | return dict((host.id, host) for host in hosts) |
| 328 | |
| 329 | |
| 330 | @staticmethod |
| 331 | def _get_sql_id_list(id_list): |
| 332 | return ','.join(str(item_id) for item_id in id_list) |
| 333 | |
| 334 | |
| 335 | @classmethod |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 336 | def _get_many2many_dict(cls, query, id_list, flip=False): |
mbligh | 849a0f6 | 2008-08-28 20:12:19 +0000 | [diff] [blame] | 337 | if not id_list: |
| 338 | return {} |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 339 | query %= cls._get_sql_id_list(id_list) |
| 340 | rows = _db.execute(query) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 341 | return cls._process_many2many_dict(rows, flip) |
| 342 | |
| 343 | |
| 344 | @staticmethod |
| 345 | def _process_many2many_dict(rows, flip=False): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 346 | result = {} |
| 347 | for row in rows: |
| 348 | left_id, right_id = long(row[0]), long(row[1]) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 349 | if flip: |
| 350 | left_id, right_id = right_id, left_id |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 351 | result.setdefault(left_id, set()).add(right_id) |
| 352 | return result |
| 353 | |
| 354 | |
| 355 | @classmethod |
| 356 | def _get_job_acl_groups(cls, job_ids): |
| 357 | query = """ |
| 358 | SELECT jobs.id, acl_groups_users.acl_group_id |
| 359 | FROM jobs |
| 360 | INNER JOIN users ON users.login = jobs.owner |
| 361 | INNER JOIN acl_groups_users ON acl_groups_users.user_id = users.id |
| 362 | WHERE jobs.id IN (%s) |
| 363 | """ |
| 364 | return cls._get_many2many_dict(query, job_ids) |
| 365 | |
| 366 | |
| 367 | @classmethod |
| 368 | def _get_job_ineligible_hosts(cls, job_ids): |
| 369 | query = """ |
| 370 | SELECT job_id, host_id |
| 371 | FROM ineligible_host_queues |
| 372 | WHERE job_id IN (%s) |
| 373 | """ |
| 374 | return cls._get_many2many_dict(query, job_ids) |
| 375 | |
| 376 | |
| 377 | @classmethod |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 378 | def _get_job_dependencies(cls, job_ids): |
| 379 | query = """ |
| 380 | SELECT job_id, label_id |
| 381 | FROM jobs_dependency_labels |
| 382 | WHERE job_id IN (%s) |
| 383 | """ |
| 384 | return cls._get_many2many_dict(query, job_ids) |
| 385 | |
| 386 | |
| 387 | @classmethod |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 388 | def _get_host_acls(cls, host_ids): |
| 389 | query = """ |
| 390 | SELECT host_id, acl_group_id |
| 391 | FROM acl_groups_hosts |
| 392 | WHERE host_id IN (%s) |
| 393 | """ |
| 394 | return cls._get_many2many_dict(query, host_ids) |
| 395 | |
| 396 | |
| 397 | @classmethod |
| 398 | def _get_label_hosts(cls, host_ids): |
| 399 | query = """ |
| 400 | SELECT label_id, host_id |
| 401 | FROM hosts_labels |
| 402 | WHERE host_id IN (%s) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 403 | """ % cls._get_sql_id_list(host_ids) |
| 404 | rows = _db.execute(query) |
| 405 | labels_to_hosts = cls._process_many2many_dict(rows) |
| 406 | hosts_to_labels = cls._process_many2many_dict(rows, flip=True) |
| 407 | return labels_to_hosts, hosts_to_labels |
| 408 | |
| 409 | |
| 410 | @classmethod |
| 411 | def _get_labels(cls): |
| 412 | return dict((label.id, label) for label in Label.fetch()) |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 413 | |
| 414 | |
| 415 | def refresh(self, pending_queue_entries): |
| 416 | self._hosts_available = self._get_ready_hosts() |
| 417 | |
| 418 | relevant_jobs = [queue_entry.job_id |
| 419 | for queue_entry in pending_queue_entries] |
| 420 | self._job_acls = self._get_job_acl_groups(relevant_jobs) |
| 421 | self._ineligible_hosts = self._get_job_ineligible_hosts(relevant_jobs) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 422 | self._job_dependencies = self._get_job_dependencies(relevant_jobs) |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 423 | |
| 424 | host_ids = self._hosts_available.keys() |
| 425 | self._host_acls = self._get_host_acls(host_ids) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 426 | self._label_hosts, self._host_labels = self._get_label_hosts(host_ids) |
| 427 | |
| 428 | self._labels = self._get_labels() |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 429 | |
| 430 | |
| 431 | def _is_acl_accessible(self, host_id, queue_entry): |
| 432 | job_acls = self._job_acls.get(queue_entry.job_id, set()) |
| 433 | host_acls = self._host_acls.get(host_id, set()) |
| 434 | return len(host_acls.intersection(job_acls)) > 0 |
| 435 | |
| 436 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 437 | def _check_job_dependencies(self, job_dependencies, host_labels): |
| 438 | missing = job_dependencies - host_labels |
| 439 | return len(job_dependencies - host_labels) == 0 |
| 440 | |
| 441 | |
| 442 | def _check_only_if_needed_labels(self, job_dependencies, host_labels, |
| 443 | queue_entry): |
| 444 | for label_id in host_labels: |
| 445 | label = self._labels[label_id] |
| 446 | if not label.only_if_needed: |
| 447 | # we don't care about non-only_if_needed labels |
| 448 | continue |
| 449 | if queue_entry.meta_host == label_id: |
| 450 | # if the label was requested in a metahost it's OK |
| 451 | continue |
| 452 | if label_id not in job_dependencies: |
| 453 | return False |
| 454 | return True |
| 455 | |
| 456 | |
| 457 | def _is_host_eligible_for_job(self, host_id, queue_entry): |
| 458 | job_dependencies = self._job_dependencies.get(queue_entry.job_id, set()) |
| 459 | host_labels = self._host_labels.get(host_id, set()) |
mbligh | c993bee | 2008-10-03 03:42:34 +0000 | [diff] [blame^] | 460 | |
| 461 | acl = self._is_acl_accessible(host_id, queue_entry) |
| 462 | deps = self._check_job_dependencies(job_dependencies, host_labels) |
| 463 | only_if = self._check_only_if_needed_labels(job_dependencies, |
| 464 | host_labels, queue_entry) |
| 465 | return acl and deps and only_if |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 466 | |
| 467 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 468 | def _schedule_non_metahost(self, queue_entry): |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 469 | if not self._is_host_eligible_for_job(queue_entry.host_id, queue_entry): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 470 | return None |
| 471 | return self._hosts_available.pop(queue_entry.host_id, None) |
| 472 | |
| 473 | |
| 474 | def _is_host_usable(self, host_id): |
| 475 | if host_id not in self._hosts_available: |
| 476 | # host was already used during this scheduling cycle |
| 477 | return False |
| 478 | if self._hosts_available[host_id].invalid: |
| 479 | # Invalid hosts cannot be used for metahosts. They're included in |
| 480 | # the original query because they can be used by non-metahosts. |
| 481 | return False |
| 482 | return True |
| 483 | |
| 484 | |
| 485 | def _schedule_metahost(self, queue_entry): |
| 486 | label_id = queue_entry.meta_host |
| 487 | hosts_in_label = self._label_hosts.get(label_id, set()) |
| 488 | ineligible_host_ids = self._ineligible_hosts.get(queue_entry.job_id, |
| 489 | set()) |
| 490 | |
| 491 | # must iterate over a copy so we can mutate the original while iterating |
| 492 | for host_id in list(hosts_in_label): |
| 493 | if not self._is_host_usable(host_id): |
| 494 | hosts_in_label.remove(host_id) |
| 495 | continue |
| 496 | if host_id in ineligible_host_ids: |
| 497 | continue |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 498 | if not self._is_host_eligible_for_job(host_id, queue_entry): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 499 | continue |
| 500 | |
| 501 | hosts_in_label.remove(host_id) |
| 502 | return self._hosts_available.pop(host_id) |
| 503 | return None |
| 504 | |
| 505 | |
| 506 | def find_eligible_host(self, queue_entry): |
| 507 | if not queue_entry.meta_host: |
| 508 | return self._schedule_non_metahost(queue_entry) |
| 509 | return self._schedule_metahost(queue_entry) |
| 510 | |
| 511 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 512 | class Dispatcher: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 513 | autoserv_procs_cache = None |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 514 | max_running_processes = global_config.global_config.get_config_value( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 515 | _global_config_section, 'max_running_jobs', type=int) |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 516 | max_processes_started_per_cycle = ( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 517 | global_config.global_config.get_config_value( |
| 518 | _global_config_section, 'max_jobs_started_per_cycle', type=int)) |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 519 | clean_interval = ( |
| 520 | global_config.global_config.get_config_value( |
| 521 | _global_config_section, 'clean_interval_minutes', type=int)) |
showard | 970a6db | 2008-09-03 20:02:39 +0000 | [diff] [blame] | 522 | max_parse_processes = ( |
| 523 | global_config.global_config.get_config_value( |
| 524 | _global_config_section, 'max_parse_processes', type=int)) |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 525 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 526 | def __init__(self): |
| 527 | self._agents = [] |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 528 | self._last_clean_time = time.time() |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 529 | self._host_scheduler = HostScheduler() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 530 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 531 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 532 | def do_initial_recovery(self, recover_hosts=True): |
| 533 | # always recover processes |
| 534 | self._recover_processes() |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 535 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 536 | if recover_hosts: |
| 537 | self._recover_hosts() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 538 | |
| 539 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 540 | def tick(self): |
| 541 | Dispatcher.autoserv_procs_cache = None |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 542 | if self._last_clean_time + self.clean_interval * 60 < time.time(): |
| 543 | self._abort_timed_out_jobs() |
| 544 | self._clear_inactive_blocks() |
| 545 | self._last_clean_time = time.time() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 546 | self._find_aborting() |
| 547 | self._schedule_new_jobs() |
| 548 | self._handle_agents() |
showard | 970a6db | 2008-09-03 20:02:39 +0000 | [diff] [blame] | 549 | self._run_final_parses() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 550 | email_manager.send_queued_emails() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 551 | |
| 552 | |
showard | 970a6db | 2008-09-03 20:02:39 +0000 | [diff] [blame] | 553 | def _run_final_parses(self): |
| 554 | process_count = 0 |
| 555 | try: |
| 556 | for line in utils.system_output('ps -e').splitlines(): |
| 557 | if 'parse.py' in line: |
| 558 | process_count += 1 |
| 559 | except Exception: |
| 560 | # We'll try again in a bit. This is a work-around for one time |
| 561 | # when the scheduler crashed due to a "Interrupted system call" |
| 562 | return |
| 563 | |
| 564 | if process_count: |
| 565 | print "%d parses currently running" % process_count |
| 566 | |
| 567 | while (process_count < self.max_parse_processes and |
| 568 | _parse_command_queue): |
| 569 | cmd = _parse_command_queue.pop(0) |
| 570 | print "Starting another final parse with cmd %s" % cmd |
| 571 | os.system(cmd) |
| 572 | process_count += 1 |
| 573 | |
| 574 | if _parse_command_queue: |
| 575 | print ("%d cmds still in final parse queue" % |
| 576 | len(_parse_command_queue)) |
| 577 | |
| 578 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 579 | def add_agent(self, agent): |
| 580 | self._agents.append(agent) |
| 581 | agent.dispatcher = self |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 582 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 583 | # Find agent corresponding to the specified queue_entry |
| 584 | def get_agents(self, queue_entry): |
| 585 | res_agents = [] |
| 586 | for agent in self._agents: |
| 587 | if queue_entry.id in agent.queue_entry_ids: |
| 588 | res_agents.append(agent) |
| 589 | return res_agents |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 590 | |
| 591 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 592 | def remove_agent(self, agent): |
| 593 | self._agents.remove(agent) |
showard | ec11316 | 2008-05-08 00:52:49 +0000 | [diff] [blame] | 594 | |
| 595 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 596 | def num_running_processes(self): |
| 597 | return sum(agent.num_processes for agent in self._agents |
| 598 | if agent.is_running()) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 599 | |
| 600 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 601 | @classmethod |
| 602 | def find_autoservs(cls, orphans_only=False): |
| 603 | """\ |
| 604 | Returns a dict mapping pids to command lines for root autoserv |
| 605 | processes. If orphans_only=True, return only processes that |
| 606 | have been orphaned (i.e. parent pid = 1). |
| 607 | """ |
| 608 | if cls.autoserv_procs_cache is not None: |
| 609 | return cls.autoserv_procs_cache |
| 610 | |
| 611 | proc = subprocess.Popen( |
| 612 | ['/bin/ps', 'x', '-o', 'pid,pgid,ppid,comm,args'], |
| 613 | stdout=subprocess.PIPE) |
| 614 | # split each line into the four columns output by ps |
| 615 | procs = [line.split(None, 4) for line in |
| 616 | proc.communicate()[0].splitlines()] |
| 617 | autoserv_procs = {} |
| 618 | for proc in procs: |
| 619 | # check ppid == 1 for orphans |
| 620 | if orphans_only and proc[2] != 1: |
| 621 | continue |
| 622 | # only root autoserv processes have pgid == pid |
| 623 | if (proc[3] == 'autoserv' and # comm |
| 624 | proc[1] == proc[0]): # pgid == pid |
| 625 | # map pid to args |
| 626 | autoserv_procs[int(proc[0])] = proc[4] |
| 627 | cls.autoserv_procs_cache = autoserv_procs |
| 628 | return autoserv_procs |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 629 | |
| 630 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 631 | def recover_queue_entry(self, queue_entry, run_monitor): |
| 632 | job = queue_entry.job |
| 633 | if job.is_synchronous(): |
| 634 | all_queue_entries = job.get_host_queue_entries() |
| 635 | else: |
| 636 | all_queue_entries = [queue_entry] |
| 637 | all_queue_entry_ids = [queue_entry.id for queue_entry |
| 638 | in all_queue_entries] |
| 639 | queue_task = RecoveryQueueTask( |
| 640 | job=queue_entry.job, |
| 641 | queue_entries=all_queue_entries, |
| 642 | run_monitor=run_monitor) |
| 643 | self.add_agent(Agent(tasks=[queue_task], |
| 644 | queue_entry_ids=all_queue_entry_ids)) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 645 | |
| 646 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 647 | def _recover_processes(self): |
| 648 | orphans = self.find_autoservs(orphans_only=True) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 649 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 650 | # first, recover running queue entries |
| 651 | rows = _db.execute("""SELECT * FROM host_queue_entries |
| 652 | WHERE status = 'Running'""") |
| 653 | queue_entries = [HostQueueEntry(row=i) for i in rows] |
| 654 | requeue_entries = [] |
| 655 | recovered_entry_ids = set() |
| 656 | for queue_entry in queue_entries: |
| 657 | run_monitor = PidfileRunMonitor( |
| 658 | queue_entry.results_dir()) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 659 | pid, exit_code = run_monitor.get_pidfile_info() |
| 660 | if pid is None: |
| 661 | # autoserv apparently never got run, so requeue |
| 662 | requeue_entries.append(queue_entry) |
| 663 | continue |
| 664 | if queue_entry.id in recovered_entry_ids: |
| 665 | # synchronous job we've already recovered |
| 666 | continue |
| 667 | print 'Recovering queue entry %d (pid %d)' % ( |
| 668 | queue_entry.id, pid) |
| 669 | job = queue_entry.job |
| 670 | if job.is_synchronous(): |
| 671 | for entry in job.get_host_queue_entries(): |
| 672 | assert entry.active |
| 673 | recovered_entry_ids.add(entry.id) |
| 674 | self.recover_queue_entry(queue_entry, |
| 675 | run_monitor) |
| 676 | orphans.pop(pid, None) |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 677 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 678 | # and requeue other active queue entries |
| 679 | rows = _db.execute("""SELECT * FROM host_queue_entries |
| 680 | WHERE active AND NOT complete |
| 681 | AND status != 'Running' |
| 682 | AND status != 'Pending' |
| 683 | AND status != 'Abort' |
| 684 | AND status != 'Aborting'""") |
| 685 | queue_entries = [HostQueueEntry(row=i) for i in rows] |
| 686 | for queue_entry in queue_entries + requeue_entries: |
| 687 | print 'Requeuing running QE %d' % queue_entry.id |
| 688 | queue_entry.clear_results_dir(dont_delete_files=True) |
| 689 | queue_entry.requeue() |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 690 | |
| 691 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 692 | # now kill any remaining autoserv processes |
| 693 | for pid in orphans.keys(): |
| 694 | print 'Killing orphan %d (%s)' % (pid, orphans[pid]) |
| 695 | kill_autoserv(pid) |
| 696 | |
| 697 | # recover aborting tasks |
| 698 | rebooting_host_ids = set() |
| 699 | rows = _db.execute("""SELECT * FROM host_queue_entries |
| 700 | WHERE status='Abort' or status='Aborting'""") |
| 701 | queue_entries = [HostQueueEntry(row=i) for i in rows] |
| 702 | for queue_entry in queue_entries: |
| 703 | print 'Recovering aborting QE %d' % queue_entry.id |
| 704 | queue_host = queue_entry.get_host() |
| 705 | reboot_task = RebootTask(queue_host) |
| 706 | verify_task = VerifyTask(host = queue_host) |
| 707 | self.add_agent(Agent(tasks=[reboot_task, |
| 708 | verify_task], |
| 709 | queue_entry_ids=[queue_entry.id])) |
| 710 | queue_entry.set_status('Aborted') |
| 711 | # Secure the host from being picked up |
| 712 | queue_host.set_status('Rebooting') |
| 713 | rebooting_host_ids.add(queue_host.id) |
| 714 | |
| 715 | # reverify hosts that were in the middle of verify, repair or |
| 716 | # reboot |
| 717 | self._reverify_hosts_where("""(status = 'Repairing' OR |
| 718 | status = 'Verifying' OR |
| 719 | status = 'Rebooting')""", |
| 720 | exclude_ids=rebooting_host_ids) |
| 721 | |
| 722 | # finally, recover "Running" hosts with no active queue entries, |
| 723 | # although this should never happen |
| 724 | message = ('Recovering running host %s - this probably ' |
| 725 | 'indicates a scheduler bug') |
| 726 | self._reverify_hosts_where("""status = 'Running' AND |
| 727 | id NOT IN (SELECT host_id |
| 728 | FROM host_queue_entries |
| 729 | WHERE active)""", |
| 730 | print_message=message) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 731 | |
| 732 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 733 | def _reverify_hosts_where(self, where, |
| 734 | print_message='Reverifying host %s', |
| 735 | exclude_ids=set()): |
| 736 | rows = _db.execute('SELECT * FROM hosts WHERE locked = 0 AND ' |
| 737 | 'invalid = 0 AND ' + where) |
| 738 | hosts = [Host(row=i) for i in rows] |
| 739 | for host in hosts: |
| 740 | if host.id in exclude_ids: |
| 741 | continue |
| 742 | if print_message is not None: |
| 743 | print print_message % host.hostname |
| 744 | verify_task = VerifyTask(host = host) |
| 745 | self.add_agent(Agent(tasks = [verify_task])) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 746 | |
| 747 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 748 | def _recover_hosts(self): |
| 749 | # recover "Repair Failed" hosts |
| 750 | message = 'Reverifying dead host %s' |
| 751 | self._reverify_hosts_where("status = 'Repair Failed'", |
| 752 | print_message=message) |
mbligh | 62ba2ed | 2008-04-30 17:09:25 +0000 | [diff] [blame] | 753 | |
| 754 | |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 755 | def _abort_timed_out_jobs(self): |
| 756 | """ |
| 757 | Aborts all jobs that have timed out and not completed |
| 758 | """ |
| 759 | update = """ |
| 760 | UPDATE host_queue_entries INNER JOIN jobs |
| 761 | ON host_queue_entries.job_id = jobs.id""" |
mbligh | 7e26d62 | 2008-07-29 21:04:42 +0000 | [diff] [blame] | 762 | timed_out = ' AND jobs.created_on + INTERVAL jobs.timeout HOUR < NOW()' |
showard | 3bb499f | 2008-07-03 19:42:20 +0000 | [diff] [blame] | 763 | |
| 764 | _db.execute(update + """ |
| 765 | SET host_queue_entries.status = 'Abort' |
| 766 | WHERE host_queue_entries.active IS TRUE""" + timed_out) |
| 767 | |
| 768 | _db.execute(update + """ |
| 769 | SET host_queue_entries.status = 'Aborted', |
| 770 | host_queue_entries.active = FALSE, |
| 771 | host_queue_entries.complete = TRUE |
| 772 | WHERE host_queue_entries.active IS FALSE |
| 773 | AND host_queue_entries.complete IS FALSE""" + timed_out) |
| 774 | |
| 775 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 776 | def _clear_inactive_blocks(self): |
| 777 | """ |
| 778 | Clear out blocks for all completed jobs. |
| 779 | """ |
| 780 | # this would be simpler using NOT IN (subquery), but MySQL |
| 781 | # treats all IN subqueries as dependent, so this optimizes much |
| 782 | # better |
| 783 | _db.execute(""" |
| 784 | DELETE ihq FROM ineligible_host_queues ihq |
showard | 4eaaf52 | 2008-06-06 22:28:07 +0000 | [diff] [blame] | 785 | LEFT JOIN (SELECT DISTINCT job_id FROM host_queue_entries |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 786 | WHERE NOT complete) hqe |
| 787 | USING (job_id) WHERE hqe.job_id IS NULL""") |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 788 | |
| 789 | |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 790 | def _get_pending_queue_entries(self): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 791 | # prioritize by job priority, then non-metahost over metahost, then FIFO |
| 792 | return list(HostQueueEntry.fetch( |
| 793 | where='NOT complete AND NOT active', |
| 794 | order_by='priority DESC, meta_host, id')) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 795 | |
| 796 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 797 | def _schedule_new_jobs(self): |
| 798 | print "finding work" |
| 799 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 800 | queue_entries = self._get_pending_queue_entries() |
| 801 | if not queue_entries: |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 802 | return |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 803 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 804 | self._host_scheduler.refresh(queue_entries) |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 805 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 806 | for queue_entry in queue_entries: |
| 807 | assigned_host = self._host_scheduler.find_eligible_host(queue_entry) |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 808 | if not assigned_host: |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 809 | continue |
showard | b95b1bd | 2008-08-15 18:11:04 +0000 | [diff] [blame] | 810 | self._run_queue_entry(queue_entry, assigned_host) |
| 811 | |
| 812 | |
| 813 | def _run_queue_entry(self, queue_entry, host): |
| 814 | agent = queue_entry.run(assigned_host=host) |
| 815 | self.add_agent(agent) |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 816 | |
| 817 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 818 | def _find_aborting(self): |
| 819 | num_aborted = 0 |
| 820 | # Find jobs that are aborting |
| 821 | for entry in queue_entries_to_abort(): |
| 822 | agents_to_abort = self.get_agents(entry) |
| 823 | entry_host = entry.get_host() |
| 824 | reboot_task = RebootTask(entry_host) |
| 825 | verify_task = VerifyTask(host = entry_host) |
| 826 | tasks = [reboot_task, verify_task] |
| 827 | if agents_to_abort: |
| 828 | abort_task = AbortTask(entry, agents_to_abort) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 829 | for agent in agents_to_abort: |
| 830 | self.remove_agent(agent) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 831 | tasks.insert(0, abort_task) |
| 832 | else: |
| 833 | entry.set_status('Aborted') |
| 834 | # just to make sure this host does not get |
| 835 | # taken away |
| 836 | entry_host.set_status('Rebooting') |
| 837 | self.add_agent(Agent(tasks=tasks, |
| 838 | queue_entry_ids = [entry.id])) |
| 839 | num_aborted += 1 |
| 840 | if num_aborted >= 50: |
| 841 | break |
| 842 | |
| 843 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 844 | def _can_start_agent(self, agent, num_running_processes, |
| 845 | num_started_this_cycle, have_reached_limit): |
| 846 | # always allow zero-process agents to run |
| 847 | if agent.num_processes == 0: |
| 848 | return True |
| 849 | # don't allow any nonzero-process agents to run after we've reached a |
| 850 | # limit (this avoids starvation of many-process agents) |
| 851 | if have_reached_limit: |
| 852 | return False |
| 853 | # total process throttling |
| 854 | if (num_running_processes + agent.num_processes > |
| 855 | self.max_running_processes): |
| 856 | return False |
| 857 | # if a single agent exceeds the per-cycle throttling, still allow it to |
| 858 | # run when it's the first agent in the cycle |
| 859 | if num_started_this_cycle == 0: |
| 860 | return True |
| 861 | # per-cycle throttling |
| 862 | if (num_started_this_cycle + agent.num_processes > |
| 863 | self.max_processes_started_per_cycle): |
| 864 | return False |
| 865 | return True |
| 866 | |
| 867 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 868 | def _handle_agents(self): |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 869 | num_running_processes = self.num_running_processes() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 870 | num_started_this_cycle = 0 |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 871 | have_reached_limit = False |
| 872 | # iterate over copy, so we can remove agents during iteration |
| 873 | for agent in list(self._agents): |
| 874 | if agent.is_done(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 875 | print "agent finished" |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 876 | self._agents.remove(agent) |
| 877 | num_running_processes -= agent.num_processes |
| 878 | continue |
| 879 | if not agent.is_running(): |
| 880 | if not self._can_start_agent(agent, num_running_processes, |
| 881 | num_started_this_cycle, |
| 882 | have_reached_limit): |
| 883 | have_reached_limit = True |
| 884 | continue |
| 885 | num_running_processes += agent.num_processes |
| 886 | num_started_this_cycle += agent.num_processes |
| 887 | agent.tick() |
| 888 | print num_running_processes, 'running processes' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 889 | |
| 890 | |
| 891 | class RunMonitor(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 892 | def __init__(self, cmd, nice_level = None, log_file = None): |
| 893 | self.nice_level = nice_level |
| 894 | self.log_file = log_file |
| 895 | self.cmd = cmd |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 896 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 897 | def run(self): |
| 898 | if self.nice_level: |
| 899 | nice_cmd = ['nice','-n', str(self.nice_level)] |
| 900 | nice_cmd.extend(self.cmd) |
| 901 | self.cmd = nice_cmd |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 902 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 903 | out_file = None |
| 904 | if self.log_file: |
| 905 | try: |
| 906 | os.makedirs(os.path.dirname(self.log_file)) |
| 907 | except OSError, exc: |
| 908 | if exc.errno != errno.EEXIST: |
| 909 | log_stacktrace( |
| 910 | 'Unexpected error creating logfile ' |
| 911 | 'directory for %s' % self.log_file) |
| 912 | try: |
| 913 | out_file = open(self.log_file, 'a') |
| 914 | out_file.write("\n%s\n" % ('*'*80)) |
| 915 | out_file.write("%s> %s\n" % |
| 916 | (time.strftime("%X %x"), |
| 917 | self.cmd)) |
| 918 | out_file.write("%s\n" % ('*'*80)) |
| 919 | except (OSError, IOError): |
| 920 | log_stacktrace('Error opening log file %s' % |
| 921 | self.log_file) |
mbligh | cadb353 | 2008-04-15 17:46:26 +0000 | [diff] [blame] | 922 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 923 | if not out_file: |
| 924 | out_file = open('/dev/null', 'w') |
mbligh | cadb353 | 2008-04-15 17:46:26 +0000 | [diff] [blame] | 925 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 926 | in_devnull = open('/dev/null', 'r') |
| 927 | print "cmd = %s" % self.cmd |
| 928 | print "path = %s" % os.getcwd() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 929 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 930 | self.proc = subprocess.Popen(self.cmd, stdout=out_file, |
| 931 | stderr=subprocess.STDOUT, |
| 932 | stdin=in_devnull) |
| 933 | out_file.close() |
| 934 | in_devnull.close() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 935 | |
| 936 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 937 | def get_pid(self): |
| 938 | return self.proc.pid |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 939 | |
| 940 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 941 | def kill(self): |
| 942 | kill_autoserv(self.get_pid(), self.exit_code) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 943 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 944 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 945 | def exit_code(self): |
| 946 | return self.proc.poll() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 947 | |
| 948 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 949 | class PidfileException(Exception): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 950 | """\ |
| 951 | Raised when there's some unexpected behavior with the pid file. |
| 952 | """ |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 953 | |
| 954 | |
| 955 | class PidfileRunMonitor(RunMonitor): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 956 | def __init__(self, results_dir, cmd=None, nice_level=None, |
| 957 | log_file=None): |
| 958 | self.results_dir = os.path.abspath(results_dir) |
| 959 | self.pid_file = os.path.join(results_dir, AUTOSERV_PID_FILE) |
| 960 | self.lost_process = False |
| 961 | self.start_time = time.time() |
showard | b376bc5 | 2008-06-13 20:48:45 +0000 | [diff] [blame] | 962 | super(PidfileRunMonitor, self).__init__(cmd, nice_level, log_file) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 963 | |
| 964 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 965 | def get_pid(self): |
| 966 | pid, exit_status = self.get_pidfile_info() |
| 967 | assert pid is not None |
| 968 | return pid |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 969 | |
| 970 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 971 | def _check_command_line(self, command_line, spacer=' ', |
| 972 | print_error=False): |
| 973 | results_dir_arg = spacer.join(('', '-r', self.results_dir, '')) |
| 974 | match = results_dir_arg in command_line |
| 975 | if print_error and not match: |
| 976 | print '%s not found in %s' % (repr(results_dir_arg), |
| 977 | repr(command_line)) |
| 978 | return match |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 979 | |
| 980 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 981 | def _check_proc_fs(self, pid): |
| 982 | cmdline_path = os.path.join('/proc', str(pid), 'cmdline') |
| 983 | try: |
| 984 | cmdline_file = open(cmdline_path, 'r') |
| 985 | cmdline = cmdline_file.read().strip() |
| 986 | cmdline_file.close() |
| 987 | except IOError: |
| 988 | return False |
| 989 | # /proc/.../cmdline has \x00 separating args |
| 990 | return self._check_command_line(cmdline, spacer='\x00', |
| 991 | print_error=True) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 992 | |
| 993 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 994 | def read_pidfile(self): |
| 995 | if not os.path.exists(self.pid_file): |
| 996 | return None, None |
| 997 | file_obj = open(self.pid_file, 'r') |
| 998 | lines = file_obj.readlines() |
| 999 | file_obj.close() |
| 1000 | assert 1 <= len(lines) <= 2 |
| 1001 | try: |
| 1002 | pid = int(lines[0]) |
| 1003 | exit_status = None |
| 1004 | if len(lines) == 2: |
| 1005 | exit_status = int(lines[1]) |
| 1006 | except ValueError, exc: |
| 1007 | raise PidfileException('Corrupt pid file: ' + |
| 1008 | str(exc.args)) |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1009 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1010 | return pid, exit_status |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1011 | |
| 1012 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1013 | def _find_autoserv_proc(self): |
| 1014 | autoserv_procs = Dispatcher.find_autoservs() |
| 1015 | for pid, args in autoserv_procs.iteritems(): |
| 1016 | if self._check_command_line(args): |
| 1017 | return pid, args |
| 1018 | return None, None |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1019 | |
| 1020 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1021 | def get_pidfile_info(self): |
| 1022 | """\ |
| 1023 | Returns: |
| 1024 | None, None if autoserv has not yet run |
| 1025 | pid, None if autoserv is running |
| 1026 | pid, exit_status if autoserv has completed |
| 1027 | """ |
| 1028 | if self.lost_process: |
| 1029 | return self.pid, self.exit_status |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1030 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1031 | pid, exit_status = self.read_pidfile() |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1032 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1033 | if pid is None: |
| 1034 | return self._handle_no_pid() |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1035 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1036 | if exit_status is None: |
| 1037 | # double check whether or not autoserv is running |
| 1038 | proc_running = self._check_proc_fs(pid) |
| 1039 | if proc_running: |
| 1040 | return pid, exit_status |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1041 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1042 | # pid but no process - maybe process *just* exited |
| 1043 | pid, exit_status = self.read_pidfile() |
| 1044 | if exit_status is None: |
| 1045 | # autoserv exited without writing an exit code |
| 1046 | # to the pidfile |
| 1047 | error = ('autoserv died without writing exit ' |
| 1048 | 'code') |
| 1049 | message = error + '\nPid: %s\nPidfile: %s' % ( |
| 1050 | pid, self.pid_file) |
| 1051 | print message |
| 1052 | email_manager.enqueue_notify_email(error, |
| 1053 | message) |
| 1054 | self.on_lost_process(pid) |
| 1055 | return self.pid, self.exit_status |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1056 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1057 | return pid, exit_status |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1058 | |
| 1059 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1060 | def _handle_no_pid(self): |
| 1061 | """\ |
| 1062 | Called when no pidfile is found or no pid is in the pidfile. |
| 1063 | """ |
| 1064 | # is autoserv running? |
| 1065 | pid, args = self._find_autoserv_proc() |
| 1066 | if pid is None: |
| 1067 | # no autoserv process running |
| 1068 | message = 'No pid found at ' + self.pid_file |
| 1069 | else: |
| 1070 | message = ("Process %d (%s) hasn't written pidfile %s" % |
| 1071 | (pid, args, self.pid_file)) |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1072 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1073 | print message |
| 1074 | if time.time() - self.start_time > PIDFILE_TIMEOUT: |
| 1075 | email_manager.enqueue_notify_email( |
| 1076 | 'Process has failed to write pidfile', message) |
| 1077 | if pid is not None: |
| 1078 | kill_autoserv(pid) |
| 1079 | else: |
| 1080 | pid = 0 |
| 1081 | self.on_lost_process(pid) |
| 1082 | return self.pid, self.exit_status |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1083 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1084 | return None, None |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1085 | |
| 1086 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1087 | def on_lost_process(self, pid): |
| 1088 | """\ |
| 1089 | Called when autoserv has exited without writing an exit status, |
| 1090 | or we've timed out waiting for autoserv to write a pid to the |
| 1091 | pidfile. In either case, we just return failure and the caller |
| 1092 | should signal some kind of warning. |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1093 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1094 | pid is unimportant here, as it shouldn't be used by anyone. |
| 1095 | """ |
| 1096 | self.lost_process = True |
| 1097 | self.pid = pid |
| 1098 | self.exit_status = 1 |
mbligh | 90a549d | 2008-03-25 23:52:34 +0000 | [diff] [blame] | 1099 | |
| 1100 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1101 | def exit_code(self): |
| 1102 | pid, exit_code = self.get_pidfile_info() |
| 1103 | return exit_code |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1104 | |
| 1105 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1106 | class Agent(object): |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 1107 | def __init__(self, tasks, queue_entry_ids=[], num_processes=1): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1108 | self.active_task = None |
| 1109 | self.queue = Queue.Queue(0) |
| 1110 | self.dispatcher = None |
| 1111 | self.queue_entry_ids = queue_entry_ids |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 1112 | self.num_processes = num_processes |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1113 | |
| 1114 | for task in tasks: |
| 1115 | self.add_task(task) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1116 | |
| 1117 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1118 | def add_task(self, task): |
| 1119 | self.queue.put_nowait(task) |
| 1120 | task.agent = self |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1121 | |
| 1122 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1123 | def tick(self): |
| 1124 | print "agent tick" |
| 1125 | if self.active_task and not self.active_task.is_done(): |
| 1126 | self.active_task.poll() |
| 1127 | else: |
| 1128 | self._next_task(); |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1129 | |
| 1130 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1131 | def _next_task(self): |
| 1132 | print "agent picking task" |
| 1133 | if self.active_task: |
| 1134 | assert self.active_task.is_done() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1135 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1136 | if not self.active_task.success: |
| 1137 | self.on_task_failure() |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1138 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1139 | self.active_task = None |
| 1140 | if not self.is_done(): |
| 1141 | self.active_task = self.queue.get_nowait() |
| 1142 | if self.active_task: |
| 1143 | self.active_task.start() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1144 | |
| 1145 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1146 | def on_task_failure(self): |
| 1147 | self.queue = Queue.Queue(0) |
| 1148 | for task in self.active_task.failure_tasks: |
| 1149 | self.add_task(task) |
mbligh | 16c722d | 2008-03-05 00:58:44 +0000 | [diff] [blame] | 1150 | |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1151 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 1152 | def is_running(self): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1153 | return self.active_task is not None |
showard | ec11316 | 2008-05-08 00:52:49 +0000 | [diff] [blame] | 1154 | |
| 1155 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1156 | def is_done(self): |
| 1157 | return self.active_task == None and self.queue.empty() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1158 | |
| 1159 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1160 | def start(self): |
| 1161 | assert self.dispatcher |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1162 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1163 | self._next_task() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1164 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1165 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1166 | class AgentTask(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1167 | def __init__(self, cmd, failure_tasks = []): |
| 1168 | self.done = False |
| 1169 | self.failure_tasks = failure_tasks |
| 1170 | self.started = False |
| 1171 | self.cmd = cmd |
| 1172 | self.task = None |
| 1173 | self.agent = None |
| 1174 | self.monitor = None |
| 1175 | self.success = None |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1176 | |
| 1177 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1178 | def poll(self): |
| 1179 | print "poll" |
| 1180 | if self.monitor: |
| 1181 | self.tick(self.monitor.exit_code()) |
| 1182 | else: |
| 1183 | self.finished(False) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1184 | |
| 1185 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1186 | def tick(self, exit_code): |
| 1187 | if exit_code==None: |
| 1188 | return |
| 1189 | # print "exit_code was %d" % exit_code |
| 1190 | if exit_code == 0: |
| 1191 | success = True |
| 1192 | else: |
| 1193 | success = False |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1194 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1195 | self.finished(success) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1196 | |
| 1197 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1198 | def is_done(self): |
| 1199 | return self.done |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1200 | |
| 1201 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1202 | def finished(self, success): |
| 1203 | self.done = True |
| 1204 | self.success = success |
| 1205 | self.epilog() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1206 | |
| 1207 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1208 | def prolog(self): |
| 1209 | pass |
mbligh | d64e570 | 2008-04-04 21:39:28 +0000 | [diff] [blame] | 1210 | |
| 1211 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1212 | def create_temp_resultsdir(self, suffix=''): |
| 1213 | self.temp_results_dir = tempfile.mkdtemp(suffix=suffix) |
mbligh | d64e570 | 2008-04-04 21:39:28 +0000 | [diff] [blame] | 1214 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1215 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1216 | def cleanup(self): |
| 1217 | if (hasattr(self, 'temp_results_dir') and |
| 1218 | os.path.exists(self.temp_results_dir)): |
| 1219 | shutil.rmtree(self.temp_results_dir) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1220 | |
| 1221 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1222 | def epilog(self): |
| 1223 | self.cleanup() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1224 | |
| 1225 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1226 | def start(self): |
| 1227 | assert self.agent |
| 1228 | |
| 1229 | if not self.started: |
| 1230 | self.prolog() |
| 1231 | self.run() |
| 1232 | |
| 1233 | self.started = True |
| 1234 | |
| 1235 | |
| 1236 | def abort(self): |
| 1237 | if self.monitor: |
| 1238 | self.monitor.kill() |
| 1239 | self.done = True |
| 1240 | self.cleanup() |
| 1241 | |
| 1242 | |
| 1243 | def run(self): |
| 1244 | if self.cmd: |
| 1245 | print "agent starting monitor" |
| 1246 | log_file = None |
| 1247 | if hasattr(self, 'host'): |
| 1248 | log_file = os.path.join(RESULTS_DIR, 'hosts', |
| 1249 | self.host.hostname) |
| 1250 | self.monitor = RunMonitor( |
| 1251 | self.cmd, nice_level = AUTOSERV_NICE_LEVEL, |
| 1252 | log_file = log_file) |
| 1253 | self.monitor.run() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1254 | |
| 1255 | |
| 1256 | class RepairTask(AgentTask): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1257 | def __init__(self, host, fail_queue_entry=None): |
| 1258 | """\ |
| 1259 | fail_queue_entry: queue entry to mark failed if this repair |
| 1260 | fails. |
| 1261 | """ |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 1262 | protection = host_protections.Protection.get_string(host.protection) |
jadmanski | 542537f | 2008-07-24 14:14:56 +0000 | [diff] [blame] | 1263 | # normalize the protection name |
| 1264 | protection = host_protections.Protection.get_attr_name(protection) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1265 | self.create_temp_resultsdir('.repair') |
| 1266 | cmd = [_autoserv_path , '-R', '-m', host.hostname, |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 1267 | '-r', self.temp_results_dir, '--host-protection', protection] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1268 | self.host = host |
| 1269 | self.fail_queue_entry = fail_queue_entry |
| 1270 | super(RepairTask, self).__init__(cmd) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1271 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1272 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1273 | def prolog(self): |
| 1274 | print "repair_task starting" |
| 1275 | self.host.set_status('Repairing') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1276 | |
| 1277 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1278 | def epilog(self): |
| 1279 | super(RepairTask, self).epilog() |
| 1280 | if self.success: |
| 1281 | self.host.set_status('Ready') |
| 1282 | else: |
| 1283 | self.host.set_status('Repair Failed') |
| 1284 | if self.fail_queue_entry: |
| 1285 | self.fail_queue_entry.handle_host_failure() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1286 | |
| 1287 | |
| 1288 | class VerifyTask(AgentTask): |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 1289 | def __init__(self, queue_entry=None, host=None, run_verify=True): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1290 | assert bool(queue_entry) != bool(host) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1291 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1292 | self.host = host or queue_entry.host |
| 1293 | self.queue_entry = queue_entry |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1294 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1295 | self.create_temp_resultsdir('.verify') |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 1296 | |
| 1297 | # TODO: |
| 1298 | # While it is rediculous to instantiate a verify task object |
| 1299 | # that doesnt actually run the verify task, this is hopefully a |
| 1300 | # temporary hack and will have a cleaner way to skip this |
| 1301 | # step later. (while ensuring that the original semantics don't change) |
| 1302 | if not run_verify: |
| 1303 | cmd = ["true"] |
| 1304 | else: |
| 1305 | cmd = [_autoserv_path,'-v','-m',self.host.hostname, |
| 1306 | '-r', self.temp_results_dir] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1307 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1308 | fail_queue_entry = None |
| 1309 | if queue_entry and not queue_entry.meta_host: |
| 1310 | fail_queue_entry = queue_entry |
| 1311 | failure_tasks = [RepairTask(self.host, fail_queue_entry)] |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1312 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1313 | super(VerifyTask, self).__init__(cmd, |
| 1314 | failure_tasks=failure_tasks) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1315 | |
| 1316 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1317 | def prolog(self): |
| 1318 | print "starting verify on %s" % (self.host.hostname) |
| 1319 | if self.queue_entry: |
| 1320 | self.queue_entry.set_status('Verifying') |
| 1321 | self.queue_entry.clear_results_dir( |
| 1322 | self.queue_entry.verify_results_dir()) |
| 1323 | self.host.set_status('Verifying') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1324 | |
| 1325 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1326 | def cleanup(self): |
| 1327 | if not os.path.exists(self.temp_results_dir): |
| 1328 | return |
| 1329 | if self.queue_entry and (self.success or |
| 1330 | not self.queue_entry.meta_host): |
| 1331 | self.move_results() |
| 1332 | super(VerifyTask, self).cleanup() |
mbligh | d64e570 | 2008-04-04 21:39:28 +0000 | [diff] [blame] | 1333 | |
| 1334 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1335 | def epilog(self): |
| 1336 | super(VerifyTask, self).epilog() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1337 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1338 | if self.success: |
| 1339 | self.host.set_status('Ready') |
| 1340 | elif self.queue_entry: |
| 1341 | self.queue_entry.requeue() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1342 | |
| 1343 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1344 | def move_results(self): |
| 1345 | assert self.queue_entry is not None |
| 1346 | target_dir = self.queue_entry.verify_results_dir() |
| 1347 | if not os.path.exists(target_dir): |
| 1348 | os.makedirs(target_dir) |
| 1349 | files = os.listdir(self.temp_results_dir) |
| 1350 | for filename in files: |
| 1351 | if filename == AUTOSERV_PID_FILE: |
| 1352 | continue |
| 1353 | self.force_move(os.path.join(self.temp_results_dir, |
| 1354 | filename), |
| 1355 | os.path.join(target_dir, filename)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1356 | |
| 1357 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1358 | @staticmethod |
| 1359 | def force_move(source, dest): |
| 1360 | """\ |
| 1361 | Replacement for shutil.move() that will delete the destination |
| 1362 | if it exists, even if it's a directory. |
| 1363 | """ |
| 1364 | if os.path.exists(dest): |
| 1365 | print ('Warning: removing existing destination file ' + |
| 1366 | dest) |
| 1367 | remove_file_or_dir(dest) |
| 1368 | shutil.move(source, dest) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1369 | |
| 1370 | |
mbligh | dffd637 | 2008-02-29 22:47:33 +0000 | [diff] [blame] | 1371 | class VerifySynchronousTask(VerifyTask): |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 1372 | def __init__(self, queue_entry, run_verify=True): |
| 1373 | super(VerifySynchronousTask, self).__init__(queue_entry=queue_entry, |
| 1374 | run_verify=run_verify) |
mbligh | dffd637 | 2008-02-29 22:47:33 +0000 | [diff] [blame] | 1375 | |
| 1376 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1377 | def epilog(self): |
| 1378 | super(VerifySynchronousTask, self).epilog() |
| 1379 | if self.success: |
| 1380 | if self.queue_entry.job.num_complete() > 0: |
| 1381 | # some other entry failed verify, and we've |
| 1382 | # already been marked as stopped |
| 1383 | return |
mbligh | dffd637 | 2008-02-29 22:47:33 +0000 | [diff] [blame] | 1384 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1385 | self.queue_entry.set_status('Pending') |
| 1386 | job = self.queue_entry.job |
| 1387 | if job.is_ready(): |
| 1388 | agent = job.run(self.queue_entry) |
| 1389 | self.agent.dispatcher.add_agent(agent) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1390 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1391 | class QueueTask(AgentTask): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1392 | def __init__(self, job, queue_entries, cmd): |
| 1393 | super(QueueTask, self).__init__(cmd) |
| 1394 | self.job = job |
| 1395 | self.queue_entries = queue_entries |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1396 | |
| 1397 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1398 | @staticmethod |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1399 | def _write_keyval(keyval_dir, field, value, keyval_filename='keyval'): |
| 1400 | key_path = os.path.join(keyval_dir, keyval_filename) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1401 | keyval_file = open(key_path, 'a') |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1402 | print >> keyval_file, '%s=%s' % (field, str(value)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1403 | keyval_file.close() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1404 | |
| 1405 | |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1406 | def _host_keyval_dir(self): |
| 1407 | return os.path.join(self.results_dir(), 'host_keyvals') |
| 1408 | |
| 1409 | |
| 1410 | def _write_host_keyval(self, host): |
| 1411 | labels = ','.join(host.labels()) |
| 1412 | self._write_keyval(self._host_keyval_dir(), 'labels', labels, |
| 1413 | keyval_filename=host.hostname) |
| 1414 | |
| 1415 | def _create_host_keyval_dir(self): |
| 1416 | directory = self._host_keyval_dir() |
| 1417 | if not os.path.exists(directory): |
| 1418 | os.makedirs(directory) |
| 1419 | |
| 1420 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1421 | def results_dir(self): |
| 1422 | return self.queue_entries[0].results_dir() |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1423 | |
| 1424 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1425 | def run(self): |
| 1426 | """\ |
| 1427 | Override AgentTask.run() so we can use a PidfileRunMonitor. |
| 1428 | """ |
| 1429 | self.monitor = PidfileRunMonitor(self.results_dir(), |
| 1430 | cmd=self.cmd, |
| 1431 | nice_level=AUTOSERV_NICE_LEVEL) |
| 1432 | self.monitor.run() |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1433 | |
| 1434 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1435 | def prolog(self): |
| 1436 | # write some job timestamps into the job keyval file |
| 1437 | queued = time.mktime(self.job.created_on.timetuple()) |
| 1438 | started = time.time() |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1439 | self._write_keyval(self.results_dir(), "job_queued", int(queued)) |
| 1440 | self._write_keyval(self.results_dir(), "job_started", int(started)) |
| 1441 | self._create_host_keyval_dir() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1442 | for queue_entry in self.queue_entries: |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1443 | self._write_host_keyval(queue_entry.host) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1444 | print "starting queue_task on %s/%s" % (queue_entry.host.hostname, queue_entry.id) |
| 1445 | queue_entry.set_status('Running') |
| 1446 | queue_entry.host.set_status('Running') |
| 1447 | if (not self.job.is_synchronous() and |
| 1448 | self.job.num_machines() > 1): |
| 1449 | assert len(self.queue_entries) == 1 |
| 1450 | self.job.write_to_machines_file(self.queue_entries[0]) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1451 | |
| 1452 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1453 | def _finish_task(self): |
| 1454 | # write out the finished time into the results keyval |
| 1455 | finished = time.time() |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 1456 | self._write_keyval(self.results_dir(), "job_finished", int(finished)) |
jadmanski | c2ac77f | 2008-05-16 21:44:04 +0000 | [diff] [blame] | 1457 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1458 | # parse the results of the job |
| 1459 | if self.job.is_synchronous() or self.job.num_machines() == 1: |
| 1460 | parse_results(self.job.results_dir()) |
| 1461 | else: |
| 1462 | for queue_entry in self.queue_entries: |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 1463 | parse_results(queue_entry.results_dir(), flags="-l 2") |
| 1464 | |
| 1465 | |
| 1466 | def _log_abort(self): |
| 1467 | # build up sets of all the aborted_by and aborted_on values |
| 1468 | aborted_by, aborted_on = set(), set() |
| 1469 | for queue_entry in self.queue_entries: |
| 1470 | if queue_entry.aborted_by: |
| 1471 | aborted_by.add(queue_entry.aborted_by) |
| 1472 | t = int(time.mktime(queue_entry.aborted_on.timetuple())) |
| 1473 | aborted_on.add(t) |
| 1474 | |
| 1475 | # extract some actual, unique aborted by value and write it out |
| 1476 | assert len(aborted_by) <= 1 |
| 1477 | if len(aborted_by) == 1: |
| 1478 | results_dir = self.results_dir() |
| 1479 | self._write_keyval(results_dir, "aborted_by", aborted_by.pop()) |
| 1480 | self._write_keyval(results_dir, "aborted_on", max(aborted_on)) |
jadmanski | c2ac77f | 2008-05-16 21:44:04 +0000 | [diff] [blame] | 1481 | |
| 1482 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1483 | def abort(self): |
| 1484 | super(QueueTask, self).abort() |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 1485 | self._log_abort() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1486 | self._finish_task() |
jadmanski | c2ac77f | 2008-05-16 21:44:04 +0000 | [diff] [blame] | 1487 | |
| 1488 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1489 | def epilog(self): |
| 1490 | super(QueueTask, self).epilog() |
| 1491 | if self.success: |
| 1492 | status = 'Completed' |
| 1493 | else: |
| 1494 | status = 'Failed' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1495 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1496 | for queue_entry in self.queue_entries: |
| 1497 | queue_entry.set_status(status) |
| 1498 | queue_entry.host.set_status('Ready') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1499 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1500 | self._finish_task() |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1501 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1502 | print "queue_task finished with %s/%s" % (status, self.success) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1503 | |
| 1504 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1505 | class RecoveryQueueTask(QueueTask): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1506 | def __init__(self, job, queue_entries, run_monitor): |
| 1507 | super(RecoveryQueueTask, self).__init__(job, |
| 1508 | queue_entries, cmd=None) |
| 1509 | self.run_monitor = run_monitor |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1510 | |
| 1511 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1512 | def run(self): |
| 1513 | self.monitor = self.run_monitor |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1514 | |
| 1515 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1516 | def prolog(self): |
| 1517 | # recovering an existing process - don't do prolog |
| 1518 | pass |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 1519 | |
| 1520 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1521 | class RebootTask(AgentTask): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1522 | def __init__(self, host): |
| 1523 | global _autoserv_path |
| 1524 | |
| 1525 | # Current implementation of autoserv requires control file |
| 1526 | # to be passed on reboot action request. TODO: remove when no |
| 1527 | # longer appropriate. |
| 1528 | self.create_temp_resultsdir('.reboot') |
| 1529 | self.cmd = [_autoserv_path, '-b', '-m', host.hostname, |
| 1530 | '-r', self.temp_results_dir, '/dev/null'] |
| 1531 | self.host = host |
| 1532 | super(RebootTask, self).__init__(self.cmd, |
| 1533 | failure_tasks=[RepairTask(host)]) |
mbligh | 16c722d | 2008-03-05 00:58:44 +0000 | [diff] [blame] | 1534 | |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 1535 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1536 | def prolog(self): |
| 1537 | print "starting reboot task for host: %s" % self.host.hostname |
| 1538 | self.host.set_status("Rebooting") |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 1539 | |
mbligh | d5c9580 | 2008-03-05 00:33:46 +0000 | [diff] [blame] | 1540 | |
| 1541 | class AbortTask(AgentTask): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1542 | def __init__(self, queue_entry, agents_to_abort): |
| 1543 | self.queue_entry = queue_entry |
| 1544 | self.agents_to_abort = agents_to_abort |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1545 | super(AbortTask, self).__init__('') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1546 | |
| 1547 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1548 | def prolog(self): |
| 1549 | print "starting abort on host %s, job %s" % ( |
| 1550 | self.queue_entry.host_id, self.queue_entry.job_id) |
| 1551 | self.queue_entry.set_status('Aborting') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1552 | |
mbligh | d64e570 | 2008-04-04 21:39:28 +0000 | [diff] [blame] | 1553 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1554 | def epilog(self): |
| 1555 | super(AbortTask, self).epilog() |
| 1556 | self.queue_entry.set_status('Aborted') |
| 1557 | self.success = True |
| 1558 | |
| 1559 | |
| 1560 | def run(self): |
| 1561 | for agent in self.agents_to_abort: |
| 1562 | if (agent.active_task): |
| 1563 | agent.active_task.abort() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1564 | |
| 1565 | |
| 1566 | class DBObject(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1567 | def __init__(self, id=None, row=None, new_record=False): |
| 1568 | assert (bool(id) != bool(row)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1569 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1570 | self.__table = self._get_table() |
| 1571 | fields = self._fields() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1572 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1573 | self.__new_record = new_record |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1574 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1575 | if row is None: |
| 1576 | sql = 'SELECT * FROM %s WHERE ID=%%s' % self.__table |
| 1577 | rows = _db.execute(sql, (id,)) |
| 1578 | if len(rows) == 0: |
| 1579 | raise "row not found (table=%s, id=%s)" % \ |
| 1580 | (self.__table, id) |
| 1581 | row = rows[0] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1582 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1583 | assert len(row) == self.num_cols(), ( |
| 1584 | "table = %s, row = %s/%d, fields = %s/%d" % ( |
| 1585 | self.__table, row, len(row), fields, self.num_cols())) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1586 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1587 | self.__valid_fields = {} |
| 1588 | for i,value in enumerate(row): |
| 1589 | self.__dict__[fields[i]] = value |
| 1590 | self.__valid_fields[fields[i]] = True |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1591 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1592 | del self.__valid_fields['id'] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1593 | |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1594 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1595 | @classmethod |
| 1596 | def _get_table(cls): |
| 1597 | raise NotImplementedError('Subclasses must override this') |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1598 | |
| 1599 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1600 | @classmethod |
| 1601 | def _fields(cls): |
| 1602 | raise NotImplementedError('Subclasses must override this') |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1603 | |
| 1604 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1605 | @classmethod |
| 1606 | def num_cols(cls): |
| 1607 | return len(cls._fields()) |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1608 | |
| 1609 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1610 | def count(self, where, table = None): |
| 1611 | if not table: |
| 1612 | table = self.__table |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1613 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1614 | rows = _db.execute(""" |
| 1615 | SELECT count(*) FROM %s |
| 1616 | WHERE %s |
| 1617 | """ % (table, where)) |
mbligh | 6f8bab4 | 2008-02-29 22:45:14 +0000 | [diff] [blame] | 1618 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1619 | assert len(rows) == 1 |
| 1620 | |
| 1621 | return int(rows[0][0]) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1622 | |
| 1623 | |
mbligh | f8c624d | 2008-07-03 16:58:45 +0000 | [diff] [blame] | 1624 | def update_field(self, field, value, condition=''): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1625 | assert self.__valid_fields[field] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1626 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1627 | if self.__dict__[field] == value: |
| 1628 | return |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1629 | |
mbligh | f8c624d | 2008-07-03 16:58:45 +0000 | [diff] [blame] | 1630 | query = "UPDATE %s SET %s = %%s WHERE id = %%s" % (self.__table, field) |
| 1631 | if condition: |
| 1632 | query += ' AND (%s)' % condition |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1633 | _db.execute(query, (value, self.id)) |
| 1634 | |
| 1635 | self.__dict__[field] = value |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1636 | |
| 1637 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1638 | def save(self): |
| 1639 | if self.__new_record: |
| 1640 | keys = self._fields()[1:] # avoid id |
| 1641 | columns = ','.join([str(key) for key in keys]) |
| 1642 | values = ['"%s"' % self.__dict__[key] for key in keys] |
| 1643 | values = ','.join(values) |
| 1644 | query = """INSERT INTO %s (%s) VALUES (%s)""" % \ |
| 1645 | (self.__table, columns, values) |
| 1646 | _db.execute(query) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1647 | |
| 1648 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1649 | def delete(self): |
| 1650 | query = 'DELETE FROM %s WHERE id=%%s' % self.__table |
| 1651 | _db.execute(query, (self.id,)) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1652 | |
| 1653 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 1654 | @staticmethod |
| 1655 | def _prefix_with(string, prefix): |
| 1656 | if string: |
| 1657 | string = prefix + string |
| 1658 | return string |
| 1659 | |
| 1660 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1661 | @classmethod |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 1662 | def fetch(cls, where='', params=(), joins='', order_by=''): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 1663 | table = cls._get_table() |
| 1664 | order_by = cls._prefix_with(order_by, 'ORDER BY ') |
| 1665 | where = cls._prefix_with(where, 'WHERE ') |
| 1666 | query = ('SELECT %(table)s.* FROM %(table)s %(joins)s ' |
| 1667 | '%(where)s %(order_by)s' % {'table' : cls._get_table(), |
| 1668 | 'joins' : joins, |
| 1669 | 'where' : where, |
| 1670 | 'order_by' : order_by}) |
| 1671 | rows = _db.execute(query, params) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1672 | for row in rows: |
| 1673 | yield cls(row=row) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1674 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1675 | |
| 1676 | class IneligibleHostQueue(DBObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1677 | def __init__(self, id=None, row=None, new_record=None): |
| 1678 | super(IneligibleHostQueue, self).__init__(id=id, row=row, |
| 1679 | new_record=new_record) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1680 | |
| 1681 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1682 | @classmethod |
| 1683 | def _get_table(cls): |
| 1684 | return 'ineligible_host_queues' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1685 | |
| 1686 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1687 | @classmethod |
| 1688 | def _fields(cls): |
| 1689 | return ['id', 'job_id', 'host_id'] |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1690 | |
| 1691 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 1692 | class Label(DBObject): |
| 1693 | @classmethod |
| 1694 | def _get_table(cls): |
| 1695 | return 'labels' |
| 1696 | |
| 1697 | |
| 1698 | @classmethod |
| 1699 | def _fields(cls): |
| 1700 | return ['id', 'name', 'kernel_config', 'platform', 'invalid', |
| 1701 | 'only_if_needed'] |
| 1702 | |
| 1703 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1704 | class Host(DBObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1705 | def __init__(self, id=None, row=None): |
| 1706 | super(Host, self).__init__(id=id, row=row) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1707 | |
| 1708 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1709 | @classmethod |
| 1710 | def _get_table(cls): |
| 1711 | return 'hosts' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1712 | |
| 1713 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1714 | @classmethod |
| 1715 | def _fields(cls): |
| 1716 | return ['id', 'hostname', 'locked', 'synch_id','status', |
showard | fb2a7fa | 2008-07-17 17:04:12 +0000 | [diff] [blame] | 1717 | 'invalid', 'protection', 'locked_by_id', 'lock_time'] |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1718 | |
| 1719 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1720 | def current_task(self): |
| 1721 | rows = _db.execute(""" |
| 1722 | SELECT * FROM host_queue_entries WHERE host_id=%s AND NOT complete AND active |
| 1723 | """, (self.id,)) |
| 1724 | |
| 1725 | if len(rows) == 0: |
| 1726 | return None |
| 1727 | else: |
| 1728 | assert len(rows) == 1 |
| 1729 | results = rows[0]; |
mbligh | f8c624d | 2008-07-03 16:58:45 +0000 | [diff] [blame] | 1730 | # print "current = %s" % results |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1731 | return HostQueueEntry(row=results) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1732 | |
| 1733 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1734 | def yield_work(self): |
| 1735 | print "%s yielding work" % self.hostname |
| 1736 | if self.current_task(): |
| 1737 | self.current_task().requeue() |
| 1738 | |
| 1739 | def set_status(self,status): |
| 1740 | print '%s -> %s' % (self.hostname, status) |
| 1741 | self.update_field('status',status) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1742 | |
| 1743 | |
showard | d8e548a | 2008-09-09 03:04:57 +0000 | [diff] [blame] | 1744 | def labels(self): |
| 1745 | """ |
| 1746 | Fetch a list of names of all non-platform labels associated with this |
| 1747 | host. |
| 1748 | """ |
| 1749 | rows = _db.execute(""" |
| 1750 | SELECT labels.name |
| 1751 | FROM labels |
| 1752 | INNER JOIN hosts_labels ON labels.id = hosts_labels.label_id |
| 1753 | WHERE NOT labels.platform AND hosts_labels.host_id = %s |
| 1754 | ORDER BY labels.name |
| 1755 | """, (self.id,)) |
| 1756 | return [row[0] for row in rows] |
| 1757 | |
| 1758 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1759 | class HostQueueEntry(DBObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1760 | def __init__(self, id=None, row=None): |
| 1761 | assert id or row |
| 1762 | super(HostQueueEntry, self).__init__(id=id, row=row) |
| 1763 | self.job = Job(self.job_id) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1764 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1765 | if self.host_id: |
| 1766 | self.host = Host(self.host_id) |
| 1767 | else: |
| 1768 | self.host = None |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1769 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1770 | self.queue_log_path = os.path.join(self.job.results_dir(), |
| 1771 | 'queue.log.' + str(self.id)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1772 | |
| 1773 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1774 | @classmethod |
| 1775 | def _get_table(cls): |
| 1776 | return 'host_queue_entries' |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1777 | |
| 1778 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1779 | @classmethod |
| 1780 | def _fields(cls): |
| 1781 | return ['id', 'job_id', 'host_id', 'priority', 'status', |
showard | b8471e3 | 2008-07-03 19:51:08 +0000 | [diff] [blame] | 1782 | 'meta_host', 'active', 'complete', 'deleted'] |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1783 | |
| 1784 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1785 | def set_host(self, host): |
| 1786 | if host: |
| 1787 | self.queue_log_record('Assigning host ' + host.hostname) |
| 1788 | self.update_field('host_id', host.id) |
| 1789 | self.update_field('active', True) |
| 1790 | self.block_host(host.id) |
| 1791 | else: |
| 1792 | self.queue_log_record('Releasing host') |
| 1793 | self.unblock_host(self.host.id) |
| 1794 | self.update_field('host_id', None) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1795 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1796 | self.host = host |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1797 | |
| 1798 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1799 | def get_host(self): |
| 1800 | return self.host |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1801 | |
| 1802 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1803 | def queue_log_record(self, log_line): |
| 1804 | now = str(datetime.datetime.now()) |
| 1805 | queue_log = open(self.queue_log_path, 'a', 0) |
| 1806 | queue_log.write(now + ' ' + log_line + '\n') |
| 1807 | queue_log.close() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1808 | |
| 1809 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1810 | def block_host(self, host_id): |
| 1811 | print "creating block %s/%s" % (self.job.id, host_id) |
| 1812 | row = [0, self.job.id, host_id] |
| 1813 | block = IneligibleHostQueue(row=row, new_record=True) |
| 1814 | block.save() |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1815 | |
| 1816 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1817 | def unblock_host(self, host_id): |
| 1818 | print "removing block %s/%s" % (self.job.id, host_id) |
| 1819 | blocks = IneligibleHostQueue.fetch( |
| 1820 | 'job_id=%d and host_id=%d' % (self.job.id, host_id)) |
| 1821 | for block in blocks: |
| 1822 | block.delete() |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1823 | |
| 1824 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1825 | def results_dir(self): |
| 1826 | if self.job.is_synchronous() or self.job.num_machines() == 1: |
| 1827 | return self.job.job_dir |
| 1828 | else: |
| 1829 | assert self.host |
| 1830 | return os.path.join(self.job.job_dir, |
| 1831 | self.host.hostname) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1832 | |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1833 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1834 | def verify_results_dir(self): |
| 1835 | if self.job.is_synchronous() or self.job.num_machines() > 1: |
| 1836 | assert self.host |
| 1837 | return os.path.join(self.job.job_dir, |
| 1838 | self.host.hostname) |
| 1839 | else: |
| 1840 | return self.job.job_dir |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1841 | |
| 1842 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1843 | def set_status(self, status): |
mbligh | f8c624d | 2008-07-03 16:58:45 +0000 | [diff] [blame] | 1844 | abort_statuses = ['Abort', 'Aborting', 'Aborted'] |
| 1845 | if status not in abort_statuses: |
| 1846 | condition = ' AND '.join(['status <> "%s"' % x |
| 1847 | for x in abort_statuses]) |
| 1848 | else: |
| 1849 | condition = '' |
| 1850 | self.update_field('status', status, condition=condition) |
| 1851 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1852 | if self.host: |
| 1853 | hostname = self.host.hostname |
| 1854 | else: |
| 1855 | hostname = 'no host' |
| 1856 | print "%s/%d status -> %s" % (hostname, self.id, self.status) |
mbligh | f8c624d | 2008-07-03 16:58:45 +0000 | [diff] [blame] | 1857 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1858 | if status in ['Queued']: |
| 1859 | self.update_field('complete', False) |
| 1860 | self.update_field('active', False) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1861 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1862 | if status in ['Pending', 'Running', 'Verifying', 'Starting', |
| 1863 | 'Abort', 'Aborting']: |
| 1864 | self.update_field('complete', False) |
| 1865 | self.update_field('active', True) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1866 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1867 | if status in ['Failed', 'Completed', 'Stopped', 'Aborted']: |
| 1868 | self.update_field('complete', True) |
| 1869 | self.update_field('active', False) |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 1870 | self._email_on_job_complete() |
| 1871 | |
| 1872 | |
| 1873 | def _email_on_job_complete(self): |
| 1874 | url = "%s#tab_id=view_job&object_id=%s" % (_base_url, self.job.id) |
| 1875 | |
| 1876 | if self.job.is_finished(): |
| 1877 | subject = "Autotest: Job ID: %s \"%s\" Completed" % ( |
| 1878 | self.job.id, self.job.name) |
| 1879 | body = "Job ID: %s\nJob Name: %s\n%s\n" % ( |
| 1880 | self.job.id, self.job.name, url) |
| 1881 | send_email(_email_from, self.job.email_list, subject, body) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1882 | |
| 1883 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1884 | def run(self,assigned_host=None): |
| 1885 | if self.meta_host: |
| 1886 | assert assigned_host |
| 1887 | # ensure results dir exists for the queue log |
| 1888 | self.job.create_results_dir() |
| 1889 | self.set_host(assigned_host) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1890 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1891 | print "%s/%s scheduled on %s, status=%s" % (self.job.name, |
| 1892 | self.meta_host, self.host.hostname, self.status) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1893 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1894 | return self.job.run(queue_entry=self) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1895 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1896 | def requeue(self): |
| 1897 | self.set_status('Queued') |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1898 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1899 | if self.meta_host: |
| 1900 | self.set_host(None) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1901 | |
| 1902 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1903 | def handle_host_failure(self): |
| 1904 | """\ |
| 1905 | Called when this queue entry's host has failed verification and |
| 1906 | repair. |
| 1907 | """ |
| 1908 | assert not self.meta_host |
| 1909 | self.set_status('Failed') |
| 1910 | if self.job.is_synchronous(): |
| 1911 | self.job.stop_all_entries() |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1912 | |
| 1913 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1914 | def clear_results_dir(self, results_dir=None, dont_delete_files=False): |
| 1915 | results_dir = results_dir or self.results_dir() |
| 1916 | if not os.path.exists(results_dir): |
| 1917 | return |
| 1918 | if dont_delete_files: |
| 1919 | temp_dir = tempfile.mkdtemp(suffix='.clear_results') |
| 1920 | print 'Moving results from %s to %s' % (results_dir, |
| 1921 | temp_dir) |
| 1922 | for filename in os.listdir(results_dir): |
| 1923 | path = os.path.join(results_dir, filename) |
| 1924 | if dont_delete_files: |
| 1925 | shutil.move(path, |
| 1926 | os.path.join(temp_dir, filename)) |
| 1927 | else: |
| 1928 | remove_file_or_dir(path) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1929 | |
| 1930 | |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 1931 | @property |
| 1932 | def aborted_by(self): |
| 1933 | self._load_abort_info() |
| 1934 | return self._aborted_by |
| 1935 | |
| 1936 | |
| 1937 | @property |
| 1938 | def aborted_on(self): |
| 1939 | self._load_abort_info() |
| 1940 | return self._aborted_on |
| 1941 | |
| 1942 | |
| 1943 | def _load_abort_info(self): |
| 1944 | """ Fetch info about who aborted the job. """ |
| 1945 | if hasattr(self, "_aborted_by"): |
| 1946 | return |
| 1947 | rows = _db.execute(""" |
| 1948 | SELECT users.login, aborted_host_queue_entries.aborted_on |
| 1949 | FROM aborted_host_queue_entries |
| 1950 | INNER JOIN users |
| 1951 | ON users.id = aborted_host_queue_entries.aborted_by_id |
| 1952 | WHERE aborted_host_queue_entries.queue_entry_id = %s |
| 1953 | """, (self.id,)) |
| 1954 | if rows: |
| 1955 | self._aborted_by, self._aborted_on = rows[0] |
| 1956 | else: |
| 1957 | self._aborted_by = self._aborted_on = None |
| 1958 | |
| 1959 | |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1960 | class Job(DBObject): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1961 | def __init__(self, id=None, row=None): |
| 1962 | assert id or row |
| 1963 | super(Job, self).__init__(id=id, row=row) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1964 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1965 | self.job_dir = os.path.join(RESULTS_DIR, "%s-%s" % (self.id, |
| 1966 | self.owner)) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 1967 | |
| 1968 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1969 | @classmethod |
| 1970 | def _get_table(cls): |
| 1971 | return 'jobs' |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1972 | |
| 1973 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1974 | @classmethod |
| 1975 | def _fields(cls): |
| 1976 | return ['id', 'owner', 'name', 'priority', 'control_file', |
| 1977 | 'control_type', 'created_on', 'synch_type', |
showard | 542e840 | 2008-09-19 20:16:18 +0000 | [diff] [blame] | 1978 | 'synch_count', 'synchronizing', 'timeout', |
| 1979 | 'run_verify', 'email_list'] |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 1980 | |
| 1981 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1982 | def is_server_job(self): |
| 1983 | return self.control_type != 2 |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1984 | |
| 1985 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1986 | def get_host_queue_entries(self): |
| 1987 | rows = _db.execute(""" |
| 1988 | SELECT * FROM host_queue_entries |
| 1989 | WHERE job_id= %s |
| 1990 | """, (self.id,)) |
| 1991 | entries = [HostQueueEntry(row=i) for i in rows] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1992 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1993 | assert len(entries)>0 |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1994 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1995 | return entries |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 1996 | |
| 1997 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1998 | def set_status(self, status, update_queues=False): |
| 1999 | self.update_field('status',status) |
| 2000 | |
| 2001 | if update_queues: |
| 2002 | for queue_entry in self.get_host_queue_entries(): |
| 2003 | queue_entry.set_status(status) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2004 | |
| 2005 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2006 | def is_synchronous(self): |
| 2007 | return self.synch_type == 2 |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2008 | |
| 2009 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2010 | def is_ready(self): |
| 2011 | if not self.is_synchronous(): |
| 2012 | return True |
| 2013 | sql = "job_id=%s AND status='Pending'" % self.id |
| 2014 | count = self.count(sql, table='host_queue_entries') |
| 2015 | return (count == self.synch_count) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2016 | |
| 2017 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2018 | def ready_to_synchronize(self): |
| 2019 | # heuristic |
| 2020 | queue_entries = self.get_host_queue_entries() |
| 2021 | count = 0 |
| 2022 | for queue_entry in queue_entries: |
| 2023 | if queue_entry.status == 'Pending': |
| 2024 | count += 1 |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2025 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2026 | return (count/self.synch_count >= 0.5) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2027 | |
| 2028 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2029 | def start_synchronizing(self): |
| 2030 | self.update_field('synchronizing', True) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2031 | |
| 2032 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2033 | def results_dir(self): |
| 2034 | return self.job_dir |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2035 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2036 | def num_machines(self, clause = None): |
| 2037 | sql = "job_id=%s" % self.id |
| 2038 | if clause: |
| 2039 | sql += " AND (%s)" % clause |
| 2040 | return self.count(sql, table='host_queue_entries') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2041 | |
| 2042 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2043 | def num_queued(self): |
| 2044 | return self.num_machines('not complete') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2045 | |
| 2046 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2047 | def num_active(self): |
| 2048 | return self.num_machines('active') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2049 | |
| 2050 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2051 | def num_complete(self): |
| 2052 | return self.num_machines('complete') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2053 | |
| 2054 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2055 | def is_finished(self): |
| 2056 | left = self.num_queued() |
| 2057 | print "%s: %s machines left" % (self.name, left) |
| 2058 | return left==0 |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2059 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2060 | def stop_synchronizing(self): |
| 2061 | self.update_field('synchronizing', False) |
| 2062 | self.set_status('Queued', update_queues = False) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2063 | |
| 2064 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2065 | def stop_all_entries(self): |
| 2066 | for child_entry in self.get_host_queue_entries(): |
| 2067 | if not child_entry.complete: |
| 2068 | child_entry.set_status('Stopped') |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 2069 | |
| 2070 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2071 | def write_to_machines_file(self, queue_entry): |
| 2072 | hostname = queue_entry.get_host().hostname |
| 2073 | print "writing %s to job %s machines file" % (hostname, self.id) |
| 2074 | file_path = os.path.join(self.job_dir, '.machines') |
| 2075 | mf = open(file_path, 'a') |
| 2076 | mf.write("%s\n" % queue_entry.get_host().hostname) |
| 2077 | mf.close() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2078 | |
| 2079 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2080 | def create_results_dir(self, queue_entry=None): |
| 2081 | print "create: active: %s complete %s" % (self.num_active(), |
| 2082 | self.num_complete()) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2083 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2084 | if not os.path.exists(self.job_dir): |
| 2085 | os.makedirs(self.job_dir) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2086 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2087 | if queue_entry: |
| 2088 | return queue_entry.results_dir() |
| 2089 | return self.job_dir |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2090 | |
| 2091 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2092 | def run(self, queue_entry): |
| 2093 | results_dir = self.create_results_dir(queue_entry) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2094 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2095 | if self.is_synchronous(): |
| 2096 | if not self.is_ready(): |
| 2097 | return Agent([VerifySynchronousTask( |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 2098 | queue_entry=queue_entry, |
| 2099 | run_verify=self.run_verify)], |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2100 | [queue_entry.id]) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2101 | |
showard | ccb86d7 | 2008-08-22 18:16:22 +0000 | [diff] [blame] | 2102 | queue_entry.set_status('Starting') |
| 2103 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2104 | ctrl = open(os.tmpnam(), 'w') |
| 2105 | if self.control_file: |
| 2106 | ctrl.write(self.control_file) |
| 2107 | else: |
| 2108 | ctrl.write("") |
| 2109 | ctrl.flush() |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2110 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2111 | if self.is_synchronous(): |
| 2112 | queue_entries = self.get_host_queue_entries() |
| 2113 | else: |
| 2114 | assert queue_entry |
| 2115 | queue_entries = [queue_entry] |
| 2116 | hostnames = ','.join([entry.get_host().hostname |
| 2117 | for entry in queue_entries]) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2118 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2119 | # determine the job tag |
| 2120 | if self.is_synchronous() or self.num_machines() == 1: |
| 2121 | job_name = "%s-%s" % (self.id, self.owner) |
| 2122 | else: |
| 2123 | job_name = "%s-%s/%s" % (self.id, self.owner, |
| 2124 | hostnames) |
mbligh | 6437ff5 | 2008-04-17 15:24:38 +0000 | [diff] [blame] | 2125 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2126 | params = [_autoserv_path, '-P', job_name, '-p', '-n', |
| 2127 | '-r', os.path.abspath(results_dir), |
| 2128 | '-b', '-u', self.owner, '-l', self.name, |
| 2129 | '-m', hostnames, ctrl.name] |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2130 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2131 | if not self.is_server_job(): |
| 2132 | params.append('-c') |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2133 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2134 | tasks = [] |
| 2135 | if not self.is_synchronous(): |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 2136 | tasks.append(VerifyTask(queue_entry, run_verify=self.run_verify)) |
mbligh | e258668 | 2008-02-29 22:45:46 +0000 | [diff] [blame] | 2137 | |
showard | 3d9899a | 2008-07-31 02:11:58 +0000 | [diff] [blame] | 2138 | tasks.append(QueueTask(job=self, |
| 2139 | queue_entries=queue_entries, |
| 2140 | cmd=params)) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2141 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2142 | ids = [] |
| 2143 | for entry in queue_entries: |
| 2144 | ids.append(entry.id) |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2145 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 2146 | agent = Agent(tasks, ids, num_processes=len(queue_entries)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2147 | |
| 2148 | return agent |
mbligh | 36768f0 | 2008-02-22 18:28:33 +0000 | [diff] [blame] | 2149 | |
| 2150 | |
| 2151 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 2152 | main() |