| #!/usr/bin/python -u |
| import os, sys, signal, time, subprocess, logging, logging.config |
| from optparse import OptionParser |
| import common |
| from autotest_lib.client.common_lib import error, global_config, utils |
| |
| PAUSE_LENGTH = 60 |
| STALL_TIMEOUT = 2*60*60 |
| |
| parser = OptionParser() |
| parser.add_option("-r", action="store_true", dest="recover") |
| (options, args) = parser.parse_args() |
| |
| autodir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) |
| results_dir = os.path.join(autodir, 'results') |
| monitor_db = os.path.join(autodir, 'scheduler/monitor_db.py') |
| recover = (options.recover == True) |
| |
| # load logging settings |
| scheduler_dir = os.path.join(autodir, 'scheduler') |
| AUTOTEST_BABYSITTER_LOG_DIR = os.path.join(autodir, 'logs') |
| os.environ['AUTOTEST_BABYSITTER_LOG_DIR'] = AUTOTEST_BABYSITTER_LOG_DIR |
| # Here we export the log name, using the same convention as autoserv's results |
| # directory. |
| babysitter_log_name = 'babysitter.log.%s' % time.strftime('%Y-%m-%d-%H.%M.%S') |
| os.environ['AUTOTEST_BABYSITTER_LOG_NAME'] = babysitter_log_name |
| logging.config.fileConfig(os.path.join(scheduler_dir, 'debug_babysitter.ini')) |
| |
| if len(args) != 0: |
| print "Usage: %s [options]" % __file__ |
| print " -r Run recovery mode. (Note: recovery is implicit after" |
| print " any crash!)" |
| print |
| sys.exit(1) |
| |
| |
| def run_banner_output(cmd): |
| """Returns ------ CMD ------\nCMD_OUTPUT in a string""" |
| banner_output = '%s\n%%s\n\n' % cmd.center(60, '-') |
| command_output = '' |
| try: |
| cmd_out = utils.run(cmd, ignore_status=True, timeout=30) |
| command_output = cmd_out.stdout + cmd_out.stderr |
| except error.CmdError: |
| command_output = 'Timed out' |
| |
| return banner_output % command_output |
| |
| |
| def kill_all_monitors(): |
| logging.info("Killing all monitor_dbs") |
| # try shutdown first |
| status = os.system("killall -2 monitor_db.py") |
| if status == 0: # were any killed? |
| # give them some time to shutdown |
| time.sleep(30) |
| # kill any that are left |
| os.system("killall monitor_db.py") |
| |
| |
| def handle_sigterm(signum, frame): |
| logging.info('Caught SIGTERM') |
| kill_all_monitors() |
| sys.exit(1) |
| |
| signal.signal(signal.SIGTERM, handle_sigterm) |
| |
| |
| class MonitorProc: |
| def __init__(self, do_recovery=False): |
| args = [monitor_db] |
| if do_recovery: |
| args.append("--recover-hosts") |
| args.append(results_dir) |
| |
| kill_all_monitors() |
| environ = os.environ |
| log_name = 'scheduler.log.%s' % time.strftime('%Y-%m-%d-%H.%M.%S') |
| os.environ['AUTOTEST_SCHEDULER_LOG_NAME'] = log_name |
| os.environ['AUTOTEST_SCHEDULER_LOG_DIR'] = AUTOTEST_BABYSITTER_LOG_DIR |
| self.log_path = os.path.join(autodir, 'logs', log_name) |
| |
| self.log_size = 0 |
| self.last_log_change = time.time() |
| |
| logging.info("STARTING monitor_db with log file %s" % self.log_path) |
| devnull = open(os.devnull, 'w') |
| self.proc = subprocess.Popen(args, stdout=devnull) |
| |
| |
| def is_running(self): |
| if self.proc.poll() is not None: |
| logging.info("monitor_db DIED") |
| return False |
| |
| old_size = self.log_size |
| new_size = os.path.getsize(self.log_path) |
| if old_size != new_size: |
| logging.info("Log was touched") |
| self.log_size = new_size |
| self.last_log_change = time.time() |
| elif self.last_log_change + STALL_TIMEOUT < time.time(): |
| logging.info("monitor_db STALLED") |
| self.collect_stalled_info() |
| return False |
| |
| return True |
| |
| |
| def collect_stalled_info(self): |
| INFO_TO_COLLECT = ['uptime', |
| 'ps auxwww', |
| 'iostat -k -x 2 4', |
| ] |
| db_cmd = '/usr/bin/mysqladmin --verbose processlist -u%s -p%s' |
| config = global_config.global_config |
| try: |
| user = config.get_config_value("BACKUP", "user") |
| password = config.get_config_value("BACKUP", "password") |
| db_cmd %= (user, password) |
| INFO_TO_COLLECT.append(db_cmd) |
| except global_config.ConfigError: |
| pass |
| stall_log_path = self.log_path + '.stall_info' |
| log = open(stall_log_path, "w") |
| for cmd in INFO_TO_COLLECT: |
| log.write(run_banner_output(cmd)) |
| |
| log.close() |
| |
| |
| logging.info("initializing") |
| |
| if os.getuid() == 0: |
| logging.critical("running as root, aborting!") |
| sys.exit(1) |
| |
| utils.write_pid("monitor_db_babysitter") |
| |
| while True: |
| proc = MonitorProc(do_recovery=recover) |
| time.sleep(PAUSE_LENGTH) |
| while proc.is_running(): |
| logging.info("Tick") |
| time.sleep(PAUSE_LENGTH) |
| recover = False |