mbligh | 6203ace | 2007-10-04 21:54:24 +0000 | [diff] [blame] | 1 | #!/usr/bin/python -u |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 2 | # Copyright 2007-2008 Martin J. Bligh <mbligh@google.com>, Google Inc. |
mbligh | 82648e5 | 2008-11-20 16:54:25 +0000 | [diff] [blame] | 3 | # Released under the GPL v2 |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 4 | |
mbligh | c8949b8 | 2007-07-23 16:33:58 +0000 | [diff] [blame] | 5 | """ |
Aviv Keshet | de6bb19 | 2013-01-30 16:17:22 -0800 | [diff] [blame] | 6 | Run a control file through the server side engine |
mbligh | dcd57a8 | 2007-07-11 23:06:47 +0000 | [diff] [blame] | 7 | """ |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 8 | |
mbligh | a5f5e54 | 2009-12-30 16:57:49 +0000 | [diff] [blame] | 9 | import sys, os, re, traceback, signal, time, logging, getpass |
mbligh | 1ffd5dc | 2008-11-25 13:24:05 +0000 | [diff] [blame] | 10 | |
mbligh | f5427bb | 2008-04-09 15:55:57 +0000 | [diff] [blame] | 11 | import common |
mbligh | 9ff89cd | 2009-09-03 20:28:17 +0000 | [diff] [blame] | 12 | |
mbligh | cb8cb33 | 2009-09-03 21:08:56 +0000 | [diff] [blame] | 13 | from autotest_lib.client.common_lib.global_config import global_config |
| 14 | require_atfork = global_config.get_config_value( |
| 15 | 'AUTOSERV', 'require_atfork_module', type=bool, default=True) |
| 16 | |
| 17 | try: |
| 18 | import atfork |
| 19 | atfork.monkeypatch_os_fork_functions() |
| 20 | import atfork.stdlib_fixer |
| 21 | # Fix the Python standard library for threading+fork safety with its |
| 22 | # internal locks. http://code.google.com/p/python-atfork/ |
| 23 | import warnings |
| 24 | warnings.filterwarnings('ignore', 'logging module already imported') |
| 25 | atfork.stdlib_fixer.fix_logging_module() |
| 26 | except ImportError, e: |
| 27 | from autotest_lib.client.common_lib import global_config |
| 28 | if global_config.global_config.get_config_value( |
| 29 | 'AUTOSERV', 'require_atfork_module', type=bool, default=False): |
| 30 | print >>sys.stderr, 'Please run utils/build_externals.py' |
| 31 | print e |
| 32 | sys.exit(1) |
mbligh | 9ff89cd | 2009-09-03 20:28:17 +0000 | [diff] [blame] | 33 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 34 | from autotest_lib.server import server_logging_config |
showard | 043c62a | 2009-06-10 19:48:57 +0000 | [diff] [blame] | 35 | from autotest_lib.server import server_job, utils, autoserv_parser, autotest |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 36 | from autotest_lib.client.common_lib import pidfile, logging_manager |
mbligh | 92c0fc2 | 2008-11-20 16:52:23 +0000 | [diff] [blame] | 37 | |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 38 | def log_alarm(signum, frame): |
| 39 | logging.error("Received SIGALARM. Ignoring and continuing on.") |
Alex Miller | 0528d6f | 2013-01-11 10:49:48 -0800 | [diff] [blame] | 40 | sys.exit(1) |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 41 | |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 42 | def run_autoserv(pid_file_manager, results, parser): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 43 | # send stdin to /dev/null |
| 44 | dev_null = os.open(os.devnull, os.O_RDONLY) |
| 45 | os.dup2(dev_null, sys.stdin.fileno()) |
| 46 | os.close(dev_null) |
mbligh | dbf3761 | 2007-11-24 19:38:11 +0000 | [diff] [blame] | 47 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 48 | # Create separate process group |
| 49 | os.setpgrp() |
mbligh | 1d42d4e | 2007-11-05 22:42:00 +0000 | [diff] [blame] | 50 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 51 | # Implement SIGTERM handler |
mbligh | c229956 | 2009-07-02 19:00:36 +0000 | [diff] [blame] | 52 | def handle_sigterm(signum, frame): |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 53 | if pid_file_manager: |
| 54 | pid_file_manager.close_file(1, signal.SIGTERM) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 55 | os.killpg(os.getpgrp(), signal.SIGKILL) |
mbligh | faf0cd4 | 2007-11-19 16:00:24 +0000 | [diff] [blame] | 56 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 57 | # Set signal handler |
mbligh | c229956 | 2009-07-02 19:00:36 +0000 | [diff] [blame] | 58 | signal.signal(signal.SIGTERM, handle_sigterm) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 59 | |
David Rochberg | 8a60d1e | 2011-02-01 14:22:07 -0500 | [diff] [blame] | 60 | # Ignore SIGTTOU's generated by output from forked children. |
| 61 | signal.signal(signal.SIGTTOU, signal.SIG_IGN) |
| 62 | |
Alex Miller | f1af17e | 2013-01-09 22:50:32 -0800 | [diff] [blame] | 63 | # If we received a SIGALARM, let's be loud about it. |
| 64 | signal.signal(signal.SIGALRM, log_alarm) |
| 65 | |
mbligh | a5f5e54 | 2009-12-30 16:57:49 +0000 | [diff] [blame] | 66 | # Server side tests that call shell scripts often depend on $USER being set |
| 67 | # but depending on how you launch your autotest scheduler it may not be set. |
| 68 | os.environ['USER'] = getpass.getuser() |
| 69 | |
mbligh | cce191f | 2008-09-19 20:31:03 +0000 | [diff] [blame] | 70 | if parser.options.machines: |
| 71 | machines = parser.options.machines.replace(',', ' ').strip().split() |
| 72 | else: |
| 73 | machines = [] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 74 | machines_file = parser.options.machines_file |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 75 | label = parser.options.label |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 76 | group_name = parser.options.group_name |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 77 | user = parser.options.user |
| 78 | client = parser.options.client |
| 79 | server = parser.options.server |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 80 | install_before = parser.options.install_before |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 81 | install_after = parser.options.install_after |
| 82 | verify = parser.options.verify |
| 83 | repair = parser.options.repair |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 84 | cleanup = parser.options.cleanup |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 85 | provision = parser.options.provision |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame^] | 86 | reset = parser.options.reset |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 87 | no_tee = parser.options.no_tee |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 88 | parse_job = parser.options.parse_job |
mbligh | e7d9c60 | 2009-07-02 19:02:33 +0000 | [diff] [blame] | 89 | execution_tag = parser.options.execution_tag |
| 90 | if not execution_tag: |
| 91 | execution_tag = parse_job |
jadmanski | fbc1f0a | 2008-07-09 14:12:54 +0000 | [diff] [blame] | 92 | host_protection = parser.options.host_protection |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 93 | ssh_user = parser.options.ssh_user |
| 94 | ssh_port = parser.options.ssh_port |
| 95 | ssh_pass = parser.options.ssh_pass |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 96 | collect_crashinfo = parser.options.collect_crashinfo |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 97 | control_filename = parser.options.control_filename |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 98 | test_retry = parser.options.test_retry |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 99 | verify_job_repo_url = parser.options.verify_job_repo_url |
Christopher Wiley | f594c5e | 2013-07-03 18:25:30 -0700 | [diff] [blame] | 100 | skip_crash_collection = parser.options.skip_crash_collection |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 101 | |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 102 | # can't be both a client and a server side test |
| 103 | if client and server: |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 104 | parser.parser.error("Can not specify a test as both server and client!") |
mbligh | b2bea30 | 2008-07-24 20:25:57 +0000 | [diff] [blame] | 105 | |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 106 | if provision and client: |
| 107 | parser.parser.error("Cannot specify provisioning and client!") |
| 108 | |
| 109 | is_special_task = (verify or repair or cleanup or collect_crashinfo or |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame^] | 110 | provision or reset) |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 111 | if len(parser.args) < 1 and not is_special_task: |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 112 | parser.parser.error("Missing argument: control file") |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 113 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 114 | # We have a control file unless it's just a verify/repair/cleanup job |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 115 | if len(parser.args) > 0: |
| 116 | control = parser.args[0] |
| 117 | else: |
| 118 | control = None |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 119 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 120 | if machines_file: |
| 121 | machines = [] |
| 122 | for m in open(machines_file, 'r').readlines(): |
| 123 | # remove comments, spaces |
| 124 | m = re.sub('#.*', '', m).strip() |
| 125 | if m: |
| 126 | machines.append(m) |
| 127 | print "Read list of machines from file: %s" % machines_file |
| 128 | print ','.join(machines) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 129 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 130 | if machines: |
| 131 | for machine in machines: |
| 132 | if not machine or re.search('\s', machine): |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 133 | parser.parser.error("Invalid machine: %s" % str(machine)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 134 | machines = list(set(machines)) |
| 135 | machines.sort() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 136 | |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 137 | if group_name and len(machines) < 2: |
Eric Li | 861b2d5 | 2011-02-04 14:50:35 -0800 | [diff] [blame] | 138 | parser.parser.error("-G %r may only be supplied with more than one machine." |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 139 | % group_name) |
mbligh | 374f341 | 2009-05-13 21:29:45 +0000 | [diff] [blame] | 140 | |
mbligh | e0cbc91 | 2010-03-11 18:03:07 +0000 | [diff] [blame] | 141 | kwargs = {'group_name': group_name, 'tag': execution_tag} |
| 142 | if control_filename: |
| 143 | kwargs['control_filename'] = control_filename |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 144 | job = server_job.server_job(control, parser.args[1:], results, label, |
| 145 | user, machines, client, parse_job, |
Scott Zawalski | 91493c8 | 2013-01-25 16:15:20 -0500 | [diff] [blame] | 146 | ssh_user, ssh_port, ssh_pass, test_retry, |
| 147 | **kwargs) |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 148 | job.logging.start_logging() |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 149 | job.init_parser() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 150 | |
mbligh | 161fe6f | 2008-06-19 16:26:04 +0000 | [diff] [blame] | 151 | # perform checks |
| 152 | job.precheck() |
| 153 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 154 | # run the job |
| 155 | exit_code = 0 |
| 156 | try: |
mbligh | 332000a | 2009-06-08 16:47:28 +0000 | [diff] [blame] | 157 | try: |
| 158 | if repair: |
| 159 | job.repair(host_protection) |
| 160 | elif verify: |
| 161 | job.verify() |
Alex Miller | cb79ba7 | 2013-05-29 14:43:00 -0700 | [diff] [blame] | 162 | elif provision: |
| 163 | job.provision(provision) |
Dan Shi | 07e09af | 2013-04-12 09:31:29 -0700 | [diff] [blame^] | 164 | elif reset: |
| 165 | job.reset() |
mbligh | 332000a | 2009-06-08 16:47:28 +0000 | [diff] [blame] | 166 | else: |
jadmanski | def0c3c | 2009-03-25 20:07:10 +0000 | [diff] [blame] | 167 | job.run(cleanup, install_before, install_after, |
beeps | cb6f1e2 | 2013-06-28 19:14:10 -0700 | [diff] [blame] | 168 | verify_job_repo_url=verify_job_repo_url, |
Christopher Wiley | f594c5e | 2013-07-03 18:25:30 -0700 | [diff] [blame] | 169 | only_collect_crashinfo=collect_crashinfo, |
| 170 | skip_crash_collection=skip_crash_collection) |
mbligh | 332000a | 2009-06-08 16:47:28 +0000 | [diff] [blame] | 171 | finally: |
| 172 | while job.hosts: |
| 173 | host = job.hosts.pop() |
| 174 | host.close() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 175 | except: |
jadmanski | 27b37ea | 2008-10-29 23:54:31 +0000 | [diff] [blame] | 176 | exit_code = 1 |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 177 | traceback.print_exc() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 178 | |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 179 | if pid_file_manager: |
| 180 | pid_file_manager.num_tests_failed = job.num_tests_failed |
| 181 | pid_file_manager.close_file(exit_code) |
jadmanski | e0dffc3 | 2008-12-15 17:30:30 +0000 | [diff] [blame] | 182 | job.cleanup_parser() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 183 | |
jadmanski | 27b37ea | 2008-10-29 23:54:31 +0000 | [diff] [blame] | 184 | sys.exit(exit_code) |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 185 | |
| 186 | |
| 187 | def main(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 188 | # grab the parser |
| 189 | parser = autoserv_parser.autoserv_parser |
mbligh | a5cb406 | 2009-02-17 15:53:39 +0000 | [diff] [blame] | 190 | parser.parse_args() |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 191 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 192 | if len(sys.argv) == 1: |
| 193 | parser.parser.print_help() |
| 194 | sys.exit(1) |
mbligh | a6f1308 | 2008-06-05 23:53:46 +0000 | [diff] [blame] | 195 | |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 196 | if parser.options.no_logging: |
| 197 | results = None |
| 198 | else: |
| 199 | results = parser.options.results |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 200 | if not results: |
| 201 | results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') |
| 202 | results = os.path.abspath(results) |
showard | 566d3c0 | 2010-01-12 18:57:01 +0000 | [diff] [blame] | 203 | resultdir_exists = False |
| 204 | for filename in ('control.srv', 'status.log', '.autoserv_execute'): |
| 205 | if os.path.exists(os.path.join(results, filename)): |
| 206 | resultdir_exists = True |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 207 | if not parser.options.use_existing_results and resultdir_exists: |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 208 | error = "Error: results directory already exists: %s\n" % results |
| 209 | sys.stderr.write(error) |
| 210 | sys.exit(1) |
mbligh | a788dc4 | 2009-03-26 21:10:16 +0000 | [diff] [blame] | 211 | |
| 212 | # Now that we certified that there's no leftover results dir from |
| 213 | # previous jobs, lets create the result dir since the logging system |
| 214 | # needs to create the log file in there. |
| 215 | if not os.path.isdir(results): |
| 216 | os.makedirs(results) |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 217 | |
| 218 | logging_manager.configure_logging( |
| 219 | server_logging_config.ServerLoggingConfig(), results_dir=results, |
showard | 10d8417 | 2009-06-18 23:16:50 +0000 | [diff] [blame] | 220 | use_console=not parser.options.no_tee, |
| 221 | verbose=parser.options.verbose, |
| 222 | no_console_prefix=parser.options.no_console_prefix) |
showard | 75cdfee | 2009-06-10 17:40:41 +0000 | [diff] [blame] | 223 | if results: |
mbligh | a788dc4 | 2009-03-26 21:10:16 +0000 | [diff] [blame] | 224 | logging.info("Results placed in %s" % results) |
mbligh | 1071763 | 2008-11-19 00:21:57 +0000 | [diff] [blame] | 225 | |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 226 | # wait until now to perform this check, so it get properly logged |
| 227 | if parser.options.use_existing_results and not resultdir_exists: |
| 228 | logging.error("No existing results directory found: %s", results) |
| 229 | sys.exit(1) |
| 230 | |
| 231 | |
mbligh | 80e1eba | 2008-11-19 00:26:18 +0000 | [diff] [blame] | 232 | if parser.options.write_pidfile: |
mbligh | 4608b00 | 2010-01-05 18:22:35 +0000 | [diff] [blame] | 233 | pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, |
| 234 | results) |
jadmanski | d5ab8c5 | 2008-12-03 16:27:07 +0000 | [diff] [blame] | 235 | pid_file_manager.open_file() |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 236 | else: |
| 237 | pid_file_manager = None |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 238 | |
jadmanski | f22fea8 | 2008-11-26 20:57:07 +0000 | [diff] [blame] | 239 | autotest.BaseAutotest.set_install_in_tmpdir( |
| 240 | parser.options.install_in_tmpdir) |
| 241 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 242 | exit_code = 0 |
| 243 | try: |
| 244 | try: |
mbligh | 1071763 | 2008-11-19 00:21:57 +0000 | [diff] [blame] | 245 | run_autoserv(pid_file_manager, results, parser) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 246 | except SystemExit, e: |
| 247 | exit_code = e.code |
| 248 | except: |
| 249 | traceback.print_exc() |
| 250 | # If we don't know what happened, we'll classify it as |
| 251 | # an 'abort' and return 1. |
| 252 | exit_code = 1 |
| 253 | finally: |
mbligh | ff7d61f | 2008-12-22 14:53:35 +0000 | [diff] [blame] | 254 | if pid_file_manager: |
| 255 | pid_file_manager.close_file(exit_code) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 256 | sys.exit(exit_code) |
mbligh | faf0cd4 | 2007-11-19 16:00:24 +0000 | [diff] [blame] | 257 | |
mbligh | bb42185 | 2008-03-11 22:36:16 +0000 | [diff] [blame] | 258 | |
mbligh | a46678d | 2008-05-01 20:00:01 +0000 | [diff] [blame] | 259 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 260 | main() |