Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 1 | # Copyright 2017 The Chromium Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Allen Li | 464220f | 2017-09-12 17:14:22 -0700 | [diff] [blame] | 5 | """Run a job against Autotest. |
| 6 | |
| 7 | See http://goto.google.com/monitor_db_per_job_refactor |
Allen Li | 2678d36 | 2017-10-11 16:59:07 -0700 | [diff] [blame] | 8 | |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 9 | See also https://chromium.googlesource.com/chromiumos/infra/lucifer |
Allen Li | 2678d36 | 2017-10-11 16:59:07 -0700 | [diff] [blame] | 10 | |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 11 | job_reporter is a thin wrapper around lucifer and only updates the |
Allen Li | 2678d36 | 2017-10-11 16:59:07 -0700 | [diff] [blame] | 12 | Autotest database according to status events. |
Allen Li | 464220f | 2017-09-12 17:14:22 -0700 | [diff] [blame] | 13 | """ |
| 14 | |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 15 | from __future__ import absolute_import |
| 16 | from __future__ import division |
| 17 | from __future__ import print_function |
| 18 | |
Allen Li | ff162b7 | 2017-11-13 12:10:33 -0800 | [diff] [blame] | 19 | import atexit |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 20 | import argparse |
| 21 | import logging |
Allen Li | 3f9f622 | 2017-10-18 16:12:08 -0700 | [diff] [blame] | 22 | import os |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 23 | import sys |
| 24 | |
Allen Li | 464220f | 2017-09-12 17:14:22 -0700 | [diff] [blame] | 25 | from lucifer import autotest |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 26 | from lucifer import eventlib |
Allen Li | d31d732 | 2017-11-20 12:48:57 -0800 | [diff] [blame] | 27 | from lucifer import handlers |
Allen Li | 7b88e93 | 2018-02-09 18:26:32 -0800 | [diff] [blame] | 28 | from lucifer import jobx |
Allen Li | d64e4c8 | 2017-11-14 16:34:34 -0800 | [diff] [blame] | 29 | from lucifer import leasing |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 30 | from lucifer import loglib |
| 31 | |
| 32 | logger = logging.getLogger(__name__) |
| 33 | |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 34 | |
Allen Li | 0b80a4b | 2018-03-01 15:22:40 -0800 | [diff] [blame] | 35 | def main(argv): |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 36 | """Main function |
| 37 | |
Allen Li | 0b80a4b | 2018-03-01 15:22:40 -0800 | [diff] [blame] | 38 | @param argv: command line args |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 39 | """ |
Allen Li | 0b80a4b | 2018-03-01 15:22:40 -0800 | [diff] [blame] | 40 | print('job_reporter: Running with argv: %r' % argv, file=sys.stderr) |
| 41 | args = _parse_args_and_configure_logging(argv[1:]) |
| 42 | logger.info('Running with parsed args: %r', args) |
Allen Li | bda3236 | 2018-01-24 17:49:34 -0800 | [diff] [blame] | 43 | with leasing.obtain_lease(_lease_path(args.jobdir, args.job_id)): |
Allen Li | 75f8808 | 2017-11-15 15:24:33 -0800 | [diff] [blame] | 44 | autotest.monkeypatch() |
Allen Li | bda3236 | 2018-01-24 17:49:34 -0800 | [diff] [blame] | 45 | ret = _main(args) |
| 46 | logger.info('Exiting normally with: %r', ret) |
| 47 | return ret |
Allen Li | 58e01ce | 2017-11-10 13:17:53 -0800 | [diff] [blame] | 48 | |
| 49 | |
| 50 | def _parse_args_and_configure_logging(args): |
Allen Li | 464220f | 2017-09-12 17:14:22 -0700 | [diff] [blame] | 51 | parser = argparse.ArgumentParser(prog='job_reporter', description=__doc__) |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 52 | loglib.add_logging_options(parser) |
Allen Li | 45c2fdf | 2018-02-14 18:47:40 -0800 | [diff] [blame] | 53 | |
| 54 | # General configuration |
Allen Li | 057be2c | 2017-11-08 13:51:24 -0800 | [diff] [blame] | 55 | parser.add_argument('--jobdir', default='/usr/local/autotest/leases', |
Allen Li | d64e4c8 | 2017-11-14 16:34:34 -0800 | [diff] [blame] | 56 | help='Path to job leases directory.') |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 57 | parser.add_argument('--lucifer-path', default='/usr/bin/lucifer', |
| 58 | help='Path to lucifer binary') |
Allen Li | 45c2fdf | 2018-02-14 18:47:40 -0800 | [diff] [blame] | 59 | |
| 60 | # Job specific |
Allen Li | 8bb357b | 2018-02-28 17:17:57 -0800 | [diff] [blame] | 61 | |
| 62 | # General |
Allen Li | bc6599c | 2018-02-22 12:43:21 -0800 | [diff] [blame] | 63 | parser.add_argument('--lucifer-level', required=True, |
Allen Li | 3ac0007 | 2018-05-24 14:25:09 -0700 | [diff] [blame] | 64 | help='Lucifer level', choices=['STARTING']) |
Allen Li | 8bb357b | 2018-02-28 17:17:57 -0800 | [diff] [blame] | 65 | parser.add_argument('--job-id', type=int, required=True, |
| 66 | help='Autotest Job ID') |
| 67 | parser.add_argument('--results-dir', required=True, |
Allen Li | deb5d3b | 2018-03-05 16:44:50 -0800 | [diff] [blame] | 68 | help='Path to job results directory.') |
Allen Li | 8bb357b | 2018-02-28 17:17:57 -0800 | [diff] [blame] | 69 | |
| 70 | # STARTING flags |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 71 | parser.add_argument('--execution-tag', default=None, |
| 72 | help='Autotest execution tag.') |
Allen Li | 03b9150 | 2018-05-24 14:19:39 -0700 | [diff] [blame] | 73 | parser.add_argument('--parsing-only', action='store_true', |
| 74 | help='Whether to only do parsing' |
| 75 | ' (only with --lucifer-level STARTING)') |
Allen Li | 8bb357b | 2018-02-28 17:17:57 -0800 | [diff] [blame] | 76 | |
Allen Li | dbdaab5 | 2017-11-20 12:41:00 -0800 | [diff] [blame] | 77 | args = parser.parse_args(args) |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 78 | loglib.configure_logging_with_args(parser, args) |
Allen Li | 58e01ce | 2017-11-10 13:17:53 -0800 | [diff] [blame] | 79 | return args |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 80 | |
| 81 | |
Allen Li | ff162b7 | 2017-11-13 12:10:33 -0800 | [diff] [blame] | 82 | def _main(args): |
| 83 | """Main program body, running under a lease file. |
| 84 | |
| 85 | @param args: Namespace object containing parsed arguments |
| 86 | """ |
| 87 | ts_mon_config = autotest.chromite_load('ts_mon_config') |
| 88 | metrics = autotest.chromite_load('metrics') |
| 89 | with ts_mon_config.SetupTsMonGlobalState( |
Allen Li | 9c86070 | 2018-02-16 15:25:15 -0800 | [diff] [blame] | 90 | 'job_reporter', short_lived=True): |
Allen Li | 3d089a1 | 2017-11-20 12:06:24 -0800 | [diff] [blame] | 91 | atexit.register(metrics.Flush) |
Allen Li | 0d4322b | 2018-02-09 18:44:57 -0800 | [diff] [blame] | 92 | return _run_autotest_job(args) |
| 93 | |
| 94 | |
| 95 | def _run_autotest_job(args): |
| 96 | """Run a job as seen from Autotest. |
| 97 | |
| 98 | This include some Autotest setup and cleanup around lucifer starting |
| 99 | proper. |
| 100 | """ |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 101 | models = autotest.load('frontend.afe.models') |
| 102 | job = models.Job.objects.get(id=args.job_id) |
Allen Li | 3ac0007 | 2018-05-24 14:25:09 -0700 | [diff] [blame] | 103 | _prepare_autotest_job_files(args, job) |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 104 | handler = _make_handler(args, job) |
| 105 | ret = _run_lucifer_job(handler, args, job) |
Allen Li | 0d4322b | 2018-02-09 18:44:57 -0800 | [diff] [blame] | 106 | if handler.completed: |
| 107 | _mark_handoff_completed(args.job_id) |
| 108 | return ret |
Allen Li | ff162b7 | 2017-11-13 12:10:33 -0800 | [diff] [blame] | 109 | |
| 110 | |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 111 | def _prepare_autotest_job_files(args, job): |
| 112 | jobx.prepare_control_file(job, args.results_dir) |
| 113 | jobx.prepare_keyvals_files(job, args.results_dir) |
| 114 | |
| 115 | |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 116 | def _make_handler(args, job): |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 117 | """Make event handler for lucifer.""" |
Allen Li | d31d732 | 2017-11-20 12:48:57 -0800 | [diff] [blame] | 118 | return handlers.EventHandler( |
Allen Li | d31d732 | 2017-11-20 12:48:57 -0800 | [diff] [blame] | 119 | metrics=handlers.Metrics(), |
Allen Li | ff162b7 | 2017-11-13 12:10:33 -0800 | [diff] [blame] | 120 | job=job, |
Allen Li | 3ac0007 | 2018-05-24 14:25:09 -0700 | [diff] [blame] | 121 | autoserv_exit=None, |
Allen Li | 5afdf6d | 2018-03-23 15:02:25 -0700 | [diff] [blame] | 122 | results_dir=args.results_dir, |
Allen Li | ff162b7 | 2017-11-13 12:10:33 -0800 | [diff] [blame] | 123 | ) |
Allen Li | 63e21c3 | 2017-11-10 13:20:28 -0800 | [diff] [blame] | 124 | |
| 125 | |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 126 | def _run_lucifer_job(event_handler, args, job): |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 127 | """Run lucifer test. |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 128 | |
Allen Li | 902c6e9 | 2017-10-18 15:46:43 -0700 | [diff] [blame] | 129 | Issued events will be handled by event_handler. |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 130 | |
Allen Li | 464220f | 2017-09-12 17:14:22 -0700 | [diff] [blame] | 131 | @param event_handler: callable that takes an Event |
Allen Li | 5cebb2a | 2017-09-21 14:04:43 -0700 | [diff] [blame] | 132 | @param args: parsed arguments |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 133 | @returns: exit status of lucifer |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 134 | """ |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 135 | command_args = [args.lucifer_path] |
Allen Li | e7dcbb6 | 2018-02-09 18:38:28 -0800 | [diff] [blame] | 136 | command_args.extend([ |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 137 | 'test', |
Allen Li | 58d29e9 | 2018-04-03 14:43:47 -0700 | [diff] [blame] | 138 | '-autotestdir', '/usr/local/autotest', |
Allen Li | 45c2fdf | 2018-02-14 18:47:40 -0800 | [diff] [blame] | 139 | |
Allen Li | e7dcbb6 | 2018-02-09 18:38:28 -0800 | [diff] [blame] | 140 | '-abortsock', _abort_sock_path(args.jobdir, args.job_id), |
Allen Li | a6458b3 | 2018-02-14 14:56:34 -0800 | [diff] [blame] | 141 | '-hosts', ','.join(jobx.hostnames(job)), |
Allen Li | 45c2fdf | 2018-02-14 18:47:40 -0800 | [diff] [blame] | 142 | |
Allen Li | a22a974 | 2018-02-20 17:44:24 -0800 | [diff] [blame] | 143 | '-x-level', args.lucifer_level, |
Allen Li | 58d7c91 | 2018-05-24 15:36:04 -0700 | [diff] [blame] | 144 | '-resultsdir', args.results_dir, |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 145 | ]) |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 146 | _add_level_specific_args(command_args, args, job) |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 147 | return eventlib.run_event_command( |
| 148 | event_handler=event_handler, args=command_args) |
| 149 | |
| 150 | |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 151 | def _add_level_specific_args(command_args, args, job): |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 152 | """Add level specific arguments for lucifer test. |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 153 | |
| 154 | command_args is modified in place. |
| 155 | """ |
| 156 | if args.lucifer_level == 'STARTING': |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 157 | _add_starting_args(command_args, args, job) |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 158 | else: |
| 159 | raise Exception('Invalid lucifer level %s' % args.lucifer_level) |
| 160 | |
| 161 | |
Allen Li | 6c17f1d | 2018-03-06 12:20:11 -0800 | [diff] [blame] | 162 | def _add_starting_args(command_args, args, job): |
Allen Li | 08fd214 | 2018-06-28 13:18:34 -0700 | [diff] [blame] | 163 | """Add STARTING level arguments for lucifer test. |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 164 | |
| 165 | command_args is modified in place. |
| 166 | """ |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 167 | RebootAfter = autotest.load('frontend.afe.model_attributes').RebootAfter |
| 168 | command_args.extend([ |
| 169 | '-x-control-file', jobx.control_file_path(args.results_dir), |
| 170 | ]) |
Allen Li | ec2422b | 2018-07-06 16:30:10 -0700 | [diff] [blame] | 171 | if args.execution_tag is not None: |
| 172 | command_args.extend(['-x-execution-tag', args.execution_tag]) |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 173 | command_args.extend(['-x-job-owner', job.owner]) |
| 174 | command_args.extend(['-x-job-name', job.name]) |
| 175 | command_args.extend( |
| 176 | ['-x-reboot-after', |
| 177 | RebootAfter.get_string(job.reboot_after).lower()]) |
Allen Li | 03b9150 | 2018-05-24 14:19:39 -0700 | [diff] [blame] | 178 | if args.parsing_only: |
Allen Li | 1929fb3 | 2018-07-16 15:50:08 -0700 | [diff] [blame] | 179 | command_args.append('-x-parse-only') |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 180 | if job.run_reset: |
| 181 | command_args.append('-x-run-reset') |
Allen Li | 164e8eb | 2018-02-28 17:11:19 -0800 | [diff] [blame] | 182 | if jobx.is_client_job(job): |
| 183 | command_args.append('-x-client-test') |
| 184 | if jobx.needs_ssp(job): |
| 185 | command_args.append('-x-require-ssp') |
| 186 | test_source_build = job.keyval_dict().get('test_source_build', None) |
| 187 | if test_source_build: |
| 188 | command_args.extend(['-x-test-source-build', test_source_build]) |
| 189 | if job.parent_job_id: |
| 190 | command_args.extend(['-x-parent-job-id', str(job.parent_job_id)]) |
Allen Li | 292b4ca | 2018-02-28 17:28:10 -0800 | [diff] [blame] | 191 | |
| 192 | |
Allen Li | 5cca818 | 2017-11-20 13:12:51 -0800 | [diff] [blame] | 193 | def _mark_handoff_completed(job_id): |
| 194 | models = autotest.load('frontend.afe.models') |
| 195 | handoff = models.JobHandoff.objects.get(job_id=job_id) |
| 196 | handoff.completed = True |
| 197 | handoff.save() |
| 198 | |
| 199 | |
Allen Li | d64e4c8 | 2017-11-14 16:34:34 -0800 | [diff] [blame] | 200 | def _abort_sock_path(jobdir, job_id): |
| 201 | return _lease_path(jobdir, job_id) + '.sock' |
| 202 | |
| 203 | |
| 204 | def _lease_path(jobdir, job_id): |
| 205 | return os.path.join(jobdir, str(job_id)) |
| 206 | |
| 207 | |
Allen Li | 20dd90f | 2017-07-17 12:27:10 -0700 | [diff] [blame] | 208 | if __name__ == '__main__': |
Allen Li | 0b80a4b | 2018-03-01 15:22:40 -0800 | [diff] [blame] | 209 | sys.exit(main(sys.argv)) |