mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 1 | # Copyright Martin J. Bligh, Google Inc 2008 |
| 2 | # Released under the GPL v2 |
| 3 | |
| 4 | """ |
| 5 | This class allows you to communicate with the frontend to submit jobs etc |
| 6 | It is designed for writing more sophisiticated server-side control files that |
| 7 | can recursively add and manage other jobs. |
| 8 | |
| 9 | We turn the JSON dictionaries into real objects that are more idiomatic |
| 10 | |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 11 | For docs, see: |
Aviv Keshet | 2c709f6 | 2013-05-07 12:52:15 -0700 | [diff] [blame] | 12 | http://www.chromium.org/chromium-os/testing/afe-rpc-infrastructure |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 13 | http://docs.djangoproject.com/en/dev/ref/models/querysets/#queryset-api |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 14 | """ |
| 15 | |
mbligh | db59e3c | 2009-11-21 01:45:18 +0000 | [diff] [blame] | 16 | import getpass, os, time, traceback, re |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 17 | import common |
| 18 | from autotest_lib.frontend.afe import rpc_client_lib |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 19 | from autotest_lib.client.common_lib import global_config |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 20 | from autotest_lib.client.common_lib import utils |
Aviv Keshet | 3dd8beb | 2013-05-13 17:36:04 -0700 | [diff] [blame] | 21 | from autotest_lib.client.common_lib import control_data |
Scott Zawalski | 63470dd | 2012-09-05 00:49:43 -0400 | [diff] [blame] | 22 | from autotest_lib.tko import db |
| 23 | |
| 24 | |
mbligh | 4e57661 | 2008-12-22 14:56:36 +0000 | [diff] [blame] | 25 | try: |
| 26 | from autotest_lib.server.site_common import site_utils as server_utils |
| 27 | except: |
| 28 | from autotest_lib.server import utils as server_utils |
| 29 | form_ntuples_from_machines = server_utils.form_ntuples_from_machines |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 30 | |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 31 | GLOBAL_CONFIG = global_config.global_config |
| 32 | DEFAULT_SERVER = 'autotest' |
| 33 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 34 | def dump_object(header, obj): |
| 35 | """ |
| 36 | Standard way to print out the frontend objects (eg job, host, acl, label) |
| 37 | in a human-readable fashion for debugging |
| 38 | """ |
| 39 | result = header + '\n' |
| 40 | for key in obj.hash: |
| 41 | if key == 'afe' or key == 'hash': |
| 42 | continue |
| 43 | result += '%20s: %s\n' % (key, obj.hash[key]) |
| 44 | return result |
| 45 | |
| 46 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 47 | class RpcClient(object): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 48 | """ |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 49 | Abstract RPC class for communicating with the autotest frontend |
| 50 | Inherited for both TKO and AFE uses. |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 51 | |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 52 | All the constructors go in the afe / tko class. |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 53 | Manipulating methods go in the object classes themselves |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 54 | """ |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 55 | def __init__(self, path, user, server, print_log, debug, reply_debug): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 56 | """ |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 57 | Create a cached instance of a connection to the frontend |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 58 | |
| 59 | user: username to connect as |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 60 | server: frontend server to connect to |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 61 | print_log: pring a logging message to stdout on every operation |
| 62 | debug: print out all RPC traffic |
| 63 | """ |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 64 | if not user: |
mbligh | db59e3c | 2009-11-21 01:45:18 +0000 | [diff] [blame] | 65 | user = getpass.getuser() |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 66 | if not server: |
mbligh | 475f776 | 2009-01-30 00:34:04 +0000 | [diff] [blame] | 67 | if 'AUTOTEST_WEB' in os.environ: |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 68 | server = os.environ['AUTOTEST_WEB'] |
mbligh | 475f776 | 2009-01-30 00:34:04 +0000 | [diff] [blame] | 69 | else: |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 70 | server = GLOBAL_CONFIG.get_config_value('SERVER', 'hostname', |
| 71 | default=DEFAULT_SERVER) |
| 72 | self.server = server |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 73 | self.user = user |
| 74 | self.print_log = print_log |
| 75 | self.debug = debug |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 76 | self.reply_debug = reply_debug |
Scott Zawalski | 347aaf4 | 2012-04-03 16:33:00 -0400 | [diff] [blame] | 77 | headers = {'AUTHORIZATION': self.user} |
| 78 | rpc_server = 'http://' + server + path |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 79 | if debug: |
| 80 | print 'SERVER: %s' % rpc_server |
| 81 | print 'HEADERS: %s' % headers |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 82 | self.proxy = rpc_client_lib.get_proxy(rpc_server, headers=headers) |
| 83 | |
| 84 | |
| 85 | def run(self, call, **dargs): |
| 86 | """ |
| 87 | Make a RPC call to the AFE server |
| 88 | """ |
| 89 | rpc_call = getattr(self.proxy, call) |
| 90 | if self.debug: |
| 91 | print 'DEBUG: %s %s' % (call, dargs) |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 92 | try: |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 93 | result = utils.strip_unicode(rpc_call(**dargs)) |
| 94 | if self.reply_debug: |
| 95 | print result |
| 96 | return result |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 97 | except Exception: |
| 98 | print 'FAILED RPC CALL: %s %s' % (call, dargs) |
| 99 | raise |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 100 | |
| 101 | |
| 102 | def log(self, message): |
| 103 | if self.print_log: |
| 104 | print message |
| 105 | |
| 106 | |
jamesren | c394022 | 2010-02-19 21:57:37 +0000 | [diff] [blame] | 107 | class Planner(RpcClient): |
| 108 | def __init__(self, user=None, server=None, print_log=True, debug=False, |
| 109 | reply_debug=False): |
| 110 | super(Planner, self).__init__(path='/planner/server/rpc/', |
| 111 | user=user, |
| 112 | server=server, |
| 113 | print_log=print_log, |
| 114 | debug=debug, |
| 115 | reply_debug=reply_debug) |
| 116 | |
| 117 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 118 | class TKO(RpcClient): |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 119 | def __init__(self, user=None, server=None, print_log=True, debug=False, |
| 120 | reply_debug=False): |
Scott Zawalski | 347aaf4 | 2012-04-03 16:33:00 -0400 | [diff] [blame] | 121 | super(TKO, self).__init__(path='/new_tko/server/noauth/rpc/', |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 122 | user=user, |
| 123 | server=server, |
| 124 | print_log=print_log, |
| 125 | debug=debug, |
| 126 | reply_debug=reply_debug) |
Scott Zawalski | 63470dd | 2012-09-05 00:49:43 -0400 | [diff] [blame] | 127 | self._db = None |
| 128 | |
| 129 | |
| 130 | def get_job_test_statuses_from_db(self, job_id): |
| 131 | """Get job test statuses from the database. |
| 132 | |
| 133 | Retrieve a set of fields from a job that reflect the status of each test |
| 134 | run within a job. |
| 135 | fields retrieved: status, test_name, reason, test_started_time, |
| 136 | test_finished_time, afe_job_id, job_owner, hostname. |
| 137 | |
| 138 | @param job_id: The afe job id to look up. |
| 139 | @returns a TestStatus object of the resulting information. |
| 140 | """ |
| 141 | if self._db is None: |
| 142 | self._db = db.db() |
Fang Deng | 5c50833 | 2014-03-19 10:26:00 -0700 | [diff] [blame] | 143 | fields = ['status', 'test_name', 'subdir', 'reason', |
| 144 | 'test_started_time', 'test_finished_time', 'afe_job_id', |
| 145 | 'job_owner', 'hostname', 'job_tag'] |
Scott Zawalski | 63470dd | 2012-09-05 00:49:43 -0400 | [diff] [blame] | 146 | table = 'tko_test_view_2' |
| 147 | where = 'job_tag like "%s-%%"' % job_id |
| 148 | test_status = [] |
| 149 | # Run commit before we query to ensure that we are pulling the latest |
| 150 | # results. |
| 151 | self._db.commit() |
| 152 | for entry in self._db.select(','.join(fields), table, (where, None)): |
| 153 | status_dict = {} |
| 154 | for key,value in zip(fields, entry): |
| 155 | # All callers expect values to be a str object. |
| 156 | status_dict[key] = str(value) |
| 157 | # id is used by TestStatus to uniquely identify each Test Status |
| 158 | # obj. |
| 159 | status_dict['id'] = [status_dict['reason'], status_dict['hostname'], |
| 160 | status_dict['test_name']] |
| 161 | test_status.append(status_dict) |
| 162 | |
| 163 | return [TestStatus(self, e) for e in test_status] |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 164 | |
| 165 | |
| 166 | def get_status_counts(self, job, **data): |
| 167 | entries = self.run('get_status_counts', |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 168 | group_by=['hostname', 'test_name', 'reason'], |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 169 | job_tag__startswith='%s-' % job, **data) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 170 | return [TestStatus(self, e) for e in entries['groups']] |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 171 | |
| 172 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 173 | class AFE(RpcClient): |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 174 | def __init__(self, user=None, server=None, print_log=True, debug=False, |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 175 | reply_debug=False, job=None): |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 176 | self.job = job |
Scott Zawalski | 347aaf4 | 2012-04-03 16:33:00 -0400 | [diff] [blame] | 177 | super(AFE, self).__init__(path='/afe/server/noauth/rpc/', |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 178 | user=user, |
| 179 | server=server, |
| 180 | print_log=print_log, |
| 181 | debug=debug, |
| 182 | reply_debug=reply_debug) |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 183 | |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 184 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 185 | def host_statuses(self, live=None): |
jamesren | 121eee6 | 2010-04-13 19:10:12 +0000 | [diff] [blame] | 186 | dead_statuses = ['Repair Failed', 'Repairing'] |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 187 | statuses = self.run('get_static_data')['host_statuses'] |
| 188 | if live == True: |
mbligh | c2847b7 | 2009-03-25 19:32:20 +0000 | [diff] [blame] | 189 | return list(set(statuses) - set(dead_statuses)) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 190 | if live == False: |
| 191 | return dead_statuses |
| 192 | else: |
| 193 | return statuses |
| 194 | |
| 195 | |
mbligh | 7109401 | 2009-12-19 05:35:21 +0000 | [diff] [blame] | 196 | @staticmethod |
| 197 | def _dict_for_host_query(hostnames=(), status=None, label=None): |
| 198 | query_args = {} |
mbligh | 4e545a5 | 2009-12-19 05:30:39 +0000 | [diff] [blame] | 199 | if hostnames: |
| 200 | query_args['hostname__in'] = hostnames |
| 201 | if status: |
| 202 | query_args['status'] = status |
| 203 | if label: |
| 204 | query_args['labels__name'] = label |
mbligh | 7109401 | 2009-12-19 05:35:21 +0000 | [diff] [blame] | 205 | return query_args |
| 206 | |
| 207 | |
| 208 | def get_hosts(self, hostnames=(), status=None, label=None, **dargs): |
| 209 | query_args = dict(dargs) |
| 210 | query_args.update(self._dict_for_host_query(hostnames=hostnames, |
| 211 | status=status, |
| 212 | label=label)) |
| 213 | hosts = self.run('get_hosts', **query_args) |
| 214 | return [Host(self, h) for h in hosts] |
| 215 | |
| 216 | |
| 217 | def get_hostnames(self, status=None, label=None, **dargs): |
| 218 | """Like get_hosts() but returns hostnames instead of Host objects.""" |
| 219 | # This implementation can be replaced with a more efficient one |
| 220 | # that does not query for entire host objects in the future. |
| 221 | return [host_obj.hostname for host_obj in |
| 222 | self.get_hosts(status=status, label=label, **dargs)] |
| 223 | |
| 224 | |
| 225 | def reverify_hosts(self, hostnames=(), status=None, label=None): |
| 226 | query_args = dict(locked=False, |
| 227 | aclgroup__users__login=self.user) |
| 228 | query_args.update(self._dict_for_host_query(hostnames=hostnames, |
| 229 | status=status, |
| 230 | label=label)) |
mbligh | 4e545a5 | 2009-12-19 05:30:39 +0000 | [diff] [blame] | 231 | return self.run('reverify_hosts', **query_args) |
| 232 | |
| 233 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 234 | def create_host(self, hostname, **dargs): |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 235 | id = self.run('add_host', hostname=hostname, **dargs) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 236 | return self.get_hosts(id=id)[0] |
| 237 | |
| 238 | |
MK Ryu | acf3592 | 2014-10-03 14:56:49 -0700 | [diff] [blame] | 239 | def get_host_attribute(self, attr, **dargs): |
| 240 | host_attrs = self.run('get_host_attribute', attribute=attr, **dargs) |
| 241 | return [HostAttribute(self, a) for a in host_attrs] |
| 242 | |
| 243 | |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 244 | def set_host_attribute(self, attr, val, **dargs): |
| 245 | self.run('set_host_attribute', attribute=attr, value=val, **dargs) |
| 246 | |
| 247 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 248 | def get_labels(self, **dargs): |
| 249 | labels = self.run('get_labels', **dargs) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 250 | return [Label(self, l) for l in labels] |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 251 | |
| 252 | |
| 253 | def create_label(self, name, **dargs): |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 254 | id = self.run('add_label', name=name, **dargs) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 255 | return self.get_labels(id=id)[0] |
| 256 | |
| 257 | |
| 258 | def get_acls(self, **dargs): |
| 259 | acls = self.run('get_acl_groups', **dargs) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 260 | return [Acl(self, a) for a in acls] |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 261 | |
| 262 | |
| 263 | def create_acl(self, name, **dargs): |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 264 | id = self.run('add_acl_group', name=name, **dargs) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 265 | return self.get_acls(id=id)[0] |
| 266 | |
| 267 | |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 268 | def get_users(self, **dargs): |
| 269 | users = self.run('get_users', **dargs) |
| 270 | return [User(self, u) for u in users] |
| 271 | |
| 272 | |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 273 | def generate_control_file(self, tests, **dargs): |
| 274 | ret = self.run('generate_control_file', tests=tests, **dargs) |
| 275 | return ControlFile(self, ret) |
| 276 | |
| 277 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 278 | def get_jobs(self, summary=False, **dargs): |
| 279 | if summary: |
| 280 | jobs_data = self.run('get_jobs_summary', **dargs) |
| 281 | else: |
| 282 | jobs_data = self.run('get_jobs', **dargs) |
mbligh | afbba0c | 2009-06-08 16:44:45 +0000 | [diff] [blame] | 283 | jobs = [] |
| 284 | for j in jobs_data: |
| 285 | job = Job(self, j) |
| 286 | # Set up some extra information defaults |
| 287 | job.testname = re.sub('\s.*', '', job.name) # arbitrary default |
| 288 | job.platform_results = {} |
| 289 | job.platform_reasons = {} |
| 290 | jobs.append(job) |
| 291 | return jobs |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 292 | |
| 293 | |
| 294 | def get_host_queue_entries(self, **data): |
| 295 | entries = self.run('get_host_queue_entries', **data) |
mbligh | f9e3586 | 2009-02-26 01:03:11 +0000 | [diff] [blame] | 296 | job_statuses = [JobStatus(self, e) for e in entries] |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 297 | |
| 298 | # Sadly, get_host_queue_entries doesn't return platforms, we have |
| 299 | # to get those back from an explicit get_hosts queury, then patch |
| 300 | # the new host objects back into the host list. |
| 301 | hostnames = [s.host.hostname for s in job_statuses if s.host] |
| 302 | host_hash = {} |
| 303 | for host in self.get_hosts(hostname__in=hostnames): |
| 304 | host_hash[host.hostname] = host |
| 305 | for status in job_statuses: |
| 306 | if status.host: |
| 307 | status.host = host_hash[status.host.hostname] |
mbligh | f9e3586 | 2009-02-26 01:03:11 +0000 | [diff] [blame] | 308 | # filter job statuses that have either host or meta_host |
| 309 | return [status for status in job_statuses if (status.host or |
| 310 | status.meta_host)] |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 311 | |
| 312 | |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 313 | def create_job_by_test(self, tests, kernel=None, use_container=False, |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 314 | kernel_cmdline=None, **dargs): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 315 | """ |
| 316 | Given a test name, fetch the appropriate control file from the server |
mbligh | 4e57661 | 2008-12-22 14:56:36 +0000 | [diff] [blame] | 317 | and submit it. |
| 318 | |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 319 | @param kernel: A comma separated list of kernel versions to boot. |
| 320 | @param kernel_cmdline: The command line used to boot all kernels listed |
| 321 | in the kernel parameter. |
| 322 | |
mbligh | 4e57661 | 2008-12-22 14:56:36 +0000 | [diff] [blame] | 323 | Returns a list of job objects |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 324 | """ |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 325 | assert ('hosts' in dargs or |
| 326 | 'atomic_group_name' in dargs and 'synch_count' in dargs) |
showard | a2cd72b | 2009-10-01 18:43:53 +0000 | [diff] [blame] | 327 | if kernel: |
| 328 | kernel_list = re.split('[\s,]+', kernel.strip()) |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 329 | kernel_info = [] |
| 330 | for version in kernel_list: |
| 331 | kernel_dict = {'version': version} |
| 332 | if kernel_cmdline is not None: |
| 333 | kernel_dict['cmdline'] = kernel_cmdline |
| 334 | kernel_info.append(kernel_dict) |
showard | a2cd72b | 2009-10-01 18:43:53 +0000 | [diff] [blame] | 335 | else: |
| 336 | kernel_info = None |
| 337 | control_file = self.generate_control_file( |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 338 | tests=tests, kernel=kernel_info, use_container=use_container) |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 339 | if control_file.is_server: |
Aviv Keshet | 3dd8beb | 2013-05-13 17:36:04 -0700 | [diff] [blame] | 340 | dargs['control_type'] = control_data.CONTROL_TYPE_NAMES.SERVER |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 341 | else: |
Aviv Keshet | 3dd8beb | 2013-05-13 17:36:04 -0700 | [diff] [blame] | 342 | dargs['control_type'] = control_data.CONTROL_TYPE_NAMES.CLIENT |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 343 | dargs['dependencies'] = dargs.get('dependencies', []) + \ |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 344 | control_file.dependencies |
| 345 | dargs['control_file'] = control_file.control_file |
mbligh | 672666c | 2009-07-28 23:22:13 +0000 | [diff] [blame] | 346 | if not dargs.get('synch_count', None): |
mbligh | c99fccf | 2009-07-11 00:59:33 +0000 | [diff] [blame] | 347 | dargs['synch_count'] = control_file.synch_count |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 348 | if 'hosts' in dargs and len(dargs['hosts']) < dargs['synch_count']: |
| 349 | # will not be able to satisfy this request |
mbligh | 38b0915 | 2009-04-28 18:34:25 +0000 | [diff] [blame] | 350 | return None |
| 351 | return self.create_job(**dargs) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 352 | |
| 353 | |
| 354 | def create_job(self, control_file, name=' ', priority='Medium', |
Aviv Keshet | 3dd8beb | 2013-05-13 17:36:04 -0700 | [diff] [blame] | 355 | control_type=control_data.CONTROL_TYPE_NAMES.CLIENT, **dargs): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 356 | id = self.run('create_job', name=name, priority=priority, |
| 357 | control_file=control_file, control_type=control_type, **dargs) |
| 358 | return self.get_jobs(id=id)[0] |
| 359 | |
| 360 | |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 361 | def run_test_suites(self, pairings, kernel, kernel_label=None, |
| 362 | priority='Medium', wait=True, poll_interval=10, |
Simran Basi | 7e60574 | 2013-11-12 13:43:36 -0800 | [diff] [blame] | 363 | email_from=None, email_to=None, timeout_mins=10080, |
Simran Basi | 3421702 | 2012-11-06 13:43:15 -0800 | [diff] [blame] | 364 | max_runtime_mins=10080, kernel_cmdline=None): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 365 | """ |
| 366 | Run a list of test suites on a particular kernel. |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 367 | |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 368 | Poll for them to complete, and return whether they worked or not. |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 369 | |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 370 | @param pairings: List of MachineTestPairing objects to invoke. |
| 371 | @param kernel: Name of the kernel to run. |
| 372 | @param kernel_label: Label (string) of the kernel to run such as |
| 373 | '<kernel-version> : <config> : <date>' |
| 374 | If any pairing object has its job_label attribute set it |
| 375 | will override this value for that particular job. |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 376 | @param kernel_cmdline: The command line to boot the kernel(s) with. |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 377 | @param wait: boolean - Wait for the results to come back? |
| 378 | @param poll_interval: Interval between polling for job results (in mins) |
| 379 | @param email_from: Send notification email upon completion from here. |
| 380 | @param email_from: Send notification email upon completion to here. |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 381 | """ |
| 382 | jobs = [] |
| 383 | for pairing in pairings: |
mbligh | 0c4f8d7 | 2009-05-12 20:52:18 +0000 | [diff] [blame] | 384 | try: |
| 385 | new_job = self.invoke_test(pairing, kernel, kernel_label, |
Simran Basi | 7e60574 | 2013-11-12 13:43:36 -0800 | [diff] [blame] | 386 | priority, timeout_mins=timeout_mins, |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 387 | kernel_cmdline=kernel_cmdline, |
Simran Basi | 3421702 | 2012-11-06 13:43:15 -0800 | [diff] [blame] | 388 | max_runtime_mins=max_runtime_mins) |
mbligh | 0c4f8d7 | 2009-05-12 20:52:18 +0000 | [diff] [blame] | 389 | if not new_job: |
| 390 | continue |
mbligh | 0c4f8d7 | 2009-05-12 20:52:18 +0000 | [diff] [blame] | 391 | jobs.append(new_job) |
| 392 | except Exception, e: |
| 393 | traceback.print_exc() |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 394 | if not wait or not jobs: |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 395 | return |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 396 | tko = TKO() |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 397 | while True: |
| 398 | time.sleep(60 * poll_interval) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 399 | result = self.poll_all_jobs(tko, jobs, email_from, email_to) |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 400 | if result is not None: |
| 401 | return result |
| 402 | |
| 403 | |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 404 | def result_notify(self, job, email_from, email_to): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 405 | """ |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 406 | Notify about the result of a job. Will always print, if email data |
| 407 | is provided, will send email for it as well. |
| 408 | |
| 409 | job: job object to notify about |
| 410 | email_from: send notification email upon completion from here |
| 411 | email_from: send notification email upon completion to here |
| 412 | """ |
| 413 | if job.result == True: |
| 414 | subject = 'Testing PASSED: ' |
| 415 | else: |
| 416 | subject = 'Testing FAILED: ' |
| 417 | subject += '%s : %s\n' % (job.name, job.id) |
| 418 | text = [] |
| 419 | for platform in job.results_platform_map: |
| 420 | for status in job.results_platform_map[platform]: |
| 421 | if status == 'Total': |
| 422 | continue |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 423 | for host in job.results_platform_map[platform][status]: |
| 424 | text.append('%20s %10s %10s' % (platform, status, host)) |
| 425 | if status == 'Failed': |
| 426 | for test_status in job.test_status[host].fail: |
| 427 | text.append('(%s, %s) : %s' % \ |
| 428 | (host, test_status.test_name, |
| 429 | test_status.reason)) |
| 430 | text.append('') |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 431 | |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 432 | base_url = 'http://' + self.server |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 433 | |
| 434 | params = ('columns=test', |
| 435 | 'rows=machine_group', |
| 436 | "condition=tag~'%s-%%25'" % job.id, |
| 437 | 'title=Report') |
| 438 | query_string = '&'.join(params) |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 439 | url = '%s/tko/compose_query.cgi?%s' % (base_url, query_string) |
| 440 | text.append(url + '\n') |
| 441 | url = '%s/afe/#tab_id=view_job&object_id=%s' % (base_url, job.id) |
| 442 | text.append(url + '\n') |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 443 | |
| 444 | body = '\n'.join(text) |
| 445 | print '---------------------------------------------------' |
| 446 | print 'Subject: ', subject |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 447 | print body |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 448 | print '---------------------------------------------------' |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 449 | if email_from and email_to: |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 450 | print 'Sending email ...' |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 451 | utils.send_email(email_from, email_to, subject, body) |
| 452 | print |
mbligh | 37eceaa | 2008-12-15 22:56:37 +0000 | [diff] [blame] | 453 | |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 454 | |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 455 | def print_job_result(self, job): |
| 456 | """ |
| 457 | Print the result of a single job. |
| 458 | job: a job object |
| 459 | """ |
| 460 | if job.result is None: |
| 461 | print 'PENDING', |
| 462 | elif job.result == True: |
| 463 | print 'PASSED', |
| 464 | elif job.result == False: |
| 465 | print 'FAILED', |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 466 | elif job.result == "Abort": |
| 467 | print 'ABORT', |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 468 | print ' %s : %s' % (job.id, job.name) |
| 469 | |
| 470 | |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 471 | def poll_all_jobs(self, tko, jobs, email_from=None, email_to=None): |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 472 | """ |
| 473 | Poll all jobs in a list. |
| 474 | jobs: list of job objects to poll |
| 475 | email_from: send notification email upon completion from here |
| 476 | email_from: send notification email upon completion to here |
| 477 | |
| 478 | Returns: |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 479 | a) All complete successfully (return True) |
| 480 | b) One or more has failed (return False) |
| 481 | c) Cannot tell yet (return None) |
| 482 | """ |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 483 | results = [] |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 484 | for job in jobs: |
mbligh | 676dcbe | 2009-06-15 21:57:27 +0000 | [diff] [blame] | 485 | if getattr(job, 'result', None) is None: |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 486 | job.result = self.poll_job_results(tko, job) |
mbligh | 676dcbe | 2009-06-15 21:57:27 +0000 | [diff] [blame] | 487 | if job.result is not None: |
| 488 | self.result_notify(job, email_from, email_to) |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 489 | |
mbligh | 676dcbe | 2009-06-15 21:57:27 +0000 | [diff] [blame] | 490 | results.append(job.result) |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 491 | self.print_job_result(job) |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 492 | |
| 493 | if None in results: |
| 494 | return None |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 495 | elif False in results or "Abort" in results: |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 496 | return False |
| 497 | else: |
| 498 | return True |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 499 | |
| 500 | |
mbligh | 1f23f36 | 2008-12-22 14:46:12 +0000 | [diff] [blame] | 501 | def _included_platform(self, host, platforms): |
| 502 | """ |
| 503 | See if host's platforms matches any of the patterns in the included |
| 504 | platforms list. |
| 505 | """ |
| 506 | if not platforms: |
| 507 | return True # No filtering of platforms |
| 508 | for platform in platforms: |
| 509 | if re.search(platform, host.platform): |
| 510 | return True |
| 511 | return False |
| 512 | |
| 513 | |
mbligh | 7b31228 | 2009-01-07 16:45:43 +0000 | [diff] [blame] | 514 | def invoke_test(self, pairing, kernel, kernel_label, priority='Medium', |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 515 | kernel_cmdline=None, **dargs): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 516 | """ |
| 517 | Given a pairing of a control file to a machine label, find all machines |
| 518 | with that label, and submit that control file to them. |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 519 | |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 520 | @param kernel_label: Label (string) of the kernel to run such as |
| 521 | '<kernel-version> : <config> : <date>' |
| 522 | If any pairing object has its job_label attribute set it |
| 523 | will override this value for that particular job. |
| 524 | |
| 525 | @returns A list of job objects. |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 526 | """ |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 527 | # The pairing can override the job label. |
| 528 | if pairing.job_label: |
| 529 | kernel_label = pairing.job_label |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 530 | job_name = '%s : %s' % (pairing.machine_label, kernel_label) |
| 531 | hosts = self.get_hosts(multiple_labels=[pairing.machine_label]) |
mbligh | 1f23f36 | 2008-12-22 14:46:12 +0000 | [diff] [blame] | 532 | platforms = pairing.platforms |
| 533 | hosts = [h for h in hosts if self._included_platform(h, platforms)] |
mbligh | c2847b7 | 2009-03-25 19:32:20 +0000 | [diff] [blame] | 534 | dead_statuses = self.host_statuses(live=False) |
| 535 | host_list = [h.hostname for h in hosts if h.status not in dead_statuses] |
mbligh | 1f23f36 | 2008-12-22 14:46:12 +0000 | [diff] [blame] | 536 | print 'HOSTS: %s' % host_list |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 537 | if pairing.atomic_group_sched: |
mbligh | c99fccf | 2009-07-11 00:59:33 +0000 | [diff] [blame] | 538 | dargs['synch_count'] = pairing.synch_count |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 539 | dargs['atomic_group_name'] = pairing.machine_label |
| 540 | else: |
| 541 | dargs['hosts'] = host_list |
mbligh | 38b0915 | 2009-04-28 18:34:25 +0000 | [diff] [blame] | 542 | new_job = self.create_job_by_test(name=job_name, |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 543 | dependencies=[pairing.machine_label], |
| 544 | tests=[pairing.control_file], |
| 545 | priority=priority, |
| 546 | kernel=kernel, |
Eric Li | e0493a4 | 2010-11-15 13:05:43 -0800 | [diff] [blame] | 547 | kernel_cmdline=kernel_cmdline, |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 548 | use_container=pairing.container, |
| 549 | **dargs) |
mbligh | 38b0915 | 2009-04-28 18:34:25 +0000 | [diff] [blame] | 550 | if new_job: |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 551 | if pairing.testname: |
| 552 | new_job.testname = pairing.testname |
mbligh | 4e57661 | 2008-12-22 14:56:36 +0000 | [diff] [blame] | 553 | print 'Invoked test %s : %s' % (new_job.id, job_name) |
mbligh | 38b0915 | 2009-04-28 18:34:25 +0000 | [diff] [blame] | 554 | return new_job |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 555 | |
| 556 | |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 557 | def _job_test_results(self, tko, job, debug, tests=[]): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 558 | """ |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 559 | Retrieve test results for a job |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 560 | """ |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 561 | job.test_status = {} |
| 562 | try: |
| 563 | test_statuses = tko.get_status_counts(job=job.id) |
| 564 | except Exception: |
| 565 | print "Ignoring exception on poll job; RPC interface is flaky" |
| 566 | traceback.print_exc() |
| 567 | return |
| 568 | |
| 569 | for test_status in test_statuses: |
mbligh | 7479a18 | 2009-01-07 16:46:24 +0000 | [diff] [blame] | 570 | # SERVER_JOB is buggy, and often gives false failures. Ignore it. |
| 571 | if test_status.test_name == 'SERVER_JOB': |
| 572 | continue |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 573 | # if tests is not empty, restrict list of test_statuses to tests |
| 574 | if tests and test_status.test_name not in tests: |
| 575 | continue |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 576 | if debug: |
| 577 | print test_status |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 578 | hostname = test_status.hostname |
| 579 | if hostname not in job.test_status: |
| 580 | job.test_status[hostname] = TestResults() |
| 581 | job.test_status[hostname].add(test_status) |
| 582 | |
| 583 | |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 584 | def _job_results_platform_map(self, job, debug): |
mbligh | c9e427e | 2009-04-28 18:35:06 +0000 | [diff] [blame] | 585 | # Figure out which hosts passed / failed / aborted in a job |
| 586 | # Creates a 2-dimensional hash, stored as job.results_platform_map |
| 587 | # 1st index - platform type (string) |
| 588 | # 2nd index - Status (string) |
| 589 | # 'Completed' / 'Failed' / 'Aborted' |
| 590 | # Data indexed by this hash is a list of hostnames (text strings) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 591 | job.results_platform_map = {} |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 592 | try: |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 593 | job_statuses = self.get_host_queue_entries(job=job.id) |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 594 | except Exception: |
| 595 | print "Ignoring exception on poll job; RPC interface is flaky" |
| 596 | traceback.print_exc() |
| 597 | return None |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 598 | |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 599 | platform_map = {} |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 600 | job.job_status = {} |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 601 | job.metahost_index = {} |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 602 | for job_status in job_statuses: |
mbligh | c9e427e | 2009-04-28 18:35:06 +0000 | [diff] [blame] | 603 | # This is basically "for each host / metahost in the job" |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 604 | if job_status.host: |
| 605 | hostname = job_status.host.hostname |
| 606 | else: # This is a metahost |
| 607 | metahost = job_status.meta_host |
| 608 | index = job.metahost_index.get(metahost, 1) |
| 609 | job.metahost_index[metahost] = index + 1 |
| 610 | hostname = '%s.%s' % (metahost, index) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 611 | job.job_status[hostname] = job_status.status |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 612 | status = job_status.status |
mbligh | 0ecbe63 | 2009-05-13 21:34:56 +0000 | [diff] [blame] | 613 | # Skip hosts that failed verify or repair: |
| 614 | # that's a machine failure, not a job failure |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 615 | if hostname in job.test_status: |
| 616 | verify_failed = False |
| 617 | for failure in job.test_status[hostname].fail: |
mbligh | 0ecbe63 | 2009-05-13 21:34:56 +0000 | [diff] [blame] | 618 | if (failure.test_name == 'verify' or |
| 619 | failure.test_name == 'repair'): |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 620 | verify_failed = True |
| 621 | break |
| 622 | if verify_failed: |
| 623 | continue |
mbligh | c9e427e | 2009-04-28 18:35:06 +0000 | [diff] [blame] | 624 | if hostname in job.test_status and job.test_status[hostname].fail: |
| 625 | # If the any tests failed in the job, we want to mark the |
| 626 | # job result as failed, overriding the default job status. |
| 627 | if status != "Aborted": # except if it's an aborted job |
| 628 | status = 'Failed' |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 629 | if job_status.host: |
| 630 | platform = job_status.host.platform |
| 631 | else: # This is a metahost |
| 632 | platform = job_status.meta_host |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 633 | if platform not in platform_map: |
| 634 | platform_map[platform] = {'Total' : [hostname]} |
| 635 | else: |
| 636 | platform_map[platform]['Total'].append(hostname) |
| 637 | new_host_list = platform_map[platform].get(status, []) + [hostname] |
| 638 | platform_map[platform][status] = new_host_list |
mbligh | 45ffc43 | 2008-12-09 23:35:17 +0000 | [diff] [blame] | 639 | job.results_platform_map = platform_map |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 640 | |
| 641 | |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 642 | def set_platform_results(self, test_job, platform, result): |
| 643 | """ |
| 644 | Result must be None, 'FAIL', 'WARN' or 'GOOD' |
| 645 | """ |
| 646 | if test_job.platform_results[platform] is not None: |
| 647 | # We're already done, and results recorded. This can't change later. |
| 648 | return |
| 649 | test_job.platform_results[platform] = result |
| 650 | # Note that self.job refers to the metajob we're IN, not the job |
| 651 | # that we're excuting from here. |
| 652 | testname = '%s.%s' % (test_job.testname, platform) |
| 653 | if self.job: |
| 654 | self.job.record(result, None, testname, status='') |
| 655 | |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 656 | def poll_job_results(self, tko, job, enough=1, debug=False): |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 657 | """ |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 658 | Analyse all job results by platform |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 659 | |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 660 | params: |
| 661 | tko: a TKO object representing the results DB. |
| 662 | job: the job to be examined. |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 663 | enough: the acceptable delta between the number of completed |
| 664 | tests and the total number of tests. |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 665 | debug: enable debugging output. |
| 666 | |
| 667 | returns: |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 668 | False: if any platform has more than |enough| failures |
| 669 | None: if any platform has less than |enough| machines |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 670 | not yet Good. |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 671 | True: if all platforms have at least |enough| machines |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 672 | Good. |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 673 | """ |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 674 | self._job_test_results(tko, job, debug) |
mbligh | e7fcf56 | 2009-05-21 01:43:17 +0000 | [diff] [blame] | 675 | if job.test_status == {}: |
| 676 | return None |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 677 | self._job_results_platform_map(job, debug) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 678 | |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 679 | good_platforms = [] |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 680 | failed_platforms = [] |
| 681 | aborted_platforms = [] |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 682 | unknown_platforms = [] |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 683 | platform_map = job.results_platform_map |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 684 | for platform in platform_map: |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 685 | if not job.platform_results.has_key(platform): |
| 686 | # record test start, but there's no way to do this right now |
| 687 | job.platform_results[platform] = None |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 688 | total = len(platform_map[platform]['Total']) |
| 689 | completed = len(platform_map[platform].get('Completed', [])) |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 690 | failed = len(platform_map[platform].get('Failed', [])) |
| 691 | aborted = len(platform_map[platform].get('Aborted', [])) |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 692 | |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 693 | # We set up what we want to record here, but don't actually do |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 694 | # it yet, until we have a decisive answer for this platform |
| 695 | if aborted or failed: |
| 696 | bad = aborted + failed |
| 697 | if (bad > 1) or (bad * 2 >= total): |
| 698 | platform_test_result = 'FAIL' |
| 699 | else: |
| 700 | platform_test_result = 'WARN' |
| 701 | |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 702 | if aborted > enough: |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 703 | aborted_platforms.append(platform) |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 704 | self.set_platform_results(job, platform, platform_test_result) |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 705 | elif (failed * 2 >= total) or (failed > enough): |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 706 | failed_platforms.append(platform) |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 707 | self.set_platform_results(job, platform, platform_test_result) |
Chris Masone | 6fed646 | 2011-10-20 16:36:43 -0700 | [diff] [blame] | 708 | elif (completed >= enough) and (completed + enough >= total): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 709 | good_platforms.append(platform) |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 710 | self.set_platform_results(job, platform, 'GOOD') |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 711 | else: |
| 712 | unknown_platforms.append(platform) |
| 713 | detail = [] |
| 714 | for status in platform_map[platform]: |
| 715 | if status == 'Total': |
| 716 | continue |
| 717 | detail.append('%s=%s' % (status,platform_map[platform][status])) |
| 718 | if debug: |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 719 | print '%20s %d/%d %s' % (platform, completed, total, |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 720 | ' '.join(detail)) |
| 721 | print |
mbligh | 1ef218d | 2009-08-03 16:57:56 +0000 | [diff] [blame] | 722 | |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 723 | if len(aborted_platforms) > 0: |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 724 | if debug: |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 725 | print 'Result aborted - platforms: ', |
| 726 | print ' '.join(aborted_platforms) |
mbligh | 912c3f3 | 2009-03-25 19:31:30 +0000 | [diff] [blame] | 727 | return "Abort" |
| 728 | if len(failed_platforms) > 0: |
| 729 | if debug: |
| 730 | print 'Result bad - platforms: ' + ' '.join(failed_platforms) |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 731 | return False |
| 732 | if len(unknown_platforms) > 0: |
| 733 | if debug: |
| 734 | platform_list = ' '.join(unknown_platforms) |
| 735 | print 'Result unknown - platforms: ', platform_list |
| 736 | return None |
| 737 | if debug: |
| 738 | platform_list = ' '.join(good_platforms) |
| 739 | print 'Result good - all platforms passed: ', platform_list |
| 740 | return True |
| 741 | |
| 742 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 743 | class TestResults(object): |
| 744 | """ |
| 745 | Container class used to hold the results of the tests for a job |
| 746 | """ |
| 747 | def __init__(self): |
| 748 | self.good = [] |
| 749 | self.fail = [] |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 750 | self.pending = [] |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 751 | |
| 752 | |
| 753 | def add(self, result): |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 754 | if result.complete_count > result.pass_count: |
| 755 | self.fail.append(result) |
| 756 | elif result.incomplete_count > 0: |
| 757 | self.pending.append(result) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 758 | else: |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 759 | self.good.append(result) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 760 | |
| 761 | |
| 762 | class RpcObject(object): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 763 | """ |
| 764 | Generic object used to construct python objects from rpc calls |
| 765 | """ |
| 766 | def __init__(self, afe, hash): |
| 767 | self.afe = afe |
| 768 | self.hash = hash |
| 769 | self.__dict__.update(hash) |
| 770 | |
| 771 | |
| 772 | def __str__(self): |
| 773 | return dump_object(self.__repr__(), self) |
| 774 | |
| 775 | |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 776 | class ControlFile(RpcObject): |
| 777 | """ |
| 778 | AFE control file object |
| 779 | |
| 780 | Fields: synch_count, dependencies, control_file, is_server |
| 781 | """ |
| 782 | def __repr__(self): |
| 783 | return 'CONTROL FILE: %s' % self.control_file |
| 784 | |
| 785 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 786 | class Label(RpcObject): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 787 | """ |
| 788 | AFE label object |
| 789 | |
| 790 | Fields: |
| 791 | name, invalid, platform, kernel_config, id, only_if_needed |
| 792 | """ |
| 793 | def __repr__(self): |
| 794 | return 'LABEL: %s' % self.name |
| 795 | |
| 796 | |
| 797 | def add_hosts(self, hosts): |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 798 | return self.afe.run('label_add_hosts', id=self.id, hosts=hosts) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 799 | |
| 800 | |
| 801 | def remove_hosts(self, hosts): |
Chris Masone | 3a560bd | 2011-11-14 16:53:56 -0800 | [diff] [blame] | 802 | return self.afe.run('label_remove_hosts', id=self.id, hosts=hosts) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 803 | |
| 804 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 805 | class Acl(RpcObject): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 806 | """ |
| 807 | AFE acl object |
| 808 | |
| 809 | Fields: |
| 810 | users, hosts, description, name, id |
| 811 | """ |
| 812 | def __repr__(self): |
| 813 | return 'ACL: %s' % self.name |
| 814 | |
| 815 | |
| 816 | def add_hosts(self, hosts): |
| 817 | self.afe.log('Adding hosts %s to ACL %s' % (hosts, self.name)) |
| 818 | return self.afe.run('acl_group_add_hosts', self.id, hosts) |
| 819 | |
| 820 | |
| 821 | def remove_hosts(self, hosts): |
| 822 | self.afe.log('Removing hosts %s from ACL %s' % (hosts, self.name)) |
| 823 | return self.afe.run('acl_group_remove_hosts', self.id, hosts) |
| 824 | |
| 825 | |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 826 | def add_users(self, users): |
| 827 | self.afe.log('Adding users %s to ACL %s' % (users, self.name)) |
| 828 | return self.afe.run('acl_group_add_users', id=self.name, users=users) |
| 829 | |
| 830 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 831 | class Job(RpcObject): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 832 | """ |
| 833 | AFE job object |
| 834 | |
| 835 | Fields: |
| 836 | name, control_file, control_type, synch_count, reboot_before, |
| 837 | run_verify, priority, email_list, created_on, dependencies, |
| 838 | timeout, owner, reboot_after, id |
| 839 | """ |
| 840 | def __repr__(self): |
| 841 | return 'JOB: %s' % self.id |
| 842 | |
| 843 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 844 | class JobStatus(RpcObject): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 845 | """ |
| 846 | AFE job_status object |
| 847 | |
| 848 | Fields: |
| 849 | status, complete, deleted, meta_host, host, active, execution_subdir, id |
| 850 | """ |
| 851 | def __init__(self, afe, hash): |
| 852 | # This should call super |
| 853 | self.afe = afe |
| 854 | self.hash = hash |
| 855 | self.__dict__.update(hash) |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 856 | self.job = Job(afe, self.job) |
Dale Curtis | 8adf789 | 2011-09-08 16:13:36 -0700 | [diff] [blame] | 857 | if getattr(self, 'host'): |
mbligh | 99b24f4 | 2009-06-08 16:45:55 +0000 | [diff] [blame] | 858 | self.host = Host(afe, self.host) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 859 | |
| 860 | |
| 861 | def __repr__(self): |
mbligh | 451ede1 | 2009-02-12 21:54:03 +0000 | [diff] [blame] | 862 | if self.host and self.host.hostname: |
| 863 | hostname = self.host.hostname |
| 864 | else: |
| 865 | hostname = 'None' |
| 866 | return 'JOB STATUS: %s-%s' % (self.job.id, hostname) |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 867 | |
| 868 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 869 | class Host(RpcObject): |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 870 | """ |
| 871 | AFE host object |
| 872 | |
| 873 | Fields: |
| 874 | status, lock_time, locked_by, locked, hostname, invalid, |
| 875 | synch_id, labels, platform, protection, dirty, id |
| 876 | """ |
| 877 | def __repr__(self): |
| 878 | return 'HOST OBJECT: %s' % self.hostname |
| 879 | |
| 880 | |
| 881 | def show(self): |
| 882 | labels = list(set(self.labels) - set([self.platform])) |
| 883 | print '%-6s %-7s %-7s %-16s %s' % (self.hostname, self.status, |
| 884 | self.locked, self.platform, |
| 885 | ', '.join(labels)) |
| 886 | |
| 887 | |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 888 | def delete(self): |
| 889 | return self.afe.run('delete_host', id=self.id) |
| 890 | |
| 891 | |
mbligh | 6463c4b | 2009-01-30 00:33:37 +0000 | [diff] [blame] | 892 | def modify(self, **dargs): |
| 893 | return self.afe.run('modify_host', id=self.id, **dargs) |
| 894 | |
| 895 | |
mbligh | 6764715 | 2008-11-19 00:18:14 +0000 | [diff] [blame] | 896 | def get_acls(self): |
| 897 | return self.afe.get_acls(hosts__hostname=self.hostname) |
| 898 | |
| 899 | |
| 900 | def add_acl(self, acl_name): |
| 901 | self.afe.log('Adding ACL %s to host %s' % (acl_name, self.hostname)) |
| 902 | return self.afe.run('acl_group_add_hosts', id=acl_name, |
| 903 | hosts=[self.hostname]) |
| 904 | |
| 905 | |
| 906 | def remove_acl(self, acl_name): |
| 907 | self.afe.log('Removing ACL %s from host %s' % (acl_name, self.hostname)) |
| 908 | return self.afe.run('acl_group_remove_hosts', id=acl_name, |
| 909 | hosts=[self.hostname]) |
| 910 | |
| 911 | |
| 912 | def get_labels(self): |
| 913 | return self.afe.get_labels(host__hostname__in=[self.hostname]) |
| 914 | |
| 915 | |
| 916 | def add_labels(self, labels): |
| 917 | self.afe.log('Adding labels %s to host %s' % (labels, self.hostname)) |
| 918 | return self.afe.run('host_add_labels', id=self.id, labels=labels) |
| 919 | |
| 920 | |
| 921 | def remove_labels(self, labels): |
| 922 | self.afe.log('Removing labels %s from host %s' % (labels,self.hostname)) |
| 923 | return self.afe.run('host_remove_labels', id=self.id, labels=labels) |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 924 | |
| 925 | |
mbligh | 54459c7 | 2009-01-21 19:26:44 +0000 | [diff] [blame] | 926 | class User(RpcObject): |
| 927 | def __repr__(self): |
| 928 | return 'USER: %s' % self.login |
| 929 | |
| 930 | |
mbligh | 5280e3b | 2008-12-22 14:39:28 +0000 | [diff] [blame] | 931 | class TestStatus(RpcObject): |
mbligh | c31e402 | 2008-12-11 19:32:30 +0000 | [diff] [blame] | 932 | """ |
| 933 | TKO test status object |
| 934 | |
| 935 | Fields: |
| 936 | test_idx, hostname, testname, id |
| 937 | complete_count, incomplete_count, group_count, pass_count |
| 938 | """ |
| 939 | def __repr__(self): |
| 940 | return 'TEST STATUS: %s' % self.id |
| 941 | |
| 942 | |
MK Ryu | acf3592 | 2014-10-03 14:56:49 -0700 | [diff] [blame] | 943 | class HostAttribute(RpcObject): |
| 944 | """ |
| 945 | AFE host attribute object |
| 946 | |
| 947 | Fields: |
| 948 | id, host, attribute, value |
| 949 | """ |
| 950 | def __repr__(self): |
| 951 | return 'HOST ATTRIBUTE %d' % self.id |
| 952 | |
| 953 | |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 954 | class MachineTestPairing(object): |
| 955 | """ |
| 956 | Object representing the pairing of a machine label with a control file |
mbligh | 1f23f36 | 2008-12-22 14:46:12 +0000 | [diff] [blame] | 957 | |
| 958 | machine_label: use machines from this label |
| 959 | control_file: use this control file (by name in the frontend) |
| 960 | platforms: list of rexeps to filter platforms by. [] => no filtering |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 961 | job_label: The label (name) to give to the autotest job launched |
| 962 | to run this pairing. '<kernel-version> : <config> : <date>' |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 963 | """ |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 964 | def __init__(self, machine_label, control_file, platforms=[], |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 965 | container=False, atomic_group_sched=False, synch_count=0, |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 966 | testname=None, job_label=None): |
mbligh | 5b61838 | 2008-12-03 15:24:01 +0000 | [diff] [blame] | 967 | self.machine_label = machine_label |
| 968 | self.control_file = control_file |
mbligh | 1f23f36 | 2008-12-22 14:46:12 +0000 | [diff] [blame] | 969 | self.platforms = platforms |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 970 | self.container = container |
mbligh | b9db516 | 2009-04-17 22:21:41 +0000 | [diff] [blame] | 971 | self.atomic_group_sched = atomic_group_sched |
| 972 | self.synch_count = synch_count |
mbligh | 17c75e6 | 2009-06-08 16:18:21 +0000 | [diff] [blame] | 973 | self.testname = testname |
mbligh | 282ce89 | 2010-01-06 18:40:17 +0000 | [diff] [blame] | 974 | self.job_label = job_label |
mbligh | 1354c9d | 2008-12-22 14:56:13 +0000 | [diff] [blame] | 975 | |
| 976 | |
| 977 | def __repr__(self): |
| 978 | return '%s %s %s %s' % (self.machine_label, self.control_file, |
| 979 | self.platforms, self.container) |