mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1 | # Copyright 2008 Google Inc. All Rights Reserved. |
| 2 | |
| 3 | """ |
| 4 | The host module contains the objects and method used to |
| 5 | manage a host in Autotest. |
| 6 | |
| 7 | The valid actions are: |
| 8 | create: adds host(s) |
| 9 | delete: deletes host(s) |
| 10 | list: lists host(s) |
| 11 | stat: displays host(s) information |
| 12 | mod: modifies host(s) |
| 13 | jobs: lists all jobs that ran on host(s) |
| 14 | |
| 15 | The common options are: |
| 16 | -M|--mlist: file containing a list of machines |
| 17 | |
| 18 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 19 | See topic_common.py for a High Level Design and Algorithm. |
| 20 | |
| 21 | """ |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 22 | import common |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 23 | import json |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 24 | import random |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 25 | import re |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 26 | import socket |
Gregory Nisbet | 93fa5b5 | 2019-07-19 14:56:35 -0700 | [diff] [blame] | 27 | import time |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 28 | |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 29 | from autotest_lib.cli import action_common, rpc, topic_common, skylab_utils, skylab_migration |
Gregory Nisbet | 93fa5b5 | 2019-07-19 14:56:35 -0700 | [diff] [blame] | 30 | from autotest_lib.cli import fair_partition |
Xixuan Wu | 06f7e23 | 2019-07-12 23:33:28 +0000 | [diff] [blame] | 31 | from autotest_lib.client.bin import utils as bin_utils |
Gregory Nisbet | 93fa5b5 | 2019-07-19 14:56:35 -0700 | [diff] [blame] | 32 | from autotest_lib.cli.skylab_json_utils import process_labels |
Gregory Nisbet | ac7f7f7 | 2019-09-10 16:25:20 -0700 | [diff] [blame^] | 33 | from autotest_lib.cli import skylab_rollback |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 34 | from autotest_lib.client.common_lib import error, host_protections |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 35 | from autotest_lib.server import frontend, hosts |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 36 | from autotest_lib.server.hosts import host_info |
Gregory Nisbet | 93fa5b5 | 2019-07-19 14:56:35 -0700 | [diff] [blame] | 37 | from autotest_lib.server.lib.status_history import HostJobHistory |
| 38 | from autotest_lib.server.lib.status_history import UNUSED, WORKING |
| 39 | from autotest_lib.server.lib.status_history import BROKEN, UNKNOWN |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 40 | |
| 41 | |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 42 | try: |
| 43 | from skylab_inventory import text_manager |
| 44 | from skylab_inventory.lib import device |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 45 | from skylab_inventory.lib import server as skylab_server |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 46 | except ImportError: |
| 47 | pass |
| 48 | |
| 49 | |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 50 | MIGRATED_HOST_SUFFIX = '-migrated-do-not-use' |
| 51 | |
| 52 | |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 53 | ID_AUTOGEN_MESSAGE = ("[IGNORED]. Do not edit (crbug.com/950553). ID is " |
| 54 | "auto-generated.") |
| 55 | |
| 56 | |
| 57 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 58 | class host(topic_common.atest): |
| 59 | """Host class |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 60 | atest host [create|delete|list|stat|mod|jobs|rename|migrate|skylab_migrate|statjson] <options>""" |
| 61 | usage_action = '[create|delete|list|stat|mod|jobs|rename|migrate|skylab_migrate|statjson]' |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 62 | topic = msg_topic = 'host' |
| 63 | msg_items = '<hosts>' |
| 64 | |
mbligh | aed47e8 | 2009-03-17 19:06:18 +0000 | [diff] [blame] | 65 | protections = host_protections.Protection.names |
| 66 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 67 | |
| 68 | def __init__(self): |
| 69 | """Add to the parser the options common to all the |
| 70 | host actions""" |
| 71 | super(host, self).__init__() |
| 72 | |
| 73 | self.parser.add_option('-M', '--mlist', |
| 74 | help='File listing the machines', |
| 75 | type='string', |
| 76 | default=None, |
| 77 | metavar='MACHINE_FLIST') |
| 78 | |
mbligh | 9deeefa | 2009-05-01 23:11:08 +0000 | [diff] [blame] | 79 | self.topic_parse_info = topic_common.item_parse_info( |
| 80 | attribute_name='hosts', |
| 81 | filename_option='mlist', |
| 82 | use_leftover=True) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 83 | |
| 84 | |
| 85 | def _parse_lock_options(self, options): |
| 86 | if options.lock and options.unlock: |
| 87 | self.invalid_syntax('Only specify one of ' |
| 88 | '--lock and --unlock.') |
| 89 | |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 90 | self.lock = options.lock |
| 91 | self.unlock = options.unlock |
| 92 | self.lock_reason = options.lock_reason |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 93 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 94 | if options.lock: |
| 95 | self.data['locked'] = True |
| 96 | self.messages.append('Locked host') |
| 97 | elif options.unlock: |
| 98 | self.data['locked'] = False |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 99 | self.data['lock_reason'] = '' |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 100 | self.messages.append('Unlocked host') |
| 101 | |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 102 | if options.lock and options.lock_reason: |
| 103 | self.data['lock_reason'] = options.lock_reason |
| 104 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 105 | |
| 106 | def _cleanup_labels(self, labels, platform=None): |
| 107 | """Removes the platform label from the overall labels""" |
| 108 | if platform: |
| 109 | return [label for label in labels |
| 110 | if label != platform] |
| 111 | else: |
| 112 | try: |
| 113 | return [label for label in labels |
| 114 | if not label['platform']] |
| 115 | except TypeError: |
| 116 | # This is a hack - the server will soon |
| 117 | # do this, so all this code should be removed. |
| 118 | return labels |
| 119 | |
| 120 | |
| 121 | def get_items(self): |
| 122 | return self.hosts |
| 123 | |
| 124 | |
| 125 | class host_help(host): |
| 126 | """Just here to get the atest logic working. |
| 127 | Usage is set by its parent""" |
| 128 | pass |
| 129 | |
| 130 | |
| 131 | class host_list(action_common.atest_list, host): |
| 132 | """atest host list [--mlist <file>|<hosts>] [--label <label>] |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 133 | [--status <status1,status2>] [--acl <ACL>] [--user <user>]""" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 134 | |
| 135 | def __init__(self): |
| 136 | super(host_list, self).__init__() |
| 137 | |
| 138 | self.parser.add_option('-b', '--label', |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 139 | default='', |
| 140 | help='Only list hosts with all these labels ' |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 141 | '(comma separated). When --skylab is provided, ' |
| 142 | 'a label must be in the format of ' |
| 143 | 'label-key:label-value (e.g., board:lumpy).') |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 144 | self.parser.add_option('-s', '--status', |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 145 | default='', |
| 146 | help='Only list hosts with any of these ' |
| 147 | 'statuses (comma separated)') |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 148 | self.parser.add_option('-a', '--acl', |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 149 | default='', |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 150 | help=('Only list hosts within this ACL. %s' % |
| 151 | skylab_utils.MSG_INVALID_IN_SKYLAB)) |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 152 | self.parser.add_option('-u', '--user', |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 153 | default='', |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 154 | help=('Only list hosts available to this user. ' |
| 155 | '%s' % skylab_utils.MSG_INVALID_IN_SKYLAB)) |
mbligh | 70b9bf4 | 2008-11-18 15:00:46 +0000 | [diff] [blame] | 156 | self.parser.add_option('-N', '--hostnames-only', help='Only return ' |
| 157 | 'hostnames for the machines queried.', |
| 158 | action='store_true') |
mbligh | 91e0efd | 2009-02-26 01:02:16 +0000 | [diff] [blame] | 159 | self.parser.add_option('--locked', |
| 160 | default=False, |
| 161 | help='Only list locked hosts', |
| 162 | action='store_true') |
| 163 | self.parser.add_option('--unlocked', |
| 164 | default=False, |
| 165 | help='Only list unlocked hosts', |
| 166 | action='store_true') |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 167 | self.parser.add_option('--full-output', |
| 168 | default=False, |
| 169 | help=('Print out the full content of the hosts. ' |
| 170 | 'Only supported with --skylab.'), |
| 171 | action='store_true', |
| 172 | dest='full_output') |
mbligh | 91e0efd | 2009-02-26 01:02:16 +0000 | [diff] [blame] | 173 | |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 174 | self.add_skylab_options() |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 175 | |
| 176 | |
| 177 | def parse(self): |
| 178 | """Consume the specific options""" |
jamesren | c286316 | 2010-07-12 21:20:51 +0000 | [diff] [blame] | 179 | label_info = topic_common.item_parse_info(attribute_name='labels', |
| 180 | inline_option='label') |
| 181 | |
| 182 | (options, leftover) = super(host_list, self).parse([label_info]) |
| 183 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 184 | self.status = options.status |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 185 | self.acl = options.acl |
| 186 | self.user = options.user |
mbligh | 70b9bf4 | 2008-11-18 15:00:46 +0000 | [diff] [blame] | 187 | self.hostnames_only = options.hostnames_only |
mbligh | 91e0efd | 2009-02-26 01:02:16 +0000 | [diff] [blame] | 188 | |
| 189 | if options.locked and options.unlocked: |
| 190 | self.invalid_syntax('--locked and --unlocked are ' |
| 191 | 'mutually exclusive') |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 192 | |
mbligh | 91e0efd | 2009-02-26 01:02:16 +0000 | [diff] [blame] | 193 | self.locked = options.locked |
| 194 | self.unlocked = options.unlocked |
Ningning Xia | 64ced00 | 2018-05-16 17:36:20 -0700 | [diff] [blame] | 195 | self.label_map = None |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 196 | |
| 197 | if self.skylab: |
| 198 | if options.user or options.acl or options.status: |
| 199 | self.invalid_syntax('--user, --acl or --status is not ' |
| 200 | 'supported with --skylab.') |
| 201 | self.full_output = options.full_output |
| 202 | if self.full_output and self.hostnames_only: |
| 203 | self.invalid_syntax('--full-output is conflicted with ' |
| 204 | '--hostnames-only.') |
Ningning Xia | 64ced00 | 2018-05-16 17:36:20 -0700 | [diff] [blame] | 205 | |
| 206 | if self.labels: |
| 207 | self.label_map = device.convert_to_label_map(self.labels) |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 208 | else: |
| 209 | if options.full_output: |
| 210 | self.invalid_syntax('--full_output is only supported with ' |
| 211 | '--skylab.') |
| 212 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 213 | return (options, leftover) |
| 214 | |
| 215 | |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 216 | def execute_skylab(self): |
| 217 | """Execute 'atest host list' with --skylab.""" |
| 218 | inventory_repo = skylab_utils.InventoryRepo(self.inventory_repo_dir) |
| 219 | inventory_repo.initialize() |
| 220 | lab = text_manager.load_lab(inventory_repo.get_data_dir()) |
| 221 | |
| 222 | # TODO(nxia): support filtering on run-time labels and status. |
| 223 | return device.get_devices( |
| 224 | lab, |
| 225 | 'duts', |
| 226 | self.environment, |
Ningning Xia | 64ced00 | 2018-05-16 17:36:20 -0700 | [diff] [blame] | 227 | label_map=self.label_map, |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 228 | hostnames=self.hosts, |
| 229 | locked=self.locked, |
| 230 | unlocked=self.unlocked) |
| 231 | |
| 232 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 233 | def execute(self): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 234 | """Execute 'atest host list'.""" |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 235 | if self.skylab: |
| 236 | return self.execute_skylab() |
| 237 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 238 | filters = {} |
| 239 | check_results = {} |
| 240 | if self.hosts: |
| 241 | filters['hostname__in'] = self.hosts |
| 242 | check_results['hostname__in'] = 'hostname' |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 243 | |
| 244 | if self.labels: |
jamesren | c286316 | 2010-07-12 21:20:51 +0000 | [diff] [blame] | 245 | if len(self.labels) == 1: |
mbligh | f703fb4 | 2009-01-30 00:35:05 +0000 | [diff] [blame] | 246 | # This is needed for labels with wildcards (x86*) |
jamesren | c286316 | 2010-07-12 21:20:51 +0000 | [diff] [blame] | 247 | filters['labels__name__in'] = self.labels |
mbligh | f703fb4 | 2009-01-30 00:35:05 +0000 | [diff] [blame] | 248 | check_results['labels__name__in'] = None |
| 249 | else: |
jamesren | c286316 | 2010-07-12 21:20:51 +0000 | [diff] [blame] | 250 | filters['multiple_labels'] = self.labels |
mbligh | f703fb4 | 2009-01-30 00:35:05 +0000 | [diff] [blame] | 251 | check_results['multiple_labels'] = None |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 252 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 253 | if self.status: |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 254 | statuses = self.status.split(',') |
| 255 | statuses = [status.strip() for status in statuses |
| 256 | if status.strip()] |
| 257 | |
| 258 | filters['status__in'] = statuses |
mbligh | cd8eb97 | 2008-08-25 19:20:39 +0000 | [diff] [blame] | 259 | check_results['status__in'] = None |
mbligh | 6444c6b | 2008-10-27 20:55:13 +0000 | [diff] [blame] | 260 | |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 261 | if self.acl: |
showard | d9ac445 | 2009-02-07 02:04:37 +0000 | [diff] [blame] | 262 | filters['aclgroup__name'] = self.acl |
| 263 | check_results['aclgroup__name'] = None |
mbligh | 536a524 | 2008-10-18 14:35:54 +0000 | [diff] [blame] | 264 | if self.user: |
showard | d9ac445 | 2009-02-07 02:04:37 +0000 | [diff] [blame] | 265 | filters['aclgroup__users__login'] = self.user |
| 266 | check_results['aclgroup__users__login'] = None |
mbligh | 91e0efd | 2009-02-26 01:02:16 +0000 | [diff] [blame] | 267 | |
| 268 | if self.locked or self.unlocked: |
| 269 | filters['locked'] = self.locked |
| 270 | check_results['locked'] = None |
| 271 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 272 | return super(host_list, self).execute(op='get_hosts', |
| 273 | filters=filters, |
| 274 | check_results=check_results) |
| 275 | |
| 276 | |
| 277 | def output(self, results): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 278 | """Print output of 'atest host list'. |
| 279 | |
| 280 | @param results: the results to be printed. |
| 281 | """ |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 282 | if results and not self.skylab: |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 283 | # Remove the platform from the labels. |
| 284 | for result in results: |
| 285 | result['labels'] = self._cleanup_labels(result['labels'], |
| 286 | result['platform']) |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 287 | if self.skylab and self.full_output: |
| 288 | print results |
| 289 | return |
| 290 | |
| 291 | if self.skylab: |
| 292 | results = device.convert_to_autotest_hosts(results) |
| 293 | |
mbligh | 70b9bf4 | 2008-11-18 15:00:46 +0000 | [diff] [blame] | 294 | if self.hostnames_only: |
mbligh | df75f8b | 2008-11-18 19:07:42 +0000 | [diff] [blame] | 295 | self.print_list(results, key='hostname') |
mbligh | 70b9bf4 | 2008-11-18 15:00:46 +0000 | [diff] [blame] | 296 | else: |
Ningning Xia | ea02ab1 | 2018-05-11 15:08:57 -0700 | [diff] [blame] | 297 | keys = ['hostname', 'status', 'shard', 'locked', 'lock_reason', |
| 298 | 'locked_by', 'platform', 'labels'] |
Prashanth Balasubramanian | a504856 | 2014-12-12 10:14:11 -0800 | [diff] [blame] | 299 | super(host_list, self).output(results, keys=keys) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 300 | |
| 301 | |
| 302 | class host_stat(host): |
| 303 | """atest host stat --mlist <file>|<hosts>""" |
| 304 | usage_action = 'stat' |
| 305 | |
| 306 | def execute(self): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 307 | """Execute 'atest host stat'.""" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 308 | results = [] |
| 309 | # Convert wildcards into real host stats. |
| 310 | existing_hosts = [] |
| 311 | for host in self.hosts: |
| 312 | if host.endswith('*'): |
| 313 | stats = self.execute_rpc('get_hosts', |
| 314 | hostname__startswith=host.rstrip('*')) |
| 315 | if len(stats) == 0: |
| 316 | self.failure('No hosts matching %s' % host, item=host, |
| 317 | what_failed='Failed to stat') |
| 318 | continue |
| 319 | else: |
| 320 | stats = self.execute_rpc('get_hosts', hostname=host) |
| 321 | if len(stats) == 0: |
| 322 | self.failure('Unknown host %s' % host, item=host, |
| 323 | what_failed='Failed to stat') |
| 324 | continue |
| 325 | existing_hosts.extend(stats) |
| 326 | |
| 327 | for stat in existing_hosts: |
| 328 | host = stat['hostname'] |
| 329 | # The host exists, these should succeed |
| 330 | acls = self.execute_rpc('get_acl_groups', hosts__hostname=host) |
| 331 | |
| 332 | labels = self.execute_rpc('get_labels', host__hostname=host) |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 333 | results.append([[stat], acls, labels, stat['attributes']]) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 334 | return results |
| 335 | |
| 336 | |
| 337 | def output(self, results): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 338 | """Print output of 'atest host stat'. |
| 339 | |
| 340 | @param results: the results to be printed. |
| 341 | """ |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 342 | for stats, acls, labels, attributes in results: |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 343 | print '-'*5 |
| 344 | self.print_fields(stats, |
Aviv Keshet | a40d127 | 2017-11-16 00:56:35 -0800 | [diff] [blame] | 345 | keys=['hostname', 'id', 'platform', |
mbligh | e163b03 | 2008-10-18 14:30:27 +0000 | [diff] [blame] | 346 | 'status', 'locked', 'locked_by', |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 347 | 'lock_time', 'lock_reason', 'protection',]) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 348 | self.print_by_ids(acls, 'ACLs', line_before=True) |
| 349 | labels = self._cleanup_labels(labels) |
| 350 | self.print_by_ids(labels, 'Labels', line_before=True) |
Simran Basi | 0739d68 | 2015-02-25 16:22:56 -0800 | [diff] [blame] | 351 | self.print_dict(attributes, 'Host Attributes', line_before=True) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 352 | |
| 353 | |
Gregory Nisbet | 93fa5b5 | 2019-07-19 14:56:35 -0700 | [diff] [blame] | 354 | class host_get_migration_plan(host_stat): |
| 355 | """atest host get_migration_plan --mlist <file>|<hosts>""" |
| 356 | usage_action = "get_migration_plan" |
| 357 | |
| 358 | def __init__(self): |
| 359 | super(host_get_migration_plan, self).__init__() |
| 360 | self.parser.add_option("--ratio", default=0.5, type=float, dest="ratio") |
| 361 | self.add_skylab_options() |
| 362 | |
| 363 | def parse(self): |
| 364 | (options, leftover) = super(host_get_migration_plan, self).parse() |
| 365 | self.ratio = options.ratio |
| 366 | return (options, leftover) |
| 367 | |
| 368 | def execute(self): |
| 369 | afe = frontend.AFE() |
| 370 | results = super(host_get_migration_plan, self).execute() |
| 371 | working = [] |
| 372 | non_working = [] |
| 373 | for stats, _, _, _ in results: |
| 374 | assert len(stats) == 1 |
| 375 | stats = stats[0] |
| 376 | hostname = stats["hostname"] |
| 377 | now = time.time() |
| 378 | history = HostJobHistory.get_host_history( |
| 379 | afe=afe, |
| 380 | hostname=hostname, |
| 381 | start_time=now, |
| 382 | end_time=now - 24 * 60 * 60, |
| 383 | ) |
| 384 | dut_status, _ = history.last_diagnosis() |
| 385 | if dut_status in [UNUSED, WORKING]: |
| 386 | working.append(hostname) |
| 387 | elif dut_status == BROKEN: |
| 388 | non_working.append(hostname) |
| 389 | elif dut_status == UNKNOWN: |
| 390 | # if it's unknown, randomly assign it to working or |
| 391 | # nonworking, since we don't know. |
| 392 | # The two choices aren't actually equiprobable, but it |
| 393 | # should be fine. |
| 394 | random.choice([working, non_working]).append(hostname) |
| 395 | else: |
| 396 | raise ValueError("unknown status %s" % dut_status) |
| 397 | working_transfer, working_retain = fair_partition.partition(working, self.ratio) |
| 398 | non_working_transfer, non_working_retain = \ |
| 399 | fair_partition.partition(non_working, self.ratio) |
| 400 | return { |
| 401 | "transfer": working_transfer + non_working_transfer, |
| 402 | "retain": working_retain + non_working_retain, |
| 403 | } |
| 404 | |
| 405 | def output(self, results): |
| 406 | print json.dumps(results, indent=4, sort_keys=True) |
| 407 | |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 408 | |
| 409 | class host_statjson(host_stat): |
| 410 | """atest host statjson --mlist <file>|<hosts> |
| 411 | |
| 412 | exposes the same information that 'atest host stat' does, but in the json |
| 413 | format that 'skylab add-dut' expects |
| 414 | """ |
| 415 | |
| 416 | usage_action = "statjson" |
| 417 | |
| 418 | |
| 419 | def output(self, results): |
| 420 | """Print output of 'atest host stat-skylab-json'""" |
| 421 | for row in results: |
| 422 | stats, acls, labels, attributes = row |
| 423 | # TODO(gregorynisbet): under what circumstances is stats |
| 424 | # not a list of length 1? |
| 425 | assert len(stats) == 1 |
| 426 | stats_map = stats[0] |
Gregory Nisbet | ce77cef | 2019-07-29 12:27:21 -0700 | [diff] [blame] | 427 | |
| 428 | # Stripping the MIGRATED_HOST_SUFFIX makes it possible to |
| 429 | # migrate a DUT from autotest to skylab even after its hostname |
| 430 | # has been changed. |
| 431 | # This enables the steps (renaming the host, |
| 432 | # copying the inventory information to skylab) to be doable in |
| 433 | # either order. |
Gregory Nisbet | 48f1eea | 2019-08-05 16:18:56 -0700 | [diff] [blame] | 434 | hostname = _remove_hostname_suffix_if_present( |
| 435 | stats_map["hostname"], |
| 436 | MIGRATED_HOST_SUFFIX |
| 437 | ) |
| 438 | |
Gregory Nisbet | 932f5a9 | 2019-09-05 14:02:31 -0700 | [diff] [blame] | 439 | # TODO(gregorynisbet): clean up servo information |
| 440 | if "servo_host" not in attributes: |
| 441 | attributes["servo_host"] = "dummy_host" |
| 442 | if "servo_port" not in attributes: |
| 443 | attributes["servo_port"] = "dummy_port" |
| 444 | |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 445 | labels = self._cleanup_labels(labels) |
| 446 | attrs = [{"key": k, "value": v} for k, v in attributes.iteritems()] |
| 447 | out_labels = process_labels(labels, platform=stats_map["platform"]) |
| 448 | skylab_json = { |
| 449 | "common": { |
| 450 | "attributes": attrs, |
Gregory Nisbet | 23a06b9 | 2019-07-11 16:44:43 -0700 | [diff] [blame] | 451 | "environment": "ENVIRONMENT_PROD", |
Gregory Nisbet | ce77cef | 2019-07-29 12:27:21 -0700 | [diff] [blame] | 452 | "hostname": hostname, |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 453 | "id": ID_AUTOGEN_MESSAGE, |
| 454 | "labels": out_labels, |
Gregory Nisbet | cc0941f | 2019-08-29 13:56:05 -0700 | [diff] [blame] | 455 | "serialNumber": attributes.get("serial_number", None), |
Gregory Nisbet | 99b6194 | 2019-07-08 09:43:06 -0700 | [diff] [blame] | 456 | } |
| 457 | } |
| 458 | print json.dumps(skylab_json, indent=4, sort_keys=True) |
| 459 | |
| 460 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 461 | class host_jobs(host): |
mbligh | 1494eca | 2008-08-13 21:24:22 +0000 | [diff] [blame] | 462 | """atest host jobs [--max-query] --mlist <file>|<hosts>""" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 463 | usage_action = 'jobs' |
| 464 | |
mbligh | 6996fe8 | 2008-08-13 00:32:27 +0000 | [diff] [blame] | 465 | def __init__(self): |
| 466 | super(host_jobs, self).__init__() |
| 467 | self.parser.add_option('-q', '--max-query', |
| 468 | help='Limits the number of results ' |
| 469 | '(20 by default)', |
| 470 | type='int', default=20) |
| 471 | |
| 472 | |
| 473 | def parse(self): |
| 474 | """Consume the specific options""" |
mbligh | 9deeefa | 2009-05-01 23:11:08 +0000 | [diff] [blame] | 475 | (options, leftover) = super(host_jobs, self).parse() |
mbligh | 6996fe8 | 2008-08-13 00:32:27 +0000 | [diff] [blame] | 476 | self.max_queries = options.max_query |
| 477 | return (options, leftover) |
| 478 | |
| 479 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 480 | def execute(self): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 481 | """Execute 'atest host jobs'.""" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 482 | results = [] |
| 483 | real_hosts = [] |
| 484 | for host in self.hosts: |
| 485 | if host.endswith('*'): |
| 486 | stats = self.execute_rpc('get_hosts', |
| 487 | hostname__startswith=host.rstrip('*')) |
| 488 | if len(stats) == 0: |
| 489 | self.failure('No host matching %s' % host, item=host, |
| 490 | what_failed='Failed to stat') |
| 491 | [real_hosts.append(stat['hostname']) for stat in stats] |
| 492 | else: |
| 493 | real_hosts.append(host) |
| 494 | |
| 495 | for host in real_hosts: |
| 496 | queue_entries = self.execute_rpc('get_host_queue_entries', |
mbligh | 6996fe8 | 2008-08-13 00:32:27 +0000 | [diff] [blame] | 497 | host__hostname=host, |
mbligh | 1494eca | 2008-08-13 21:24:22 +0000 | [diff] [blame] | 498 | query_limit=self.max_queries, |
showard | 6958c72 | 2009-09-23 20:03:16 +0000 | [diff] [blame] | 499 | sort_by=['-job__id']) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 500 | jobs = [] |
| 501 | for entry in queue_entries: |
| 502 | job = {'job_id': entry['job']['id'], |
| 503 | 'job_owner': entry['job']['owner'], |
| 504 | 'job_name': entry['job']['name'], |
| 505 | 'status': entry['status']} |
| 506 | jobs.append(job) |
| 507 | results.append((host, jobs)) |
| 508 | return results |
| 509 | |
| 510 | |
| 511 | def output(self, results): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 512 | """Print output of 'atest host jobs'. |
| 513 | |
| 514 | @param results: the results to be printed. |
| 515 | """ |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 516 | for host, jobs in results: |
| 517 | print '-'*5 |
| 518 | print 'Hostname: %s' % host |
| 519 | self.print_table(jobs, keys_header=['job_id', |
showard | be0d869 | 2009-08-20 23:42:44 +0000 | [diff] [blame] | 520 | 'job_owner', |
| 521 | 'job_name', |
| 522 | 'status']) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 523 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 524 | class BaseHostModCreate(host): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 525 | """The base class for host_mod and host_create""" |
Justin Giorgi | 9261f9f | 2016-07-11 17:48:52 -0700 | [diff] [blame] | 526 | # Matches one attribute=value pair |
| 527 | attribute_regex = r'(?P<attribute>\w+)=(?P<value>.+)?' |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 528 | |
| 529 | def __init__(self): |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 530 | """Add the options shared between host mod and host create actions.""" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 531 | self.messages = [] |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 532 | self.host_ids = {} |
| 533 | super(BaseHostModCreate, self).__init__() |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 534 | self.parser.add_option('-l', '--lock', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 535 | help='Lock hosts.', |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 536 | action='store_true') |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 537 | self.parser.add_option('-r', '--lock_reason', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 538 | help='Reason for locking hosts.', |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 539 | default='') |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 540 | self.parser.add_option('-u', '--unlock', |
| 541 | help='Unlock hosts.', |
| 542 | action='store_true') |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 543 | |
mbligh | e163b03 | 2008-10-18 14:30:27 +0000 | [diff] [blame] | 544 | self.parser.add_option('-p', '--protection', type='choice', |
mbligh | aed47e8 | 2009-03-17 19:06:18 +0000 | [diff] [blame] | 545 | help=('Set the protection level on a host. ' |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 546 | 'Must be one of: %s. %s' % |
| 547 | (', '.join('"%s"' % p |
| 548 | for p in self.protections), |
| 549 | skylab_utils.MSG_INVALID_IN_SKYLAB)), |
mbligh | aed47e8 | 2009-03-17 19:06:18 +0000 | [diff] [blame] | 550 | choices=self.protections) |
Justin Giorgi | 9261f9f | 2016-07-11 17:48:52 -0700 | [diff] [blame] | 551 | self._attributes = [] |
| 552 | self.parser.add_option('--attribute', '-i', |
| 553 | help=('Host attribute to add or change. Format ' |
| 554 | 'is <attribute>=<value>. Multiple ' |
| 555 | 'attributes can be set by passing the ' |
| 556 | 'argument multiple times. Attributes can ' |
| 557 | 'be unset by providing an empty value.'), |
| 558 | action='append') |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 559 | self.parser.add_option('-b', '--labels', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 560 | help=('Comma separated list of labels. ' |
| 561 | 'When --skylab is provided, a label must ' |
| 562 | 'be in the format of label-key:label-value' |
| 563 | ' (e.g., board:lumpy).')) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 564 | self.parser.add_option('-B', '--blist', |
| 565 | help='File listing the labels', |
| 566 | type='string', |
| 567 | metavar='LABEL_FLIST') |
| 568 | self.parser.add_option('-a', '--acls', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 569 | help=('Comma separated list of ACLs. %s' % |
| 570 | skylab_utils.MSG_INVALID_IN_SKYLAB)) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 571 | self.parser.add_option('-A', '--alist', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 572 | help=('File listing the acls. %s' % |
| 573 | skylab_utils.MSG_INVALID_IN_SKYLAB), |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 574 | type='string', |
| 575 | metavar='ACL_FLIST') |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 576 | self.parser.add_option('-t', '--platform', |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 577 | help=('Sets the platform label. %s Please set ' |
| 578 | 'platform in labels (e.g., -b ' |
| 579 | 'platform:platform_name) with --skylab.' % |
| 580 | skylab_utils.MSG_INVALID_IN_SKYLAB)) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 581 | |
| 582 | |
| 583 | def parse(self): |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 584 | """Consume the options common to host create and host mod. |
| 585 | """ |
mbligh | 9deeefa | 2009-05-01 23:11:08 +0000 | [diff] [blame] | 586 | label_info = topic_common.item_parse_info(attribute_name='labels', |
| 587 | inline_option='labels', |
| 588 | filename_option='blist') |
| 589 | acl_info = topic_common.item_parse_info(attribute_name='acls', |
| 590 | inline_option='acls', |
| 591 | filename_option='alist') |
| 592 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 593 | (options, leftover) = super(BaseHostModCreate, self).parse([label_info, |
mbligh | 9deeefa | 2009-05-01 23:11:08 +0000 | [diff] [blame] | 594 | acl_info], |
| 595 | req_items='hosts') |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 596 | |
| 597 | self._parse_lock_options(options) |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 598 | |
Ningning Xia | 64ced00 | 2018-05-16 17:36:20 -0700 | [diff] [blame] | 599 | self.label_map = None |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 600 | if self.allow_skylab and self.skylab: |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 601 | # TODO(nxia): drop these flags when all hosts are migrated to skylab |
Ningning Xia | 64ced00 | 2018-05-16 17:36:20 -0700 | [diff] [blame] | 602 | if (options.protection or options.acls or options.alist or |
| 603 | options.platform): |
| 604 | self.invalid_syntax( |
| 605 | '--protection, --acls, --alist or --platform is not ' |
| 606 | 'supported with --skylab.') |
| 607 | |
| 608 | if self.labels: |
| 609 | self.label_map = device.convert_to_label_map(self.labels) |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 610 | |
mbligh | aed47e8 | 2009-03-17 19:06:18 +0000 | [diff] [blame] | 611 | if options.protection: |
| 612 | self.data['protection'] = options.protection |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 613 | self.messages.append('Protection set to "%s"' % options.protection) |
| 614 | |
| 615 | self.attributes = {} |
Justin Giorgi | 9261f9f | 2016-07-11 17:48:52 -0700 | [diff] [blame] | 616 | if options.attribute: |
| 617 | for pair in options.attribute: |
| 618 | m = re.match(self.attribute_regex, pair) |
| 619 | if not m: |
| 620 | raise topic_common.CliError('Attribute must be in key=value ' |
| 621 | 'syntax.') |
| 622 | elif m.group('attribute') in self.attributes: |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 623 | raise topic_common.CliError( |
| 624 | 'Multiple values provided for attribute ' |
| 625 | '%s.' % m.group('attribute')) |
Justin Giorgi | 9261f9f | 2016-07-11 17:48:52 -0700 | [diff] [blame] | 626 | self.attributes[m.group('attribute')] = m.group('value') |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 627 | |
| 628 | self.platform = options.platform |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 629 | return (options, leftover) |
| 630 | |
| 631 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 632 | def _set_acls(self, hosts, acls): |
| 633 | """Add hosts to acls (and remove from all other acls). |
| 634 | |
| 635 | @param hosts: list of hostnames |
| 636 | @param acls: list of acl names |
| 637 | """ |
| 638 | # Remove from all ACLs except 'Everyone' and ACLs in list |
| 639 | # Skip hosts that don't exist |
| 640 | for host in hosts: |
| 641 | if host not in self.host_ids: |
| 642 | continue |
| 643 | host_id = self.host_ids[host] |
| 644 | for a in self.execute_rpc('get_acl_groups', hosts=host_id): |
| 645 | if a['name'] not in self.acls and a['id'] != 1: |
| 646 | self.execute_rpc('acl_group_remove_hosts', id=a['id'], |
| 647 | hosts=self.hosts) |
| 648 | |
| 649 | # Add hosts to the ACLs |
| 650 | self.check_and_create_items('get_acl_groups', 'add_acl_group', |
| 651 | self.acls) |
| 652 | for a in acls: |
| 653 | self.execute_rpc('acl_group_add_hosts', id=a, hosts=hosts) |
| 654 | |
| 655 | |
| 656 | def _remove_labels(self, host, condition): |
| 657 | """Remove all labels from host that meet condition(label). |
| 658 | |
| 659 | @param host: hostname |
| 660 | @param condition: callable that returns bool when given a label |
| 661 | """ |
| 662 | if host in self.host_ids: |
| 663 | host_id = self.host_ids[host] |
| 664 | labels_to_remove = [] |
| 665 | for l in self.execute_rpc('get_labels', host=host_id): |
| 666 | if condition(l): |
| 667 | labels_to_remove.append(l['id']) |
| 668 | if labels_to_remove: |
| 669 | self.execute_rpc('host_remove_labels', id=host_id, |
| 670 | labels=labels_to_remove) |
| 671 | |
| 672 | |
| 673 | def _set_labels(self, host, labels): |
| 674 | """Apply labels to host (and remove all other labels). |
| 675 | |
| 676 | @param host: hostname |
| 677 | @param labels: list of label names |
| 678 | """ |
| 679 | condition = lambda l: l['name'] not in labels and not l['platform'] |
| 680 | self._remove_labels(host, condition) |
| 681 | self.check_and_create_items('get_labels', 'add_label', labels) |
| 682 | self.execute_rpc('host_add_labels', id=host, labels=labels) |
| 683 | |
| 684 | |
| 685 | def _set_platform_label(self, host, platform_label): |
| 686 | """Apply the platform label to host (and remove existing). |
| 687 | |
| 688 | @param host: hostname |
| 689 | @param platform_label: platform label's name |
| 690 | """ |
| 691 | self._remove_labels(host, lambda l: l['platform']) |
| 692 | self.check_and_create_items('get_labels', 'add_label', [platform_label], |
| 693 | platform=True) |
| 694 | self.execute_rpc('host_add_labels', id=host, labels=[platform_label]) |
| 695 | |
| 696 | |
| 697 | def _set_attributes(self, host, attributes): |
| 698 | """Set attributes on host. |
| 699 | |
| 700 | @param host: hostname |
| 701 | @param attributes: attribute dictionary |
| 702 | """ |
| 703 | for attr, value in self.attributes.iteritems(): |
| 704 | self.execute_rpc('set_host_attribute', attribute=attr, |
| 705 | value=value, hostname=host) |
| 706 | |
| 707 | |
| 708 | class host_mod(BaseHostModCreate): |
| 709 | """atest host mod [--lock|--unlock --force_modify_locking |
| 710 | --platform <arch> |
| 711 | --labels <labels>|--blist <label_file> |
| 712 | --acls <acls>|--alist <acl_file> |
| 713 | --protection <protection_type> |
| 714 | --attributes <attr>=<value>;<attr>=<value> |
| 715 | --mlist <mach_file>] <hosts>""" |
| 716 | usage_action = 'mod' |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 717 | |
| 718 | def __init__(self): |
| 719 | """Add the options specific to the mod action""" |
| 720 | super(host_mod, self).__init__() |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 721 | self.parser.add_option('--unlock-lock-id', |
| 722 | help=('Unlock the lock with the lock-id. %s' % |
| 723 | skylab_utils.MSG_ONLY_VALID_IN_SKYLAB), |
| 724 | default=None) |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 725 | self.parser.add_option('-f', '--force_modify_locking', |
| 726 | help='Forcefully lock\unlock a host', |
| 727 | action='store_true') |
| 728 | self.parser.add_option('--remove_acls', |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 729 | help=('Remove all active acls. %s' % |
| 730 | skylab_utils.MSG_INVALID_IN_SKYLAB), |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 731 | action='store_true') |
| 732 | self.parser.add_option('--remove_labels', |
| 733 | help='Remove all labels.', |
| 734 | action='store_true') |
| 735 | |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 736 | self.add_skylab_options() |
Ningning Xia | 1dd82ee | 2018-06-08 11:41:03 -0700 | [diff] [blame] | 737 | self.parser.add_option('--new-env', |
| 738 | dest='new_env', |
| 739 | choices=['staging', 'prod'], |
| 740 | help=('The new environment ("staging" or ' |
| 741 | '"prod") of the hosts. %s' % |
| 742 | skylab_utils.MSG_ONLY_VALID_IN_SKYLAB), |
| 743 | default=None) |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 744 | |
| 745 | |
| 746 | def _parse_unlock_options(self, options): |
| 747 | """Parse unlock related options.""" |
| 748 | if self.skylab and options.unlock and options.unlock_lock_id is None: |
| 749 | self.invalid_syntax('Must provide --unlock-lock-id with "--skylab ' |
| 750 | '--unlock".') |
| 751 | |
| 752 | if (not (self.skylab and options.unlock) and |
| 753 | options.unlock_lock_id is not None): |
| 754 | self.invalid_syntax('--unlock-lock-id is only valid with ' |
| 755 | '"--skylab --unlock".') |
| 756 | |
| 757 | self.unlock_lock_id = options.unlock_lock_id |
| 758 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 759 | |
| 760 | def parse(self): |
| 761 | """Consume the specific options""" |
| 762 | (options, leftover) = super(host_mod, self).parse() |
| 763 | |
Ningning Xia | 5c0e8f3 | 2018-05-21 16:26:50 -0700 | [diff] [blame] | 764 | self._parse_unlock_options(options) |
| 765 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 766 | if options.force_modify_locking: |
| 767 | self.data['force_modify_locking'] = True |
| 768 | |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 769 | if self.skylab and options.remove_acls: |
| 770 | # TODO(nxia): drop the flag when all hosts are migrated to skylab |
| 771 | self.invalid_syntax('--remove_acls is not supported with --skylab.') |
| 772 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 773 | self.remove_acls = options.remove_acls |
| 774 | self.remove_labels = options.remove_labels |
Ningning Xia | 1dd82ee | 2018-06-08 11:41:03 -0700 | [diff] [blame] | 775 | self.new_env = options.new_env |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 776 | |
| 777 | return (options, leftover) |
| 778 | |
| 779 | |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 780 | def execute_skylab(self): |
| 781 | """Execute atest host mod with --skylab. |
| 782 | |
| 783 | @return A list of hostnames which have been successfully modified. |
| 784 | """ |
| 785 | inventory_repo = skylab_utils.InventoryRepo(self.inventory_repo_dir) |
| 786 | inventory_repo.initialize() |
| 787 | data_dir = inventory_repo.get_data_dir() |
| 788 | lab = text_manager.load_lab(data_dir) |
| 789 | |
| 790 | locked_by = None |
| 791 | if self.lock: |
| 792 | locked_by = inventory_repo.git_repo.config('user.email') |
| 793 | |
| 794 | successes = [] |
| 795 | for hostname in self.hosts: |
| 796 | try: |
| 797 | device.modify( |
| 798 | lab, |
| 799 | 'duts', |
| 800 | hostname, |
| 801 | self.environment, |
| 802 | lock=self.lock, |
| 803 | locked_by=locked_by, |
| 804 | lock_reason = self.lock_reason, |
| 805 | unlock=self.unlock, |
| 806 | unlock_lock_id=self.unlock_lock_id, |
| 807 | attributes=self.attributes, |
| 808 | remove_labels=self.remove_labels, |
Ningning Xia | 1dd82ee | 2018-06-08 11:41:03 -0700 | [diff] [blame] | 809 | label_map=self.label_map, |
| 810 | new_env=self.new_env) |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 811 | successes.append(hostname) |
| 812 | except device.SkylabDeviceActionError as e: |
| 813 | print('Cannot modify host %s: %s' % (hostname, e)) |
| 814 | |
| 815 | if successes: |
| 816 | text_manager.dump_lab(data_dir, lab) |
| 817 | |
| 818 | status = inventory_repo.git_repo.status() |
| 819 | if not status: |
| 820 | print('Nothing is changed for hosts %s.' % successes) |
| 821 | return [] |
| 822 | |
| 823 | message = skylab_utils.construct_commit_message( |
| 824 | 'Modify %d hosts.\n\n%s' % (len(successes), successes)) |
| 825 | self.change_number = inventory_repo.upload_change( |
| 826 | message, draft=self.draft, dryrun=self.dryrun, |
| 827 | submit=self.submit) |
| 828 | |
| 829 | return successes |
| 830 | |
| 831 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 832 | def execute(self): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 833 | """Execute 'atest host mod'.""" |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 834 | if self.skylab: |
| 835 | return self.execute_skylab() |
| 836 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 837 | successes = [] |
| 838 | for host in self.execute_rpc('get_hosts', hostname__in=self.hosts): |
| 839 | self.host_ids[host['hostname']] = host['id'] |
| 840 | for host in self.hosts: |
| 841 | if host not in self.host_ids: |
| 842 | self.failure('Cannot modify non-existant host %s.' % host) |
| 843 | continue |
| 844 | host_id = self.host_ids[host] |
| 845 | |
| 846 | try: |
| 847 | if self.data: |
| 848 | self.execute_rpc('modify_host', item=host, |
| 849 | id=host, **self.data) |
| 850 | |
| 851 | if self.attributes: |
| 852 | self._set_attributes(host, self.attributes) |
| 853 | |
| 854 | if self.labels or self.remove_labels: |
| 855 | self._set_labels(host, self.labels) |
| 856 | |
| 857 | if self.platform: |
| 858 | self._set_platform_label(host, self.platform) |
| 859 | |
| 860 | # TODO: Make the AFE return True or False, |
| 861 | # especially for lock |
| 862 | successes.append(host) |
| 863 | except topic_common.CliError, full_error: |
| 864 | # Already logged by execute_rpc() |
| 865 | pass |
| 866 | |
| 867 | if self.acls or self.remove_acls: |
| 868 | self._set_acls(self.hosts, self.acls) |
| 869 | |
| 870 | return successes |
| 871 | |
| 872 | |
| 873 | def output(self, hosts): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 874 | """Print output of 'atest host mod'. |
| 875 | |
| 876 | @param hosts: the host list to be printed. |
| 877 | """ |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 878 | for msg in self.messages: |
| 879 | self.print_wrapped(msg, hosts) |
| 880 | |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 881 | if hosts and self.skylab: |
Ningning Xia | c8b430d | 2018-05-17 15:10:25 -0700 | [diff] [blame] | 882 | print('Modified hosts: %s.' % ', '.join(hosts)) |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 883 | if self.skylab and not self.dryrun and not self.submit: |
Ningning Xia | c8b430d | 2018-05-17 15:10:25 -0700 | [diff] [blame] | 884 | print(skylab_utils.get_cl_message(self.change_number)) |
Ningning Xia | ef35cb5 | 2018-05-04 17:58:20 -0700 | [diff] [blame] | 885 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 886 | |
| 887 | class HostInfo(object): |
| 888 | """Store host information so we don't have to keep looking it up.""" |
| 889 | def __init__(self, hostname, platform, labels): |
| 890 | self.hostname = hostname |
| 891 | self.platform = platform |
| 892 | self.labels = labels |
| 893 | |
| 894 | |
| 895 | class host_create(BaseHostModCreate): |
| 896 | """atest host create [--lock|--unlock --platform <arch> |
| 897 | --labels <labels>|--blist <label_file> |
| 898 | --acls <acls>|--alist <acl_file> |
| 899 | --protection <protection_type> |
| 900 | --attributes <attr>=<value>;<attr>=<value> |
| 901 | --mlist <mach_file>] <hosts>""" |
| 902 | usage_action = 'create' |
| 903 | |
| 904 | def parse(self): |
| 905 | """Option logic specific to create action. |
| 906 | """ |
| 907 | (options, leftovers) = super(host_create, self).parse() |
| 908 | self.locked = options.lock |
| 909 | if 'serials' in self.attributes: |
| 910 | if len(self.hosts) > 1: |
| 911 | raise topic_common.CliError('Can not specify serials with ' |
| 912 | 'multiple hosts.') |
| 913 | |
| 914 | |
| 915 | @classmethod |
| 916 | def construct_without_parse( |
| 917 | cls, web_server, hosts, platform=None, |
| 918 | locked=False, lock_reason='', labels=[], acls=[], |
| 919 | protection=host_protections.Protection.NO_PROTECTION): |
| 920 | """Construct a host_create object and fill in data from args. |
| 921 | |
| 922 | Do not need to call parse after the construction. |
| 923 | |
| 924 | Return an object of site_host_create ready to execute. |
| 925 | |
| 926 | @param web_server: A string specifies the autotest webserver url. |
| 927 | It is needed to setup comm to make rpc. |
| 928 | @param hosts: A list of hostnames as strings. |
| 929 | @param platform: A string or None. |
| 930 | @param locked: A boolean. |
| 931 | @param lock_reason: A string. |
| 932 | @param labels: A list of labels as strings. |
| 933 | @param acls: A list of acls as strings. |
| 934 | @param protection: An enum defined in host_protections. |
| 935 | """ |
| 936 | obj = cls() |
| 937 | obj.web_server = web_server |
| 938 | try: |
| 939 | # Setup stuff needed for afe comm. |
| 940 | obj.afe = rpc.afe_comm(web_server) |
| 941 | except rpc.AuthError, s: |
| 942 | obj.failure(str(s), fatal=True) |
| 943 | obj.hosts = hosts |
| 944 | obj.platform = platform |
| 945 | obj.locked = locked |
| 946 | if locked and lock_reason.strip(): |
| 947 | obj.data['lock_reason'] = lock_reason.strip() |
| 948 | obj.labels = labels |
| 949 | obj.acls = acls |
| 950 | if protection: |
| 951 | obj.data['protection'] = protection |
| 952 | obj.attributes = {} |
| 953 | return obj |
| 954 | |
| 955 | |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 956 | def _detect_host_info(self, host): |
| 957 | """Detect platform and labels from the host. |
| 958 | |
| 959 | @param host: hostname |
| 960 | |
| 961 | @return: HostInfo object |
| 962 | """ |
| 963 | # Mock an afe_host object so that the host is constructed as if the |
| 964 | # data was already in afe |
| 965 | data = {'attributes': self.attributes, 'labels': self.labels} |
| 966 | afe_host = frontend.Host(None, data) |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 967 | store = host_info.InMemoryHostInfoStore( |
| 968 | host_info.HostInfo(labels=self.labels, |
| 969 | attributes=self.attributes)) |
| 970 | machine = { |
| 971 | 'hostname': host, |
| 972 | 'afe_host': afe_host, |
| 973 | 'host_info_store': store |
| 974 | } |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 975 | try: |
Allen Li | 2c32d6b | 2017-02-03 15:28:10 -0800 | [diff] [blame] | 976 | if bin_utils.ping(host, tries=1, deadline=1) == 0: |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 977 | serials = self.attributes.get('serials', '').split(',') |
Richard Barnette | 9db8068 | 2018-04-26 00:55:15 +0000 | [diff] [blame] | 978 | adb_serial = self.attributes.get('serials') |
| 979 | host_dut = hosts.create_host(machine, |
| 980 | adb_serial=adb_serial) |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 981 | |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 982 | info = HostInfo(host, host_dut.get_platform(), |
| 983 | host_dut.get_labels()) |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 984 | # Clean host to make sure nothing left after calling it, |
| 985 | # e.g. tunnels. |
| 986 | if hasattr(host_dut, 'close'): |
| 987 | host_dut.close() |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 988 | else: |
| 989 | # Can't ping the host, use default information. |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 990 | info = HostInfo(host, None, []) |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 991 | except (socket.gaierror, error.AutoservRunError, |
| 992 | error.AutoservSSHTimeout): |
| 993 | # We may be adding a host that does not exist yet or we can't |
| 994 | # reach due to hostname/address issues or if the host is down. |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 995 | info = HostInfo(host, None, []) |
| 996 | return info |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 997 | |
| 998 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 999 | def _execute_add_one_host(self, host): |
mbligh | 719e14a | 2008-12-04 01:17:08 +0000 | [diff] [blame] | 1000 | # Always add the hosts as locked to avoid the host |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 1001 | # being picked up by the scheduler before it's ACL'ed. |
mbligh | 719e14a | 2008-12-04 01:17:08 +0000 | [diff] [blame] | 1002 | self.data['locked'] = True |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 1003 | if not self.locked: |
| 1004 | self.data['lock_reason'] = 'Forced lock on device creation' |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1005 | self.execute_rpc('add_host', hostname=host, status="Ready", **self.data) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1006 | |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1007 | # If there are labels avaliable for host, use them. |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 1008 | info = self._detect_host_info(host) |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1009 | labels = set(self.labels) |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 1010 | if info.labels: |
| 1011 | labels.update(info.labels) |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1012 | |
| 1013 | if labels: |
| 1014 | self._set_labels(host, list(labels)) |
| 1015 | |
| 1016 | # Now add the platform label. |
| 1017 | # If a platform was not provided and we were able to retrieve it |
| 1018 | # from the host, use the retrieved platform. |
Prathmesh Prabhu | d252d26 | 2017-05-31 11:46:34 -0700 | [diff] [blame] | 1019 | platform = self.platform if self.platform else info.platform |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1020 | if platform: |
| 1021 | self._set_platform_label(host, platform) |
| 1022 | |
| 1023 | if self.attributes: |
| 1024 | self._set_attributes(host, self.attributes) |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1025 | |
| 1026 | |
Justin Giorgi | 5208eaa | 2016-07-02 20:12:12 -0700 | [diff] [blame] | 1027 | def execute(self): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 1028 | """Execute 'atest host create'.""" |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1029 | successful_hosts = [] |
| 1030 | for host in self.hosts: |
| 1031 | try: |
| 1032 | self._execute_add_one_host(host) |
| 1033 | successful_hosts.append(host) |
| 1034 | except topic_common.CliError: |
| 1035 | pass |
Jiaxi Luo | c342f9f | 2014-05-19 16:22:03 -0700 | [diff] [blame] | 1036 | |
| 1037 | if successful_hosts: |
Justin Giorgi | 16bba56 | 2016-06-22 10:42:13 -0700 | [diff] [blame] | 1038 | self._set_acls(successful_hosts, self.acls) |
Jiaxi Luo | c342f9f | 2014-05-19 16:22:03 -0700 | [diff] [blame] | 1039 | |
| 1040 | if not self.locked: |
| 1041 | for host in successful_hosts: |
Matthew Sartori | 6818633 | 2015-04-27 17:19:53 -0700 | [diff] [blame] | 1042 | self.execute_rpc('modify_host', id=host, locked=False, |
| 1043 | lock_reason='') |
Jiaxi Luo | c342f9f | 2014-05-19 16:22:03 -0700 | [diff] [blame] | 1044 | return successful_hosts |
| 1045 | |
| 1046 | |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1047 | def output(self, hosts): |
xixuan | d6011f1 | 2016-12-08 15:01:58 -0800 | [diff] [blame] | 1048 | """Print output of 'atest host create'. |
| 1049 | |
| 1050 | @param hosts: the added host list to be printed. |
| 1051 | """ |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1052 | self.print_wrapped('Added host', hosts) |
| 1053 | |
| 1054 | |
| 1055 | class host_delete(action_common.atest_delete, host): |
| 1056 | """atest host delete [--mlist <mach_file>] <hosts>""" |
Ningning Xia | c8b430d | 2018-05-17 15:10:25 -0700 | [diff] [blame] | 1057 | |
| 1058 | def __init__(self): |
| 1059 | super(host_delete, self).__init__() |
| 1060 | |
| 1061 | self.add_skylab_options() |
| 1062 | |
| 1063 | |
| 1064 | def execute_skylab(self): |
| 1065 | """Execute 'atest host delete' with '--skylab'. |
| 1066 | |
| 1067 | @return A list of hostnames which have been successfully deleted. |
| 1068 | """ |
| 1069 | inventory_repo = skylab_utils.InventoryRepo(self.inventory_repo_dir) |
| 1070 | inventory_repo.initialize() |
| 1071 | data_dir = inventory_repo.get_data_dir() |
| 1072 | lab = text_manager.load_lab(data_dir) |
| 1073 | |
| 1074 | successes = [] |
| 1075 | for hostname in self.hosts: |
| 1076 | try: |
| 1077 | device.delete( |
| 1078 | lab, |
| 1079 | 'duts', |
| 1080 | hostname, |
| 1081 | self.environment) |
| 1082 | successes.append(hostname) |
| 1083 | except device.SkylabDeviceActionError as e: |
| 1084 | print('Cannot delete host %s: %s' % (hostname, e)) |
| 1085 | |
| 1086 | if successes: |
| 1087 | text_manager.dump_lab(data_dir, lab) |
| 1088 | message = skylab_utils.construct_commit_message( |
| 1089 | 'Delete %d hosts.\n\n%s' % (len(successes), successes)) |
| 1090 | self.change_number = inventory_repo.upload_change( |
| 1091 | message, draft=self.draft, dryrun=self.dryrun, |
| 1092 | submit=self.submit) |
| 1093 | |
| 1094 | return successes |
| 1095 | |
| 1096 | |
| 1097 | def execute(self): |
| 1098 | """Execute 'atest host delete'. |
| 1099 | |
| 1100 | @return A list of hostnames which have been successfully deleted. |
| 1101 | """ |
| 1102 | if self.skylab: |
| 1103 | return self.execute_skylab() |
| 1104 | |
| 1105 | return super(host_delete, self).execute() |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1106 | |
| 1107 | |
| 1108 | class InvalidHostnameError(Exception): |
| 1109 | """Cannot perform actions on the host because of invalid hostname.""" |
| 1110 | |
| 1111 | |
| 1112 | def _add_hostname_suffix(hostname, suffix): |
| 1113 | """Add the suffix to the hostname.""" |
| 1114 | if hostname.endswith(suffix): |
| 1115 | raise InvalidHostnameError( |
| 1116 | 'Cannot add "%s" as it already contains the suffix.' % suffix) |
| 1117 | |
| 1118 | return hostname + suffix |
| 1119 | |
| 1120 | |
Gregory Nisbet | 48f1eea | 2019-08-05 16:18:56 -0700 | [diff] [blame] | 1121 | def _remove_hostname_suffix_if_present(hostname, suffix): |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1122 | """Remove the suffix from the hostname.""" |
Gregory Nisbet | 48f1eea | 2019-08-05 16:18:56 -0700 | [diff] [blame] | 1123 | if hostname.endswith(suffix): |
| 1124 | return hostname[:len(hostname) - len(suffix)] |
| 1125 | else: |
| 1126 | return hostname |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1127 | |
| 1128 | |
| 1129 | class host_rename(host): |
| 1130 | """Host rename is only for migrating hosts between skylab and AFE DB.""" |
| 1131 | |
| 1132 | usage_action = 'rename' |
| 1133 | |
| 1134 | def __init__(self): |
| 1135 | """Add the options specific to the rename action.""" |
| 1136 | super(host_rename, self).__init__() |
| 1137 | |
| 1138 | self.parser.add_option('--for-migration', |
| 1139 | help=('Rename hostnames for migration. Rename ' |
| 1140 | 'each "hostname" to "hostname%s". ' |
| 1141 | 'The original "hostname" must not contain ' |
| 1142 | 'suffix.' % MIGRATED_HOST_SUFFIX), |
| 1143 | action='store_true', |
| 1144 | default=False) |
| 1145 | self.parser.add_option('--for-rollback', |
| 1146 | help=('Rename hostnames for migration rollback. ' |
| 1147 | 'Rename each "hostname%s" to its original ' |
| 1148 | '"hostname".' % MIGRATED_HOST_SUFFIX), |
| 1149 | action='store_true', |
| 1150 | default=False) |
| 1151 | self.parser.add_option('--dryrun', |
| 1152 | help='Execute the action as a dryrun.', |
| 1153 | action='store_true', |
| 1154 | default=False) |
Gregory Nisbet | ee457f6 | 2019-07-08 15:47:26 -0700 | [diff] [blame] | 1155 | self.parser.add_option('--non-interactive', |
| 1156 | help='run non-interactively', |
Gregory Nisbet | 9744ea9 | 2019-07-02 12:36:59 -0700 | [diff] [blame] | 1157 | action='store_true', |
| 1158 | default=False) |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1159 | |
| 1160 | |
| 1161 | def parse(self): |
| 1162 | """Consume the options common to host rename.""" |
| 1163 | (options, leftovers) = super(host_rename, self).parse() |
| 1164 | self.for_migration = options.for_migration |
| 1165 | self.for_rollback = options.for_rollback |
| 1166 | self.dryrun = options.dryrun |
Gregory Nisbet | ee457f6 | 2019-07-08 15:47:26 -0700 | [diff] [blame] | 1167 | self.interactive = not options.non_interactive |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1168 | self.host_ids = {} |
| 1169 | |
| 1170 | if not (self.for_migration ^ self.for_rollback): |
| 1171 | self.invalid_syntax('--for-migration and --for-rollback are ' |
| 1172 | 'exclusive, and one of them must be enabled.') |
| 1173 | |
| 1174 | if not self.hosts: |
| 1175 | self.invalid_syntax('Must provide hostname(s).') |
| 1176 | |
| 1177 | if self.dryrun: |
| 1178 | print('This will be a dryrun and will not rename hostnames.') |
| 1179 | |
| 1180 | return (options, leftovers) |
| 1181 | |
| 1182 | |
| 1183 | def execute(self): |
| 1184 | """Execute 'atest host rename'.""" |
Gregory Nisbet | ee457f6 | 2019-07-08 15:47:26 -0700 | [diff] [blame] | 1185 | if self.interactive: |
| 1186 | if self.prompt_confirmation(): |
| 1187 | pass |
| 1188 | else: |
| 1189 | return |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1190 | |
| 1191 | successes = [] |
| 1192 | for host in self.execute_rpc('get_hosts', hostname__in=self.hosts): |
| 1193 | self.host_ids[host['hostname']] = host['id'] |
| 1194 | for host in self.hosts: |
| 1195 | if host not in self.host_ids: |
| 1196 | self.failure('Cannot rename non-existant host %s.' % host, |
| 1197 | item=host, what_failed='Failed to rename') |
| 1198 | continue |
| 1199 | try: |
Ningning Xia | b261b16 | 2018-06-07 17:24:59 -0700 | [diff] [blame] | 1200 | host_id = self.host_ids[host] |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1201 | if self.for_migration: |
| 1202 | new_hostname = _add_hostname_suffix( |
| 1203 | host, MIGRATED_HOST_SUFFIX) |
| 1204 | else: |
| 1205 | #for_rollback |
Gregory Nisbet | 917ee33 | 2019-09-05 11:41:23 -0700 | [diff] [blame] | 1206 | new_hostname = _remove_hostname_suffix_if_present( |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1207 | host, MIGRATED_HOST_SUFFIX) |
| 1208 | |
| 1209 | if not self.dryrun: |
Ningning Xia | 423c796 | 2018-06-07 15:27:18 -0700 | [diff] [blame] | 1210 | # TODO(crbug.com/850737): delete and abort HQE. |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1211 | data = {'hostname': new_hostname} |
Ningning Xia | b261b16 | 2018-06-07 17:24:59 -0700 | [diff] [blame] | 1212 | self.execute_rpc('modify_host', item=host, id=host_id, |
| 1213 | **data) |
Ningning Xia | c46bdd1 | 2018-05-29 11:24:14 -0700 | [diff] [blame] | 1214 | successes.append((host, new_hostname)) |
| 1215 | except InvalidHostnameError as e: |
| 1216 | self.failure('Cannot rename host %s: %s' % (host, e), item=host, |
| 1217 | what_failed='Failed to rename') |
| 1218 | except topic_common.CliError, full_error: |
| 1219 | # Already logged by execute_rpc() |
| 1220 | pass |
| 1221 | |
| 1222 | return successes |
| 1223 | |
| 1224 | |
| 1225 | def output(self, results): |
| 1226 | """Print output of 'atest host rename'.""" |
| 1227 | if results: |
| 1228 | print('Successfully renamed:') |
| 1229 | for old_hostname, new_hostname in results: |
| 1230 | print('%s to %s' % (old_hostname, new_hostname)) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1231 | |
| 1232 | |
| 1233 | class host_migrate(action_common.atest_list, host): |
| 1234 | """'atest host migrate' to migrate or rollback hosts.""" |
| 1235 | |
| 1236 | usage_action = 'migrate' |
| 1237 | |
| 1238 | def __init__(self): |
| 1239 | super(host_migrate, self).__init__() |
| 1240 | |
| 1241 | self.parser.add_option('--migration', |
| 1242 | dest='migration', |
| 1243 | help='Migrate the hosts to skylab.', |
| 1244 | action='store_true', |
| 1245 | default=False) |
| 1246 | self.parser.add_option('--rollback', |
| 1247 | dest='rollback', |
| 1248 | help='Rollback the hosts migrated to skylab.', |
| 1249 | action='store_true', |
| 1250 | default=False) |
| 1251 | self.parser.add_option('--model', |
| 1252 | help='Model of the hosts to migrate.', |
| 1253 | dest='model', |
| 1254 | default=None) |
Xixuan Wu | 84621e5 | 2018-08-28 14:29:53 -0700 | [diff] [blame] | 1255 | self.parser.add_option('--board', |
| 1256 | help='Board of the hosts to migrate.', |
| 1257 | dest='board', |
| 1258 | default=None) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1259 | self.parser.add_option('--pool', |
| 1260 | help=('Pool of the hosts to migrate. Must ' |
| 1261 | 'specify --model for the pool.'), |
| 1262 | dest='pool', |
| 1263 | default=None) |
| 1264 | |
| 1265 | self.add_skylab_options(enforce_skylab=True) |
| 1266 | |
| 1267 | |
| 1268 | def parse(self): |
| 1269 | """Consume the specific options""" |
| 1270 | (options, leftover) = super(host_migrate, self).parse() |
| 1271 | |
| 1272 | self.migration = options.migration |
| 1273 | self.rollback = options.rollback |
| 1274 | self.model = options.model |
| 1275 | self.pool = options.pool |
Xixuan Wu | 84621e5 | 2018-08-28 14:29:53 -0700 | [diff] [blame] | 1276 | self.board = options.board |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1277 | self.host_ids = {} |
| 1278 | |
| 1279 | if not (self.migration ^ self.rollback): |
| 1280 | self.invalid_syntax('--migration and --rollback are exclusive, ' |
| 1281 | 'and one of them must be enabled.') |
| 1282 | |
Xixuan Wu | 84621e5 | 2018-08-28 14:29:53 -0700 | [diff] [blame] | 1283 | if self.pool is not None and (self.model is None and |
| 1284 | self.board is None): |
| 1285 | self.invalid_syntax('Must provide --model or --board with --pool.') |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1286 | |
Xixuan Wu | 84621e5 | 2018-08-28 14:29:53 -0700 | [diff] [blame] | 1287 | if not self.hosts and not (self.model or self.board): |
| 1288 | self.invalid_syntax('Must provide hosts or --model or --board.') |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1289 | |
| 1290 | return (options, leftover) |
| 1291 | |
| 1292 | |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1293 | def _remove_invalid_hostnames(self, hostnames, log_failure=False): |
| 1294 | """Remove hostnames with MIGRATED_HOST_SUFFIX. |
| 1295 | |
| 1296 | @param hostnames: A list of hostnames. |
| 1297 | @param log_failure: Bool indicating whether to log invalid hostsnames. |
| 1298 | |
| 1299 | @return A list of valid hostnames. |
| 1300 | """ |
| 1301 | invalid_hostnames = set() |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1302 | for hostname in hostnames: |
| 1303 | if hostname.endswith(MIGRATED_HOST_SUFFIX): |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1304 | if log_failure: |
| 1305 | self.failure('Cannot migrate host with suffix "%s" %s.' % |
| 1306 | (MIGRATED_HOST_SUFFIX, hostname), |
| 1307 | item=hostname, what_failed='Failed to rename') |
| 1308 | invalid_hostnames.add(hostname) |
| 1309 | |
| 1310 | hostnames = list(set(hostnames) - invalid_hostnames) |
| 1311 | |
| 1312 | return hostnames |
| 1313 | |
| 1314 | |
| 1315 | def execute(self): |
| 1316 | """Execute 'atest host migrate'.""" |
| 1317 | hostnames = self._remove_invalid_hostnames(self.hosts, log_failure=True) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1318 | |
| 1319 | filters = {} |
| 1320 | check_results = {} |
| 1321 | if hostnames: |
| 1322 | check_results['hostname__in'] = 'hostname' |
| 1323 | if self.migration: |
| 1324 | filters['hostname__in'] = hostnames |
| 1325 | else: |
| 1326 | # rollback |
| 1327 | hostnames_with_suffix = [ |
| 1328 | _add_hostname_suffix(h, MIGRATED_HOST_SUFFIX) |
| 1329 | for h in hostnames] |
| 1330 | filters['hostname__in'] = hostnames_with_suffix |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1331 | else: |
| 1332 | # TODO(nxia): add exclude_filter {'hostname__endswith': |
| 1333 | # MIGRATED_HOST_SUFFIX} for --migration |
| 1334 | if self.rollback: |
| 1335 | filters['hostname__endswith'] = MIGRATED_HOST_SUFFIX |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1336 | |
| 1337 | labels = [] |
| 1338 | if self.model: |
| 1339 | labels.append('model:%s' % self.model) |
| 1340 | if self.pool: |
| 1341 | labels.append('pool:%s' % self.pool) |
Xixuan Wu | 84621e5 | 2018-08-28 14:29:53 -0700 | [diff] [blame] | 1342 | if self.board: |
| 1343 | labels.append('board:%s' % self.board) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1344 | |
| 1345 | if labels: |
| 1346 | if len(labels) == 1: |
| 1347 | filters['labels__name__in'] = labels |
| 1348 | check_results['labels__name__in'] = None |
| 1349 | else: |
| 1350 | filters['multiple_labels'] = labels |
| 1351 | check_results['multiple_labels'] = None |
| 1352 | |
| 1353 | results = super(host_migrate, self).execute( |
| 1354 | op='get_hosts', filters=filters, check_results=check_results) |
| 1355 | hostnames = [h['hostname'] for h in results] |
| 1356 | |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1357 | if self.migration: |
| 1358 | hostnames = self._remove_invalid_hostnames(hostnames) |
| 1359 | else: |
| 1360 | # rollback |
| 1361 | hostnames = [_remove_hostname_suffix(h, MIGRATED_HOST_SUFFIX) |
| 1362 | for h in hostnames] |
| 1363 | |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1364 | return self.execute_skylab_migration(hostnames) |
| 1365 | |
| 1366 | |
Xixuan Wu | a8d93c8 | 2018-08-03 16:23:26 -0700 | [diff] [blame] | 1367 | def assign_duts_to_drone(self, infra, devices, environment): |
Ningning Xia | 877817f | 2018-06-08 12:23:48 -0700 | [diff] [blame] | 1368 | """Assign uids of the devices to a random skylab drone. |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1369 | |
| 1370 | @param infra: An instance of lab_pb2.Infrastructure. |
Xixuan Wu | a8d93c8 | 2018-08-03 16:23:26 -0700 | [diff] [blame] | 1371 | @param devices: A list of device_pb2.Device to be assigned to the drone. |
| 1372 | @param environment: 'staging' or 'prod'. |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1373 | """ |
| 1374 | skylab_drones = skylab_server.get_servers( |
Xixuan Wu | a8d93c8 | 2018-08-03 16:23:26 -0700 | [diff] [blame] | 1375 | infra, environment, role='skylab_drone', status='primary') |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1376 | |
| 1377 | if len(skylab_drones) == 0: |
| 1378 | raise device.SkylabDeviceActionError( |
| 1379 | 'No skylab drone is found in primary status and staging ' |
| 1380 | 'environment. Please confirm there is at least one valid skylab' |
| 1381 | ' drone added in skylab inventory.') |
| 1382 | |
Xixuan Wu | dfc2de2 | 2018-08-06 09:29:01 -0700 | [diff] [blame] | 1383 | for device in devices: |
| 1384 | # Randomly distribute each device to a skylab_drone. |
| 1385 | skylab_drone = random.choice(skylab_drones) |
| 1386 | skylab_server.add_dut_uids(skylab_drone, [device]) |
Ningning Xia | 877817f | 2018-06-08 12:23:48 -0700 | [diff] [blame] | 1387 | |
| 1388 | |
| 1389 | def remove_duts_from_drone(self, infra, devices): |
| 1390 | """Remove uids of the devices from their skylab drones. |
| 1391 | |
| 1392 | @param infra: An instance of lab_pb2.Infrastructure. |
| 1393 | @devices: A list of device_pb2.Device to be remove from the drone. |
| 1394 | """ |
| 1395 | skylab_drones = skylab_server.get_servers( |
| 1396 | infra, 'staging', role='skylab_drone', status='primary') |
| 1397 | |
| 1398 | for skylab_drone in skylab_drones: |
| 1399 | skylab_server.remove_dut_uids(skylab_drone, devices) |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1400 | |
| 1401 | |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1402 | def execute_skylab_migration(self, hostnames): |
| 1403 | """Execute migration in skylab_inventory. |
| 1404 | |
| 1405 | @param hostnames: A list of hostnames to migrate. |
| 1406 | @return If there're hosts to migrate, return a list of the hostnames and |
| 1407 | a message instructing actions after the migration; else return |
| 1408 | None. |
| 1409 | """ |
| 1410 | if not hostnames: |
| 1411 | return |
| 1412 | |
| 1413 | inventory_repo = skylab_utils.InventoryRepo(self.inventory_repo_dir) |
| 1414 | inventory_repo.initialize() |
| 1415 | |
| 1416 | subdirs = ['skylab', 'prod', 'staging'] |
| 1417 | data_dirs = skylab_data_dir, prod_data_dir, staging_data_dir = [ |
| 1418 | inventory_repo.get_data_dir(data_subdir=d) for d in subdirs] |
| 1419 | skylab_lab, prod_lab, staging_lab = [ |
| 1420 | text_manager.load_lab(d) for d in data_dirs] |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1421 | infra = text_manager.load_infrastructure(skylab_data_dir) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1422 | |
| 1423 | label_map = None |
| 1424 | labels = [] |
Xixuan Wu | 46f8e20 | 2018-12-13 14:40:58 -0800 | [diff] [blame] | 1425 | if self.board: |
| 1426 | labels.append('board:%s' % self.board) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1427 | if self.model: |
Xixuan Wu | 46f8e20 | 2018-12-13 14:40:58 -0800 | [diff] [blame] | 1428 | labels.append('model:%s' % self.model) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1429 | if self.pool: |
| 1430 | labels.append('critical_pool:%s' % self.pool) |
| 1431 | if labels: |
| 1432 | label_map = device.convert_to_label_map(labels) |
| 1433 | |
| 1434 | if self.migration: |
| 1435 | prod_devices = device.move_devices( |
| 1436 | prod_lab, skylab_lab, 'duts', label_map=label_map, |
| 1437 | hostnames=hostnames) |
| 1438 | staging_devices = device.move_devices( |
| 1439 | staging_lab, skylab_lab, 'duts', label_map=label_map, |
| 1440 | hostnames=hostnames) |
| 1441 | |
| 1442 | all_devices = prod_devices + staging_devices |
| 1443 | # Hostnames in afe_hosts tabel. |
| 1444 | device_hostnames = [str(d.common.hostname) for d in all_devices] |
| 1445 | message = ( |
| 1446 | 'Migration: move %s hosts into skylab_inventory.\n\n' |
Ningning Xia | 423c796 | 2018-06-07 15:27:18 -0700 | [diff] [blame] | 1447 | 'Please run this command after the CL is submitted:\n' |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1448 | 'atest host rename --for-migration %s' % |
| 1449 | (len(all_devices), ' '.join(device_hostnames))) |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1450 | |
Xixuan Wu | a8d93c8 | 2018-08-03 16:23:26 -0700 | [diff] [blame] | 1451 | self.assign_duts_to_drone(infra, prod_devices, 'prod') |
| 1452 | self.assign_duts_to_drone(infra, staging_devices, 'staging') |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1453 | else: |
| 1454 | # rollback |
| 1455 | prod_devices = device.move_devices( |
| 1456 | skylab_lab, prod_lab, 'duts', environment='prod', |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1457 | label_map=label_map, hostnames=hostnames) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1458 | staging_devices = device.move_devices( |
Ningning Xia | 877817f | 2018-06-08 12:23:48 -0700 | [diff] [blame] | 1459 | skylab_lab, staging_lab, 'duts', environment='staging', |
Ningning Xia | 56d6843 | 2018-06-06 17:28:20 -0700 | [diff] [blame] | 1460 | label_map=label_map, hostnames=hostnames) |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1461 | |
| 1462 | all_devices = prod_devices + staging_devices |
| 1463 | # Hostnames in afe_hosts tabel. |
| 1464 | device_hostnames = [_add_hostname_suffix(str(d.common.hostname), |
| 1465 | MIGRATED_HOST_SUFFIX) |
| 1466 | for d in all_devices] |
| 1467 | message = ( |
| 1468 | 'Rollback: remove %s hosts from skylab_inventory.\n\n' |
Ningning Xia | 423c796 | 2018-06-07 15:27:18 -0700 | [diff] [blame] | 1469 | 'Please run this command after the CL is submitted:\n' |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1470 | 'atest host rename --for-rollback %s' % |
| 1471 | (len(all_devices), ' '.join(device_hostnames))) |
| 1472 | |
Ningning Xia | 877817f | 2018-06-08 12:23:48 -0700 | [diff] [blame] | 1473 | self.remove_duts_from_drone(infra, all_devices) |
| 1474 | |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1475 | if all_devices: |
Ningning Xia | ffb3c1f | 2018-06-07 18:42:32 -0700 | [diff] [blame] | 1476 | text_manager.dump_infrastructure(skylab_data_dir, infra) |
| 1477 | |
Ningning Xia | 9df3d15 | 2018-05-23 17:15:14 -0700 | [diff] [blame] | 1478 | if prod_devices: |
| 1479 | text_manager.dump_lab(prod_data_dir, prod_lab) |
| 1480 | |
| 1481 | if staging_devices: |
| 1482 | text_manager.dump_lab(staging_data_dir, staging_lab) |
| 1483 | |
| 1484 | text_manager.dump_lab(skylab_data_dir, skylab_lab) |
| 1485 | |
| 1486 | self.change_number = inventory_repo.upload_change( |
| 1487 | message, draft=self.draft, dryrun=self.dryrun, |
| 1488 | submit=self.submit) |
| 1489 | |
| 1490 | return all_devices, message |
| 1491 | |
| 1492 | |
| 1493 | def output(self, result): |
| 1494 | """Print output of 'atest host list'. |
| 1495 | |
| 1496 | @param result: the result to be printed. |
| 1497 | """ |
| 1498 | if result: |
| 1499 | devices, message = result |
| 1500 | |
| 1501 | if devices: |
| 1502 | hostnames = [h.common.hostname for h in devices] |
| 1503 | if self.migration: |
| 1504 | print('Migrating hosts: %s' % ','.join(hostnames)) |
| 1505 | else: |
| 1506 | # rollback |
| 1507 | print('Rolling back hosts: %s' % ','.join(hostnames)) |
| 1508 | |
| 1509 | if not self.dryrun: |
| 1510 | if not self.submit: |
| 1511 | print(skylab_utils.get_cl_message(self.change_number)) |
| 1512 | else: |
| 1513 | # Print the instruction command for renaming hosts. |
| 1514 | print('%s' % message) |
| 1515 | else: |
| 1516 | print('No hosts were migrated.') |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1517 | |
| 1518 | |
Gregory Nisbet | 8235023 | 2019-09-10 11:12:47 -0700 | [diff] [blame] | 1519 | |
| 1520 | def _host_skylab_migrate_get_hostnames(obj, class_, model=None, pool=None, board=None): |
| 1521 | """ |
| 1522 | @params : in 'model', 'pool', 'board' |
| 1523 | |
| 1524 | """ |
| 1525 | # TODO(gregorynisbet) |
| 1526 | # this just gets all the hostnames, it doesn't filter by |
| 1527 | # presence or absence of migrated-do-not-use. |
| 1528 | labels = [] |
| 1529 | for key, value in ({'model': model, 'board': board, 'pool': pool}).items(): |
| 1530 | if value: |
| 1531 | labels.append(key + ":" + value) |
| 1532 | filters = {} |
| 1533 | check_results = {} |
| 1534 | # Copy the filter and check_results initialization logic from |
| 1535 | # the 'execute' method of the class 'host_migrate'. |
| 1536 | if not labels: |
| 1537 | return [] |
| 1538 | elif len(labels) == 1: |
| 1539 | filters['labels__name__in'] = labels |
| 1540 | check_results['labels__name__in'] = None |
| 1541 | elif len(labels) > 1: |
| 1542 | filters['multiple_labels'] = labels |
| 1543 | check_results['multiple_labels'] = None |
| 1544 | else: |
| 1545 | assert False |
| 1546 | |
| 1547 | results = super(class_, obj).execute( |
| 1548 | op='get_hosts', filters=filters, check_results=check_results) |
| 1549 | return [result['hostname'] for result in results] |
| 1550 | |
| 1551 | |
| 1552 | |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1553 | class host_skylab_migrate(action_common.atest_list, host): |
| 1554 | usage_action = 'skylab_migrate' |
| 1555 | |
| 1556 | def __init__(self): |
| 1557 | super(host_skylab_migrate, self).__init__() |
| 1558 | self.parser.add_option('--dry-run', |
| 1559 | help='Dry run. Show only candidate hosts.', |
| 1560 | action='store_true', |
| 1561 | dest='dry_run') |
| 1562 | self.parser.add_option('--ratio', |
| 1563 | help='ratio of hosts to migrate as number from 0 to 1.', |
| 1564 | type=float, |
| 1565 | dest='ratio', |
| 1566 | default=1) |
| 1567 | self.parser.add_option('--bug-number', |
| 1568 | help='bug number for tracking purposes.', |
| 1569 | dest='bug_number', |
| 1570 | default=None) |
| 1571 | self.parser.add_option('--board', |
| 1572 | help='Board of the hosts to migrate', |
| 1573 | dest='board', |
| 1574 | default=None) |
| 1575 | self.parser.add_option('--model', |
| 1576 | help='Model of the hosts to migrate', |
| 1577 | dest='model', |
| 1578 | default=None) |
| 1579 | self.parser.add_option('--pool', |
| 1580 | help='Pool of the hosts to migrate', |
| 1581 | dest='pool', |
| 1582 | default=None) |
Gregory Nisbet | 917ee33 | 2019-09-05 11:41:23 -0700 | [diff] [blame] | 1583 | # TODO(gregorynisbet): remove this flag and make quick-add-duts default. |
| 1584 | self.parser.add_option('-q', |
| 1585 | '--use-quick-add', |
| 1586 | help='whether to use "skylab quick-add-duts"', |
| 1587 | dest='use_quick_add', |
| 1588 | action='store_true') |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1589 | def parse(self): |
| 1590 | (options, leftover) = super(host_skylab_migrate, self).parse() |
| 1591 | self.dry_run = options.dry_run |
| 1592 | self.ratio = options.ratio |
| 1593 | self.bug_number = options.bug_number |
| 1594 | self.model = options.model |
| 1595 | self.pool = options.pool |
| 1596 | self.board = options.board |
| 1597 | self._reason = "migration to skylab: %s" % self.bug_number |
Gregory Nisbet | 917ee33 | 2019-09-05 11:41:23 -0700 | [diff] [blame] | 1598 | self.use_quick_add = options.use_quick_add |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1599 | return (options, leftover) |
| 1600 | |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1601 | def _validate_one_hostname_source(self): |
| 1602 | """Validate that hostname source is explicit hostnames or valid query. |
| 1603 | |
| 1604 | Hostnames must either be provided explicitly or be the result of a |
| 1605 | query defined by 'model', 'board', and 'pool'. |
| 1606 | |
| 1607 | @returns : whether the hostnames come from exactly one valid source. |
| 1608 | """ |
| 1609 | has_criteria = any([(self.model and self.board), self.board, self.pool]) |
| 1610 | has_command_line_hosts = bool(self.hosts) |
| 1611 | if has_criteria != has_command_line_hosts: |
| 1612 | # all good, one data source |
| 1613 | return True |
| 1614 | if has_criteria and has_command_line_hosts: |
| 1615 | self.failure( |
| 1616 | '--model/host/board and explicit hostnames are alternatives. Provide exactly one.', |
| 1617 | item='cli', |
| 1618 | what_failed='user') |
| 1619 | return False |
| 1620 | self.failure( |
| 1621 | 'no explicit hosts and no criteria provided.', |
| 1622 | item='cli', |
| 1623 | what_failed='user') |
| 1624 | return False |
| 1625 | |
| 1626 | |
| 1627 | def execute(self): |
| 1628 | if not self._validate_one_hostname_source(): |
| 1629 | return None |
| 1630 | if self.hosts: |
| 1631 | hostnames = self.hosts |
| 1632 | else: |
Gregory Nisbet | 8235023 | 2019-09-10 11:12:47 -0700 | [diff] [blame] | 1633 | hostnames = _host_skylab_migrate_get_hostnames( |
| 1634 | obj=self, |
| 1635 | class_=host_skylab_migrate, |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1636 | model=self.model, |
| 1637 | board=self.board, |
| 1638 | pool=self.pool, |
| 1639 | ) |
| 1640 | if self.dry_run: |
| 1641 | return hostnames |
| 1642 | if not hostnames: |
| 1643 | return {'error': 'no hosts to migrate'} |
| 1644 | res = skylab_migration.migrate( |
| 1645 | ratio=self.ratio, |
| 1646 | reason=self._reason, |
| 1647 | hostnames=hostnames, |
| 1648 | max_duration=10 * 60, |
| 1649 | interval_len=2, |
| 1650 | min_ready_intervals=10, |
| 1651 | immediately=True, |
Gregory Nisbet | 917ee33 | 2019-09-05 11:41:23 -0700 | [diff] [blame] | 1652 | use_quick_add=self.use_quick_add, |
Gregory Nisbet | c4d63ca | 2019-08-08 10:46:40 -0700 | [diff] [blame] | 1653 | ) |
| 1654 | return res |
| 1655 | |
| 1656 | |
| 1657 | def output(self, result): |
| 1658 | if result is not None: |
| 1659 | print json.dumps(result, indent=4, sort_keys=True) |
Gregory Nisbet | ac7f7f7 | 2019-09-10 16:25:20 -0700 | [diff] [blame^] | 1660 | |
| 1661 | |
| 1662 | class skylab_rollback(): |
| 1663 | usage_action = "skylab_rollback" |
| 1664 | |
| 1665 | def __init__(self): |
| 1666 | super(skylab_rollback, self).__init__() |
| 1667 | self.parser.add_option('--bug-number', |
| 1668 | help='bug number for tracking purposes.', |
| 1669 | dest='bug_number', |
| 1670 | default=None) |
| 1671 | |
| 1672 | def parse(self): |
| 1673 | (options, leftover) = super(host_skylab_migrate, self).parse() |
| 1674 | self.bug_number = options.bug_number |
| 1675 | return (options, leftover) |
| 1676 | |
| 1677 | def execute(self): |
| 1678 | if self.hosts: |
| 1679 | hostnames = self.hosts |
| 1680 | else: |
| 1681 | hostnames = _host_skylab_migrate_get_hostnames( |
| 1682 | obj=self, |
| 1683 | class_=host_skylab_migrate, |
| 1684 | model=self.model, |
| 1685 | board=self.board, |
| 1686 | pool=self.pool, |
| 1687 | ) |
| 1688 | if not hostnames: |
| 1689 | return {'error': 'no hosts to migrate'} |
| 1690 | res = skylab_rollback.rollback( |
| 1691 | hosts=hostnames, |
| 1692 | bug=self.bug_number, |
| 1693 | dry_run=False, |
| 1694 | ) |
| 1695 | return res |