blob: 0416484cee5c3c1c1ceb05f83d49c59bfff530a1 [file] [log] [blame]
mblighbe630eb2008-08-01 16:41:48 +00001#
2# Copyright 2008 Google Inc. All Rights Reserved.
3#
4"""
5This module contains the generic CLI object
6
7High Level Design:
8
9The atest class contains attributes & method generic to all the CLI
10operations.
11
12The class inheritance is shown here using the command
13'atest host create ...' as an example:
14
15atest <-- host <-- host_create <-- site_host_create
16
17Note: The site_<topic>.py and its classes are only needed if you need
18to override the common <topic>.py methods with your site specific ones.
19
20
21High Level Algorithm:
22
231. atest figures out the topic and action from the 2 first arguments
24 on the command line and imports the <topic> (or site_<topic>)
25 module.
26
271. Init
28 The main atest module creates a <topic>_<action> object. The
29 __init__() function is used to setup the parser options, if this
30 <action> has some specific options to add to its <topic>.
31
32 If it exists, the child __init__() method must call its parent
33 class __init__() before adding its own parser arguments.
34
352. Parsing
36 If the child wants to validate the parsing (e.g. make sure that
37 there are hosts in the arguments), or if it wants to check the
38 options it added in its __init__(), it should implement a parse()
39 method.
40
41 The child parser must call its parent parser and gets back the
42 options dictionary and the rest of the command line arguments
43 (leftover). Each level gets to see all the options, but the
44 leftovers can be deleted as they can be consumed by only one
45 object.
46
473. Execution
48 This execute() method is specific to the child and should use the
49 self.execute_rpc() to send commands to the Autotest Front-End. It
50 should return results.
51
524. Output
53 The child output() method is called with the execute() resutls as a
54 parameter. This is child-specific, but should leverage the
55 atest.print_*() methods.
56"""
57
Ningning Xia5aaca4a2018-05-16 12:18:31 -070058import logging
Dan Shi3963caa2014-11-26 12:51:25 -080059import optparse
60import os
61import re
62import sys
63import textwrap
64import traceback
65import urllib2
66
Ningning Xia84190b82018-04-16 15:01:40 -070067import common
68
mblighbe630eb2008-08-01 16:41:48 +000069from autotest_lib.cli import rpc
Ningning Xia9c188b92018-04-27 15:34:23 -070070from autotest_lib.cli import skylab_utils
mblighcd26d042010-05-03 18:58:24 +000071from autotest_lib.client.common_lib.test_utils import mock
Ningning Xia84190b82018-04-16 15:01:40 -070072from autotest_lib.client.common_lib import autotemp
Ningning Xia9c188b92018-04-27 15:34:23 -070073
Ningning Xiaa7ba0c22018-04-30 11:02:30 -070074skylab_inventory_imported = False
Ningning Xia9c188b92018-04-27 15:34:23 -070075try:
76 from skylab_inventory import translation_utils
Ningning Xiaa7ba0c22018-04-30 11:02:30 -070077 skylab_inventory_imported = True
Ningning Xia9c188b92018-04-27 15:34:23 -070078except ImportError:
Ningning Xiaa7ba0c22018-04-30 11:02:30 -070079 pass
mblighbe630eb2008-08-01 16:41:48 +000080
81
82# Maps the AFE keys to printable names.
83KEYS_TO_NAMES_EN = {'hostname': 'Host',
84 'platform': 'Platform',
85 'status': 'Status',
86 'locked': 'Locked',
87 'locked_by': 'Locked by',
mblighe163b032008-10-18 14:30:27 +000088 'lock_time': 'Locked time',
Matthew Sartori68186332015-04-27 17:19:53 -070089 'lock_reason': 'Lock Reason',
mblighbe630eb2008-08-01 16:41:48 +000090 'labels': 'Labels',
91 'description': 'Description',
92 'hosts': 'Hosts',
93 'users': 'Users',
94 'id': 'Id',
95 'name': 'Name',
96 'invalid': 'Valid',
97 'login': 'Login',
98 'access_level': 'Access Level',
99 'job_id': 'Job Id',
100 'job_owner': 'Job Owner',
101 'job_name': 'Job Name',
102 'test_type': 'Test Type',
103 'test_class': 'Test Class',
104 'path': 'Path',
105 'owner': 'Owner',
106 'status_counts': 'Status Counts',
107 'hosts_status': 'Host Status',
mblighfca5ed12009-11-06 02:59:56 +0000108 'hosts_selected_status': 'Hosts filtered by Status',
mblighbe630eb2008-08-01 16:41:48 +0000109 'priority': 'Priority',
110 'control_type': 'Control Type',
111 'created_on': 'Created On',
mblighbe630eb2008-08-01 16:41:48 +0000112 'control_file': 'Control File',
showard989f25d2008-10-01 11:38:11 +0000113 'only_if_needed': 'Use only if needed',
mblighe163b032008-10-18 14:30:27 +0000114 'protection': 'Protection',
showard21baa452008-10-21 00:08:39 +0000115 'run_verify': 'Run verify',
116 'reboot_before': 'Pre-job reboot',
117 'reboot_after': 'Post-job reboot',
mbligh140a23c2008-10-29 16:55:21 +0000118 'experimental': 'Experimental',
mbligh8fadff32009-03-09 21:19:59 +0000119 'synch_count': 'Sync Count',
showardfb64e6a2009-04-22 21:01:18 +0000120 'max_number_of_machines': 'Max. hosts to use',
showarda1e74b32009-05-12 17:32:04 +0000121 'parse_failed_repair': 'Include failed repair results',
Prashanth Balasubramaniana5048562014-12-12 10:14:11 -0800122 'shard': 'Shard',
mblighbe630eb2008-08-01 16:41:48 +0000123 }
124
125# In the failure, tag that will replace the item.
126FAIL_TAG = '<XYZ>'
127
mbligh8c7b04c2009-03-25 18:01:56 +0000128# Global socket timeout: uploading kernels can take much,
129# much longer than the default
mblighbe630eb2008-08-01 16:41:48 +0000130UPLOAD_SOCKET_TIMEOUT = 60*30
131
Ningning Xia5aaca4a2018-05-16 12:18:31 -0700132LOGGING_LEVEL_MAP = {
133 'CRITICAL': logging.CRITICAL,
134 'ERROR': logging.ERROR,
135 'WARNING': logging.WARNING,
136 'INFO': logging.INFO,
137 'DEBUG': logging.DEBUG,
138}
139
mblighbe630eb2008-08-01 16:41:48 +0000140
141# Convertion functions to be called for printing,
142# e.g. to print True/False for booleans.
143def __convert_platform(field):
mbligh0887d402009-01-30 00:50:29 +0000144 if field is None:
mblighbe630eb2008-08-01 16:41:48 +0000145 return ""
mbligh0887d402009-01-30 00:50:29 +0000146 elif isinstance(field, int):
mblighbe630eb2008-08-01 16:41:48 +0000147 # Can be 0/1 for False/True
148 return str(bool(field))
149 else:
150 # Can be a platform name
151 return field
152
153
showard989f25d2008-10-01 11:38:11 +0000154def _int_2_bool_string(value):
155 return str(bool(value))
156
157KEYS_CONVERT = {'locked': _int_2_bool_string,
mblighbe630eb2008-08-01 16:41:48 +0000158 'invalid': lambda flag: str(bool(not flag)),
showard989f25d2008-10-01 11:38:11 +0000159 'only_if_needed': _int_2_bool_string,
mblighbe630eb2008-08-01 16:41:48 +0000160 'platform': __convert_platform,
Prashanth Balasubramaniana5048562014-12-12 10:14:11 -0800161 'labels': lambda labels: ', '.join(labels),
162 'shards': lambda shard: shard.hostname if shard else ''}
mblighbe630eb2008-08-01 16:41:48 +0000163
showard088b8262009-07-01 22:12:35 +0000164
165def _get_item_key(item, key):
166 """Allow for lookups in nested dictionaries using '.'s within a key."""
167 if key in item:
168 return item[key]
169 nested_item = item
170 for subkey in key.split('.'):
171 if not subkey:
172 raise ValueError('empty subkey in %r' % key)
173 try:
174 nested_item = nested_item[subkey]
175 except KeyError, e:
176 raise KeyError('%r - looking up key %r in %r' %
177 (e, key, nested_item))
178 else:
179 return nested_item
180
181
mblighbe630eb2008-08-01 16:41:48 +0000182class CliError(Exception):
Dan Shi3963caa2014-11-26 12:51:25 -0800183 """Error raised by cli calls.
184 """
mblighbe630eb2008-08-01 16:41:48 +0000185 pass
186
187
mbligh9deeefa2009-05-01 23:11:08 +0000188class item_parse_info(object):
Dan Shi3963caa2014-11-26 12:51:25 -0800189 """Object keeping track of the parsing options.
190 """
191
mbligh9deeefa2009-05-01 23:11:08 +0000192 def __init__(self, attribute_name, inline_option='',
193 filename_option='', use_leftover=False):
194 """Object keeping track of the parsing options that will
195 make up the content of the atest attribute:
Jakob Juelich8b110ee2014-09-15 16:13:42 -0700196 attribute_name: the atest attribute name to populate (label)
mbligh9deeefa2009-05-01 23:11:08 +0000197 inline_option: the option containing the items (--label)
198 filename_option: the option containing the filename (--blist)
199 use_leftover: whether to add the leftover arguments or not."""
200 self.attribute_name = attribute_name
201 self.filename_option = filename_option
202 self.inline_option = inline_option
203 self.use_leftover = use_leftover
204
205
206 def get_values(self, options, leftover=[]):
207 """Returns the value for that attribute by accumualting all
208 the values found through the inline option, the parsing of the
209 file and the leftover"""
jamesrenc2863162010-07-12 21:20:51 +0000210
211 def __get_items(input, split_spaces=True):
212 """Splits a string of comma separated items. Escaped commas will not
213 be split. I.e. Splitting 'a, b\,c, d' will yield ['a', 'b,c', 'd'].
214 If split_spaces is set to False spaces will not be split. I.e.
215 Splitting 'a b, c\,d, e' will yield ['a b', 'c,d', 'e']"""
216
217 # Replace escaped slashes with null characters so we don't misparse
218 # proceeding commas.
219 input = input.replace(r'\\', '\0')
220
221 # Split on commas which are not preceded by a slash.
222 if not split_spaces:
223 split = re.split(r'(?<!\\),', input)
224 else:
225 split = re.split(r'(?<!\\),|\s', input)
226
227 # Convert null characters to single slashes and escaped commas to
228 # just plain commas.
229 return (item.strip().replace('\0', '\\').replace(r'\,', ',') for
230 item in split if item.strip())
mbligh9deeefa2009-05-01 23:11:08 +0000231
232 if self.use_leftover:
233 add_on = leftover
234 leftover = []
235 else:
236 add_on = []
237
238 # Start with the add_on
239 result = set()
240 for items in add_on:
241 # Don't split on space here because the add-on
242 # may have some spaces (like the job name)
jamesrenc2863162010-07-12 21:20:51 +0000243 result.update(__get_items(items, split_spaces=False))
mbligh9deeefa2009-05-01 23:11:08 +0000244
245 # Process the inline_option, if any
246 try:
247 items = getattr(options, self.inline_option)
248 result.update(__get_items(items))
249 except (AttributeError, TypeError):
250 pass
251
252 # Process the file list, if any and not empty
253 # The file can contain space and/or comma separated items
254 try:
255 flist = getattr(options, self.filename_option)
256 file_content = []
257 for line in open(flist).readlines():
258 file_content += __get_items(line)
259 if len(file_content) == 0:
260 raise CliError("Empty file %s" % flist)
261 result.update(file_content)
262 except (AttributeError, TypeError):
263 pass
264 except IOError:
265 raise CliError("Could not open file %s" % flist)
266
267 return list(result), leftover
268
269
mblighbe630eb2008-08-01 16:41:48 +0000270class atest(object):
271 """Common class for generic processing
272 Should only be instantiated by itself for usage
273 references, otherwise, the <topic> objects should
274 be used."""
Richard Barnetteaeb79662018-04-18 13:51:40 -0700275 msg_topic = '[acl|host|job|label|shard|test|user|server]'
Dan Shi25e1fd42014-12-19 14:36:42 -0800276 usage_action = '[action]'
mblighbe630eb2008-08-01 16:41:48 +0000277 msg_items = ''
278
279 def invalid_arg(self, header, follow_up=''):
Dan Shi3963caa2014-11-26 12:51:25 -0800280 """Fail the command with error that command line has invalid argument.
281
282 @param header: Header of the error message.
283 @param follow_up: Extra error message, default to empty string.
284 """
mblighbe630eb2008-08-01 16:41:48 +0000285 twrap = textwrap.TextWrapper(initial_indent=' ',
286 subsequent_indent=' ')
287 rest = twrap.fill(follow_up)
288
289 if self.kill_on_failure:
290 self.invalid_syntax(header + rest)
291 else:
292 print >> sys.stderr, header + rest
293
294
295 def invalid_syntax(self, msg):
Dan Shi3963caa2014-11-26 12:51:25 -0800296 """Fail the command with error that the command line syntax is wrong.
297
298 @param msg: Error message.
299 """
mblighbe630eb2008-08-01 16:41:48 +0000300 print
301 print >> sys.stderr, msg
302 print
303 print "usage:",
304 print self._get_usage()
305 print
306 sys.exit(1)
307
308
309 def generic_error(self, msg):
Dan Shi3963caa2014-11-26 12:51:25 -0800310 """Fail the command with a generic error.
311
312 @param msg: Error message.
313 """
showardfb64e6a2009-04-22 21:01:18 +0000314 if self.debug:
315 traceback.print_exc()
mblighbe630eb2008-08-01 16:41:48 +0000316 print >> sys.stderr, msg
317 sys.exit(1)
318
319
mbligh7a3ebe32008-12-01 17:10:33 +0000320 def parse_json_exception(self, full_error):
321 """Parses the JSON exception to extract the bad
322 items and returns them
323 This is very kludgy for the moment, but we would need
324 to refactor the exceptions sent from the front end
Dan Shi3963caa2014-11-26 12:51:25 -0800325 to make this better.
326
327 @param full_error: The complete error message.
328 """
mbligh7a3ebe32008-12-01 17:10:33 +0000329 errmsg = str(full_error).split('Traceback')[0].rstrip('\n')
330 parts = errmsg.split(':')
331 # Kludge: If there are 2 colons the last parts contains
332 # the items that failed.
333 if len(parts) != 3:
334 return []
335 return [item.strip() for item in parts[2].split(',') if item.strip()]
336
337
mblighb68405d2010-03-11 18:32:39 +0000338 def failure(self, full_error, item=None, what_failed='', fatal=False):
mblighbe630eb2008-08-01 16:41:48 +0000339 """If kill_on_failure, print this error and die,
340 otherwise, queue the error and accumulate all the items
Dan Shi3963caa2014-11-26 12:51:25 -0800341 that triggered the same error.
342
343 @param full_error: The complete error message.
344 @param item: Name of the actionable item, e.g., hostname.
345 @param what_failed: Name of the failed item.
346 @param fatal: True to exit the program with failure.
347 """
mblighbe630eb2008-08-01 16:41:48 +0000348
349 if self.debug:
350 errmsg = str(full_error)
351 else:
352 errmsg = str(full_error).split('Traceback')[0].rstrip('\n')
353
mblighb68405d2010-03-11 18:32:39 +0000354 if self.kill_on_failure or fatal:
mblighbe630eb2008-08-01 16:41:48 +0000355 print >> sys.stderr, "%s\n %s" % (what_failed, errmsg)
356 sys.exit(1)
357
358 # Build a dictionary with the 'what_failed' as keys. The
359 # values are dictionaries with the errmsg as keys and a set
360 # of items as values.
mbligh1ef218d2009-08-03 16:57:56 +0000361 # self.failed =
mblighbe630eb2008-08-01 16:41:48 +0000362 # {'Operation delete_host_failed': {'AclAccessViolation:
363 # set('host0', 'host1')}}
364 # Try to gather all the same error messages together,
365 # even if they contain the 'item'
366 if item and item in errmsg:
367 errmsg = errmsg.replace(item, FAIL_TAG)
368 if self.failed.has_key(what_failed):
369 self.failed[what_failed].setdefault(errmsg, set()).add(item)
370 else:
371 self.failed[what_failed] = {errmsg: set([item])}
372
373
374 def show_all_failures(self):
Dan Shi3963caa2014-11-26 12:51:25 -0800375 """Print all failure information.
376 """
mblighbe630eb2008-08-01 16:41:48 +0000377 if not self.failed:
378 return 0
379 for what_failed in self.failed.keys():
380 print >> sys.stderr, what_failed + ':'
381 for (errmsg, items) in self.failed[what_failed].iteritems():
382 if len(items) == 0:
383 print >> sys.stderr, errmsg
384 elif items == set(['']):
385 print >> sys.stderr, ' ' + errmsg
386 elif len(items) == 1:
387 # Restore the only item
388 if FAIL_TAG in errmsg:
389 errmsg = errmsg.replace(FAIL_TAG, items.pop())
390 else:
391 errmsg = '%s (%s)' % (errmsg, items.pop())
392 print >> sys.stderr, ' ' + errmsg
393 else:
394 print >> sys.stderr, ' ' + errmsg + ' with <XYZ> in:'
395 twrap = textwrap.TextWrapper(initial_indent=' ',
396 subsequent_indent=' ')
397 items = list(items)
398 items.sort()
399 print >> sys.stderr, twrap.fill(', '.join(items))
400 return 1
401
402
403 def __init__(self):
404 """Setup the parser common options"""
405 # Initialized for unit tests.
406 self.afe = None
407 self.failed = {}
408 self.data = {}
409 self.debug = False
mbligh47dc4d22009-02-12 21:48:34 +0000410 self.parse_delim = '|'
mblighbe630eb2008-08-01 16:41:48 +0000411 self.kill_on_failure = False
412 self.web_server = ''
413 self.verbose = False
Dan Shi25e1fd42014-12-19 14:36:42 -0800414 self.no_confirmation = False
Ningning Xia84190b82018-04-16 15:01:40 -0700415 # Whether the topic or command supports skylab inventory repo.
416 self.allow_skylab = False
Ningning Xia9df3d152018-05-23 17:15:14 -0700417 self.enforce_skylab = False
mbligh9deeefa2009-05-01 23:11:08 +0000418 self.topic_parse_info = item_parse_info(attribute_name='not_used')
mblighbe630eb2008-08-01 16:41:48 +0000419
420 self.parser = optparse.OptionParser(self._get_usage())
421 self.parser.add_option('-g', '--debug',
422 help='Print debugging information',
423 action='store_true', default=False)
424 self.parser.add_option('--kill-on-failure',
425 help='Stop at the first failure',
426 action='store_true', default=False)
427 self.parser.add_option('--parse',
mbligh47dc4d22009-02-12 21:48:34 +0000428 help='Print the output using | '
mblighbe630eb2008-08-01 16:41:48 +0000429 'separated key=value fields',
430 action='store_true', default=False)
mbligh47dc4d22009-02-12 21:48:34 +0000431 self.parser.add_option('--parse-delim',
432 help='Delimiter to use to separate the '
433 'key=value fields', default='|')
Dan Shi25e1fd42014-12-19 14:36:42 -0800434 self.parser.add_option('--no-confirmation',
435 help=('Skip all confirmation in when function '
436 'require_confirmation is called.'),
437 action='store_true', default=False)
mblighbe630eb2008-08-01 16:41:48 +0000438 self.parser.add_option('-v', '--verbose',
439 action='store_true', default=False)
440 self.parser.add_option('-w', '--web',
441 help='Specify the autotest server '
442 'to talk to',
443 action='store', type='string',
444 dest='web_server', default=None)
Ningning Xia5aaca4a2018-05-16 12:18:31 -0700445 self.parser.add_option('--log-level',
446 help=('Set the logging level. Must be one of %s.'
447 ' Default to ERROR' %
448 LOGGING_LEVEL_MAP.keys()),
449 choices=LOGGING_LEVEL_MAP.keys(),
450 default='ERROR',
451 dest='log_level')
mblighbe630eb2008-08-01 16:41:48 +0000452
mblighbe630eb2008-08-01 16:41:48 +0000453
Ningning Xia9df3d152018-05-23 17:15:14 -0700454 def add_skylab_options(self, enforce_skylab=False):
Ningning Xia84190b82018-04-16 15:01:40 -0700455 """Add options for reading and writing skylab inventory repository."""
456 self.allow_skylab = True
Ningning Xia9df3d152018-05-23 17:15:14 -0700457 self.enforce_skylab = enforce_skylab
458
Allen Li75e3d342018-08-06 19:04:31 +0000459 self.parser.add_option('--skylab',
460 help=('Use the skylab inventory as the data '
461 'source. Default to %s.' %
462 self.enforce_skylab),
463 action='store_true', dest='skylab',
464 default=self.enforce_skylab)
Ningning Xia84190b82018-04-16 15:01:40 -0700465 self.parser.add_option('--env',
Allen Li75e3d342018-08-06 19:04:31 +0000466 help=('Environment ("prod" or "staging") of the '
467 'machine. Default to "prod". %s' %
Ningning Xia3748b5f2018-05-21 16:33:08 -0700468 skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xia84190b82018-04-16 15:01:40 -0700469 dest='environment',
Ningning Xiae8714052018-04-30 18:58:54 -0700470 default='prod')
Ningning Xia84190b82018-04-16 15:01:40 -0700471 self.parser.add_option('--inventory-repo-dir',
Ningning Xiaee3e3a92018-05-22 17:38:51 -0700472 help=('The path of directory to clone skylab '
473 'inventory repo into. It can be an empty '
474 'folder or an existing clean checkout of '
Allen Li75e3d342018-08-06 19:04:31 +0000475 'infra_internal/skylab_inventory. '
Ningning Xia84190b82018-04-16 15:01:40 -0700476 'If not provided, a temporary dir will be '
Ningning Xia3748b5f2018-05-21 16:33:08 -0700477 'created and used as the repo dir. %s' %
478 skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xia84190b82018-04-16 15:01:40 -0700479 dest='inventory_repo_dir')
480 self.parser.add_option('--keep-repo-dir',
Allen Li75e3d342018-08-06 19:04:31 +0000481 help=('Keep the inventory-repo-dir after the '
482 'action completes, otherwise the dir will '
483 'be cleaned up. %s' %
Ningning Xia3748b5f2018-05-21 16:33:08 -0700484 skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xia84190b82018-04-16 15:01:40 -0700485 action='store_true',
486 dest='keep_repo_dir')
487 self.parser.add_option('--draft',
Allen Li75e3d342018-08-06 19:04:31 +0000488 help=('Upload a change CL as a draft. %s' %
Ningning Xia3748b5f2018-05-21 16:33:08 -0700489 skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xia84190b82018-04-16 15:01:40 -0700490 action='store_true',
491 dest='draft',
492 default=False)
493 self.parser.add_option('--dryrun',
Ningning Xia3748b5f2018-05-21 16:33:08 -0700494 help=('Execute the action as a dryrun. %s' %
495 skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xia84190b82018-04-16 15:01:40 -0700496 action='store_true',
497 dest='dryrun',
498 default=False)
Ningning Xiaa043aad2018-04-23 15:07:09 -0700499 self.parser.add_option('--submit',
Allen Li75e3d342018-08-06 19:04:31 +0000500 help=('Submit a change CL directly without '
501 'reviewing and submitting it in Gerrit. %s'
502 % skylab_utils.MSG_ONLY_VALID_IN_SKYLAB),
Ningning Xiaa043aad2018-04-23 15:07:09 -0700503 action='store_true',
504 dest='submit',
505 default=False)
Ningning Xia84190b82018-04-16 15:01:40 -0700506
507
mblighbe630eb2008-08-01 16:41:48 +0000508 def _get_usage(self):
509 return "atest %s %s [options] %s" % (self.msg_topic.lower(),
510 self.usage_action,
511 self.msg_items)
512
513
mbligh5a496082009-08-03 16:44:54 +0000514 def backward_compatibility(self, action, argv):
Dan Shi3963caa2014-11-26 12:51:25 -0800515 """To be overidden by subclass if their syntax changed.
516
517 @param action: Name of the action.
518 @param argv: A list of arguments.
519 """
mbligh5a496082009-08-03 16:44:54 +0000520 return action
521
522
Ningning Xia84190b82018-04-16 15:01:40 -0700523 def parse_skylab_options(self, options):
524 """Parse skylab related options.
525
526 @param: options: Option values parsed by the parser.
527 """
Allen Li75e3d342018-08-06 19:04:31 +0000528 self.skylab = options.skylab
Ningning Xiaa043aad2018-04-23 15:07:09 -0700529 if not self.skylab:
530 return
Ningning Xia84190b82018-04-16 15:01:40 -0700531
Ningning Xia9c188b92018-04-27 15:34:23 -0700532 # TODO(nxia): crbug.com/837831 Add skylab_inventory to
533 # autotest-server-deps ebuilds to remove the ImportError check.
Ningning Xiaa7ba0c22018-04-30 11:02:30 -0700534 if not skylab_inventory_imported:
Ningning Xia9c188b92018-04-27 15:34:23 -0700535 raise skylab_utils.SkylabInventoryNotImported(
536 "Please try to run utils/build_externals.py.")
537
Ningning Xiaa043aad2018-04-23 15:07:09 -0700538 self.draft = options.draft
Ningning Xia84190b82018-04-16 15:01:40 -0700539
Ningning Xiaa043aad2018-04-23 15:07:09 -0700540 self.dryrun = options.dryrun
541 if self.dryrun:
542 print('This is a dryrun. NO CL will be uploaded.\n')
Ningning Xia84190b82018-04-16 15:01:40 -0700543
Ningning Xiaa043aad2018-04-23 15:07:09 -0700544 self.submit = options.submit
545 if self.submit and (self.dryrun or self.draft):
546 self.invalid_syntax('Can not set --dryrun or --draft when '
547 '--submit is set.')
548
549 # The change number of the inventory change CL.
550 self.change_number = None
551
552 self.environment = options.environment
553 translation_utils.validate_environment(self.environment)
554
555 self.keep_repo_dir = options.keep_repo_dir
556 self.inventory_repo_dir = options.inventory_repo_dir
557 if self.inventory_repo_dir is None:
558 self.temp_dir = autotemp.tempdir(
559 prefix='inventory_repo',
560 auto_clean=not self.keep_repo_dir)
Allen Li75e3d342018-08-06 19:04:31 +0000561
Ningning Xiaa043aad2018-04-23 15:07:09 -0700562 self.inventory_repo_dir = self.temp_dir.name
Allen Li75e3d342018-08-06 19:04:31 +0000563 if self.debug or self.keep_repo_dir:
564 print('The inventory_repo_dir is created at %s' %
Ningning Xiaa043aad2018-04-23 15:07:09 -0700565 self.inventory_repo_dir)
Ningning Xia84190b82018-04-16 15:01:40 -0700566
567
mbligh9deeefa2009-05-01 23:11:08 +0000568 def parse(self, parse_info=[], req_items=None):
Dan Shi3963caa2014-11-26 12:51:25 -0800569 """Parse command arguments.
mblighbe630eb2008-08-01 16:41:48 +0000570
Dan Shi3963caa2014-11-26 12:51:25 -0800571 parse_info is a list of item_parse_info objects.
mbligh9deeefa2009-05-01 23:11:08 +0000572 There should only be one use_leftover set to True in the list.
mblighbe630eb2008-08-01 16:41:48 +0000573
Dan Shi3963caa2014-11-26 12:51:25 -0800574 Also check that the req_items is not empty after parsing.
575
576 @param parse_info: A list of item_parse_info objects.
577 @param req_items: A list of required items.
578 """
mbligh9deeefa2009-05-01 23:11:08 +0000579 (options, leftover) = self.parse_global()
mblighbe630eb2008-08-01 16:41:48 +0000580
mbligh9deeefa2009-05-01 23:11:08 +0000581 all_parse_info = parse_info[:]
582 all_parse_info.append(self.topic_parse_info)
583
584 try:
585 for item_parse_info in all_parse_info:
586 values, leftover = item_parse_info.get_values(options,
587 leftover)
588 setattr(self, item_parse_info.attribute_name, values)
589 except CliError, s:
590 self.invalid_syntax(s)
mblighbe630eb2008-08-01 16:41:48 +0000591
592 if (req_items and not getattr(self, req_items, None)):
593 self.invalid_syntax('%s %s requires at least one %s' %
594 (self.msg_topic,
595 self.usage_action,
596 self.msg_topic))
597
Ningning Xia84190b82018-04-16 15:01:40 -0700598 if self.allow_skylab:
599 self.parse_skylab_options(options)
600
Ningning Xia5aaca4a2018-05-16 12:18:31 -0700601 logging.getLogger().setLevel(LOGGING_LEVEL_MAP[options.log_level])
602
mblighbe630eb2008-08-01 16:41:48 +0000603 return (options, leftover)
604
605
mbligh9deeefa2009-05-01 23:11:08 +0000606 def parse_global(self):
607 """Parse the global arguments.
mblighbe630eb2008-08-01 16:41:48 +0000608
609 It consumes what the common object needs to know, and
610 let the children look at all the options. We could
611 remove the options that we have used, but there is no
612 harm in leaving them, and the children may need them
613 in the future.
614
615 Must be called from its children parse()"""
616 (options, leftover) = self.parser.parse_args()
617 # Handle our own options setup in __init__()
618 self.debug = options.debug
619 self.kill_on_failure = options.kill_on_failure
620
621 if options.parse:
622 suffix = '_parse'
623 else:
624 suffix = '_std'
625 for func in ['print_fields', 'print_table',
mblighdf75f8b2008-11-18 19:07:42 +0000626 'print_by_ids', 'print_list']:
mblighbe630eb2008-08-01 16:41:48 +0000627 setattr(self, func, getattr(self, func + suffix))
628
mbligh47dc4d22009-02-12 21:48:34 +0000629 self.parse_delim = options.parse_delim
630
mblighbe630eb2008-08-01 16:41:48 +0000631 self.verbose = options.verbose
Dan Shi25e1fd42014-12-19 14:36:42 -0800632 self.no_confirmation = options.no_confirmation
mblighbe630eb2008-08-01 16:41:48 +0000633 self.web_server = options.web_server
mblighb68405d2010-03-11 18:32:39 +0000634 try:
635 self.afe = rpc.afe_comm(self.web_server)
636 except rpc.AuthError, s:
637 self.failure(str(s), fatal=True)
mblighbe630eb2008-08-01 16:41:48 +0000638
639 return (options, leftover)
640
641
642 def check_and_create_items(self, op_get, op_create,
643 items, **data_create):
Dan Shi3963caa2014-11-26 12:51:25 -0800644 """Create the items if they don't exist already.
645
646 @param op_get: Name of `get` RPC.
647 @param op_create: Name of `create` RPC.
648 @param items: Actionable items specified in CLI command, e.g., hostname,
649 to be passed to each RPC.
650 @param data_create: Data to be passed to `create` RPC.
651 """
mblighbe630eb2008-08-01 16:41:48 +0000652 for item in items:
653 ret = self.execute_rpc(op_get, name=item)
654
655 if len(ret) == 0:
656 try:
657 data_create['name'] = item
658 self.execute_rpc(op_create, **data_create)
659 except CliError:
660 continue
661
662
663 def execute_rpc(self, op, item='', **data):
Dan Shi3963caa2014-11-26 12:51:25 -0800664 """Execute RPC.
665
666 @param op: Name of the RPC.
667 @param item: Actionable item specified in CLI command.
668 @param data: Data to be passed to RPC.
669 """
mblighbe630eb2008-08-01 16:41:48 +0000670 retry = 2
671 while retry:
672 try:
673 return self.afe.run(op, **data)
674 except urllib2.URLError, err:
mbligh11efd232008-11-27 00:20:46 +0000675 if hasattr(err, 'reason'):
676 if 'timed out' not in err.reason:
677 self.invalid_syntax('Invalid server name %s: %s' %
678 (self.afe.web_server, err))
679 if hasattr(err, 'code'):
showard53d91e22010-01-15 00:18:27 +0000680 error_parts = [str(err)]
681 if self.debug:
682 error_parts.append(err.read()) # read the response body
683 self.failure('\n\n'.join(error_parts), item=item,
mbligh11efd232008-11-27 00:20:46 +0000684 what_failed=("Error received from web server"))
685 raise CliError("Error from web server")
mblighbe630eb2008-08-01 16:41:48 +0000686 if self.debug:
687 print 'retrying: %r %d' % (data, retry)
688 retry -= 1
689 if retry == 0:
690 if item:
691 myerr = '%s timed out for %s' % (op, item)
692 else:
693 myerr = '%s timed out' % op
694 self.failure(myerr, item=item,
695 what_failed=("Timed-out contacting "
696 "the Autotest server"))
697 raise CliError("Timed-out contacting the Autotest server")
mblighcd26d042010-05-03 18:58:24 +0000698 except mock.CheckPlaybackError:
699 raise
mblighbe630eb2008-08-01 16:41:48 +0000700 except Exception, full_error:
701 # There are various exceptions throwns by JSON,
702 # urllib & httplib, so catch them all.
703 self.failure(full_error, item=item,
704 what_failed='Operation %s failed' % op)
705 raise CliError(str(full_error))
706
707
708 # There is no output() method in the atest object (yet?)
709 # but here are some helper functions to be used by its
710 # children
711 def print_wrapped(self, msg, values):
Dan Shi3963caa2014-11-26 12:51:25 -0800712 """Print given message and values in wrapped lines unless
713 AUTOTEST_CLI_NO_WRAP is specified in environment variables.
714
715 @param msg: Message to print.
716 @param values: A list of values to print.
717 """
mblighbe630eb2008-08-01 16:41:48 +0000718 if len(values) == 0:
719 return
720 elif len(values) == 1:
721 print msg + ': '
722 elif len(values) > 1:
723 if msg.endswith('s'):
724 print msg + ': '
725 else:
726 print msg + 's: '
727
728 values.sort()
mbligh552d2402009-09-18 19:35:23 +0000729
730 if 'AUTOTEST_CLI_NO_WRAP' in os.environ:
731 print '\n'.join(values)
732 return
733
mblighbe630eb2008-08-01 16:41:48 +0000734 twrap = textwrap.TextWrapper(initial_indent='\t',
735 subsequent_indent='\t')
736 print twrap.fill(', '.join(values))
737
738
739 def __conv_value(self, type, value):
740 return KEYS_CONVERT.get(type, str)(value)
741
742
743 def print_fields_std(self, items, keys, title=None):
Dan Shi3963caa2014-11-26 12:51:25 -0800744 """Print the keys in each item, one on each line.
745
746 @param items: Items to print.
747 @param keys: Name of the keys to look up each item in items.
748 @param title: Title of the output, default to None.
749 """
mblighbe630eb2008-08-01 16:41:48 +0000750 if not items:
mblighbe630eb2008-08-01 16:41:48 +0000751 return
752 if title:
753 print title
754 for item in items:
755 for key in keys:
756 print '%s: %s' % (KEYS_TO_NAMES_EN[key],
757 self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000758 _get_item_key(item, key)))
mblighbe630eb2008-08-01 16:41:48 +0000759
760
761 def print_fields_parse(self, items, keys, title=None):
Dan Shi3963caa2014-11-26 12:51:25 -0800762 """Print the keys in each item as comma separated name=value
763
764 @param items: Items to print.
765 @param keys: Name of the keys to look up each item in items.
766 @param title: Title of the output, default to None.
767 """
mblighbe630eb2008-08-01 16:41:48 +0000768 for item in items:
769 values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
770 self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000771 _get_item_key(item, key)))
mblighbe630eb2008-08-01 16:41:48 +0000772 for key in keys
773 if self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000774 _get_item_key(item, key)) != '']
mbligh47dc4d22009-02-12 21:48:34 +0000775 print self.parse_delim.join(values)
mblighbe630eb2008-08-01 16:41:48 +0000776
777
778 def __find_justified_fmt(self, items, keys):
Dan Shi3963caa2014-11-26 12:51:25 -0800779 """Find the max length for each field.
780
781 @param items: Items to lookup for.
782 @param keys: Name of the keys to look up each item in items.
783 """
mblighbe630eb2008-08-01 16:41:48 +0000784 lens = {}
785 # Don't justify the last field, otherwise we have blank
786 # lines when the max is overlaps but the current values
787 # are smaller
788 if not items:
789 print "No results"
790 return
791 for key in keys[:-1]:
792 lens[key] = max(len(self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000793 _get_item_key(item, key)))
mblighbe630eb2008-08-01 16:41:48 +0000794 for item in items)
795 lens[key] = max(lens[key], len(KEYS_TO_NAMES_EN[key]))
796 lens[keys[-1]] = 0
797
798 return ' '.join(["%%-%ds" % lens[key] for key in keys])
799
800
Simran Basi0739d682015-02-25 16:22:56 -0800801 def print_dict(self, items, title=None, line_before=False):
802 """Print a dictionary.
803
804 @param items: Dictionary to print.
805 @param title: Title of the output, default to None.
806 @param line_before: True to print an empty line before the output,
807 default to False.
808 """
809 if not items:
810 return
811 if line_before:
812 print
813 print title
814 for key, value in items.items():
815 print '%s : %s' % (key, value)
816
817
mbligh838c7472009-05-13 20:56:50 +0000818 def print_table_std(self, items, keys_header, sublist_keys=()):
Dan Shi3963caa2014-11-26 12:51:25 -0800819 """Print a mix of header and lists in a user readable format.
820
821 The headers are justified, the sublist_keys are wrapped.
822
823 @param items: Items to print.
824 @param keys_header: Header of the keys, use to look up in items.
825 @param sublist_keys: Keys for sublist in each item.
826 """
mblighbe630eb2008-08-01 16:41:48 +0000827 if not items:
mblighbe630eb2008-08-01 16:41:48 +0000828 return
829 fmt = self.__find_justified_fmt(items, keys_header)
830 header = tuple(KEYS_TO_NAMES_EN[key] for key in keys_header)
831 print fmt % header
832 for item in items:
showard088b8262009-07-01 22:12:35 +0000833 values = tuple(self.__conv_value(key,
834 _get_item_key(item, key))
mblighbe630eb2008-08-01 16:41:48 +0000835 for key in keys_header)
836 print fmt % values
mbligh838c7472009-05-13 20:56:50 +0000837 if sublist_keys:
mblighbe630eb2008-08-01 16:41:48 +0000838 for key in sublist_keys:
839 self.print_wrapped(KEYS_TO_NAMES_EN[key],
showard088b8262009-07-01 22:12:35 +0000840 _get_item_key(item, key))
mblighbe630eb2008-08-01 16:41:48 +0000841 print '\n'
842
843
mbligh838c7472009-05-13 20:56:50 +0000844 def print_table_parse(self, items, keys_header, sublist_keys=()):
Dan Shi3963caa2014-11-26 12:51:25 -0800845 """Print a mix of header and lists in a user readable format.
846
847 @param items: Items to print.
848 @param keys_header: Header of the keys, use to look up in items.
849 @param sublist_keys: Keys for sublist in each item.
850 """
mblighbe630eb2008-08-01 16:41:48 +0000851 for item in items:
852 values = ['%s=%s' % (KEYS_TO_NAMES_EN[key],
showard088b8262009-07-01 22:12:35 +0000853 self.__conv_value(key, _get_item_key(item, key)))
mblighbe630eb2008-08-01 16:41:48 +0000854 for key in keys_header
855 if self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000856 _get_item_key(item, key)) != '']
mblighbe630eb2008-08-01 16:41:48 +0000857
mbligh838c7472009-05-13 20:56:50 +0000858 if sublist_keys:
mblighbe630eb2008-08-01 16:41:48 +0000859 [values.append('%s=%s'% (KEYS_TO_NAMES_EN[key],
showard088b8262009-07-01 22:12:35 +0000860 ','.join(_get_item_key(item, key))))
mblighbe630eb2008-08-01 16:41:48 +0000861 for key in sublist_keys
showard088b8262009-07-01 22:12:35 +0000862 if len(_get_item_key(item, key))]
mblighbe630eb2008-08-01 16:41:48 +0000863
mbligh47dc4d22009-02-12 21:48:34 +0000864 print self.parse_delim.join(values)
mblighbe630eb2008-08-01 16:41:48 +0000865
866
867 def print_by_ids_std(self, items, title=None, line_before=False):
Dan Shi3963caa2014-11-26 12:51:25 -0800868 """Prints ID & names of items in a user readable form.
869
870 @param items: Items to print.
871 @param title: Title of the output, default to None.
872 @param line_before: True to print an empty line before the output,
873 default to False.
874 """
mblighbe630eb2008-08-01 16:41:48 +0000875 if not items:
876 return
877 if line_before:
878 print
879 if title:
880 print title + ':'
881 self.print_table_std(items, keys_header=['id', 'name'])
882
883
884 def print_by_ids_parse(self, items, title=None, line_before=False):
Dan Shi3963caa2014-11-26 12:51:25 -0800885 """Prints ID & names of items in a parseable format.
886
887 @param items: Items to print.
888 @param title: Title of the output, default to None.
889 @param line_before: True to print an empty line before the output,
890 default to False.
891 """
mblighbe630eb2008-08-01 16:41:48 +0000892 if not items:
mblighbe630eb2008-08-01 16:41:48 +0000893 return
Dan Shi3963caa2014-11-26 12:51:25 -0800894 if line_before:
895 print
mblighbe630eb2008-08-01 16:41:48 +0000896 if title:
897 print title + '=',
898 values = []
899 for item in items:
900 values += ['%s=%s' % (KEYS_TO_NAMES_EN[key],
901 self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000902 _get_item_key(item, key)))
mblighbe630eb2008-08-01 16:41:48 +0000903 for key in ['id', 'name']
904 if self.__conv_value(key,
showard088b8262009-07-01 22:12:35 +0000905 _get_item_key(item, key)) != '']
mbligh47dc4d22009-02-12 21:48:34 +0000906 print self.parse_delim.join(values)
mblighdf75f8b2008-11-18 19:07:42 +0000907
908
909 def print_list_std(self, items, key):
Dan Shi3963caa2014-11-26 12:51:25 -0800910 """Print a wrapped list of results
911
912 @param items: Items to to lookup for given key, could be a nested
913 dictionary.
914 @param key: Name of the key to look up for value.
915 """
mblighdf75f8b2008-11-18 19:07:42 +0000916 if not items:
mblighdf75f8b2008-11-18 19:07:42 +0000917 return
showard088b8262009-07-01 22:12:35 +0000918 print ' '.join(_get_item_key(item, key) for item in items)
mblighdf75f8b2008-11-18 19:07:42 +0000919
920
921 def print_list_parse(self, items, key):
Dan Shi3963caa2014-11-26 12:51:25 -0800922 """Print a wrapped list of results.
923
924 @param items: Items to to lookup for given key, could be a nested
925 dictionary.
926 @param key: Name of the key to look up for value.
927 """
mblighdf75f8b2008-11-18 19:07:42 +0000928 if not items:
mblighdf75f8b2008-11-18 19:07:42 +0000929 return
930 print '%s=%s' % (KEYS_TO_NAMES_EN[key],
showard088b8262009-07-01 22:12:35 +0000931 ','.join(_get_item_key(item, key) for item in items))
Dan Shi25e1fd42014-12-19 14:36:42 -0800932
933
934 @staticmethod
935 def prompt_confirmation(message=None):
936 """Prompt a question for user to confirm the action before proceeding.
937
938 @param message: A detailed message to explain possible impact of the
939 action.
940
941 @return: True to proceed or False to abort.
942 """
943 if message:
944 print message
945 sys.stdout.write('Continue? [y/N] ')
946 read = raw_input().lower()
947 if read == 'y':
948 return True
949 else:
950 print 'User did not confirm. Aborting...'
951 return False
952
953
954 @staticmethod
955 def require_confirmation(message=None):
956 """Decorator to prompt a question for user to confirm action before
957 proceeding.
958
959 If user chooses not to proceed, do not call the function.
960
961 @param message: A detailed message to explain possible impact of the
962 action.
963
964 @return: A decorator wrapper for calling the actual function.
965 """
966 def deco_require_confirmation(func):
967 """Wrapper for the decorator.
968
969 @param func: Function to be called.
970
971 @return: the actual decorator to call the function.
972 """
973 def func_require_confirmation(*args, **kwargs):
974 """Decorator to prompt a question for user to confirm.
975
976 @param message: A detailed message to explain possible impact of
977 the action.
978 """
979 if (args[0].no_confirmation or
980 atest.prompt_confirmation(message)):
981 func(*args, **kwargs)
982
983 return func_require_confirmation
984 return deco_require_confirmation