mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 1 | # |
| 2 | # Copyright 2008 Google Inc. All Rights Reserved. |
| 3 | # |
| 4 | """ |
| 5 | This module contains the generic CLI object |
| 6 | |
| 7 | High Level Design: |
| 8 | |
| 9 | The atest class contains attributes & method generic to all the CLI |
| 10 | operations. |
| 11 | |
| 12 | The class inheritance is shown here using the command |
| 13 | 'atest host create ...' as an example: |
| 14 | |
| 15 | atest <-- host <-- host_create <-- site_host_create |
| 16 | |
| 17 | Note: The site_<topic>.py and its classes are only needed if you need |
| 18 | to override the common <topic>.py methods with your site specific ones. |
| 19 | |
| 20 | |
| 21 | High Level Algorithm: |
| 22 | |
| 23 | 1. atest figures out the topic and action from the 2 first arguments |
| 24 | on the command line and imports the <topic> (or site_<topic>) |
| 25 | module. |
| 26 | |
| 27 | 1. Init |
| 28 | The main atest module creates a <topic>_<action> object. The |
| 29 | __init__() function is used to setup the parser options, if this |
| 30 | <action> has some specific options to add to its <topic>. |
| 31 | |
| 32 | If it exists, the child __init__() method must call its parent |
| 33 | class __init__() before adding its own parser arguments. |
| 34 | |
| 35 | 2. Parsing |
| 36 | If the child wants to validate the parsing (e.g. make sure that |
| 37 | there are hosts in the arguments), or if it wants to check the |
| 38 | options it added in its __init__(), it should implement a parse() |
| 39 | method. |
| 40 | |
| 41 | The child parser must call its parent parser and gets back the |
| 42 | options dictionary and the rest of the command line arguments |
| 43 | (leftover). Each level gets to see all the options, but the |
| 44 | leftovers can be deleted as they can be consumed by only one |
| 45 | object. |
| 46 | |
| 47 | 3. Execution |
| 48 | This execute() method is specific to the child and should use the |
| 49 | self.execute_rpc() to send commands to the Autotest Front-End. It |
| 50 | should return results. |
| 51 | |
| 52 | 4. Output |
| 53 | The child output() method is called with the execute() resutls as a |
| 54 | parameter. This is child-specific, but should leverage the |
| 55 | atest.print_*() methods. |
| 56 | """ |
| 57 | |
| 58 | import os, sys, pwd, optparse, re, textwrap, urllib2, getpass, socket |
| 59 | from autotest_lib.cli import rpc |
| 60 | from autotest_lib.frontend.afe.json_rpc import proxy |
| 61 | |
| 62 | |
| 63 | # Maps the AFE keys to printable names. |
| 64 | KEYS_TO_NAMES_EN = {'hostname': 'Host', |
| 65 | 'platform': 'Platform', |
| 66 | 'status': 'Status', |
| 67 | 'locked': 'Locked', |
| 68 | 'locked_by': 'Locked by', |
| 69 | 'labels': 'Labels', |
| 70 | 'description': 'Description', |
| 71 | 'hosts': 'Hosts', |
| 72 | 'users': 'Users', |
| 73 | 'id': 'Id', |
| 74 | 'name': 'Name', |
| 75 | 'invalid': 'Valid', |
| 76 | 'login': 'Login', |
| 77 | 'access_level': 'Access Level', |
| 78 | 'job_id': 'Job Id', |
| 79 | 'job_owner': 'Job Owner', |
| 80 | 'job_name': 'Job Name', |
| 81 | 'test_type': 'Test Type', |
| 82 | 'test_class': 'Test Class', |
| 83 | 'path': 'Path', |
| 84 | 'owner': 'Owner', |
| 85 | 'status_counts': 'Status Counts', |
| 86 | 'hosts_status': 'Host Status', |
| 87 | 'priority': 'Priority', |
| 88 | 'control_type': 'Control Type', |
| 89 | 'created_on': 'Created On', |
| 90 | 'synch_type': 'Synch Type', |
| 91 | 'control_file': 'Control File', |
| 92 | } |
| 93 | |
| 94 | # In the failure, tag that will replace the item. |
| 95 | FAIL_TAG = '<XYZ>' |
| 96 | |
| 97 | # Global socket timeout |
| 98 | DEFAULT_SOCKET_TIMEOUT = 5 |
| 99 | # For list commands, can take longer |
| 100 | LIST_SOCKET_TIMEOUT = 30 |
| 101 | # For uploading kernels, can take much, much longer |
| 102 | UPLOAD_SOCKET_TIMEOUT = 60*30 |
| 103 | |
| 104 | |
| 105 | # Convertion functions to be called for printing, |
| 106 | # e.g. to print True/False for booleans. |
| 107 | def __convert_platform(field): |
| 108 | if not field: |
| 109 | # Can be None |
| 110 | return "" |
| 111 | elif type(field) == int: |
| 112 | # Can be 0/1 for False/True |
| 113 | return str(bool(field)) |
| 114 | else: |
| 115 | # Can be a platform name |
| 116 | return field |
| 117 | |
| 118 | |
| 119 | KEYS_CONVERT = {'locked': lambda flag: str(bool(flag)), |
| 120 | 'invalid': lambda flag: str(bool(not flag)), |
| 121 | 'platform': __convert_platform, |
| 122 | 'labels': lambda labels: ', '.join(labels)} |
| 123 | |
| 124 | class CliError(Exception): |
| 125 | pass |
| 126 | |
| 127 | |
| 128 | class atest(object): |
| 129 | """Common class for generic processing |
| 130 | Should only be instantiated by itself for usage |
| 131 | references, otherwise, the <topic> objects should |
| 132 | be used.""" |
mbligh | 76dced8 | 2008-08-11 23:46:57 +0000 | [diff] [blame^] | 133 | msg_topic = "[acl|host|job|label|test|user]" |
mbligh | be630eb | 2008-08-01 16:41:48 +0000 | [diff] [blame] | 134 | usage_action = "[action]" |
| 135 | msg_items = '' |
| 136 | |
| 137 | def invalid_arg(self, header, follow_up=''): |
| 138 | twrap = textwrap.TextWrapper(initial_indent=' ', |
| 139 | subsequent_indent=' ') |
| 140 | rest = twrap.fill(follow_up) |
| 141 | |
| 142 | if self.kill_on_failure: |
| 143 | self.invalid_syntax(header + rest) |
| 144 | else: |
| 145 | print >> sys.stderr, header + rest |
| 146 | |
| 147 | |
| 148 | def invalid_syntax(self, msg): |
| 149 | print |
| 150 | print >> sys.stderr, msg |
| 151 | print |
| 152 | print "usage:", |
| 153 | print self._get_usage() |
| 154 | print |
| 155 | sys.exit(1) |
| 156 | |
| 157 | |
| 158 | def generic_error(self, msg): |
| 159 | print >> sys.stderr, msg |
| 160 | sys.exit(1) |
| 161 | |
| 162 | |
| 163 | def failure(self, full_error, item=None, what_failed=''): |
| 164 | """If kill_on_failure, print this error and die, |
| 165 | otherwise, queue the error and accumulate all the items |
| 166 | that triggered the same error.""" |
| 167 | |
| 168 | if self.debug: |
| 169 | errmsg = str(full_error) |
| 170 | else: |
| 171 | errmsg = str(full_error).split('Traceback')[0].rstrip('\n') |
| 172 | |
| 173 | if self.kill_on_failure: |
| 174 | print >> sys.stderr, "%s\n %s" % (what_failed, errmsg) |
| 175 | sys.exit(1) |
| 176 | |
| 177 | # Build a dictionary with the 'what_failed' as keys. The |
| 178 | # values are dictionaries with the errmsg as keys and a set |
| 179 | # of items as values. |
| 180 | # self.failed = |
| 181 | # {'Operation delete_host_failed': {'AclAccessViolation: |
| 182 | # set('host0', 'host1')}} |
| 183 | # Try to gather all the same error messages together, |
| 184 | # even if they contain the 'item' |
| 185 | if item and item in errmsg: |
| 186 | errmsg = errmsg.replace(item, FAIL_TAG) |
| 187 | if self.failed.has_key(what_failed): |
| 188 | self.failed[what_failed].setdefault(errmsg, set()).add(item) |
| 189 | else: |
| 190 | self.failed[what_failed] = {errmsg: set([item])} |
| 191 | |
| 192 | |
| 193 | def show_all_failures(self): |
| 194 | if not self.failed: |
| 195 | return 0 |
| 196 | for what_failed in self.failed.keys(): |
| 197 | print >> sys.stderr, what_failed + ':' |
| 198 | for (errmsg, items) in self.failed[what_failed].iteritems(): |
| 199 | if len(items) == 0: |
| 200 | print >> sys.stderr, errmsg |
| 201 | elif items == set(['']): |
| 202 | print >> sys.stderr, ' ' + errmsg |
| 203 | elif len(items) == 1: |
| 204 | # Restore the only item |
| 205 | if FAIL_TAG in errmsg: |
| 206 | errmsg = errmsg.replace(FAIL_TAG, items.pop()) |
| 207 | else: |
| 208 | errmsg = '%s (%s)' % (errmsg, items.pop()) |
| 209 | print >> sys.stderr, ' ' + errmsg |
| 210 | else: |
| 211 | print >> sys.stderr, ' ' + errmsg + ' with <XYZ> in:' |
| 212 | twrap = textwrap.TextWrapper(initial_indent=' ', |
| 213 | subsequent_indent=' ') |
| 214 | items = list(items) |
| 215 | items.sort() |
| 216 | print >> sys.stderr, twrap.fill(', '.join(items)) |
| 217 | return 1 |
| 218 | |
| 219 | |
| 220 | def __init__(self): |
| 221 | """Setup the parser common options""" |
| 222 | # Initialized for unit tests. |
| 223 | self.afe = None |
| 224 | self.failed = {} |
| 225 | self.data = {} |
| 226 | self.debug = False |
| 227 | self.kill_on_failure = False |
| 228 | self.web_server = '' |
| 229 | self.verbose = False |
| 230 | |
| 231 | self.parser = optparse.OptionParser(self._get_usage()) |
| 232 | self.parser.add_option('-g', '--debug', |
| 233 | help='Print debugging information', |
| 234 | action='store_true', default=False) |
| 235 | self.parser.add_option('--kill-on-failure', |
| 236 | help='Stop at the first failure', |
| 237 | action='store_true', default=False) |
| 238 | self.parser.add_option('--parse', |
| 239 | help='Print the output using colon ' |
| 240 | 'separated key=value fields', |
| 241 | action='store_true', default=False) |
| 242 | self.parser.add_option('-v', '--verbose', |
| 243 | action='store_true', default=False) |
| 244 | self.parser.add_option('-w', '--web', |
| 245 | help='Specify the autotest server ' |
| 246 | 'to talk to', |
| 247 | action='store', type='string', |
| 248 | dest='web_server', default=None) |
| 249 | |
| 250 | # Shorten the TCP timeout. |
| 251 | socket.setdefaulttimeout(DEFAULT_SOCKET_TIMEOUT) |
| 252 | |
| 253 | |
| 254 | def _file_list(self, options, opt_file='', opt_list='', add_on=[]): |
| 255 | """Returns a list containing the unique items from the |
| 256 | options.<opt_list>, from the file options.<opt_file>, |
| 257 | and from the space separated add_on strings. |
| 258 | The opt_list can be space or comma separated list. |
| 259 | Used for host, acls, labels... arguments""" |
| 260 | # Start with the add_on |
| 261 | result = set() |
| 262 | [result.add(item) |
| 263 | for items in add_on |
| 264 | for item in items.split(',')] |
| 265 | |
| 266 | # Process the opt_list, if any |
| 267 | try: |
| 268 | args = getattr(options, opt_list) |
| 269 | [result.add(arg) for arg in re.split(r'[\s,]', args)] |
| 270 | except (AttributeError, TypeError): |
| 271 | pass |
| 272 | |
| 273 | # Process the file list, if any and not empty |
| 274 | # The file can contain space and/or comma separated items |
| 275 | try: |
| 276 | flist = getattr(options, opt_file) |
| 277 | file_content = [] |
| 278 | for line in open(flist).readlines(): |
| 279 | if line == '\n': |
| 280 | continue |
| 281 | file_content += re.split(r'[\s,]', |
| 282 | line.rstrip('\n')) |
| 283 | if len(file_content) == 0: |
| 284 | self.invalid_syntax("Empty file %s" % flist) |
| 285 | result = result.union(file_content) |
| 286 | except (AttributeError, TypeError): |
| 287 | pass |
| 288 | except IOError: |
| 289 | self.invalid_syntax("Could not open file %s" % flist) |
| 290 | |
| 291 | return list(result) |
| 292 | |
| 293 | |
| 294 | def _get_usage(self): |
| 295 | return "atest %s %s [options] %s" % (self.msg_topic.lower(), |
| 296 | self.usage_action, |
| 297 | self.msg_items) |
| 298 | |
| 299 | |
| 300 | def parse_with_flist(self, flists, req_items): |
| 301 | """Flists is a list of tuples containing: |
| 302 | (attribute, opt_fname, opt_list, use_leftover) |
| 303 | |
| 304 | self.<atttribute> will be populated with a set |
| 305 | containing the lines of the file named |
| 306 | options.<opt_fname> and the options.<opt_list> values |
| 307 | and the leftover from the parsing if use_leftover is |
| 308 | True. There should only be one use_leftover set to |
| 309 | True in the list. |
| 310 | Also check if the req_items is not empty after parsing.""" |
| 311 | (options, leftover) = atest.parse(self) |
| 312 | if leftover == None: |
| 313 | leftover = [] |
| 314 | |
| 315 | for (attribute, opt_fname, opt_list, use_leftover) in flists: |
| 316 | if use_leftover: |
| 317 | add_on = leftover |
| 318 | leftover = [] |
| 319 | else: |
| 320 | add_on = [] |
| 321 | |
| 322 | setattr(self, attribute, |
| 323 | self._file_list(options, |
| 324 | opt_file=opt_fname, |
| 325 | opt_list=opt_list, |
| 326 | add_on=add_on)) |
| 327 | |
| 328 | if (req_items and not getattr(self, req_items, None)): |
| 329 | self.invalid_syntax('%s %s requires at least one %s' % |
| 330 | (self.msg_topic, |
| 331 | self.usage_action, |
| 332 | self.msg_topic)) |
| 333 | |
| 334 | return (options, leftover) |
| 335 | |
| 336 | |
| 337 | def parse(self): |
| 338 | """Parse all the arguments. |
| 339 | |
| 340 | It consumes what the common object needs to know, and |
| 341 | let the children look at all the options. We could |
| 342 | remove the options that we have used, but there is no |
| 343 | harm in leaving them, and the children may need them |
| 344 | in the future. |
| 345 | |
| 346 | Must be called from its children parse()""" |
| 347 | (options, leftover) = self.parser.parse_args() |
| 348 | # Handle our own options setup in __init__() |
| 349 | self.debug = options.debug |
| 350 | self.kill_on_failure = options.kill_on_failure |
| 351 | |
| 352 | if options.parse: |
| 353 | suffix = '_parse' |
| 354 | else: |
| 355 | suffix = '_std' |
| 356 | for func in ['print_fields', 'print_table', |
| 357 | 'print_by_ids']: |
| 358 | setattr(self, func, getattr(self, func + suffix)) |
| 359 | |
| 360 | self.verbose = options.verbose |
| 361 | self.web_server = options.web_server |
| 362 | self.afe = rpc.afe_comm(self.web_server) |
| 363 | |
| 364 | return (options, leftover) |
| 365 | |
| 366 | |
| 367 | def check_and_create_items(self, op_get, op_create, |
| 368 | items, **data_create): |
| 369 | """Create the items if they don't exist already""" |
| 370 | for item in items: |
| 371 | ret = self.execute_rpc(op_get, name=item) |
| 372 | |
| 373 | if len(ret) == 0: |
| 374 | try: |
| 375 | data_create['name'] = item |
| 376 | self.execute_rpc(op_create, **data_create) |
| 377 | except CliError: |
| 378 | continue |
| 379 | |
| 380 | |
| 381 | def execute_rpc(self, op, item='', **data): |
| 382 | retry = 2 |
| 383 | while retry: |
| 384 | try: |
| 385 | return self.afe.run(op, **data) |
| 386 | except urllib2.URLError, err: |
| 387 | if 'timed out' not in err.reason: |
| 388 | self.invalid_syntax('Invalid server name %s: %s' % |
| 389 | (self.afe.web_server, err)) |
| 390 | if self.debug: |
| 391 | print 'retrying: %r %d' % (data, retry) |
| 392 | retry -= 1 |
| 393 | if retry == 0: |
| 394 | if item: |
| 395 | myerr = '%s timed out for %s' % (op, item) |
| 396 | else: |
| 397 | myerr = '%s timed out' % op |
| 398 | self.failure(myerr, item=item, |
| 399 | what_failed=("Timed-out contacting " |
| 400 | "the Autotest server")) |
| 401 | raise CliError("Timed-out contacting the Autotest server") |
| 402 | except Exception, full_error: |
| 403 | # There are various exceptions throwns by JSON, |
| 404 | # urllib & httplib, so catch them all. |
| 405 | self.failure(full_error, item=item, |
| 406 | what_failed='Operation %s failed' % op) |
| 407 | raise CliError(str(full_error)) |
| 408 | |
| 409 | |
| 410 | # There is no output() method in the atest object (yet?) |
| 411 | # but here are some helper functions to be used by its |
| 412 | # children |
| 413 | def print_wrapped(self, msg, values): |
| 414 | if len(values) == 0: |
| 415 | return |
| 416 | elif len(values) == 1: |
| 417 | print msg + ': ' |
| 418 | elif len(values) > 1: |
| 419 | if msg.endswith('s'): |
| 420 | print msg + ': ' |
| 421 | else: |
| 422 | print msg + 's: ' |
| 423 | |
| 424 | values.sort() |
| 425 | twrap = textwrap.TextWrapper(initial_indent='\t', |
| 426 | subsequent_indent='\t') |
| 427 | print twrap.fill(', '.join(values)) |
| 428 | |
| 429 | |
| 430 | def __conv_value(self, type, value): |
| 431 | return KEYS_CONVERT.get(type, str)(value) |
| 432 | |
| 433 | |
| 434 | def print_fields_std(self, items, keys, title=None): |
| 435 | """Print the keys in each item, one on each line""" |
| 436 | if not items: |
| 437 | print "No results" |
| 438 | return |
| 439 | if title: |
| 440 | print title |
| 441 | for item in items: |
| 442 | for key in keys: |
| 443 | print '%s: %s' % (KEYS_TO_NAMES_EN[key], |
| 444 | self.__conv_value(key, |
| 445 | item[key])) |
| 446 | |
| 447 | |
| 448 | def print_fields_parse(self, items, keys, title=None): |
| 449 | """Print the keys in each item as comma |
| 450 | separated name=value""" |
| 451 | if not items: |
| 452 | print "No results" |
| 453 | return |
| 454 | for item in items: |
| 455 | values = ['%s=%s' % (KEYS_TO_NAMES_EN[key], |
| 456 | self.__conv_value(key, |
| 457 | item[key])) |
| 458 | for key in keys |
| 459 | if self.__conv_value(key, |
| 460 | item[key]) != ''] |
| 461 | print ':'.join(values) |
| 462 | |
| 463 | |
| 464 | def __find_justified_fmt(self, items, keys): |
| 465 | """Find the max length for each field.""" |
| 466 | lens = {} |
| 467 | # Don't justify the last field, otherwise we have blank |
| 468 | # lines when the max is overlaps but the current values |
| 469 | # are smaller |
| 470 | if not items: |
| 471 | print "No results" |
| 472 | return |
| 473 | for key in keys[:-1]: |
| 474 | lens[key] = max(len(self.__conv_value(key, |
| 475 | item[key])) |
| 476 | for item in items) |
| 477 | lens[key] = max(lens[key], len(KEYS_TO_NAMES_EN[key])) |
| 478 | lens[keys[-1]] = 0 |
| 479 | |
| 480 | return ' '.join(["%%-%ds" % lens[key] for key in keys]) |
| 481 | |
| 482 | |
| 483 | def print_table_std(self, items, keys_header, sublist_keys={}): |
| 484 | """Print a mix of header and lists in a user readable |
| 485 | format |
| 486 | The headers are justified, the sublist_keys are wrapped.""" |
| 487 | if not items: |
| 488 | print "No results" |
| 489 | return |
| 490 | fmt = self.__find_justified_fmt(items, keys_header) |
| 491 | header = tuple(KEYS_TO_NAMES_EN[key] for key in keys_header) |
| 492 | print fmt % header |
| 493 | for item in items: |
| 494 | values = tuple(self.__conv_value(key, item[key]) |
| 495 | for key in keys_header) |
| 496 | print fmt % values |
| 497 | if self.verbose and sublist_keys: |
| 498 | for key in sublist_keys: |
| 499 | self.print_wrapped(KEYS_TO_NAMES_EN[key], |
| 500 | item[key]) |
| 501 | print '\n' |
| 502 | |
| 503 | |
| 504 | def print_table_parse(self, items, keys_header, sublist_keys=[]): |
| 505 | """Print a mix of header and lists in a user readable |
| 506 | format""" |
| 507 | if not items: |
| 508 | print "No results" |
| 509 | return |
| 510 | for item in items: |
| 511 | values = ['%s=%s' % (KEYS_TO_NAMES_EN[key], |
| 512 | self.__conv_value(key, item[key])) |
| 513 | for key in keys_header |
| 514 | if self.__conv_value(key, |
| 515 | item[key]) != ''] |
| 516 | |
| 517 | if self.verbose: |
| 518 | [values.append('%s=%s'% (KEYS_TO_NAMES_EN[key], |
| 519 | ','.join(item[key]))) |
| 520 | for key in sublist_keys |
| 521 | if len(item[key])] |
| 522 | |
| 523 | print ':'.join(values) |
| 524 | |
| 525 | |
| 526 | def print_by_ids_std(self, items, title=None, line_before=False): |
| 527 | """Prints ID & names of items in a user readable form""" |
| 528 | if not items: |
| 529 | return |
| 530 | if line_before: |
| 531 | print |
| 532 | if title: |
| 533 | print title + ':' |
| 534 | self.print_table_std(items, keys_header=['id', 'name']) |
| 535 | |
| 536 | |
| 537 | def print_by_ids_parse(self, items, title=None, line_before=False): |
| 538 | """Prints ID & names of items in a parseable format""" |
| 539 | if not items: |
| 540 | print "No results" |
| 541 | return |
| 542 | if title: |
| 543 | print title + '=', |
| 544 | values = [] |
| 545 | for item in items: |
| 546 | values += ['%s=%s' % (KEYS_TO_NAMES_EN[key], |
| 547 | self.__conv_value(key, |
| 548 | item[key])) |
| 549 | for key in ['id', 'name'] |
| 550 | if self.__conv_value(key, |
| 551 | item[key]) != ''] |
| 552 | print ':'.join(values) |