autotest: Delete run_suite.py
There are no autotest instance to run suites on.
Bug: 1033823
Change-Id: Icba925fbcdd9d3ca96b8a67d13bdcc9f0b99d7dc
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/1967054
Reviewed-by: Xixuan Wu <xixuan@chromium.org>
Tested-by: Allen Li <ayatane@chromium.org>
Commit-Queue: Allen Li <ayatane@chromium.org>
diff --git a/site_utils/run_suite.py b/site_utils/run_suite.py
deleted file mode 100755
index 0e62563..0000000
--- a/site_utils/run_suite.py
+++ /dev/null
@@ -1,2312 +0,0 @@
-#!/usr/bin/python2
-#
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Tool for running suites of tests and waiting for completion.
-
-The desired test suite will be scheduled with autotest. By default,
-this tool will block until the job is complete, printing a summary
-at the end. Error conditions result in exceptions.
-
-This is intended for use only with Chrome OS test suits that leverage the
-dynamic suite infrastructure in server/cros/dynamic_suite.py.
-
-This script exits with one of the following codes:
-0 - OK: Suite finished successfully
-1 - ERROR: Test(s) failed, or hits its own timeout
-2 - WARNING: Test(s) raised a warning or passed on retry, none failed/timed out.
-3 - INFRA_FAILURE: Infrastructure related issues, e.g.
- * Lab is down
- * Too many duts (defined as a constant) in repair failed status
- * Suite job issues, like bug in dynamic suite,
- user aborted the suite, lose a drone/all devservers/rpc server,
- 0 tests ran, etc.
- * provision failed
- TODO(fdeng): crbug.com/413918, reexamine treating all provision
- failures as INFRA failures.
-4 - SUITE_TIMEOUT: Suite timed out, some tests ran,
- none failed by the time the suite job was aborted. This will cover,
- but not limited to, the following cases:
- * A devserver failure that manifests as a timeout
- * No DUTs available midway through a suite
- * Provision/Reset/Cleanup took longer time than expected for new image
- * A regression in scheduler tick time.
-5- BOARD_NOT_AVAILABLE: If there is no host for the requested board/pool.
-6- INVALID_OPTIONS: If options are not valid.
-"""
-
-import argparse
-import ast
-import collections
-from datetime import datetime
-from datetime import timedelta
-import functools
-import getpass
-import json
-import logging
-import os
-import re
-import sys
-import time
-import warnings
-
-import common
-from chromite.lib import buildbot_annotations as annotations
-from chromite.lib import cros_build_lib
-from chromite.lib import gs
-from chromite.lib import osutils
-
-from django.core import exceptions as django_exceptions
-
-try:
- from suite_scheduler import config_reader
- from suite_scheduler import skylab
-except ImportError:
- # For unittest
- config_reader = None
- skylab = None
-
-from autotest_lib.client.common_lib import control_data
-from autotest_lib.client.common_lib import error
-from autotest_lib.client.common_lib import global_config
-from autotest_lib.client.common_lib import priorities
-from autotest_lib.client.common_lib import time_utils
-from autotest_lib.client.common_lib.cros import retry
-from autotest_lib.frontend.afe import rpc_client_lib
-from autotest_lib.frontend.afe.json_rpc import proxy
-from autotest_lib.server import site_utils
-from autotest_lib.server import utils
-from autotest_lib.server.cros import provision
-from autotest_lib.server.cros.dynamic_suite import constants
-from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
-from autotest_lib.server.cros.dynamic_suite import reporting_utils
-from autotest_lib.server.cros.dynamic_suite import suite_common
-from autotest_lib.server.cros.dynamic_suite import tools
-try:
- from autotest_lib.site_utils import diagnosis_utils
-except django_exceptions.ImproperlyConfigured as e:
- if 'Error loading MySQLdb module: libmariadbclient' in str(e):
- logging.error('Unable to import a necessary MySQLdb module. This is '
- 'commonly caused by running a command inside[outside] '
- 'of the chroot but having autotest utility packages '
- 'that were build outside[inside] the chroot. '
- 'Please re-run utils/build_externals.py inside[outside] '
- 'of the chroot accordingly.')
- raise
-
-from autotest_lib.site_utils import paygen
-from autotest_lib.site_utils import run_suite_common
-
-CONFIG = global_config.global_config
-
-_DEFAULT_AUTOTEST_INSTANCE = CONFIG.get_config_value(
- 'SERVER', 'hostname', type=str)
-_URL_PATTERN = CONFIG.get_config_value('CROS', 'log_url_pattern', type=str)
-_ENABLE_RUN_SUITE_TRAMPOLINE = CONFIG.get_config_value(
- 'CROS', 'enable_run_suite_trampoline', type=bool, default=False)
-
-_SKYLAB_TOOL = '/opt/infra-tools/skylab'
-_SKYLAB_SERVICE_ACCOUNT = '/creds/service_accounts/skylab_swarming.json'
-_MIGRATION_CONFIG_FILE = 'migration_config.ini'
-_MIGRATION_CONFIG_BUCKET = 'suite-scheduler.google.com.a.appspot.com'
-_TRAMPOLINE_CONFIG = 'gs://%s/%s' % (_MIGRATION_CONFIG_BUCKET,
- _MIGRATION_CONFIG_FILE)
-
-# Minimum RPC timeout setting for calls expected to take long time, e.g.,
-# create_suite_job. If default socket time (socket.getdefaulttimeout()) is
-# None or greater than this value, the default will be used.
-# The value here is set to be the same as the timeout for the RetryingAFE object
-# so long running RPCs can wait long enough before being aborted.
-_MIN_RPC_TIMEOUT = 600
-
-# Number of days back to search for existing job.
-_SEARCH_JOB_MAX_DAYS = 14
-
-_PROVISION_SUITE = 'provision'
-
-# Only special tasks can have a priority lower than 50.
-_SKYLAB_PRIORITY_MIN = 50
-# Mandated by Swarming.
-_SKYLAB_PRIORITY_MAX = 255
-
-
-@functools.total_ordering
-class _ReturnResult(object):
- """Represents overall result of run_suite operation.
-
- _ReturnResult instances sort based on priority (the order in
- _RETURN_RESULTS).
-
- Furthermore, _ReturnResult instances can be combined by bitwise or
- ("union"), which returns the instance with the higher priority
- between the two (the instance with higher priority is a "superset"
- of the other).
-
- Do not create new instances of this; use _RETURN_RESULTS instead.
- """
-
- def __init__(self, return_code, message):
- self.return_code = return_code
- self.message = message
-
- def __repr__(self):
- return '<{cls} {key}, {this.return_code}, {this.message}>'.format(
- cls=type(self).__name__,
- key=self._getkey(),
- this=self)
-
- def __gt__(self, other):
- if isinstance(other, type(self)):
- return self._getkey() > other._getkey()
- else:
- return NotImplemented
-
- def __eq__(self, other):
- if isinstance(other, type(self)):
- return (self.return_code == other.return_code
- and self.message == other.message)
- else:
- return NotImplemented
-
- def __hash__(self):
- return hash(self.return_code) ^ hash(self.message)
-
- def __or__(self, other):
- if isinstance(other, type(self)):
- if self > other:
- return self
- else:
- return other
- else:
- return NotImplemented
-
- def _getkey(self):
- """Return sort key."""
- return _RETURN_RESULTS_LIST.index(self)
-
- def suite_result(self, output_dict=None):
- """Make a SuiteResult using this _ReturnResult.
-
- @param output_dict: output_dict to merge into SuiteResult.
- """
- if output_dict is None:
- output_dict = dict()
- else:
- output_dict = output_dict.copy()
- if self.message:
- output_dict['return_message'] = self.message
- return run_suite_common.SuiteResult(self.return_code, output_dict)
-
-
-_RETURN_RESULTS = collections.OrderedDict([
- ('ok', _ReturnResult(run_suite_common.RETURN_CODES.OK, '')),
-
- ('test_warning', _ReturnResult(
- run_suite_common.RETURN_CODES.WARNING, 'Test job raised warning.')),
- ('suite_warning', _ReturnResult(
- run_suite_common.RETURN_CODES.WARNING, 'Suite job raised warning.')),
- ('test_retry', _ReturnResult(
- run_suite_common.RETURN_CODES.WARNING, 'Tests were retried.')),
-
- ('test_aborted_prestart', _ReturnResult(
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT,
- 'Tests were aborted before running; suite must have timed out.')),
- # This really indicates a user action or an infra failure. But, suite
- # timeouts cause similar fauilres in the individual tests, so we must
- # classify these lower than suite_timeout. In case of a suite_timeout, the
- # result from the suite job will promote the result to suite_timeout.
- ('test_aborted_mystery',
- _ReturnResult(
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT,
- 'Tests were aborted after running, but before timeout; '
- 'Test was manually aborted or parsing results failed: '
- 'crbug.com/796348.')),
- ('suite_timeout', _ReturnResult(
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT, 'Suite job timed out.')),
-
- ('test_views_missing', _ReturnResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE, 'No test views found.')),
- ('suite_failed', _ReturnResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE, 'Suite job failed.')),
- ('provision_failed', _ReturnResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE, 'Provisioning failed.')),
-
- ('test_failure', _ReturnResult(
- run_suite_common.RETURN_CODES.ERROR, 'Tests failed.')),
-])
-_RETURN_RESULTS_LIST = list(_RETURN_RESULTS.values())
-
-
-def bool_str(x):
- """Boolean string type for option arguments.
-
- @param x: string representation of boolean value.
-
- """
- if x == 'True':
- return True
- elif x == 'False':
- return False
- else:
- raise argparse.ArgumentTypeError(
- '%s is not one of True or False' % (x,))
-
-
-def _get_priority_value(x):
- """Convert a priority representation to its int value.
-
- Priorities can be described either by an int value (possibly as a string)
- or a name string. This function coerces both forms to an int value.
-
- This function is intended for casting command line arguments during
- parsing.
-
- @param x: priority value as an int, int string, or name string
-
- @returns: int value of priority
- """
- try:
- return int(x)
- except ValueError:
- try:
- return priorities.Priority.get_value(x)
- except AttributeError:
- raise argparse.ArgumentTypeError(
- 'Unknown priority level %s. Try one of %s.'
- % (x, ', '.join(priorities.Priority.names)))
-
-
-def skylab_priority_for(afe_priority):
- """Convert AFE priority to Skylab priority.
-
- Args:
- afe_priority: An integer get from _get_priority_value().
-
- Returns:
- An integer representing Skylab priority.
- """
- skylab_priority = 260 - 3 * int(afe_priority)
- skylab_priority = min(skylab_priority, _SKYLAB_PRIORITY_MAX)
- skylab_priority = max(skylab_priority, _SKYLAB_PRIORITY_MIN)
- return skylab_priority
-
-
-def make_parser():
- """Make ArgumentParser instance for run_suite.py."""
- parser = argparse.ArgumentParser(
- usage="%(prog)s [options]")
- parser.add_argument("-b", "--board", dest="board")
- parser.add_argument(
- "--model",
- help="The device model to run tests against. For non-unified "
- "builds, model and board are synonymous, but board is more "
- "accurate in some cases. Only pass this option if your build "
- "is a unified build.",
- )
- parser.add_argument("-i", "--build", dest="build")
- parser.add_argument(
- "-w", "--web", dest="web", default=None,
- help="Address of a webserver to receive suite requests.")
- parser.add_argument(
- '--cheets_build', dest='cheets_build', default=None,
- help='ChromeOS Android build to be installed on dut.')
- parser.add_argument(
- '--firmware_rw_build', dest='firmware_rw_build', default=None,
- help='Firmware build to be installed in dut RW firmware.')
- parser.add_argument(
- '--firmware_ro_build', dest='firmware_ro_build', default=None,
- help='Firmware build to be installed in dut RO firmware.')
- parser.add_argument(
- '--test_source_build', dest='test_source_build', default=None,
- help=('Build that contains the test code, '
- 'e.g., it can be the value of `--build`, '
- '`--firmware_rw_build` or `--firmware_ro_build` '
- 'arguments. Default is None, that is, use the test '
- 'code from `--build` (CrOS image)'))
- # This should just be a boolean flag, but the autotest "proxy" code
- # can't handle flags that don't take arguments.
- parser.add_argument(
- "-n", "--no_wait", dest="no_wait", default=False, type=bool_str,
- help='Must pass "True" or "False" if used.')
- # If you really want no pool, --pool="" will do it. USE WITH CARE.
- parser.add_argument("-p", "--pool", dest="pool", default="suites")
- parser.add_argument("-s", "--suite_name", dest="name")
- parser.add_argument("-a", "--afe_timeout_mins", type=int,
- dest="afe_timeout_mins", default=30)
- parser.add_argument("-t", "--timeout_mins", type=int,
- dest="timeout_mins", default=1440)
- parser.add_argument("-x", "--max_runtime_mins", type=int,
- dest="max_runtime_mins", default=1440)
- parser.add_argument("-d", "--delay_sec", type=int,
- dest="delay_sec", default=10)
- parser.add_argument("-m", "--mock_job_id", dest="mock_job_id",
- help="Attach to existing job id for already running "
- "suite, and creates report.")
- # NOTE: This looks similar to --no_wait, but behaves differently.
- # --no_wait is passed in to the suite rpc itself and affects the suite,
- # while this does not.
- parser.add_argument("-c", "--create_and_return", dest="create_and_return",
- action="store_true",
- help="Create the suite and print the job id, then "
- "finish immediately.")
- parser.add_argument("-u", "--num", dest="num", type=int, default=None,
- help="Deprecated, does nothing.")
- # Same boolean flag issue applies here.
- parser.add_argument(
- "-f", "--file_bugs", dest="file_bugs", default=False, type=bool_str,
- help=('File bugs on test failures. Must pass "True" or '
- '"False" if used.'))
- parser.add_argument("-l", "--bypass_labstatus", dest="bypass_labstatus",
- action="store_true", help='Bypass lab status check.')
- # We allow either a number or a string for the priority. This way, if you
- # know what you're doing, one can specify a custom priority level between
- # other levels.
- parser.add_argument("-r", "--priority", dest="priority",
- type=_get_priority_value,
- default=priorities.Priority.DEFAULT,
- action="store",
- help="Priority of suite. Either numerical value, or "
- "one of (" + ", ".join(priorities.Priority.names)
- + ").")
- parser.add_argument(
- '--retry', dest='retry', default=False, type=bool_str, action='store',
- help='Enable test retry. Must pass "True" or "False" if used.')
- parser.add_argument('--max_retries', dest='max_retries', default=None,
- type=int, action='store', help='Maximum retries'
- 'allowed at suite level. No limit if not specified.')
- parser.add_argument('--minimum_duts', dest='minimum_duts', type=int,
- default=0, action='store',
- help='Check that the pool has at least such many '
- 'healthy machines, otherwise suite will not run. '
- 'Default to 0.')
- parser.add_argument('--suite_min_duts', dest='suite_min_duts', type=int,
- default=0, action='store',
- help='Preferred minimum number of machines. Scheduler '
- 'will prioritize on getting such many machines for '
- 'the suite when it is competing with another suite '
- 'that has a higher priority but already got minimum '
- 'machines it needs. Default to 0.')
- parser.add_argument("--suite_args", dest="suite_args",
- type=ast.literal_eval,
- default=None, action="store",
- help="A dict of args passed to the suite control file.")
- parser.add_argument("--suite_args_json", dest="suite_args_json",
- type=json.loads,
- default=None, action="store",
- help="A json-encoded string representation of args to "
- "passed to the suite control file. Overrides "
- "suite_args if specified.")
- parser.add_argument('--offload_failures_only',
- dest='offload_failures_only', type=bool_str,
- action='store', default=False,
- help='Only enable gs_offloading for failed tests. '
- 'Successful tests will be deleted. Must pass "True"'
- ' or "False" if used.')
- parser.add_argument('--use_suite_attr', dest='use_suite_attr',
- action='store_true', default=False,
- help='Advanced. Run the suite based on ATTRIBUTES of '
- 'control files, rather than SUITE.')
- parser.add_argument('--json_dump', dest='json_dump', action='store_true',
- default=False,
- help='Dump the output of run_suite to stdout as json; '
- 'silence other output.')
- parser.add_argument('--json_dump_postfix', dest='json_dump_postfix',
- action='store_true',
- help='Dump the output of run_suite to stdout as json; '
- 'do not silence other logging. Similar to '
- '--json_dump, the json payload will be wrapped in '
- 'a tag to differentiate it from logging.')
- parser.add_argument(
- '--run_prod_code', dest='run_prod_code',
- action='store_true', default=False,
- help='Run the test code that lives in prod aka the test '
- 'code currently on the lab servers.')
- parser.add_argument(
- '--delay_minutes', type=int, default=0,
- help=('Delay the creation of test jobs for a given '
- 'number of minutes. This argument can be used to '
- 'force provision jobs being delayed, which helps '
- 'to distribute loads across devservers.'))
- parser.add_argument(
- '--skip_duts_check', dest='skip_duts_check', action='store_true',
- default=False, help='If True, skip minimum available DUTs check')
- parser.add_argument(
- '--job_keyvals', dest='job_keyvals', type=ast.literal_eval,
- action='store', default=None,
- help='A dict of job keyvals to be inject to suite control file')
- parser.add_argument(
- '--test_args', dest='test_args', type=ast.literal_eval,
- action='store', default=None,
- help=('A dict of args passed all the way to each individual test that '
- 'will be actually ran.'))
- parser.add_argument(
- '--require_logfile', action='store_true',
- help=('Stream logs of run_suite.py to a local file named '
- 'run_suite-<build name>.log.'))
-
- # Used for monitoring purposes, to measure no-op swarming proxy latency.
- parser.add_argument('--do_nothing', action='store_true',
- help=argparse.SUPPRESS)
-
- # Used when lab/job status checking is needed. Currently its only user is
- # suite scheduler v2.
- parser.add_argument(
- '--pre_check', action='store_true',
- help=('Check lab and job status before kicking off a suite. Used by '
- 'suite scheduler v2.'))
-
- return parser
-
-
-def verify_and_clean_options(options):
- """Verify the validity of options.
-
- @param options: The parsed options to verify.
-
- @returns: True if verification passes, False otherwise.
-
- """
- if options.mock_job_id and (
- not options.build or not options.name or not options.board):
- print ('When using -m, need to specify build, board and suite '
- 'name which you have used for creating the original job')
- return False
- else:
- if not options.build:
- print 'Need to specify which build to use'
- return False
- if not options.board:
- print 'Need to specify board'
- return False
- if not options.name:
- print 'Need to specify suite name'
- return False
- if options.num is not None:
- warnings.warn('-u/--num option is deprecated; it does nothing.')
- del options.num
- if not options.retry and options.max_retries is not None:
- print 'max_retries can only be used with --retry=True'
- return False
- if options.use_suite_attr and options.suite_args is not None:
- print ('The new suite control file cannot parse the suite_args: %s.'
- 'Please not specify any suite_args here.' % options.suite_args)
- return False
- if options.no_wait and options.retry:
- print 'Test retry is not available when using --no_wait=True'
- if options.json_dump and options.json_dump_postfix:
- print '--json_dump and --json_dump_postfix are mutually exclusive'
- return False
- # Default to use the test code in CrOS build.
- if not options.test_source_build and options.build:
- options.test_source_build = options.build
- options.child_dependencies = _make_child_dependencies(options)
- base_dependencies = ('board:%s' % options.board,
- 'pool:%s' % options.pool)
- options.dependencies = base_dependencies + options.child_dependencies
- return True
-
-
-def change_options_for_suite_attr(options):
- """Change options to be prepared to run the suite_attr_wrapper.
-
- If specify 'use_suite_attr' from the cmd line, it indicates to run the
- new style suite control file, suite_attr_wrapper. Then, change the
- options.name to 'suite_attr_wrapper', change the options.suite_args to
- include the arguments needed by suite_attr_wrapper.
-
- @param options: The verified options.
-
- @returns: The changed options.
-
- """
- # Convert the suite_name to attribute boolean expression.
- if type(options.name) is str:
- attr_filter_val = 'suite:%s' % options.name
- else:
- attr_filter_val = ' or '.join(['suite:%s' % x for x in options.name])
-
- # change the suite_args to be a dict of arguments for suite_attr_wrapper
- # if suite_args is not None, store the values in 'other_args' of the dict
- args_dict = {}
- args_dict['attr_filter'] = attr_filter_val
- options.suite_args = args_dict
- options.name = 'suite_attr_wrapper'
-
- return options
-
-
-class TestResult(object):
-
- """Represents the result of a TestView."""
-
- def __init__(self, test_view, retry_count=0):
- """Initialize instance.
-
- @param test_view: TestView instance.
- @param retry_count: Retry count for test. Optional.
- """
- self.name = test_view.get_testname()
- self.status = test_view['status']
- self.reason = test_view['reason']
- self.retry_count = retry_count
-
- _PRETTY_STATUS_MAP = {
- 'GOOD': '[ PASSED ]',
- 'TEST_NA': '[ INFO ]',
- }
-
- @property
- def _pretty_status(self):
- """Pretty status string."""
- return self._PRETTY_STATUS_MAP.get(self.status, '[ FAILED ]')
-
- def log_using(self, log_function, name_column_width):
- """Log the test result using the given log function.
-
- @param log_function: Log function to use. Example: logging.info
- @param name_column_width: Width of name column for formatting.
- """
- padded_name = self.name.ljust(name_column_width)
- log_function('%s%s', padded_name, self._pretty_status)
- if self.status != 'GOOD':
- log_function('%s %s: %s', padded_name, self.status, self.reason)
- if self.retry_count > 0:
- log_function('%s retry_count: %s', padded_name, self.retry_count)
-
-
-def get_original_suite_name(suite_name, suite_args):
- """Get the original suite name when running suite_attr_wrapper.
-
- @param suite_name: the name of the suite launched in afe. When it is
- suite_attr_wrapper, the suite that actually running is
- specified in the suite_args.
- @param suite_args: dict of suite args from argument parsing.
-
- @returns: the original suite name.
-
- """
- if suite_name == 'suite_attr_wrapper':
- attrs = suite_args.get('attr_filter', '')
- suite_list = ([x[6:] for x in re.split('[() ]', attrs)
- if x and x.startswith('suite:')])
- return suite_list[0] if suite_list else suite_name
- return suite_name
-
-
-class LogLink(object):
- """Information needed to record a link in the logs.
-
- Depending on context and the information provided at
- construction time, the link may point to either to log files for
- a job, or to a bug filed for a failure in the job.
-
- @var anchor The link text.
- @var url The link url.
- @var bug_id Id of a bug to link to, or None.
- """
-
- # A list of tests that don't get retried so skip the dashboard.
- _SKIP_RETRY_DASHBOARD = ['provision']
-
- _BUG_LINK_PREFIX = 'Auto-Bug'
- _LOG_LINK_PREFIX = 'Test-Logs'
-
-
- def __init__(self, anchor, server, job_string, bug_info=None, reason=None,
- retry_count=0, testname=None, sponge_url=None):
- """Initialize the LogLink by generating the log URL.
-
- @param anchor The link text.
- @param server The hostname of the server this suite ran on.
- @param job_string The job whose logs we'd like to link to.
- @param bug_info Info about the bug, if one was filed.
- @param reason A string representing the reason of failure if any.
- @param retry_count How many times the test has been retried.
- @param testname Optional Arg that supplies the testname.
- @param sponge_url url to Sponge result.
- """
- self.anchor = anchor
- self.url = _URL_PATTERN % (rpc_client_lib.add_protocol(server),
- job_string)
- self.reason = reason
- self.retry_count = retry_count
- self.testname = testname
- self.sponge_url = sponge_url
- if bug_info:
- self.bug_id, self.bug_count = bug_info
- else:
- self.bug_id = None
- self.bug_count = None
-
-
- @property
- def bug_url(self):
- """URL of associated bug."""
- if self.bug_id:
- return reporting_utils.link_crbug(self.bug_id)
- else:
- return None
-
-
- @property
- def _bug_count_text(self):
- """Return bug count as human friendly text."""
- if self.bug_count is None:
- bug_info = 'unknown number of reports'
- elif self.bug_count == 1:
- bug_info = 'new report'
- else:
- bug_info = '%s reports' % self.bug_count
- return bug_info
-
-
- def GenerateBuildbotLinks(self):
- """Generate a link formatted to meet buildbot expectations.
-
- If there is a bug associated with this link, report a link to the bug
- and a link to the job logs; otherwise report a link to the job logs.
-
- @return A generator of links formatted for the buildbot log annotator.
- """
- if self.bug_url:
- yield self._get_link_to_bug()
- yield self._get_link_to_job_logs()
-
-
- def _get_link_to_bug(self):
- """Return buildbot link to bug.
-
- @return A link formatted for the buildbot log annotator.
- """
- info_strings = self._get_info_strings()
- info_strings.append(self._bug_count_text)
- anchor_text = self._format_anchor_text(self._BUG_LINK_PREFIX,
- info_strings)
- return annotations.StepLink(anchor_text, self.bug_url)
-
-
- def _get_link_to_job_logs(self):
- """Return buildbot link to job logs.
-
- @return A link formatted for the buildbot log annotator.
- """
- anchor_text = self._format_anchor_text(self._LOG_LINK_PREFIX,
- self._get_info_strings())
- return annotations.StepLink(anchor_text, self.url)
-
-
- def _get_info_strings(self):
- """Return a list of info strings for _format_anchor_text()."""
- info_strings = []
- if self.retry_count > 0:
- info_strings.append('retry_count: %d' % self.retry_count)
- if self.reason:
- info_strings.append(self.reason)
- return info_strings
-
-
- def _format_anchor_text(self, prefix, info_strings):
- """Format anchor text given a prefix and info strings.
-
- @param prefix The prefix of the anchor text.
- @param info_strings Iterable of strings.
- @return A anchor_text with the right prefix and info strings.
- """
- return '[{prefix}]: {anchor}: {info}'.format(
- prefix=prefix,
- anchor=self.anchor.strip(),
- info=', '.join(info_strings))
-
- @property
- def text_link(self):
- """Link to the job's logs, for consumption by a human.
-
- @return A link formatted for human readability.
- """
- return '%s %s' % (self.anchor, self.url)
-
- def GenerateRetryLink(self):
- """Generate a link to the retry dashboard.
-
- @return A link formatted for the buildbot log annotator.
- """
- if not self.testname or self.testname in self._SKIP_RETRY_DASHBOARD:
- return None
-
- # TODO(xixuan): Return the right flake dashboard later.
- return None
-
- def GenerateHistoryLink(self):
- """Generate a link to the test history dashboard.
-
- @return A link formatted for the buildbot log annotator.
- """
- if not self.testname or self.testname in self._SKIP_RETRY_DASHBOARD:
- return None
- return annotations.StepLink(
- text='[Test-History]: %s' % self.testname,
- url=reporting_utils.link_test_history(self.testname))
-
-
-class Timings(object):
- """Timings for important events during a suite.
-
- All timestamps are datetime.datetime objects.
-
- @var suite_job_id: the afe job id of the suite job for which
- we are recording the timing for.
- @var download_start_time: the time the devserver starts staging
- the build artifacts. Recorded in create_suite_job.
- @var payload_end_time: the time when the artifacts only necessary to start
- installsing images onto DUT's are staged.
- Recorded in create_suite_job.
- @var artifact_end_time: the remaining artifacts are downloaded after we kick
- off the reimaging job, at which point we record
- artifact_end_time. Recorded in dynamic_suite.py.
- @var suite_start_time: the time the suite started.
- @var tests_start_time: the time the first test started running.
- @var tests_end_time: the time the last test finished running.
- """
-
- def __init__(self, suite_job_id):
- self.suite_job_id = suite_job_id
- # Timings related to staging artifacts on devserver.
- self.download_start_time = None
- self.payload_end_time = None
- self.artifact_end_time = None
-
- # The test_start_time, but taken off the view that corresponds to the
- # suite instead of an individual test.
- self.suite_start_time = None
-
- # Earliest and Latest tests in the set of TestViews passed to us.
- self.tests_start_time = None
- self.tests_end_time = None
-
-
- def RecordTiming(self, view):
- """Given a test report view, extract and record pertinent time info.
-
- get_detailed_test_views() returns a list of entries that provide
- info about the various parts of a suite run. This method can take
- any one of these entries and look up timestamp info we might want
- and record it.
-
- If timestamps are unavailable, datetime.datetime.min/max will be used.
-
- @param view: A TestView object.
- """
- start_candidate = datetime.min
- end_candidate = datetime.max
- if view['test_started_time']:
- start_candidate = time_utils.time_string_to_datetime(
- view['test_started_time'])
- if view['test_finished_time']:
- end_candidate = time_utils.time_string_to_datetime(
- view['test_finished_time'])
-
- if view.get_testname() == TestView.SUITE_JOB:
- self.suite_start_time = start_candidate
- else:
- self._UpdateFirstTestStartTime(start_candidate)
- self._UpdateLastTestEndTime(end_candidate)
- if view['afe_job_id'] == self.suite_job_id and 'job_keyvals' in view:
- keyvals = view['job_keyvals']
- self.download_start_time = time_utils.time_string_to_datetime(
- keyvals.get(constants.DOWNLOAD_STARTED_TIME),
- handle_type_error=True)
-
- self.payload_end_time = time_utils.time_string_to_datetime(
- keyvals.get(constants.PAYLOAD_FINISHED_TIME),
- handle_type_error=True)
-
- self.artifact_end_time = time_utils.time_string_to_datetime(
- keyvals.get(constants.ARTIFACT_FINISHED_TIME),
- handle_type_error=True)
-
-
- def _UpdateFirstTestStartTime(self, candidate):
- """Update self.tests_start_time, iff candidate is an earlier time.
-
- @param candidate: a datetime.datetime object.
- """
- if not self.tests_start_time or candidate < self.tests_start_time:
- self.tests_start_time = candidate
-
-
- def _UpdateLastTestEndTime(self, candidate):
- """Update self.tests_end_time, iff candidate is a later time.
-
- @param candidate: a datetime.datetime object.
- """
- if not self.tests_end_time or candidate > self.tests_end_time:
- self.tests_end_time = candidate
-
-
- def __str__(self):
- return ('\n'
- 'Suite timings:\n'
- 'Downloads started at %s\n'
- 'Payload downloads ended at %s\n'
- 'Suite started at %s\n'
- 'Artifact downloads ended (at latest) at %s\n'
- 'Testing started at %s\n'
- 'Testing ended at %s\n' % (self.download_start_time,
- self.payload_end_time,
- self.suite_start_time,
- self.artifact_end_time,
- self.tests_start_time,
- self.tests_end_time))
-
-
-def instance_for_pool(pool_name):
- """
- Return the hostname of the server that should be used to service a suite
- for the specified pool.
-
- @param pool_name: The pool (without 'pool:' to schedule the suite against.
- @return: The correct host that should be used to service this suite run.
- """
- return CONFIG.get_config_value(
- 'POOL_INSTANCE_SHARDING', pool_name,
- default=_DEFAULT_AUTOTEST_INSTANCE)
-
-
-class TestView(object):
- """Represents a test view and provides a set of helper functions."""
-
-
- SUITE_JOB = 'Suite job'
-
-
- def __init__(self, view, afe_job, suite_name, build, user,
- solo_test_run=False):
- """Init a TestView object representing a tko test view.
-
- @param view: A dictionary representing a tko test view.
- @param afe_job: An instance of frontend.afe.models.Job
- representing the job that kicked off the test.
- @param suite_name: The name of the suite
- that the test belongs to.
- @param build: The build for which the test is run.
- @param user: The user for which the test is run.
- @param solo_test_run: This is a solo test run not part of a suite.
- """
- self.view = view
- self.afe_job = afe_job
- self.suite_name = suite_name
- self.build = build
- self.is_suite_view = afe_job.parent_job is None and not solo_test_run
- # This is the test name that will be shown in the output.
- self.testname = None
- self.user = user
-
- # The case that a job was aborted before it got a chance to run
- # usually indicates suite has timed out (unless aborted by user).
- # In this case, the abort reason will be None.
- # Update the reason with proper information.
- if (self.is_relevant_suite_view() and
- not self.get_testname() == self.SUITE_JOB and
- self.view['status'] == 'ABORT' and
- not self.view['reason']):
- self.view['reason'] = 'Timed out, did not run.'
-
-
- def __getitem__(self, key):
- """Overload __getitem__ so that we can still use []
-
- @param key: A key of the tko test view.
-
- @returns: The value of an attribute in the view.
-
- """
- return self.view[key]
-
-
- def __iter__(self):
- """Overload __iter__ so that it supports 'in' operator."""
- return iter(self.view)
-
-
- def get_testname(self):
- """Get test name that should be shown in the output.
-
- Formalize the test_name we got from the test view.
-
- Remove 'build/suite' prefix if any.
-
- If one runs a test in control file via the following code,
- job.runtest('my_Test', tag='tag')
- for most of the cases, view['test_name'] would look like 'my_Test.tag'.
- If this is the case, this method will just return the original
- test name, i.e. 'my_Test.tag'.
-
- There are four special cases.
- 1) A test view is for the suite job's SERVER_JOB.
- In this case, this method will return 'Suite job'.
-
- 2) A test view is of a child job or a solo test run not part of a
- suite, and for a SERVER_JOB or CLIENT_JOB.
- In this case, we will take the job name, remove the build/suite
- prefix from the job name, and append the rest to 'SERVER_JOB'
- or 'CLIENT_JOB' as a prefix. So the names returned by this
- method will look like:
- 'dummy_Pass_SERVER_JOB'
- 'dummy_Fail_SERVER_JOB'
-
- 3) A test view is of a suite job and its status is ABORT.
- In this case, the view['test_name'] is the child job's name.
- For instance,
- 'lumpy-release/R35-5712.0.0/dummy/dummy_Pass'
- 'lumpy-release/R35-5712.0.0/dummy/dummy_Fail'
- The above names will be converted to the following:
- 'dummy_Pass'
- 'dummy_Fail'
-
- 4) A test view's status is of a suite job and its status is TEST_NA.
- In this case, the view['test_name'] is the NAME field of the control
- file. For instance,
- 'dummy_Pass'
- 'dummy_Fail'
- This method will not modify these names.
-
- @returns: Test name after normalization.
-
- """
- if self.testname is not None:
- return self.testname
-
- if (self.is_suite_view and
- self.view['test_name'].startswith('SERVER_JOB')):
- # Rename suite job's SERVER_JOB to 'Suite job'.
- self.testname = self.SUITE_JOB
- return self.testname
-
- if (self.view['test_name'].startswith('SERVER_JOB') or
- self.view['test_name'].startswith('CLIENT_JOB')):
- # Append job name as a prefix for SERVER_JOB and CLIENT_JOB
- testname= '%s_%s' % (self.view['job_name'], self.view['test_name'])
- else:
- testname = self.view['test_name']
- # Remove the build and suite name from testname if any.
- self.testname = tools.get_test_name(
- self.build, self.suite_name, testname)
- return self.testname
-
-
- def is_relevant_suite_view(self):
- """Checks whether this is a suite view we should care about.
-
- @returns: True if it is relevant. False otherwise.
- """
- return (self.get_testname() == self.SUITE_JOB or
- (self.is_suite_view and
- not self.view['test_name'].startswith('CLIENT_JOB') and
- not self.view['subdir']))
-
-
- def is_test(self):
- """Return whether the view is for an actual test.
-
- @returns True if the view is for an actual test.
- False if the view is for SERVER_JOB or CLIENT_JOB.
-
- """
- return not (self.view['test_name'].startswith('SERVER_JOB') or
- self.view['test_name'].startswith('CLIENT_JOB'))
-
-
- def is_retry(self):
- """Check whether the view is for a retry.
-
- @returns: True, if the view is for a retry; False otherwise.
-
- """
- return self.view['job_keyvals'].get('retry_original_job_id') is not None
-
-
- def hit_timeout(self):
- """Check whether the corresponding job has hit its own timeout.
-
- Note this method should not be called for those test views
- that belongs to a suite job and are determined as irrelevant
- by is_relevant_suite_view. This is because they are associated
- to the suite job, whose job start/finished time make no sense
- to an irrelevant test view.
-
- @returns: True if the corresponding afe job has hit timeout.
- False otherwise.
- """
- if (self.is_relevant_suite_view() and
- self.get_testname() != self.SUITE_JOB):
- # Any relevant suite test view except SUITE_JOB
- # did not hit its own timeout because it was not ever run.
- return False
- start = (datetime.strptime(
- self.view['job_started_time'], time_utils.TIME_FMT)
- if self.view['job_started_time'] else None)
- end = (datetime.strptime(
- self.view['job_finished_time'], time_utils.TIME_FMT)
- if self.view['job_finished_time'] else None)
- if not start or not end:
- return False
- else:
- return ((end - start).total_seconds()/60.0
- > self.afe_job.max_runtime_mins)
-
-
- def is_aborted(self):
- """Check if the view was aborted.
-
- For suite job and child job test views, we check job keyval
- 'aborted_by' and test status.
-
- For relevant suite job test views, we only check test status
- because the suite job keyval won't make sense to individual
- test views.
-
- @returns: True if the test was as aborted, False otherwise.
-
- """
-
- if (self.is_relevant_suite_view() and
- self.get_testname() != self.SUITE_JOB):
- return self.view['status'] == 'ABORT'
- else:
- return (bool(self.view['job_keyvals'].get('aborted_by')) and
- self.view['status'] in ['ABORT', 'RUNNING'])
-
-
- def is_in_fail_status(self):
- """Check if the given test's status corresponds to a failure.
-
- @returns: True if the test's status is FAIL or ERROR. False otherwise.
-
- """
- # All the statuses tests can have when they fail.
- return self.view['status'] in ['FAIL', 'ERROR', 'ABORT']
-
-
- def is_provision(self):
- """Check whether this is a provision test."""
- return self.get_testname() == 'provision'
-
-
- def get_buildbot_link_reason(self):
- """Generate the buildbot link reason for the test.
-
- @returns: A string representing the reason.
-
- """
- return ('%s: %s' % (self.view['status'], self.view['reason'])
- if self.view['reason'] else self.view['status'])
-
-
- def get_job_id_owner_str(self):
- """Generate the job_id_owner string for a test.
-
- @returns: A string which looks like 135036-username
- """
- # self.user is actually the user that is executing this run_suite
- # call, which is not necessarily the same as the job-creating user.
- # The job creating user is available as a keyval; but fall back to
- # self.user in case that key is missing.
- job_user = self.view['job_keyvals'].get('user') or self.user
- return '%s-%s' % (self.view['afe_job_id'], job_user)
-
-
- def get_bug_info(self, suite_job_keyvals):
- """Get the bug info from suite_job_keyvals.
-
- If a bug has been filed for the test, its bug info (bug id and counts)
- will be stored in the suite job's keyvals. This method attempts to
- retrieve bug info of the test from |suite_job_keyvals|. It will return
- None if no bug info is found. No need to check bug info if the view is
- SUITE_JOB.
-
- @param suite_job_keyvals: The job keyval dictionary of the suite job.
- All the bug info about child jobs are stored in
- suite job's keyvals.
-
- @returns: None if there is no bug info, or a pair with the
- id of the bug, and the count of the number of
- times the bug has been seen.
-
- """
- if self.get_testname() == self.SUITE_JOB:
- return None
- if (self.view['test_name'].startswith('SERVER_JOB') or
- self.view['test_name'].startswith('CLIENT_JOB')):
- # Append job name as a prefix for SERVER_JOB and CLIENT_JOB
- testname= '%s_%s' % (self.view['job_name'], self.view['test_name'])
- else:
- testname = self.view['test_name']
-
- return tools.get_test_failure_bug_info(
- suite_job_keyvals, self.view['afe_job_id'],
- testname)
-
-
- def should_display_buildbot_link(self):
- """Check whether a buildbot link should show for this view.
-
- For suite job view, show buildbot link if it fails.
- For normal test view,
- show buildbot link if it is a retry
- show buildbot link if it hits its own timeout.
- show buildbot link if it fails. This doesn't
- include the case where it was aborted but has
- not hit its own timeout (most likely it was aborted because
- suite has timed out).
-
- @returns: True if we should show the buildbot link.
- False otherwise.
- """
- is_bad_status = (self.view['status'] != 'GOOD' and
- self.view['status'] != 'TEST_NA')
- if self.get_testname() == self.SUITE_JOB:
- return is_bad_status
- else:
- if self.is_retry():
- return True
- if is_bad_status:
- return not self.is_aborted() or self.hit_timeout()
-
-
- def get_control_file_attributes(self):
- """Get the attributes from the control file of the test.
-
- @returns: A list of test attribute or None.
- """
- control_file = self.afe_job.control_file
- attributes = None
- if control_file:
- cd = control_data.parse_control_string(control_file)
- attributes = list(cd.attributes)
- return attributes
-
-
- def override_afe_job_id(self, afe_job_id):
- """Overrides the AFE job id for the test.
-
- @param afe_job_id: The new AFE job id to use.
- """
- self.view['afe_job_id'] = afe_job_id
-
-
-def log_buildbot_links(log_func, links):
- """Output buildbot links to log.
-
- @param log_func: Logging function to use.
- @param links: Iterable of LogLink instances.
- """
- for link in links:
- for generated_link in link.GenerateBuildbotLinks():
- log_func(generated_link)
- retry_link = link.GenerateRetryLink()
- if retry_link:
- log_func(retry_link)
- history_link = link.GenerateHistoryLink()
- if history_link:
- log_func(history_link)
-
-
-class _ReturnCodeComputer(object):
- """This is responsible for returning the _ReturnResult for a suite."""
-
- def __call__(self, test_views):
- """Compute the exit code based on test results."""
- result = _RETURN_RESULTS['ok']
-
- for v in test_views:
- if v.get_testname() == TestView.SUITE_JOB:
- result |= self._get_suite_result(v)
- else:
- result |= self._get_test_result(v)
- return result
-
- def _get_suite_result(self, test_view):
- """Return the _ReturnResult for the given suite job."""
- # The order of checking each case is important.
- if test_view.is_aborted() and test_view.hit_timeout():
- return _RETURN_RESULTS['suite_timeout']
- elif test_view.is_in_fail_status():
- return _RETURN_RESULTS['suite_failed']
- elif test_view['status'] == 'WARN':
- return _RETURN_RESULTS['suite_warning']
- else:
- return _RETURN_RESULTS['ok']
-
- def _get_test_result(self, test_view):
- """Return the _ReturnResult for the given test job."""
- # The order of checking each case is important.
- if test_view.is_aborted() and test_view.is_relevant_suite_view():
- # The test was aborted before started
- # This gurantees that the suite has timed out.
- return _RETURN_RESULTS['test_aborted_prestart']
- elif test_view.is_aborted() and not test_view.hit_timeout():
- # The test was aborted, but
- # not due to a timeout. This is most likely
- # because the suite has timed out, but may
- # also because it was aborted by the user.
- # Since suite timing out is determined by checking
- # the suite job view, we simply ignore this view here.
- return _RETURN_RESULTS['test_aborted_mystery']
- elif test_view.is_in_fail_status(): # The test job failed
- if test_view.is_provision():
- return _RETURN_RESULTS['provision_failed']
- else:
- return _RETURN_RESULTS['test_failure']
- elif test_view['status'] == 'WARN':
- return _RETURN_RESULTS['test_warning']
- elif test_view.is_retry():
- # The test is a passing retry.
- return _RETURN_RESULTS['test_retry']
- else:
- return _RETURN_RESULTS['ok']
-
-
-class _ProvisionReturnCodeComputer(_ReturnCodeComputer):
- """This is used for returning the _ReturnResult for provision suites."""
-
- def __init__(self, num_required):
- """Initialize instance.
-
- num_required is the number of passing provision jobs needed.
- """
- super(_ProvisionReturnCodeComputer, self).__init__()
- self._num_required = num_required
- self._num_successful = 0
-
- def __call__(self, test_views):
- result = super(_ProvisionReturnCodeComputer, self).__call__(test_views)
- if self._num_successful >= self._num_required:
- logging.info('Return result upgraded from %r'
- ' due to enough ok provisions',
- result)
- return _RETURN_RESULTS['ok']
- else:
- return result
-
- def _get_test_result(self, test_view):
- result = (super(_ProvisionReturnCodeComputer, self)
- ._get_test_result(test_view))
- if result in {_RETURN_RESULTS[s] for s in ('ok', 'test_retry')}:
- self._num_successful += 1
- return result
-
-
-class ResultCollector(object):
- """Collect test results of a suite or a single test run.
-
- Once a suite job has finished, use this class to collect test results.
- `run` is the core method that is to be called first. Then the caller
- could retrieve information like return code, return message, is_aborted,
- and timings by accessing the collector's public attributes. And output
- the test results and links by calling the 'output_*' methods.
-
- Here is a overview of what `run` method does.
-
- 1) Collect the suite job's results from tko_test_view_2.
- For the suite job, we only pull test views without a 'subdir'.
- A NULL subdir indicates that the test was _not_ executed. This could be
- that no child job was scheduled for this test or the child job got
- aborted before starts running.
- (Note 'SERVER_JOB'/'CLIENT_JOB' are handled specially)
-
- 2) Collect the child jobs' results from tko_test_view_2.
- For child jobs, we pull all the test views associated with them.
- (Note 'SERVER_JOB'/'CLIENT_JOB' are handled specially)
-
- 3) Generate web and buildbot links.
- 4) Compute timings of the suite run.
- 5) Compute the return code based on test results.
-
- @var _instance_server: The hostname of the server that is used
- to service the suite.
- @var _afe: The afe rpc client.
- @var _tko: The tko rpc client.
- @var _build: The build for which the suite is run,
- e.g. 'lumpy-release/R35-5712.0.0'
- @var _suite_name: The suite name, e.g. 'bvt', 'dummy'.
- @var _suite_job_id: The job id of the suite for which we are going to
- collect results.
- @var _original_suite_name: The suite name we record timing would be
- different from _suite_name when running
- suite_attr_wrapper.
- @var _return_code_function: Called to return what the overall result of
- the suite is.
- @var _suite_views: A list of TestView objects, representing relevant
- test views of the suite job.
- @var _child_views: A list of TestView objects, representing test views
- of the child jobs.
- @var _test_views: A list of TestView objects, representing all test views
- from _suite_views and _child_views.
- @var _web_links: A list of web links pointing to the results of jobs.
- @var buildbot_links: A list of buildbot links for non-passing tests.
- @var _solo_test_run: True if this is a single test run.
- @var return_result: The _ReturnResult of the suite run.
- @var is_aborted: Whether the suite was aborted or not.
- True, False or None (aborting status is unknown yet)
- @var timings: A Timing object that records the suite's timings.
-
- """
-
-
- def __init__(self, instance_server, afe, tko, build,
- suite_name, suite_job_id, return_code_function,
- original_suite_name=None,
- user=None, solo_test_run=False):
- self._instance_server = instance_server
- self._afe = afe
- self._tko = tko
- self._build = build
- self._suite_name = suite_name
- self._suite_job_id = suite_job_id
- self._original_suite_name = original_suite_name or suite_name
- self._return_code_function = return_code_function
- self._suite_views = []
- self._child_views = []
- self._test_views = []
- self._retry_counts = {}
- self._missing_results = {}
- self._web_links = []
- self.buildbot_links = []
- self._num_child_jobs = 0
- self.return_result = None
- self.is_aborted = None
- self.timings = None
- self._user = user or getpass.getuser()
- self._solo_test_run = solo_test_run
-
-
- def _fetch_relevant_test_views_of_suite(self):
- """Fetch relevant test views of the suite job.
-
- For the suite job, there will be a test view for SERVER_JOB, and views
- for results of its child jobs. For example, assume we've created
- a suite job (afe_job_id: 40) that runs dummy_Pass, dummy_Fail,
- dummy_Pass.bluetooth. Assume dummy_Pass was aborted before running while
- dummy_Path.bluetooth got TEST_NA as no duts have bluetooth.
- So the suite job's test views would look like
- _____________________________________________________________________
- test_idx| job_idx|test_name |subdir |afe_job_id|status
- 10 | 1000 |SERVER_JOB |---- |40 |GOOD
- 11 | 1000 |dummy_Pass |NULL |40 |ABORT
- 12 | 1000 |dummy_Fail.Fail |41-onwer/...|40 |FAIL
- 13 | 1000 |dummy_Fail.Error |42-owner/...|40 |ERROR
- 14 | 1000 |dummy_Pass.bluetooth|NULL |40 |TEST_NA
-
- For a suite job, we only care about
- a) The test view for the suite job's SERVER_JOB
- b) The test views for real tests without a subdir. A NULL subdir
- indicates that a test didn't get executed.
- So, for the above example, we only keep test views whose test_idxs
- are 10, 11, 14.
-
- @returns: A list of TestView objects, representing relevant
- test views of the suite job.
-
- """
- suite_job = self._afe.get_jobs(id=self._suite_job_id)[0]
- views = self._tko.run(call='get_detailed_test_views',
- afe_job_id=self._suite_job_id)
- relevant_views = []
- for v in views:
- v = TestView(v, suite_job, self._suite_name, self._build, self._user,
- solo_test_run=self._solo_test_run)
- if v.is_relevant_suite_view():
- # If the test doesn't have results in TKO and is being
- # displayed in the suite view instead of the child view,
- # then afe_job_id is incorrect and from the suite.
- # Override it based on the AFE job id which was missing
- # results.
- # TODO: This is likely inaccurate if a test has multiple
- # tries which all fail TKO parse stage.
- if v['test_name'] in self._missing_results:
- v.override_afe_job_id(
- self._missing_results[v['test_name']][0])
- relevant_views.append(v)
- return relevant_views
-
-
- def _compute_retry_count(self, view):
- """Return how many times the test has been retried.
-
- @param view: A TestView instance.
- @returns: An int value indicating the retry count.
-
- """
- old_job = view['job_keyvals'].get('retry_original_job_id')
- count = 0
- while old_job:
- count += 1
- views = self._tko.run(
- call='get_detailed_test_views', afe_job_id=old_job)
- old_job = (views[0]['job_keyvals'].get('retry_original_job_id')
- if views else None)
- return count
-
-
- def _fetch_test_views_of_child_jobs(self, jobs=None):
- """Fetch test views of child jobs.
-
- @returns: A tuple (child_views, retry_counts, missing_results)
- child_views is list of TestView objects, representing
- all valid views.
- retry_counts is a dictionary that maps test_idx to retry
- counts. It only stores retry counts that are greater than 0.
- missing_results is a dictionary that maps test names to
- lists of job ids.
-
- """
- child_views = []
- retry_counts = {}
- missing_results = {}
- child_jobs = jobs or self._afe.get_jobs(parent_job_id=self._suite_job_id)
- if child_jobs:
- self._num_child_jobs = len(child_jobs)
- for job in child_jobs:
- views = [TestView(v, job, self._suite_name, self._build, self._user)
- for v in self._tko.run(
- call='get_detailed_test_views', afe_job_id=job.id,
- invalid=0)]
- if len(views) == 0:
- missing_results.setdefault(job.name, []).append(job.id)
- contains_test_failure = any(
- v.is_test() and v['status'] != 'GOOD' for v in views)
- for v in views:
- if (v.is_test() or
- v['status'] != 'GOOD' and not contains_test_failure):
- # For normal test view, just keep it.
- # For SERVER_JOB or CLIENT_JOB, only keep it
- # if it fails and no other test failure.
- child_views.append(v)
- retry_count = self._compute_retry_count(v)
- if retry_count > 0:
- retry_counts[v['test_idx']] = retry_count
- return child_views, retry_counts, missing_results
-
-
- def _generate_web_and_buildbot_links(self):
- """Generate web links and buildbot links."""
- # TODO(fdeng): If a job was aborted before it reaches Running
- # state, we read the test view from the suite job
- # and thus this method generates a link pointing to the
- # suite job's page for the aborted job. Need a fix.
- self._web_links = []
- self.buildbot_links = []
-
- # Bug info are stored in the suite job's keyvals.
- if self._solo_test_run:
- suite_job_keyvals = {}
- elif not self._suite_views:
- suite_job_keyvals = {}
- else:
- suite_job_keyvals = self._suite_views[0]['job_keyvals']
-
- for v in self._test_views:
- retry_count = self._retry_counts.get(v['test_idx'], 0)
- bug_info = v.get_bug_info(suite_job_keyvals)
- job_id_owner = v.get_job_id_owner_str()
- link = LogLink(
- anchor=v.get_testname(),
- server=self._instance_server,
- job_string=job_id_owner,
- bug_info=bug_info, retry_count=retry_count,
- testname=v.get_testname(),
- sponge_url=suite_job_keyvals.get('sponge_url'))
- self._web_links.append(link)
-
- if v.should_display_buildbot_link():
- link.reason = v.get_buildbot_link_reason()
- self.buildbot_links.append(link)
-
-
- def _record_timings(self):
- """Record suite timings."""
- self.timings = Timings(self._suite_job_id)
- for v in self._test_views:
- self.timings.RecordTiming(v)
-
-
- def _compute_return_code(self):
- """Compute the exit code based on test results."""
- self.return_result = self._return_code_function(self._test_views)
-
-
- def _make_test_results(self):
- """Make TestResults for collected tests.
-
- @returns: List of TestResult instances.
- """
- test_results = []
- for test_view in self._test_views:
- test_result = TestResult(
- test_view=test_view,
- retry_count=self._retry_counts.get(test_view['test_idx'], 0))
- test_results.append(test_result)
- return test_results
-
-
- def output_results(self):
- """Output test results, timings and web links."""
- # Output test results
- test_results = self._make_test_results()
- if len(test_results) == 0:
- max_name_length = 0
- else:
- max_name_length = max(len(t.name) for t in test_results)
- for test_result in test_results:
- test_result.log_using(logging.info, max_name_length + 3)
- # Output suite timings
- logging.info(self.timings)
- # Output links to test logs
- logging.info('\nLinks to test logs:')
- for link in self._web_links:
- logging.info(link.text_link)
- logging.info('\n')
-
-
- def get_results_dict(self):
- """Write test results, timings and web links into a dict.
-
- @returns: A dict of results in the format like:
- {
- 'tests': {
- 'test_1': {'status': 'GOOD', 'attributes': [1,2], ...}
- 'test_2': {'status': 'FAIL', 'attributes': [1],...}
- }
- 'suite_timings': {
- 'download_start': '1998-07-17 00:00:00',
- 'payload_download_end': '1998-07-17 00:00:05',
- ...
- }
- }
- """
- output_dict = {}
- tests_dict = output_dict.setdefault('tests', {})
- for v in self._test_views:
- test_name = v.get_testname()
- test_info = tests_dict.setdefault(test_name, {})
- test_info.update({
- 'status': v['status'],
- 'attributes': v.get_control_file_attributes() or list(),
- 'reason': v['reason'],
- 'retry_count': self._retry_counts.get(v['test_idx'], 0),
- 'job_id': v['afe_job_id'],
- })
- # For aborted test, the control file will not be parsed and thus
- # fail to get the attributes info. Therefore, the subsystems the
- # abort test testing will be missing. For this case, we will assume
- # the aborted test will test all subsystems, set subsystem:default.
- if (test_info['status'] == 'ABORT' and
- not any('subsystem:' in a for a in test_info['attributes'])):
- test_info['attributes'].append('subsystem:default')
-
- # Write the links to test logs into the |tests_dict| of |output_dict|.
- # For test whose status is not 'GOOD', the link is also buildbot_link.
- for link in self._web_links:
- test_name = link.anchor.strip()
- test_info = tests_dict.get(test_name)
- if test_info:
- test_info['link_to_logs'] = link.url
- test_info['sponge_url'] = link.sponge_url
- # Write the retry dashboard link into the dict.
- if link in self.buildbot_links and link.testname:
- test_info['retry_dashboard_link'] \
- = reporting_utils.link_retry_url(link.testname)
- # Always write the wmatrix link for compatibility.
- test_info['wmatrix_link'] \
- = reporting_utils.link_wmatrix_retry_url(link.testname)
- # Write the bug url into the dict.
- if link.bug_id:
- test_info['bug_url'] = link.bug_url
-
- # Write the suite timings into |output_dict|
- timings = self.timings
- if timings is not None:
- time_dict = output_dict.setdefault('suite_timings', {})
- time_dict.update({
- 'download_start' : str(timings.download_start_time),
- 'payload_download_end' : str(timings.payload_end_time),
- 'suite_start' : str(timings.suite_start_time),
- 'artifact_download_end' : str(timings.artifact_end_time),
- 'tests_start' : str(timings.tests_start_time),
- 'tests_end' : str(timings.tests_end_time),
- })
-
- output_dict['suite_job_id'] = self._suite_job_id
-
- return output_dict
-
-
- def run(self):
- """Collect test results.
-
- This method goes through the following steps:
- Fetch relevent test views of the suite job.
- Fetch test views of child jobs
- Check whether the suite was aborted.
- Generate links.
- Calculate suite timings.
- Compute return code based on the test result.
-
- """
- if self._solo_test_run:
- self._test_views, self._retry_counts, self._missing_results = (
- self._fetch_test_views_of_child_jobs(
- jobs=self._afe.get_jobs(id=self._suite_job_id)))
- else:
- self._child_views, self._retry_counts, self._missing_results = (
- self._fetch_test_views_of_child_jobs())
- self._suite_views = self._fetch_relevant_test_views_of_suite()
- self._test_views = self._suite_views + self._child_views
- # For hostless job in Starting status, there is no test view associated.
- # This can happen when a suite job in Starting status is aborted. When
- # the scheduler hits some limit, e.g., max_hostless_jobs_per_drone,
- # max_jobs_started_per_cycle, a suite job can stays in Starting status.
- if not self._test_views:
- self.return_result = _RETURN_RESULTS['test_views_missing']
- return
- self.is_aborted = any([view['job_keyvals'].get('aborted_by')
- for view in self._suite_views])
- self._generate_web_and_buildbot_links()
- self._record_timings()
- self._compute_return_code()
-
-
-def _make_child_dependencies(options):
- """Creates a list of extra dependencies for child jobs.
-
- @param options: Parsed arguments to run_suite.
-
- @returns: A list of label strings if any dependencies should be added. None
- otherwise.
- """
- if not options.model:
- return ()
- return ('model:%s' % options.model,)
-
-
-@retry.retry(error.StageControlFileFailure, timeout_min=10)
-def create_suite(afe, options):
- """Create a suite with retries.
-
- @param afe: The afe object to insert the new suite job into.
- @param options: The options to use in creating the suite.
-
- @return: The afe_job_id of the new suite job.
- """
- logging.info('%s Submitted create_suite_job rpc',
- diagnosis_utils.JobTimer.format_time(datetime.now()))
-
- return afe.run(
- 'create_suite_job',
- name=options.name,
- board=options.board,
- builds=suite_common.make_builds_from_options(options),
- test_source_build=options.test_source_build,
- check_hosts=not options.no_wait,
- pool=options.pool,
- file_bugs=options.file_bugs,
- priority=options.priority,
- suite_args=options.suite_args,
- wait_for_results=not options.no_wait,
- timeout_mins=options.timeout_mins + options.delay_minutes,
- max_runtime_mins=options.max_runtime_mins + options.delay_minutes,
- job_retry=options.retry,
- max_retries=options.max_retries,
- suite_min_duts=options.suite_min_duts,
- offload_failures_only=options.offload_failures_only,
- run_prod_code=options.run_prod_code,
- delay_minutes=options.delay_minutes,
- job_keyvals=options.job_keyvals,
- test_args=options.test_args,
- child_dependencies=options.child_dependencies,
- )
-
-
-def _run_suite(options):
- """
- run_suite script without exception handling.
-
- @param options: The parsed options.
-
- @returns: A tuple contains the return_code of run_suite and the dictionary
- of the output.
-
- """
- # If indicate to use the new style suite control file, convert the args
- if options.use_suite_attr:
- options = change_options_for_suite_attr(options)
-
- log_name = _get_log_name(options)
- utils.setup_logging(logfile=log_name)
-
- if not options.bypass_labstatus and not options.web:
- utils.check_lab_status(options.build)
-
- afe = _create_afe(options)
- instance_server = afe.server
-
- rpc_helper = diagnosis_utils.RPCHelper(afe)
- is_real_time = True
- if options.mock_job_id:
- job_id = int(options.mock_job_id)
- existing_job = afe.get_jobs(id=job_id, finished=True)
- if existing_job:
- is_real_time = False
- else:
- existing_job = afe.get_jobs(id=job_id)
- if existing_job:
- job_created_on = time_utils.date_string_to_epoch_time(
- existing_job[0].created_on)
- else:
- raise utils.TestLabException('Failed to retrieve job: %d' % job_id)
- else:
- try:
- rpc_helper.check_dut_availability(options.dependencies,
- options.minimum_duts,
- options.skip_duts_check)
- job_id = create_suite(afe, options)
- job_created_on = time.time()
- except (error.CrosDynamicSuiteException,
- error.RPCException, proxy.JSONRPCException) as e:
- logging.exception('Error Message: %s', e)
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE,
- {'return_message': str(e)})
- except AttributeError as e:
- logging.exception('Error Message: %s', e)
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INVALID_OPTIONS)
-
- job_timer = diagnosis_utils.JobTimer(
- job_created_on, float(options.timeout_mins))
- job_url = reporting_utils.link_job(job_id,
- instance_server=instance_server)
- _log_create_task(job_timer, job_url, job_id)
-
- if options.create_and_return:
- msg = '--create_and_return was specified, terminating now.'
- logging.info(msg)
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.OK,
- {'return_message': msg})
-
- if options.no_wait:
- return _handle_job_nowait(job_id, options, instance_server)
- else:
- return _handle_job_wait(afe, job_id, options, job_timer, is_real_time)
-
-
-def _get_log_name(options):
- """Return local log file's name.
-
- @param options: Parsed options.
-
- @return log_name, a string file name.
- """
- if options.require_logfile:
- # options.build is verified to exist in verify_options.
- # convert build name from containing / to containing only _.
- log_name = 'run_suite-%s.log' % options.build.replace('/', '_')
- log_dir = os.path.join(common.autotest_dir, 'logs')
- if os.path.exists(log_dir):
- log_name = os.path.join(log_dir, log_name)
-
- return log_name
- else:
- return None
-
-
-def _create_afe(options):
- """Return an afe instance based on options.
-
- @param options Parsed options.
-
- @return afe, an AFE instance.
- """
- instance_server = (options.web if options.web else
- instance_for_pool(options.pool))
- afe = frontend_wrappers.RetryingAFE(server=instance_server,
- timeout_min=options.afe_timeout_mins,
- delay_sec=options.delay_sec)
- logging.info('Autotest instance created: %s', instance_server)
- return afe
-
-
-def _handle_job_wait(afe, job_id, options, job_timer, is_real_time):
- """Handle suite job synchronously.
-
- @param afe AFE instance.
- @param job_id Suite job id.
- @param options Parsed options.
- @param job_timer JobTimer for suite job.
- @param is_real_time Whether or not to handle job timeout.
-
- @return SuiteResult of suite job.
- """
- rpc_helper = diagnosis_utils.RPCHelper(afe)
- instance_server = afe.server
- while not afe.get_jobs(id=job_id, finished=True):
- _poke_buildbot_with_output(afe, job_id, job_timer)
- if job_timer.debug_output_timer.poll():
- logging.info('The suite job has another %s till timeout.',
- job_timer.timeout_hours - job_timer.elapsed_time())
- time.sleep(10)
- logging.info('%s Suite job is finished.',
- diagnosis_utils.JobTimer.format_time(datetime.now()))
- # For most cases, ResultCollector should be able to determine whether
- # a suite has timed out by checking information in the test view.
- # However, occationally tko parser may fail on parsing the
- # job_finished time from the job's keyval file. So we add another
- # layer of timeout check in run_suite. We do the check right after
- # the suite finishes to make it as accurate as possible.
- # There is a minor race condition here where we might have aborted
- # for some reason other than a timeout, and the job_timer thinks
- # it's a timeout because of the jitter in waiting for results.
- # The consequence would be that run_suite exits with code
- # SUITE_TIMEOUT while it should have returned INFRA_FAILURE
- # instead, which should happen very rarely.
- # Note the timeout will have no sense when using -m option.
- is_suite_timeout = job_timer.is_suite_timeout()
-
- # Extract the original suite name to record timing.
- original_suite_name = get_original_suite_name(options.name,
- options.suite_args)
- # Start collecting test results.
- logging.info('%s Start collecting test results and dump them to json.',
- diagnosis_utils.JobTimer.format_time(datetime.now()))
- TKO = frontend_wrappers.RetryingTKO(server=instance_server,
- timeout_min=options.afe_timeout_mins,
- delay_sec=options.delay_sec)
- # TODO(crbug.com/672348): It needs to be possible for provision
- # suite to pass if only a few tests fail. Otherwise, a single
- # failing test will be reported as failure even if the suite reports
- # success.
- if options.name == _PROVISION_SUITE:
- # TODO(crbug.com/672348): Creating the suite job requires that
- # suite_args contains num_required.
- return_code_function = _ProvisionReturnCodeComputer(
- num_required=options.suite_args['num_required'])
- else:
- return_code_function = _ReturnCodeComputer()
- collector = ResultCollector(instance_server=instance_server,
- afe=afe, tko=TKO, build=options.build,
- suite_name=options.name,
- suite_job_id=job_id,
- return_code_function=return_code_function,
- original_suite_name=original_suite_name)
- collector.run()
- # Dump test outputs into json.
- output_dict = collector.get_results_dict()
- output_dict['autotest_instance'] = instance_server
- if not (options.json_dump or options.json_dump_postfix):
- collector.output_results()
- result = collector.return_result
- if is_real_time:
- if collector.is_aborted == True and is_suite_timeout:
- # There are two possible cases when a suite times out.
- # 1. the suite job was aborted due to timing out
- # 2. the suite job succeeded, but some child jobs
- # were already aborted before the suite job exited.
- # The case 2 was handled by ResultCollector,
- # here we handle case 1.
- result |= _RETURN_RESULTS['suite_timeout']
-
- # And output return message.
- if result.message:
- logging.info('Reason: %s', result.message)
-
- logging.info('\n %s Output below this line is for buildbot consumption:',
- diagnosis_utils.JobTimer.format_time(datetime.now()))
- log_buildbot_links(logging.info, collector.buildbot_links)
- return result.suite_result(output_dict)
-
-
-def _handle_job_nowait(job_id, options, instance_server):
- """Handle suite job asynchronously.
-
- @param job_id Suite job id.
- @param options Parsed options.
- @param instance_server Autotest instance hostname.
-
- @return SuiteResult of suite job.
- """
- logging.info('Created suite job: %r', job_id)
- link = LogLink(options.name, instance_server,
- '%s-%s' % (job_id, getpass.getuser()))
- for generate_link in link.GenerateBuildbotLinks():
- logging.info(generate_link)
- logging.info('--no_wait specified; Exiting.')
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.OK,
- {'return_message': '--no_wait specified; Exiting.'})
-
-
-def _should_run(options):
- """Check whether the suite should be run based on lab/job status checking.
-
- @param options Parsed options.
- """
- try:
- site_utils.check_lab_status(options.test_source_build)
- except site_utils.TestLabException as ex:
- logging.exception('Lab is closed or build is blocked. Skipping '
- 'suite %s, board %s, build %s: %s',
- options.name, options.board,
- options.test_source_build, str(ex))
- return False
-
- start_time = str(datetime.now() -
- timedelta(days=_SEARCH_JOB_MAX_DAYS))
- afe = _create_afe(options)
- afe_jobs = afe.get_jobs(
- name__istartswith=options.test_source_build,
- name__iendswith='control.'+options.name,
- created_on__gte=start_time,
- min_rpc_timeout=_MIN_RPC_TIMEOUT)
- if options.model:
- model_tag = 'model:%s' % options.model
- filtered_jobs = [j for j in afe_jobs if model_tag in j.control_file]
- else:
- filtered_jobs = afe_jobs
-
- if filtered_jobs:
- logging.info('Found duplicate suite %s scheduled in past.',
- filtered_jobs)
- return False
-
- return True
-
-
-def _poke_buildbot_with_output(afe, job_id, job_timer):
- """Poke buildbot so it doesn't timeout from silence.
-
- @param afe AFE instance.
- @param job_id Suite job id.
- @param job_timer JobTimer for suite job.
- """
- rpc_helper = diagnosis_utils.RPCHelper(afe)
- # Note that this call logs output, preventing buildbot's
- # 9000 second silent timeout from kicking in. Let there be no
- # doubt, this is a hack. The timeout is from upstream buildbot and
- # this is the easiest work around.
- if job_timer.first_past_halftime():
- rpc_helper.diagnose_job(job_id, afe.server)
-
-
-
-def _run_task(options):
- """Perform this script's function minus setup.
-
- Boilerplate like argument parsing, logging, output formatting happen
- elsewhere.
-
- Returns a SuiteResult instance.
-
- TODO(ayatane): The try/except should be moved into _run_suite().
- Good luck trying to figure out which function calls are supposed to
- raise which of the exceptions.
- """
- try:
- return _run_suite(options)
- except diagnosis_utils.DUTsNotAvailableError as e:
- result = run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.BOARD_NOT_AVAILABLE,
- {'return_message': 'Skipping testing: %s' % e.message})
- logging.info(result.output_dict['return_message'])
- return result
- except utils.TestLabException as e:
- result = run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE,
- {'return_message': 'TestLabException: %s' % e})
- logging.exception(result.output_dict['return_message'])
- return result
-
-
-class _ExceptionHandler(object):
- """Global exception handler replacement."""
-
- def __init__(self, dump_json):
- """Initialize instance.
-
- @param dump_json: Whether to print a JSON dump of the result dict to
- stdout.
- """
- self._should_dump_json = dump_json
-
- def __call__(self, exc_type, value, traceback):
- if self._should_dump_json:
- run_suite_common.dump_json(
- {'return_message': ('Unhandled run_suite exception: %s'
- % value)})
- sys.exit(run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
-
-def _log_create_task(job_timer, job_url, job_id):
- """Logging for task creation."""
- logging.info('%s Created suite job: %s',
- job_timer.format_time(job_timer.job_created_time),
- job_url)
- logging.info(annotations.StepLink(text='Link to suite', url=job_url))
- # For task id parsing of chromite HWTestStage.
- logging.info('Created task id: %s', job_id)
-
-
-def _if_run_in_skylab(options):
- """Detect whether to run suite in skylab.
-
- Returns:
- A tuple of (bool, string, string) to indicate
- (if_use_skylab, override_pool, override_qs_account)
- """
- # An autotest job id is a number of at least 9 digits, e.g. 296843118.
- # A skylab task id is of 16 chars, e.g. 43cabbb4e118ea10.
- if len(str(options.mock_job_id)) >= 16:
- # No override info is needed if mock_job_id is specified.
- return True, '', ''
-
- if not _ENABLE_RUN_SUITE_TRAMPOLINE:
- logging.info('trampoline to skylab is not enabled.')
- return False, '', ''
-
- task_info = 'suite:%s, board:%s, model:%s, pool:%s' % (
- options.name, options.board, options.model, options.pool)
- ctx = gs.GSContext()
- with osutils.TempDir(prefix='trampoline_') as tempdir:
- temp_file = os.path.join(tempdir, _MIGRATION_CONFIG_FILE)
- ctx.Copy(_TRAMPOLINE_CONFIG, temp_file)
- _migration_config = config_reader.MigrationConfig(
- config_reader.ConfigReader(temp_file))
-
- logging.info('Checking whether to run in skylab: Task(%s)', task_info)
- if skylab.should_run_in_skylab(_migration_config,
- options.board,
- options.model,
- options.name,
- options.pool):
- logging.info('Task (%s) Should run in skylab', task_info)
- override_pool, override_qs_account = skylab.get_override_info(
- _migration_config,
- options.board,
- options.model,
- options.name,
- options.pool)
- return True, override_pool, override_qs_account
-
- logging.info('Task (%s) Should run in autotest', task_info)
- return False, '', ''
-
-
-def _get_skylab_suite_result(child_tasks):
- """Parse skylab task result to get final result for the suite.
-
- @param child_tasks: A list of json dict of task result object, whose format
- is: {
- 'name': ...,
- 'state': ...,
- 'failure': ...
- }.
- """
- _final_suite_states = run_suite_common.get_final_skylab_suite_states()
- for ct in child_tasks:
- logging.info('Parsing test %r', ct)
- state = run_suite_common.get_final_skylab_task_state(ct)
-
- if (state not in run_suite_common.IGNORED_TEST_STATE and
- state in _final_suite_states):
- return _final_suite_states[state][1]
-
- return run_suite_common.RETURN_CODES.OK
-
-
-def _log_skylab_for_buildbot(stdout):
- """Output skylab logs to buildbot.
-
- @param stdout: A string.
- """
- logging.info('\n %s Output below this line is for buildbot consumption:',
- diagnosis_utils.JobTimer.format_time(datetime.now()))
- logging.info(stdout)
-
-
-def _run_paygen_with_skylab(options, override_pool, override_qs_account):
- """Run paygen suites with skylab."""
- builds = suite_common.make_builds_from_options(options)
- skylab_tool = os.environ.get('SKYLAB_TOOL') or _SKYLAB_TOOL
- test_source_build = suite_common.get_test_source_build(builds)
- pool = ('DUT_POOL_%s' % options.pool.upper()
- if not override_pool else override_pool)
- paygen_tests = paygen.get_paygen_tests(test_source_build, options.name)
- for test in paygen_tests:
- cmd = [skylab_tool, 'create-test']
- cmd += paygen.paygen_skylab_args(
- test, options.name, test_source_build, pool, options.board,
- options.model, options.timeout_mins,
- override_qs_account, _SKYLAB_SERVICE_ACCOUNT)
- job_created_on = time.time()
- try:
- res = cros_build_lib.RunCommand(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- logging.error(str(e))
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
- logging.info(res.output)
- job_url = res.output.split()[-1]
- job_id = job_url.split('id=')[-1]
- job_timer = diagnosis_utils.JobTimer(
- job_created_on, float(options.timeout_mins))
- _log_create_task(job_timer, job_url, job_id)
-
- return run_suite_common.SuiteResult(run_suite_common.RETURN_CODES.OK)
-
-
-def _run_with_skylab(options, override_pool, override_qs_account):
- """Run suite inside skylab."""
- if paygen.is_paygen_suite(options.name):
- return _run_paygen_with_skylab(options, override_pool,
- override_qs_account)
-
- builds = suite_common.make_builds_from_options(options)
- skylab_tool = os.environ.get('SKYLAB_TOOL') or _SKYLAB_TOOL
- pool = override_pool or options.pool
- if options.mock_job_id:
- taskID = options.mock_job_id
- cmd = [skylab_tool, 'wait-task',
- '-timeout-mins', str(options.timeout_mins),
- '-service-account-json', _SKYLAB_SERVICE_ACCOUNT,
- taskID]
- try:
- res = cros_build_lib.RunCommand(cmd, capture_output=True)
- except cros_build_lib.RunCommandError as e:
- logging.error(str(e))
- return run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
- output = json.loads(res.output)
- child_tasks = output['child-results']
- task_stdout = output['stdout']
-
- return_code = _get_skylab_suite_result(child_tasks)
- _log_skylab_for_buildbot(task_stdout)
- return run_suite_common.SuiteResult(return_code)
- else:
- cmd = [skylab_tool, 'create-suite',
- '-board', options.board,
- '-image', builds[provision.CROS_VERSION_PREFIX],
- '-pool', pool,
- '-timeout-mins', str(options.timeout_mins),
- '-priority', str(skylab_priority_for(options.priority)),
- '-service-account-json', _SKYLAB_SERVICE_ACCOUNT]
- if override_qs_account:
- cmd.extend(['-qs-account', override_qs_account])
-
- if options.max_retries is not None:
- cmd.extend(['-max-retries', str(options.max_retries)])
-
- if options.model is not None:
- cmd.extend(['-model', options.model])
-
- tags = ['skylab:run_suite_trampoline']
- for t in tags:
- cmd.extend(['-tag', t])
-
- unsupported_skylab_keyvals = ['datastore_parent_key']
- if options.job_keyvals is not None:
- for k, v in options.job_keyvals.iteritems():
- if k in unsupported_skylab_keyvals:
- continue
-
- cmd.extend(['-keyval', '%s:%s' % (k, v)])
-
- cmd.extend([options.name])
- job_created_on = time.time()
- res = cros_build_lib.RunCommand(cmd, capture_output=True)
- # TODO (xixuan): The parsing will change with crbug.com/935244.
- logging.info(res.output)
- job_url = res.output.split()[-1]
- job_id = job_url.split('id=')[-1]
- job_timer = diagnosis_utils.JobTimer(
- job_created_on, float(options.timeout_mins))
- _log_create_task(job_timer, job_url, job_id)
- return run_suite_common.SuiteResult(run_suite_common.RETURN_CODES.OK)
-
-
-def _run_with_autotest(options):
- """Run suite inside autotest."""
- if options.pre_check and not _should_run(options):
- logging.info('Suite %s-%s is terminated: Lab is closed, OR build is '
- 'blocked, OR this suite has already been kicked off '
- 'once in past %d days.',
- options.test_source_build, options.name,
- _SEARCH_JOB_MAX_DAYS)
- result = run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.ERROR,
- {'return_message': ("Lab is closed OR other reason"
- " (see code, it's complicated)")})
- else:
- result = _run_task(options)
-
- if options.json_dump or options.json_dump_postfix:
- run_suite_common.dump_json(result.output_dict)
-
- return result
-
-
-def main():
- """Entry point."""
- utils.verify_not_root_user()
-
- parser = make_parser()
- options = parser.parse_args()
- if options.do_nothing:
- return 0
-
- if options.suite_args_json and options.suite_args:
- raise ValueError("suite_args and suite_args_json may not both "
- "be specified.")
- if options.suite_args_json:
- options.suite_args = options.suite_args_json
-
- sys.exceptionhandler = _ExceptionHandler(
- dump_json=(options.json_dump or options.json_dump_postfix))
- if options.json_dump:
- # Not disabled in json_dump_postfix mode, intentionally.
- logging.disable(logging.CRITICAL)
-
- options_okay = verify_and_clean_options(options)
- # Set StreamHandler first to capture error messages if suite is not run.
- utils.setup_logging()
- if not options_okay:
- parser.print_help()
- result = run_suite_common.SuiteResult(
- run_suite_common.RETURN_CODES.INVALID_OPTIONS)
- else:
- try:
- is_skylab, ovrd_pool, ovrd_qs_account = _if_run_in_skylab(options)
- except Exception as e:
- logging.exception(str(e))
- logging.info('fall back to Autotest due to trampoline errors')
- is_skylab = False
- ovrd_pool = ''
- ovrd_qs_account = ''
-
- if is_skylab:
- result = _run_with_skylab(options, ovrd_pool, ovrd_qs_account)
- else:
- result = _run_with_autotest(options)
-
- logging.info('Will return from run_suite with status: %s',
- run_suite_common.RETURN_CODES.get_string(result.return_code))
- return result.return_code
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/site_utils/run_suite_common.py b/site_utils/run_suite_common.py
deleted file mode 100644
index ce9f21d..0000000
--- a/site_utils/run_suite_common.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright 2018 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Shared libs by run_suite.py & run_suite_skylab.py."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import json
-import sys
-import time
-
-from autotest_lib.client.common_lib import enum
-
-
-# Return code that will be sent back to callers.
-#
-# Note: Do not modify this enum; it is dependend upon by several clients.
-RETURN_CODES = enum.Enum(
- 'OK',
- 'ERROR',
- 'WARNING',
- 'INFRA_FAILURE',
- 'SUITE_TIMEOUT',
- 'BOARD_NOT_AVAILABLE',
- 'INVALID_OPTIONS',
-)
-
-# TODO (xixuan): This is duplicated from suite_tracking.py in skylab.
-# Make skylab caller also use this func.
-TASK_COMPLETED = 'COMPLETED'
-TASK_COMPLETED_SUCCESS = 'COMPLETED (SUCCESS)'
-TASK_COMPLETED_FAILURE = 'COMPLETED (FAILURE)'
-TASK_EXPIRED = 'EXPIRED'
-TASK_CANCELED = 'CANCELED'
-TASK_TIMEDOUT = 'TIMED_OUT'
-TASK_RUNNING = 'RUNNING'
-TASK_PENDING = 'PENDING'
-TASK_BOT_DIED = 'BOT_DIED'
-TASK_NO_RESOURCE = 'NO_RESOURCE'
-TASK_KILLED = 'KILLED'
-
-# Test status in _IGNORED_TEST_STATE won't be reported as test failure.
-# Or test may be reported as failure as
-# it's probably caused by the DUT is not well-provisioned.
-# TODO: Stop ignoring TASK_NO_RESOURCE if we drop TEST_NA feature.
-# Blocking issues:
-# - Not all DUT labels are in skylab yet (crbug.com/871978)
-IGNORED_TEST_STATE = [TASK_NO_RESOURCE]
-
-
-class SuiteResult(collections.namedtuple('SuiteResult',
- ['return_code', 'output_dict'])):
- """Result of running a suite to return."""
-
- def __new__(cls, return_code, output_dict=None):
- if output_dict is None:
- output_dict = dict()
- else:
- output_dict = output_dict.copy()
- output_dict['return_code'] = return_code
- return super(SuiteResult, cls).__new__(cls, return_code, output_dict)
-
- @property
- def string_code(self):
- """Return the enum string name of the numerical return_code."""
- return RETURN_CODES.get_string(self.return_code)
-
-
-def dump_json(obj):
- """Write obj JSON to stdout."""
- output_json = json.dumps(obj, sort_keys=True)
- # These sleeps and flushes are a hack around the fact that when running
- # in the autotest proxy in --json_dump_postfix mode, run_suite.py co-mingles
- # both stdout and stderr (which include both logging output and json output)
- # to a single output stream. This can cause the json output to be corrupted
- # by concurrent writes (which is particularly likely because this dump
- # occurs at the end of run_suite's execution, along with other ending
- # logging). Buffer time and forced stream flushes reduce the likelihood of
- # concurrent writes.
- sys.stderr.flush()
- time.sleep(0.5)
- sys.stdout.flush()
- sys.stdout.write('\n#JSON_START#%s#JSON_END#\n' % output_json.strip())
- sys.stdout.flush()
- time.sleep(0.5)
-
-
-# TODO (xixuan): This is duplicated from suite_tracking.py in skylab.
-# Make skylab caller also use this func.
-def get_final_skylab_task_state(task_result):
- """Get the final state of a swarming task.
-
- @param task_result: A json dict of SwarmingRpcsTaskResult object.
- """
- state = task_result['state']
- if state == TASK_COMPLETED:
- state = (TASK_COMPLETED_FAILURE if task_result['failure'] else
- TASK_COMPLETED_SUCCESS)
-
- return state
-
-
-def get_final_skylab_suite_states():
- return {
- TASK_COMPLETED_FAILURE:
- (
- TASK_COMPLETED_FAILURE,
- RETURN_CODES.ERROR,
- ),
- # Task No_Resource means no available bots to accept the task.
- # Deputy should check whether it's infra failure.
- TASK_NO_RESOURCE:
- (
- TASK_NO_RESOURCE,
- RETURN_CODES.INFRA_FAILURE,
- ),
- # Task expired means a task is not triggered, could be caused by
- # 1. No healthy DUTs/bots to run it.
- # 2. Expiration seconds are too low.
- # 3. Suite run is too slow to finish.
- # Deputy should check whether it's infra failure.
- TASK_EXPIRED:
- (
- TASK_EXPIRED,
- RETURN_CODES.INFRA_FAILURE,
- ),
- # Task canceled means a task is canceled intentionally. Deputy
- # should check whether it's infra failure.
- TASK_CANCELED:
- (
- TASK_CANCELED,
- RETURN_CODES.INFRA_FAILURE,
- ),
- TASK_TIMEDOUT:
- (
- TASK_TIMEDOUT,
- RETURN_CODES.SUITE_TIMEOUT,
- ),
- # Task pending means a task is still waiting for picking up, but
- # the suite already hits deadline. So report it as suite TIMEOUT.
- # It could also be an INFRA_FAILURE due to DUTs/bots shortage.
- TASK_PENDING:
- (
- TASK_TIMEDOUT,
- RETURN_CODES.SUITE_TIMEOUT,
- ),
- }
diff --git a/site_utils/run_suite_unittest.py b/site_utils/run_suite_unittest.py
deleted file mode 100755
index 4268e5f..0000000
--- a/site_utils/run_suite_unittest.py
+++ /dev/null
@@ -1,660 +0,0 @@
-#!/usr/bin/python2
-# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import collections
-import datetime as datetime_base
-from datetime import datetime
-import mock
-import time
-import unittest
-
-import common
-
-from autotest_lib.server.cros.dynamic_suite import constants
-from autotest_lib.site_utils import run_suite
-from autotest_lib.site_utils import run_suite_common
-from autotest_lib.site_utils import diagnosis_utils
-
-
-class ReturnResultUnittest(unittest.TestCase):
- """_ReturnResult tests."""
-
- def setUp(self):
- super(ReturnResultUnittest, self).setUp()
- patcher = mock.patch.object(run_suite, '_RETURN_RESULTS',
- collections.OrderedDict())
- self.results = results = patcher.start()
- self.addCleanup(patcher.stop)
- results['small'] = run_suite._ReturnResult(0, 'small')
- results['big'] = run_suite._ReturnResult(1, 'big')
-
- patcher = mock.patch.object(run_suite, '_RETURN_RESULTS_LIST',
- list(results.values()))
- patcher.start()
- self.addCleanup(patcher.stop)
-
- def test_equal(self):
- """Test _ReturnResult equal."""
- self.assertEqual(self.results['small'], self.results['small'])
-
- def test_unequal(self):
- """Test _ReturnResult unequal."""
- self.assertNotEqual(self.results['big'], self.results['small'])
-
- def test_greater_than(self):
- """Test _ReturnResult greater than."""
- self.assertGreater(self.results['big'], self.results['small'])
-
- def test_bitwise_or(self):
- """Test _ReturnResult bitwise or."""
- self.assertEqual(self.results['big'],
- self.results['big'] | self.results['small'])
-
-
-class ResultCollectorUnittest(unittest.TestCase):
- """Runsuite unittest"""
-
- JOB_MAX_RUNTIME_MINS = 10
-
- def setUp(self):
- """Set up test."""
- self.afe = mock.MagicMock()
- self.tko = mock.MagicMock()
-
-
- def _build_view(self, test_idx, test_name, subdir, status, afe_job_id,
- job_name='fake_job_name', reason='fake reason',
- job_keyvals=None, test_started_time=None,
- test_finished_time=None, invalidates_test_idx=None,
- job_started_time=None, job_finished_time=None):
- """Build a test view using the given fields.
-
- @param test_idx: An integer representing test_idx.
- @param test_name: A string, e.g. 'dummy_Pass'
- @param subdir: A string representing the subdir field of the test view.
- e.g. 'dummy_Pass'.
- @param status: A string representing the test status.
- e.g. 'FAIL', 'PASS'
- @param afe_job_id: An integer representing the afe job id.
- @param job_name: A string representing the job name.
- @param reason: A string representing the reason field of the test view.
- @param job_keyvals: A dictionary stroing the job keyvals.
- @param test_started_time: A string, e.g. '2014-04-12 12:35:33'
- @param test_finished_time: A string, e.g. '2014-04-12 12:35:33'
- @param invalidates_test_idx: An integer, representing the idx of the
- test that has been retried.
- @param job_started_time: A string, e.g. '2014-04-12 12:35:33'
- @param job_finished_time: A string, e.g. '2014-04-12 12:35:33'
-
- @reutrn: A dictionary representing a test view.
-
- """
- if job_keyvals is None:
- job_keyvals = {}
- return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir,
- 'status': status, 'afe_job_id': afe_job_id,
- 'job_name': job_name, 'reason': reason,
- 'job_keyvals': job_keyvals,
- 'test_started_time': test_started_time,
- 'test_finished_time': test_finished_time,
- 'invalidates_test_idx': invalidates_test_idx,
- 'job_started_time': job_started_time,
- 'job_finished_time': job_finished_time}
-
-
- def _mock_tko_get_detailed_test_views(self, test_views,
- missing_results=[]):
- """Mock tko method get_detailed_test_views call.
-
- @param test_views: A list of test views that will be returned
- by get_detailed_test_views.
- """
- return_values = {}
- for v in test_views:
- views_of_job = return_values.setdefault(
- ('get_detailed_test_views', v['afe_job_id']), [])
- views_of_job.append(v)
- for job_id in missing_results:
- views_of_job = return_values.setdefault(
- ('get_detailed_test_views', job_id), [])
-
- def side_effect(*args, **kwargs):
- """Maps args and kwargs to the mocked return values."""
- key = (kwargs['call'], kwargs['afe_job_id'])
- return return_values[key]
-
- self.tko.run = mock.MagicMock(side_effect=side_effect)
-
-
- def _mock_afe_get_jobs(self, suite_job_id, child_job_ids):
- """Mock afe get_jobs call.
-
- @param suite_job_id: The afe job id of the suite job.
- @param child_job_ids: A list of job ids of the child jobs.
-
- """
- suite_job = mock.MagicMock()
- suite_job.id = suite_job_id
- suite_job.max_runtime_mins = 10
- suite_job.parent_job = None
-
- return_values = {suite_job_id: []}
- for job_id in child_job_ids:
- new_job = mock.MagicMock()
- new_job.id = job_id
- new_job.name = 'test.%d' % job_id
- new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS
- new_job.parent_job = suite_job
- return_values[suite_job_id].append(new_job)
-
- def side_effect(*args, **kwargs):
- """Maps args and kwargs to the mocked return values."""
- if kwargs.get('id') == suite_job_id:
- return [suite_job]
- return return_values[kwargs['parent_job_id']]
-
- self.afe.get_jobs = mock.MagicMock(side_effect=side_effect)
-
-
- def testFetchSuiteTestView(self):
- """Test that it fetches the correct suite test views."""
- suite_job_id = 100
- suite_name = 'dummy'
- build = 'R23-1.1.1.1'
- server_job_view = self._build_view(
- 10, 'SERVER_JOB', '----', 'GOOD', suite_job_id)
- test_to_ignore = self._build_view(
- 11, 'dummy_Pass', '101-user/host/dummy_Pass',
- 'GOOD', suite_job_id)
- test_to_include = self._build_view(
- 12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id)
- test_missing = self._build_view(
- 13, 'dummy_Missing', None, 'ABORT', suite_job_id)
- self._mock_afe_get_jobs(suite_job_id, [])
- self._mock_tko_get_detailed_test_views(
- [server_job_view, test_to_ignore, test_to_include,
- test_missing])
- collector = run_suite.ResultCollector(
- 'fake_server', self.afe, self.tko,
- build=build, suite_name=suite_name,
- suite_job_id=suite_job_id,
- return_code_function=run_suite._ReturnCodeComputer())
- collector._missing_results = {
- test_missing['test_name']: [14, 15],
- }
- suite_views = collector._fetch_relevant_test_views_of_suite()
- suite_views = sorted(suite_views, key=lambda view: view['test_idx'])
- # Verify that SERVER_JOB is renamed to 'Suite job'
- self.assertEqual(suite_views[0].get_testname(),
- run_suite.TestView.SUITE_JOB)
- # Verify that the test with a subidr is not included.
- self.assertEqual(suite_views[0]['test_idx'], 10)
- self.assertEqual(suite_views[1]['test_idx'], 12)
- self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id)
- # Verify that the test with missing results had it's AFE job id
- # replaced.
- self.assertEqual(suite_views[2]['test_idx'], 13)
- self.assertEqual(suite_views[2]['afe_job_id'], 14)
-
-
- def testFetchTestViewOfChildJobs(self):
- """Test that it fetches the correct child test views."""
- build = 'lumpy-release/R36-5788.0.0'
- board = 'lumpy'
- suite_name = 'my_suite'
- suite_job_id = 100
- invalid_job_id = 101
- invalid_job_name = '%s/%s/test_Pass' % (build, suite_name)
- good_job_id = 102
- good_job_name = '%s/%s/test_Pass' % (build, suite_name)
- bad_job_id = 103
- bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name)
- missing_job_id = 104
-
- invalid_test = self._build_view(
- 19, 'test_Pass_Old', 'fake/subdir',
- 'FAIL', invalid_job_id, invalid_job_name)
- good_job_server_job = self._build_view(
- 20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name)
- good_job_test = self._build_view(
- 21, 'test_Pass', 'fake/subdir', 'GOOD',
- good_job_id, good_job_name,
- job_keyvals={'retry_original_job_id': invalid_job_id})
- bad_job_server_job = self._build_view(
- 22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name)
- bad_job_test = self._build_view(
- 23, 'test_ServerJobFail', 'fake/subdir', 'GOOD',
- bad_job_id, bad_job_name)
- self._mock_tko_get_detailed_test_views(
- [good_job_server_job, good_job_test,
- bad_job_server_job, bad_job_test, invalid_test],
- [missing_job_id])
- self._mock_afe_get_jobs(suite_job_id,
- [good_job_id, bad_job_id, missing_job_id])
- collector = run_suite.ResultCollector(
- 'fake_server', self.afe, self.tko,
- build, suite_name, suite_job_id,
- return_code_function=run_suite._ReturnCodeComputer())
- child_views, retry_counts, missing_results = (
- collector._fetch_test_views_of_child_jobs())
- # child_views should contain tests 21, 22, 23
- child_views = sorted(child_views, key=lambda view: view['test_idx'])
- # Verify that the SERVER_JOB has been renamed properly
- self.assertEqual(child_views[1].get_testname(),
- 'test_ServerJobFail_SERVER_JOB')
- self.assertEqual(missing_results, {'test.104': [104]})
- # Verify that failed SERVER_JOB and actual invalid tests are included,
- expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'],
- bad_job_test['test_idx']]
- child_view_ids = [v['test_idx'] for v in child_views]
- self.assertEqual(child_view_ids, expected)
- self.afe.get_jobs.assert_called_once_with(
- parent_job_id=suite_job_id)
- # Verify the retry_counts is calculated correctly
- self.assertEqual(len(retry_counts), 1)
- self.assertEqual(retry_counts[21], 1)
-
-
- def testGenerateLinks(self):
- """Test that it generates correct web and buildbot links."""
- suite_job_id = 100
- suite_name = 'my_suite'
- build = 'lumpy-release/R36-5788.0.0'
- board = 'lumpy'
- fake_job = mock.MagicMock()
- fake_job.parent = suite_job_id
- test_sponge_url = 'http://test_url'
- job_keyvals = {'sponge_url': test_sponge_url}
- suite_job_view = run_suite.TestView(
- self._build_view(
- 20, 'Suite job', '----', 'GOOD', suite_job_id,
- job_keyvals=job_keyvals),
- fake_job, suite_name, build, 'chromeos-test')
- good_test = run_suite.TestView(
- self._build_view(
- 21, 'test_Pass', 'fake/subdir', 'GOOD', 101,
- job_keyvals=job_keyvals),
- fake_job, suite_name, build, 'chromeos-test')
- bad_test = run_suite.TestView(
- self._build_view(
- 23, 'test_Fail', 'fake/subdir', 'FAIL', 102,
- job_keyvals=job_keyvals),
- fake_job, suite_name, build, 'chromeos-test')
-
- collector = run_suite.ResultCollector(
- 'fake_server', self.afe, self.tko,
- build, suite_name, suite_job_id, user='chromeos-test',
- return_code_function=run_suite._ReturnCodeComputer())
- collector._suite_views = [suite_job_view]
- collector._test_views = [suite_job_view, good_test, bad_test]
- collector._max_testname_width = max(
- [len(v.get_testname()) for v in collector._test_views]) + 3
- collector._generate_web_and_buildbot_links()
- URL_PATTERN = run_suite._URL_PATTERN
- # expected_web_links is list of (anchor, url) tuples we
- # are expecting.
- expected_web_links = [
- (v.get_testname(),
- URL_PATTERN % ('http://fake_server',
- '%s-%s' % (v['afe_job_id'], 'chromeos-test')),
- test_sponge_url)
- for v in collector._test_views]
- # Verify web links are generated correctly.
- for i in range(len(collector._web_links)):
- expect = expected_web_links[i]
- self.assertEqual(collector._web_links[i].anchor, expect[0])
- self.assertEqual(collector._web_links[i].url, expect[1])
- self.assertEqual(collector._web_links[i].sponge_url, expect[2])
-
- expected_buildbot_links = [
- (v.get_testname(),
- URL_PATTERN % ('http://fake_server',
- '%s-%s' % (v['afe_job_id'], 'chromeos-test')))
- for v in collector._test_views if v['status'] != 'GOOD']
- # Verify buildbot links are generated correctly.
- for i in range(len(collector.buildbot_links)):
- expect = expected_buildbot_links[i]
- self.assertEqual(collector.buildbot_links[i].anchor, expect[0])
- self.assertEqual(collector.buildbot_links[i].url, expect[1])
- self.assertEqual(collector.buildbot_links[i].retry_count, 0)
- # Assert that a retry dashboard link is created.
- self.assertNotEqual(
- collector.buildbot_links[i].GenerateRetryLink(), '')
- self.assertNotEqual(
- collector.buildbot_links[i].GenerateHistoryLink(), '')
-
-
- def _end_to_end_test_helper(
- self, include_bad_test=False, include_warn_test=False,
- include_timeout_test=False,
- include_self_aborted_test=False,
- include_aborted_by_suite_test=False,
- include_good_retry=False, include_bad_retry=False,
- include_good_test=True,
- suite_job_timed_out=False, suite_job_status='GOOD'):
- """A helper method for testing ResultCollector end-to-end.
-
- This method mocks the retrieving of required test views,
- and call ResultCollector.run() to collect the results.
-
- @param include_bad_test:
- If True, include a view of a test which has status 'FAIL'.
- @param include_warn_test:
- If True, include a view of a test which has status 'WARN'
- @param include_timeout_test:
- If True, include a view of a test which was aborted before
- started.
- @param include_self_aborted_test:
- If True, include a view of test which was aborted after
- started and hit hits own timeout.
- @param include_self_aborted_by_suite_test:
- If True, include a view of test which was aborted after
- started but has not hit its own timeout.
- @param include_good_retry:
- If True, include a test that passed after retry.
- @param include_bad_retry:
- If True, include a test that failed after retry.
- @param include_good_test:
- If True, include a test that passed. If False, pretend no tests
- (including the parent suite job) came back with any test
- results.
- @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING'
-
- @returns: A ResultCollector instance.
- """
- suite_job_id = 100
- good_job_id = 101
- bad_job_id = 102
- warn_job_id = 102
- timeout_job_id = 100
- self_aborted_job_id = 104
- aborted_by_suite_job_id = 105
- good_retry_job_id = 106
- bad_retry_job_id = 107
- invalid_job_id_1 = 90
- invalid_job_id_2 = 91
- suite_name = 'dummy'
- build = 'lumpy-release/R27-3888.0.0'
- suite_job_keyvals = {
- constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20',
- constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25',
- constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'}
-
- suite_job_started_time = '2014-04-29 13:14:37'
- if suite_job_timed_out:
- suite_job_keyvals['aborted_by'] = 'test_user'
- suite_job_finished_time = '2014-04-29 13:25:37'
- suite_job_status = 'ABORT'
- else:
- suite_job_finished_time = '2014-04-29 13:23:37'
-
- server_job_view = self._build_view(
- 10, 'SERVER_JOB', '----', suite_job_status, suite_job_id,
- 'lumpy-release/R27-3888.0.0-test_suites/control.dummy',
- '', suite_job_keyvals, '2014-04-29 13:14:37',
- '2014-04-29 13:20:27', job_started_time=suite_job_started_time,
- job_finished_time=suite_job_finished_time)
- good_test = self._build_view(
- 11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD',
- good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass',
- '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36')
- bad_test = self._build_view(
- 12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL',
- bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail',
- 'always fail', {}, '2014-04-29 13:16:00',
- '2014-04-29 13:16:02')
- warn_test = self._build_view(
- 13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN',
- warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn',
- 'always warn', {}, '2014-04-29 13:16:00',
- '2014-04-29 13:16:02')
- timeout_test = self._build_view(
- 15, 'dummy_Timeout', '', 'ABORT',
- timeout_job_id,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout',
- 'child job did not run', {}, '2014-04-29 13:15:37',
- '2014-04-29 13:15:38')
- self_aborted_test = self._build_view(
- 16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT',
- self_aborted_job_id,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
- 'child job aborted', {'aborted_by': 'test_user'},
- '2014-04-29 13:15:39', '2014-04-29 13:15:40',
- job_started_time='2014-04-29 13:15:39',
- job_finished_time='2014-04-29 13:25:40')
- aborted_by_suite = self._build_view(
- 17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite',
- 'RUNNING', aborted_by_suite_job_id,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',
- 'aborted by suite', {'aborted_by': 'test_user'},
- '2014-04-29 13:15:39', '2014-04-29 13:15:40',
- job_started_time='2014-04-29 13:15:39',
- job_finished_time='2014-04-29 13:15:40')
- good_retry = self._build_view(
- 18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD',
- good_retry_job_id,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
- '', {'retry_original_job_id': invalid_job_id_1},
- '2014-04-29 13:15:37',
- '2014-04-29 13:15:38', invalidates_test_idx=1)
- bad_retry = self._build_view(
- 19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL',
- bad_retry_job_id,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
- 'retry failed', {'retry_original_job_id': invalid_job_id_2},
- '2014-04-29 13:15:39', '2014-04-29 13:15:40',
- invalidates_test_idx=2)
- invalid_test_1 = self._build_view(
- 1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD',
- invalid_job_id_1,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',
- 'original test failed', {}, '2014-04-29 13:10:00',
- '2014-04-29 13:10:01')
- invalid_test_2 = self._build_view(
- 2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL',
- invalid_job_id_2,
- 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',
- 'original test failed', {},
- '2014-04-29 13:10:03', '2014-04-29 13:10:04')
-
- test_views = []
- child_jobs = set()
- missing_results = []
- if include_good_test:
- test_views.append(server_job_view)
- test_views.append(good_test)
- child_jobs.add(good_job_id)
- # Emulate missing even the parent/suite job.
- else:
- missing_results.append(suite_job_id)
- if include_bad_test:
- test_views.append(bad_test)
- child_jobs.add(bad_job_id)
- if include_warn_test:
- test_views.append(warn_test)
- child_jobs.add(warn_job_id)
- if include_timeout_test:
- test_views.append(timeout_test)
- if include_self_aborted_test:
- test_views.append(self_aborted_test)
- child_jobs.add(self_aborted_job_id)
- if include_good_retry:
- test_views.extend([good_retry, invalid_test_1])
- child_jobs.add(good_retry_job_id)
- if include_bad_retry:
- test_views.extend([bad_retry, invalid_test_2])
- child_jobs.add(bad_retry_job_id)
- if include_aborted_by_suite_test:
- test_views.append(aborted_by_suite)
- child_jobs.add(aborted_by_suite_job_id)
- self._mock_tko_get_detailed_test_views(test_views,
- missing_results=missing_results)
- self._mock_afe_get_jobs(suite_job_id, child_jobs)
- collector = run_suite.ResultCollector(
- 'fake_server', self.afe, self.tko,
- 'lumpy-release/R36-5788.0.0', 'dummy', suite_job_id,
- return_code_function=run_suite._ReturnCodeComputer())
- collector.run()
- collector.output_results()
- return collector
-
-
- def testEndToEndSuiteEmpty(self):
- """Test it returns code INFRA_FAILURE when no tests report back."""
- collector = self._end_to_end_test_helper(include_good_test=False)
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
-
- def testEndToEndSuitePass(self):
- """Test it returns code OK when all test pass."""
- collector = self._end_to_end_test_helper()
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.OK)
-
-
- def testEndToEndSuiteWarn(self):
- """Test it returns code WARNING when there is a test that warns."""
- collector = self._end_to_end_test_helper(include_warn_test=True)
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.WARNING)
-
-
- def testEndToEndSuiteFail(self):
- """Test it returns code ERROR when there is a test that fails."""
- collector = self._end_to_end_test_helper(include_bad_test=True)
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.ERROR)
-
-
- def testEndToEndSuiteJobFail(self):
- """Test it returns code SUITE_FAILURE when only the suite job failed."""
- collector = self._end_to_end_test_helper(suite_job_status='ABORT')
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
- collector = self._end_to_end_test_helper(suite_job_status='ERROR')
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.INFRA_FAILURE)
-
-
- def testEndToEndRetry(self):
- """Test it returns correct code when a test was retried."""
- collector = self._end_to_end_test_helper(include_good_retry=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.WARNING)
-
- collector = self._end_to_end_test_helper(include_good_retry=True,
- include_self_aborted_test=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.ERROR)
-
- collector = self._end_to_end_test_helper(include_good_retry=True,
- include_bad_test=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.ERROR)
-
- collector = self._end_to_end_test_helper(include_bad_retry=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.ERROR)
-
-
- def testEndToEndSuiteTimeout(self):
- """Test it returns correct code when a child job timed out."""
- # a child job timed out before started, none failed.
- collector = self._end_to_end_test_helper(include_timeout_test=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT)
-
- # a child job timed out before started, and one test failed.
- collector = self._end_to_end_test_helper(
- include_bad_test=True, include_timeout_test=True)
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.ERROR)
-
- # a child job timed out before started, and one test warned.
- collector = self._end_to_end_test_helper(
- include_warn_test=True, include_timeout_test=True)
- self.assertEqual(collector.return_result.return_code,
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT)
-
- # a child job timed out before started, and one test was retried.
- collector = self._end_to_end_test_helper(include_good_retry=True,
- include_timeout_test=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT)
-
- # a child jot was aborted because suite timed out.
- collector = self._end_to_end_test_helper(
- include_aborted_by_suite_test=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT)
-
- # suite job timed out.
- collector = self._end_to_end_test_helper(suite_job_timed_out=True)
- self.assertEqual(
- collector.return_result.return_code,
- run_suite_common.RETURN_CODES.SUITE_TIMEOUT)
-
-
-class LogLinkUnittests(unittest.TestCase):
- """Test the LogLink"""
-
- def testGenerateBuildbotLinks(self):
- """Test LogLink GenerateBuildbotLinks"""
- log_link_a = run_suite.LogLink('mock_anchor', 'mock_server',
- 'mock_job_string',
- bug_info=('mock_bug_id', 1),
- reason='mock_reason',
- retry_count=1,
- testname='mock_testname')
- # Generate a bug link and a log link when bug_info is present
- self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2)
-
- log_link_b = run_suite.LogLink('mock_anchor', 'mock_server',
- 'mock_job_string_b',
- reason='mock_reason',
- retry_count=1,
- testname='mock_testname')
- # Generate a log link when there is no bug_info
- self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1)
-
-
-class SimpleTimerUnittests(unittest.TestCase):
- """Test the simple timer."""
-
- def testPoll(self):
- """Test polling the timer."""
- interval_hours = 0.0001
- t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours)
- deadline = t.deadline
- self.assertTrue(deadline is not None and
- t.interval_hours == interval_hours)
- min_deadline = (datetime.now() +
- datetime_base.timedelta(hours=interval_hours))
- time.sleep(interval_hours * 3600)
- self.assertTrue(t.poll())
- self.assertTrue(t.deadline >= min_deadline)
-
-
- def testBadInterval(self):
- """Test a bad interval."""
- t = diagnosis_utils.SimpleTimer(interval_hours=-1)
- self.assertTrue(t.deadline is None and t.poll() == False)
- t._reset()
- self.assertTrue(t.deadline is None and t.poll() == False)
-
-
-if __name__ == '__main__':
- unittest.main()