[autotest] Suite scheduler files bugs for exceptions raised in create_suite_job
This CL is to make suite scheduler file bugs using our bug filer for
several types of exceptions raised by create_suite_job. The exceptions are:
- ControlFileNotFound
- NoControlFileList
- ControlFileEmpty
- ControlFileMalformed
To enable bug filing, use '-b' option of suite_scheduler.py
Bug will be assgined to Lab sheriff.
This CL changes the private method _create_bug_report to public in
server/cros/dynamic_suite/reporting.py, so that we can file a general
bug that is not a test failure. After crbug.com/254256 is fixed,
the bug filing logic in deduping_scheduler.py should be modified.
This CL also moves ParseBuildName() from base_event.py to site_utils,
to avoid import loop. Callers of this methods are updated.
BUG=chromium:242569
TEST=deduping_scheduler_unittest.py;driver_unittest.py;base_event_unittest.py;
ran on chromeos-lab1.hot and confirm bug are properly filed.
DEPLOY=suite_scheduler
Change-Id: Ia57a2e625b7b39dcfe51892c208613c927f3a54e
Reviewed-on: https://gerrit.chromium.org/gerrit/60158
Commit-Queue: Fang Deng <fdeng@chromium.org>
Reviewed-by: Fang Deng <fdeng@chromium.org>
Tested-by: Fang Deng <fdeng@chromium.org>
diff --git a/client/common_lib/site_utils.py b/client/common_lib/site_utils.py
index 62caf1b..580b6d5 100644
--- a/client/common_lib/site_utils.py
+++ b/client/common_lib/site_utils.py
@@ -22,6 +22,26 @@
LAB_GOOD_STATES = ('open', 'throttled')
+class ParseBuildNameException(Exception):
+ """Raised when ParseBuildName() cannot parse a build name."""
+ pass
+
+
+def ParseBuildName(name):
+ """Format a build name, given board, type, milestone, and manifest num.
+
+ @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0'
+ @return board: board the manifest is for, e.g. x86-alex.
+ @return type: one of 'release', 'factory', or 'firmware'
+ @return milestone: (numeric) milestone the manifest was associated with.
+ @return manifest: manifest number, e.g. '2015.0.0'
+ """
+ match = re.match(r'([\w-]+)-(\w+)/R(\d+)-([\d.ab-]+)', name)
+ if match and len(match.groups()) == 4:
+ return match.groups()
+ raise ParseBuildNameException('%s is a malformed build name.' % name)
+
+
def ping(host, deadline=None, tries=None, timeout=60):
"""Attempt to ping |host|.
diff --git a/global_config.ini b/global_config.ini
index 3473d40..143ee58 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -198,3 +198,4 @@
[NOTIFICATIONS]
chromium_build_url: http://build.chromium.org/p/chromiumos/
sheriffs: USE SHADOW SHERIFFS
+lab_sheriffs: USE SHADOW SHERIFFS
diff --git a/server/cros/dynamic_suite/reporting.py b/server/cros/dynamic_suite/reporting.py
index 3a1a4cf..c112a5e 100644
--- a/server/cros/dynamic_suite/reporting.py
+++ b/server/cros/dynamic_suite/reporting.py
@@ -16,7 +16,6 @@
from autotest_lib.client.common_lib import global_config
from autotest_lib.server import site_utils
from autotest_lib.server.cros.dynamic_suite import job_status
-from autotest_lib.site_utils.suite_scheduler import base_event
# Try importing the essential bug reporting libraries. Chromite and gdata_lib
# are useless unless they can import gdata too.
@@ -142,8 +141,8 @@
def get_milestone(self):
"""Parses the build string and returns a milestone."""
try:
- return 'M-%s'% base_event.ParseBuildName(self.build)[2]
- except base_event.ParseBuildNameException as e:
+ return 'M-%s'% site_utils.ParseBuildName(self.build)[2]
+ except site_utils.ParseBuildNameException as e:
logging.error(e)
return ''
@@ -363,8 +362,8 @@
kwargs['owner'] = {'name': kwargs['owner']}
return kwargs
-
- def _create_bug_report(self, description, title, name, owner, milestone='',
+ # TODO(beeps):crbug.com/254256
+ def create_bug_report(self, description, title, name, owner, milestone='',
bug_template={}, sheriffs=[]):
"""
Creates a new bug report.
@@ -377,6 +376,10 @@
Note that if either the description or title fields are missing
we won't be able to create a bug.
"""
+ if not self._check_tracker():
+ logging.error("Can't file: %s", title)
+ return None
+
issue = self._format_issue_options(bug_template, title=title,
description=description, labels=self._get_labels(name.lower()),
status='Untriaged', milestone=[milestone], owner=owner,
@@ -544,7 +547,7 @@
elif failure.suite == 'bvt':
sheriffs = site_utils.get_sheriffs()
- return self._create_bug_report(summary, failure.bug_title(),
- failure.test, self._get_owner(failure),
- failure.get_milestone(), bug_template,
- sheriffs)
+ return self.create_bug_report(summary, failure.bug_title(),
+ failure.test, self._get_owner(failure),
+ failure.get_milestone(), bug_template,
+ sheriffs)
diff --git a/server/site_utils.py b/server/site_utils.py
index 8e9010f..6b27970 100644
--- a/server/site_utils.py
+++ b/server/site_utils.py
@@ -12,6 +12,8 @@
_SHERIFF_JS = global_config.global_config.get_config_value(
'NOTIFICATIONS', 'sheriffs', default='')
+_LAB_SHERIFF_JS = global_config.global_config.get_config_value(
+ 'NOTIFICATIONS', 'lab_sheriffs', default='')
_CHROMIUM_BUILD_URL = global_config.global_config.get_config_value(
'NOTIFICATIONS', 'chromium_build_url', default='')
@@ -64,18 +66,23 @@
return get_label_from_afe(hostname, constants.VERSION_PREFIX, afe)
-def get_sheriffs():
+def get_sheriffs(lab_only=False):
"""
Polls the javascript file that holds the identity of the sheriff and
parses it's output to return a list of chromium sheriff email addresses.
The javascript file can contain the ldap of more than one sheriff, eg:
document.write('sheriff_one, sheriff_two').
- @return: A list of chroium.org sheriff email addresses to cc on the bug
- if the suite that failed was the bvt suite. An empty list otherwise.
+ @param lab_only: if True, only pulls lab sheriff.
+ @return: A list of chroium.org sheriff email addresses to cc on the bug.
+ An empty list if failed to parse the javascript.
"""
sheriff_ids = []
- for sheriff_js in _SHERIFF_JS.split(','):
+ sheriff_js_list = _LAB_SHERIFF_JS.split(',')
+ if not lab_only:
+ sheriff_js_list.extend(_SHERIFF_JS.split(','))
+
+ for sheriff_js in sheriff_js_list:
try:
url_content = base_utils.urlopen('%s%s'% (
_CHROMIUM_BUILD_URL, sheriff_js)).read()
diff --git a/site_utils/run_suite.py b/site_utils/run_suite.py
index ada6803..15bcf20 100755
--- a/site_utils/run_suite.py
+++ b/site_utils/run_suite.py
@@ -20,6 +20,7 @@
import common
from autotest_lib.client.common_lib import global_config, error, utils, enum
+from autotest_lib.client.common_lib import site_utils
from autotest_lib.server.cros.dynamic_suite import constants
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
from autotest_lib.server.cros.dynamic_suite import job_status
@@ -434,8 +435,8 @@
@return: The key used to log timing information in statsd.
"""
try:
- _board, build_type, branch = base_event.ParseBuildName(build)[:3]
- except base_event.ParseBuildNameException as e:
+ _board, build_type, branch = site_utils.ParseBuildName(build)[:3]
+ except site_utils.ParseBuildNameException as e:
logging.error(str(e))
branch = 'Unknown'
build_type = 'Unknown'
diff --git a/site_utils/suite_scheduler/base_event.py b/site_utils/suite_scheduler/base_event.py
index b46ad76..25576e4 100644
--- a/site_utils/suite_scheduler/base_event.py
+++ b/site_utils/suite_scheduler/base_event.py
@@ -2,7 +2,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-import logging, re
+import logging
import task
"""Module containing base class and methods for working with scheduler events.
@@ -15,11 +15,6 @@
_SECTION_SUFFIX = '_params'
-class ParseBuildNameException(Exception):
- """Raised when ParseBuildName() cannot parse a build name."""
- pass
-
-
def SectionName(keyword):
"""Generate a section name for a *Event config stanza."""
return keyword + _SECTION_SUFFIX
@@ -30,21 +25,6 @@
return section.endswith(_SECTION_SUFFIX)
-def ParseBuildName(name):
- """Format a build name, given board, type, milestone, and manifest num.
-
- @param name: a build name, e.g. 'x86-alex-release/R20-2015.0.0'
- @return board: board the manifest is for, e.g. x86-alex.
- @return type: one of 'release', 'factory', or 'firmware'
- @return milestone: (numeric) milestone the manifest was associated with.
- @return manifest: manifest number, e.g. '2015.0.0'
- """
- match = re.match(r'([\w-]+)-(\w+)/R(\d+)-([\d.ab-]+)', name)
- if match and len(match.groups()) == 4:
- return match.groups()
- raise ParseBuildNameException('%s is a malformed build name.' % name)
-
-
def BuildName(board, type, milestone, manifest):
"""Format a build name, given board, type, milestone, and manifest number.
diff --git a/site_utils/suite_scheduler/base_event_unittest.py b/site_utils/suite_scheduler/base_event_unittest.py
index 72661d4..23231c5 100644
--- a/site_utils/suite_scheduler/base_event_unittest.py
+++ b/site_utils/suite_scheduler/base_event_unittest.py
@@ -125,3 +125,7 @@
task.Arm()
self.mox.ReplayAll()
event.Handle(self.sched, {}, "board1")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/site_utils/suite_scheduler/deduping_scheduler.py b/site_utils/suite_scheduler/deduping_scheduler.py
index fd99196..d85519e 100644
--- a/site_utils/suite_scheduler/deduping_scheduler.py
+++ b/site_utils/suite_scheduler/deduping_scheduler.py
@@ -3,8 +3,12 @@
# found in the LICENSE file.
import logging
-from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
+import common
+from autotest_lib.client.common_lib import error
+from autotest_lib.server import site_utils
+from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
+from autotest_lib.server.cros.dynamic_suite import reporting
class DedupingSchedulerException(Exception):
"""Base class for exceptions from this module."""
@@ -31,7 +35,7 @@
"""
- def __init__(self, afe=None):
+ def __init__(self, afe=None, file_bug=False):
"""Constructor
@param afe: an instance of AFE as defined in server/frontend.py.
@@ -40,6 +44,7 @@
self._afe = afe or frontend_wrappers.RetryingAFE(timeout_min=30,
delay_sec=10,
debug=False)
+ self._file_bug = file_bug
def _ShouldScheduleSuite(self, suite, board, build):
@@ -62,6 +67,33 @@
raise DedupException(e)
+ # TODO(fdeng): After crbug.com/254256 is fixed, we
+ # should modify this method accordingly.
+ def _ReportBug(self, title, description):
+ """File a bug using bug reporter.
+
+ @param title: A string, representing the bug title.
+ @param description: A string, representing the bug description.
+ @return: The id of the issue that was created,
+ or None if bug is not filed.
+ """
+ if not self._file_bug:
+ return None
+ bug_reporter = reporting.Reporter()
+ template = {'labels': ['Suite-Scheduler-Bug'],
+ 'status': 'Available'}
+ sheriffs = site_utils.get_sheriffs(lab_only=True)
+ owner = sheriffs[0] if sheriffs else ''
+ logging.info('Filing a bug: %s', title)
+ return bug_reporter.create_bug_report(description=description,
+ title=title,
+ name='',
+ owner=owner,
+ milestone='',
+ bug_template=template,
+ sheriffs=[])
+
+
def _Schedule(self, suite, board, build, pool, num):
"""Schedule |suite|, if it hasn't already been run.
@@ -91,6 +123,16 @@
else:
raise ScheduleException(
"Can't schedule %s for %s." % (suite, build))
+ except (error.ControlFileNotFound, error.ControlFileEmpty,
+ error.ControlFileMalformed, error.NoControlFileList) as e:
+ title = ('Exception "%s" occurs when scheduling %s on '
+ '%s against %s (pool: %s)' %
+ (e.__class__.__name__, suite, build, board, pool))
+ if self._ReportBug(title, str(e)) is None:
+ # Raise the exception if not filing or failed to file a bug.
+ raise ScheduleException(e)
+ else:
+ return False
except Exception as e:
raise ScheduleException(e)
diff --git a/site_utils/suite_scheduler/deduping_scheduler_unittest.py b/site_utils/suite_scheduler/deduping_scheduler_unittest.py
index eb8c61f..9b97783 100644
--- a/site_utils/suite_scheduler/deduping_scheduler_unittest.py
+++ b/site_utils/suite_scheduler/deduping_scheduler_unittest.py
@@ -6,13 +6,15 @@
"""Unit tests for site_utils/deduping_scheduler.py."""
-import logging
import mox
import unittest
+import common
import deduping_scheduler
-from autotest_lib.server import frontend
+from autotest_lib.client.common_lib import error
+from autotest_lib.server import frontend, site_utils
+from autotest_lib.server.cros.dynamic_suite import reporting
class DedupingSchedulerTest(mox.MoxTestBase):
@@ -153,5 +155,48 @@
None)
+ def testScheduleReportsBug(self):
+ """Test that the scheduler file a bug for ControlFileNotFound."""
+ self.mox.StubOutWithMock(reporting.Reporter, '__init__')
+ self.mox.StubOutWithMock(reporting.Reporter, 'create_bug_report')
+ self.mox.StubOutWithMock(site_utils, 'get_sheriffs')
+ self.scheduler._file_bug = True
+ # A similar suite has not already been scheduled.
+ self.afe.get_jobs(name__startswith=self._BUILD,
+ name__endswith='control.'+self._SUITE).AndReturn([])
+ message = 'Control file not found.'
+ exception = error.ControlFileNotFound(message)
+ self.afe.run('create_suite_job',
+ suite_name=self._SUITE,
+ board=self._BOARD,
+ build=self._BUILD,
+ check_hosts=False,
+ pool=self._POOL,
+ num=self._NUM).AndRaise(exception)
+ reporting.Reporter.__init__()
+ title = ('Exception "%s" occurs when scheduling %s on '
+ '%s against %s (pool: %s)' %
+ (exception.__class__.__name__,
+ self._SUITE, self._BUILD, self._BOARD, self._POOL))
+ site_utils.get_sheriffs(
+ lab_only=True).AndReturn(['dummy@chromium.org'])
+ reporting.Reporter.create_bug_report(
+ description=mox.IgnoreArg(),
+ title=title,
+ name='',
+ owner='dummy@chromium.org',
+ milestone='',
+ bug_template = {'labels': ['Suite-Scheduler-Bug'],
+ 'status': 'Available'},
+ sheriffs=[]).AndReturn(1158)
+ self.mox.ReplayAll()
+ self.assertFalse(self.scheduler.ScheduleSuite(self._SUITE,
+ self._BOARD,
+ self._BUILD,
+ self._POOL,
+ self._NUM))
+ self.mox.VerifyAll()
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/site_utils/suite_scheduler/driver.py b/site_utils/suite_scheduler/driver.py
index 787dffa..669e420 100644
--- a/site_utils/suite_scheduler/driver.py
+++ b/site_utils/suite_scheduler/driver.py
@@ -8,7 +8,7 @@
import task, timed_event
import common
-from autotest_lib.client.common_lib import site_utils, error
+from autotest_lib.client.common_lib import error, site_utils
class Driver(object):
"""Implements the main loop of the suite_scheduler.
@@ -152,7 +152,7 @@
@param keywords: iterable of event keywords to force
@param build_name: instead of looking up builds to test, test this one.
"""
- board, type, milestone, manifest = base_event.ParseBuildName(build_name)
+ board, type, milestone, manifest = site_utils.ParseBuildName(build_name)
branch_builds = {task.PickBranchName(type, milestone): [build_name]}
logging.info('Testing build R%s-%s on %s', milestone, manifest, board)
diff --git a/site_utils/suite_scheduler/suite_scheduler.py b/site_utils/suite_scheduler/suite_scheduler.py
index 47d4e54..07f1f14 100755
--- a/site_utils/suite_scheduler/suite_scheduler.py
+++ b/site_utils/suite_scheduler/suite_scheduler.py
@@ -169,6 +169,10 @@
parser.add_option('-t', '--sanity', dest='sanity', action='store_true',
default=False,
help="Check the config file for any issues.")
+ parser.add_option('-b', '--file_bug', dest='file_bug', action='store_true',
+ default=False,
+ help="File bugs for known suite scheduling exceptions.")
+
options, args = parser.parse_args()
return parser, options, args
@@ -217,7 +221,7 @@
afe = frontend_wrappers.RetryingAFE(timeout_min=1, delay_sec=5, debug=False)
enumerator = board_enumerator.BoardEnumerator(afe)
- scheduler = deduping_scheduler.DedupingScheduler(afe)
+ scheduler = deduping_scheduler.DedupingScheduler(afe, options.file_bug)
mv = manifest_versions.ManifestVersions()
d = driver.Driver(scheduler, enumerator)
d.SetUpEventsAndTasks(config, mv)