[autotest] File bugs on test failure.
When a test fails as part of a suite, if |file_bugs| is true, then
file a bug on the specified bug tracker reporting the failure.
Relanding Ie7664368583f645110fc562010897fbda999b815.
TEST=./run_suite.py with version N against a dummy project and check
for bugs that are filed
TEST=./run_suite.py with version N+1 and check that comments are filed
BUG=chromium-os:29513
Change-Id: I55c72155afba5ee87bedd137d560bde2d03d2cb2
Reviewed-on: https://gerrit.chromium.org/gerrit/41763
Commit-Queue: Alex Miller <milleral@chromium.org>
Reviewed-by: Alex Miller <milleral@chromium.org>
Tested-by: Alex Miller <milleral@chromium.org>
diff --git a/server/cros/dynamic_suite/common.py b/server/cros/dynamic_suite/common.py
new file mode 100644
index 0000000..6a6c620
--- /dev/null
+++ b/server/cros/dynamic_suite/common.py
@@ -0,0 +1,8 @@
+import os, sys
+dirname = os.path.dirname(sys.modules[__name__].__file__)
+autotest_dir = os.path.abspath(os.path.join(dirname, "../../.."))
+client_dir = os.path.join(autotest_dir, "client")
+sys.path.insert(0, client_dir)
+import setup_modules
+sys.path.pop(0)
+setup_modules.setup(base_path=autotest_dir, root_module_name="autotest_lib")
diff --git a/server/cros/dynamic_suite/dynamic_suite.py b/server/cros/dynamic_suite/dynamic_suite.py
index 41c54a4..a445e4a 100644
--- a/server/cros/dynamic_suite/dynamic_suite.py
+++ b/server/cros/dynamic_suite/dynamic_suite.py
@@ -544,7 +544,8 @@
afe=afe, tko=tko, pool=spec.pool,
results_dir=spec.job.resultdir,
max_runtime_mins=spec.max_runtime_mins,
- version_prefix=reimager.version_prefix)
+ version_prefix=reimager.version_prefix,
+ file_bugs=spec.file_bugs)
# Now we get to asychronously schedule tests.
suite.schedule(spec.job.record_entry, spec.add_experimental)
diff --git a/server/cros/dynamic_suite/job_status.py b/server/cros/dynamic_suite/job_status.py
index 43e4130..c5c617b 100644
--- a/server/cros/dynamic_suite/job_status.py
+++ b/server/cros/dynamic_suite/job_status.py
@@ -575,3 +575,8 @@
@property
def owner(self):
return self._owner
+
+
+ @property
+ def reason(self):
+ return self._reason
diff --git a/server/cros/dynamic_suite/reporting.py b/server/cros/dynamic_suite/reporting.py
new file mode 100644
index 0000000..9041eed
--- /dev/null
+++ b/server/cros/dynamic_suite/reporting.py
@@ -0,0 +1,159 @@
+# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import re
+
+# We need to import common to be able to import chromite.
+import common
+from autotest_lib.client.common_lib import global_config
+try:
+ from chromite.lib import gdata_lib
+except ImportError as e:
+ gdata_lib = None
+ logging.info("Bug filing disabled. %s", e)
+
+
+BUG_CONFIG_SECTION = 'BUG_REPORTING'
+
+
+class TestFailure(object):
+ """Wrap up all information needed to make an intelligent report about a
+ test failure.
+
+ Each TestFailure has a search marker associated with it that can be used to
+ find reports of the same error."""
+
+
+ def __init__(self, build, suite, test, reason):
+ """
+ @param build The build type, of the form <board>/<milestone>-<release>.
+ ie. x86-mario-release/R25-4321.0.0
+ @param suite The name of the suite that this test run was a part of.
+ @param test The name of the test that this failure is about.
+ @param reason The reason that this test failed.
+ """
+ self.build = build
+ self.suite = suite
+ self.test = test
+ self.reason = reason
+
+
+ def bug_title(self):
+ """Converts information about a failure into a string appropriate to
+ be the title of a bug."""
+ return '[%s] %s failed on %s' % (self.suite, self.test, self.build)
+
+
+ def bug_summary(self):
+ """Converts information about this failure into a string appropriate
+ to be the summary of this bug. Includes the reason field."""
+ return ('This bug has been automatically filed to track the'
+ ' failure of %s in the %s suite on %s. It failed with a reason'
+ ' of:\n\n%s' % (self.test, self.suite, self.build, self.reason))
+
+
+ def search_marker(self):
+ """When filing a report about this failure, include the returned line in
+ the report to provide a way to search for this exact failure."""
+ return "%s(%s,%s,%s)" % ('TestFailure', self.suite,
+ self.test, self.reason)
+
+
+class Reporter(object):
+ """Files external reports about bug failures that happened inside of
+ autotest."""
+
+
+ _project_name = global_config.global_config.get_config_value(
+ BUG_CONFIG_SECTION, 'project_name', default='')
+ _username = global_config.global_config.get_config_value(
+ BUG_CONFIG_SECTION, 'username', default='')
+ _password = global_config.global_config.get_config_value(
+ BUG_CONFIG_SECTION, 'password', default='')
+ _SEARCH_MARKER = 'ANCHOR '
+
+
+ def __init__(self):
+ if gdata_lib is None:
+ logging.warning("Bug filing disabled due to missing imports.")
+ if self._project_name and self._username and self._password:
+ creds = gdata_lib.Creds()
+ creds.SetCreds(self._username, self._password)
+ self._tracker = gdata_lib.TrackerComm()
+ self._tracker.Connect(creds, self._project_name)
+ else:
+ logging.error('Tracker auth not set up in shadow_config.ini, '
+ 'cannot file bugs.')
+ self._tracker = None
+
+
+ def report(self, failure):
+ """
+ Report about a failure on the bug tracker. If this failure has already
+ happened, post a comment on the existing bug about it occurring again.
+ If this is a new failure, create a new bug about it.
+
+ @param failure A TestFailure instance about the failure.
+ @return None
+ """
+ if gdata_lib is None or self._tracker is None:
+ logging.info("Can't file %s", failure.bug_title())
+ return
+
+ issue = self._find_issue_by_marker(failure.search_marker())
+ if issue:
+ issue_comment = '%s\n\n%s\n\n%s%s' % (failure.bug_title(),
+ failure.bug_summary(),
+ self._SEARCH_MARKER,
+ failure.search_marker())
+ self._tracker.AppendTrackerIssueById(issue.id, issue_comment)
+ logging.info("Filed comment on %s", str(issue.id))
+ else:
+ summary = "%s\n\n%s%s" % (failure.bug_summary(),
+ self._SEARCH_MARKER,
+ failure.search_marker())
+ issue = gdata_lib.Issue(title=failure.bug_title(),
+ summary=summary, labels=['Test-Support'],
+ status='Untriaged', owner='')
+ bugid = self._tracker.CreateTrackerIssue(issue)
+ logging.info("Filing new bug %s", str(bugid))
+
+
+ def _find_issue_by_marker(self, marker):
+ """
+ Queries the tracker to find if there is a bug filed for this issue.
+
+ @param marker The marker string to search for to find a duplicate of
+ this issue.
+ @return A gdata_lib.Issue instance of the issue that was found, or
+ None if no issue was found.
+ """
+
+ # This will return at most 25 matches, as that's how the
+ # code.google.com API limits this query.
+ issues = self._tracker.GetTrackerIssuesByText(
+ self._SEARCH_MARKER + marker)
+
+ # TODO(milleral) The tracker doesn't support exact text searching, even
+ # with quotes around the search term. Therefore, to hack around this, we
+ # need to filter through the results we get back and search for the
+ # string ourselves.
+ # We could have gotten no results...
+ if not issues:
+ return None
+
+ # We could have gotten some results, but we need to wade through them
+ # to find if there's an actually correct one.
+ for issue in issues:
+ if marker in issue.summary:
+ return issue
+ for comment in issue.comments:
+ # Sometimes, comment.text is None...
+ if comment.text and marker in comment.text:
+ return issue
+
+ # Or, if we make it this far, we have only gotten similar, but not
+ # actually matching results.
+ return None
diff --git a/server/cros/dynamic_suite/suite.py b/server/cros/dynamic_suite/suite.py
index fc14312..cedc3de 100644
--- a/server/cros/dynamic_suite/suite.py
+++ b/server/cros/dynamic_suite/suite.py
@@ -13,6 +13,8 @@
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
from autotest_lib.server.cros.dynamic_suite import job_status
from autotest_lib.server.cros.dynamic_suite.job_status import Status
+from autotest_lib.server.cros.dynamic_suite import reporting
+from autotest_lib.server import frontend
class Suite(object):
@@ -107,7 +109,8 @@
def create_from_name(name, build, devserver, cf_getter=None, afe=None,
tko=None, pool=None, results_dir=None,
max_runtime_mins=24*60,
- version_prefix=constants.VERSION_PREFIX):
+ version_prefix=constants.VERSION_PREFIX,
+ file_bugs=False):
"""
Create a Suite using a predicate based on the SUITE control file var.
@@ -132,6 +135,8 @@
build name to form a label which the DUT needs
to be labeled with to be eligible to run this
test.
+ @param file_bugs: True if we should file bugs on test failures for
+ this suite run.
@return a Suite instance.
"""
if cf_getter is None:
@@ -139,7 +144,7 @@
return Suite(Suite.name_in_tag_predicate(name),
name, build, cf_getter, afe, tko, pool, results_dir,
- max_runtime_mins, version_prefix)
+ max_runtime_mins, version_prefix, file_bugs)
@staticmethod
@@ -147,7 +152,8 @@
cf_getter=None, afe=None, tko=None,
pool=None, results_dir=None,
max_runtime_mins=24*60,
- version_prefix=constants.VERSION_PREFIX):
+ version_prefix=constants.VERSION_PREFIX,
+ file_bugs=False):
"""
Create a Suite using a predicate based on the SUITE control file var.
@@ -173,6 +179,8 @@
build name to form a label which the DUT needs
to be labeled with to be eligible to run this
test.
+ @param file_bugs: True if we should file bugs on test failures for
+ this suite run.
@return a Suite instance.
"""
if cf_getter is None:
@@ -185,12 +193,13 @@
return Suite(in_tag_not_in_blacklist_predicate,
name, build, cf_getter, afe, tko, pool, results_dir,
- max_runtime_mins, version_prefix)
+ max_runtime_mins, version_prefix, file_bugs)
def __init__(self, predicate, tag, build, cf_getter, afe=None, tko=None,
pool=None, results_dir=None, max_runtime_mins=24*60,
- version_prefix=constants.VERSION_PREFIX):
+ version_prefix=constants.VERSION_PREFIX,
+ file_bugs=False):
"""
Constructor
@@ -228,6 +237,8 @@
add_experimental=True)
self._max_runtime_mins = max_runtime_mins
self._version_prefix = version_prefix
+ self._file_bugs = file_bugs
+
@property
def tests(self):
@@ -350,6 +361,8 @@
prototype:
record(base_job.status_log_entry)
"""
+ if self._file_bugs:
+ bug_reporter = reporting.Reporter()
try:
for result in job_status.wait_for_results(self._afe,
self._tko,
@@ -358,6 +371,19 @@
if (self._results_dir and
job_status.is_for_infrastructure_fail(result)):
self._remember_provided_job_id(result)
+
+ # I'd love to grab the actual tko test object here, as that
+ # includes almost all of the needed information: test name,
+ # status, reason, etc. However, doing so would cause a
+ # bunch of database traffic to grab data that we already
+ # have laying around in memory across several objects here.
+ worse = result.is_worse_than(job_status.Status("WARN", ""))
+ if self._file_bugs and worse:
+ failure = reporting.TestFailure(build=self._build,
+ suite=self._tag,
+ test=result.test_name,
+ reason=result.reason)
+ bug_reporter.report(failure)
except Exception: # pylint: disable=W0703
logging.error(traceback.format_exc())
Status('FAIL', self._tag,