[autotest] Create a suite for testing push to prod
Add suite push_to_prod for cbf run.
Add a tool, site_utils/test_push.py, used to test if code is ready to push.
BUG=chromium:263928,285453
TEST=unit test
Test with a trybot build: trybot-stumpy-release/R35-5526.0.0-b373
AU part is verified with latest stumpy canary build.
Change-Id: Ia221f0a1aa0f1687daab5d8a20307435baa6bdfc
Reviewed-on: https://chromium-review.googlesource.com/66460
Reviewed-by: Dan Shi <dshi@chromium.org>
Commit-Queue: Dan Shi <dshi@chromium.org>
Tested-by: Dan Shi <dshi@chromium.org>
diff --git a/server/cros/dynamic_suite/reporting.py b/server/cros/dynamic_suite/reporting.py
index 55a0860..6d7f905 100644
--- a/server/cros/dynamic_suite/reporting.py
+++ b/server/cros/dynamic_suite/reporting.py
@@ -264,10 +264,10 @@
_oauth_credentials = global_config.global_config.get_config_value(
BUG_CONFIG_SECTION, 'credentials', default='')
- # _AUTOFILED_COUNT is a label prefix used to indicate how
+ # AUTOFILED_COUNT is a label prefix used to indicate how
# many times we think we've updated an issue automatically.
- _AUTOFILED_COUNT = 'autofiled-count-'
- _PREDEFINED_LABELS = ['autofiled', '%s%d' % (_AUTOFILED_COUNT, 1),
+ AUTOFILED_COUNT = 'autofiled-count-'
+ _PREDEFINED_LABELS = ['autofiled', '%s%d' % (AUTOFILED_COUNT, 1),
'OS-Chrome', 'Type-Bug',
'Restrict-View-Google']
@@ -390,7 +390,7 @@
return filed_bug.get('id')
- def _modify_bug_report(self, issue_id, comment, label_update):
+ def modify_bug_report(self, issue_id, comment, label_update, status=''):
"""Modifies an existing bug report with a new comment.
Adds the given comment and applies the given list of label
@@ -399,23 +399,24 @@
@param issue_id Id of the issue to update with.
@param comment Comment to update the issue with.
@param label_update List with label updates.
+ @param status New status of the issue.
"""
updates = {
'content': comment,
- 'updates': { 'labels': label_update }
+ 'updates': { 'labels': label_update, 'status': status }
}
try:
self._phapi_client.update_issue(issue_id, updates)
except phapi_lib.ProjectHostingApiException as e:
logging.warning('Unable to update issue %s, comment %s, '
- 'labels %r: %s', issue_id, comment,
- label_update, e)
+ 'labels %r, status %s: %s', issue_id, comment,
+ label_update, status, e)
else:
- logging.info('Updated issue %s, comment %s, labels %r.',
- issue_id, comment, label_update)
+ logging.info('Updated issue %s, comment %s, labels %r, status %s.',
+ issue_id, comment, label_update, status)
- def _find_issue_by_marker(self, marker):
+ def find_issue_by_marker(self, marker):
"""
Queries the tracker to find if there is a bug filed for this issue.
@@ -519,7 +520,7 @@
@return An Issue instance, representing an open issue that is a
duplicate of the one being searched for.
"""
- issue = self._find_issue_by_marker(marker)
+ issue = self.find_issue_by_marker(marker)
if not issue or issue.state == constants.ISSUE_OPEN:
return issue
@@ -573,16 +574,16 @@
"""
counts = []
count_max = 1
- is_count_label = lambda l: l.startswith(self._AUTOFILED_COUNT)
+ is_count_label = lambda l: l.startswith(self.AUTOFILED_COUNT)
for label in filter(is_count_label, issue.labels):
try:
- count = int(label[len(self._AUTOFILED_COUNT):])
+ count = int(label[len(self.AUTOFILED_COUNT):])
except ValueError:
continue
count_max = max(count, count_max)
counts.append('-%s' % label)
new_count = count_max + 1
- counts.append('%s%d' % (self._AUTOFILED_COUNT, new_count))
+ counts.append('%s%d' % (self.AUTOFILED_COUNT, new_count))
return counts, new_count
@@ -626,7 +627,7 @@
comment = '%s\n\n%s' % (bug.title(), self._anchor_summary(bug))
count_update, bug_count = (
self._create_autofiled_count_update(issue))
- self._modify_bug_report(issue.id, comment, count_update)
+ self.modify_bug_report(issue.id, comment, count_update)
return issue.id, bug_count
sheriffs = []
diff --git a/server/cros/dynamic_suite/reporting_unittest.py b/server/cros/dynamic_suite/reporting_unittest.py
index 6bb3ffe..5b7f64e 100755
--- a/server/cros/dynamic_suite/reporting_unittest.py
+++ b/server/cros/dynamic_suite/reporting_unittest.py
@@ -96,14 +96,14 @@
Confirms that we call CreateTrackerIssue when an Issue search
returns None.
"""
- self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker')
+ self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
client.create_issue(mox.IgnoreArg()).AndReturn(
{'id': self._FAKE_ISSUE_ID})
- reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn(
+ reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn(
None)
reporting.TestBug.summary().AndReturn('')
@@ -120,7 +120,7 @@
Confirms that we call AppendTrackerIssueById with the same issue
returned by the issue search.
"""
- self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker')
+ self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
issue = self.mox.CreateMock(phapi_lib.Issue)
@@ -131,7 +131,7 @@
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
client.update_issue(self._FAKE_ISSUE_ID, mox.IgnoreArg())
- reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn(
+ reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn(
issue)
reporting.TestBug.summary().AndReturn('')
@@ -163,10 +163,10 @@
return False
return True
- self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker')
+ self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
- reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn(
+ reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn(
None)
reporting.TestBug.summary().AndReturn('Summary')
@@ -185,11 +185,11 @@
def testGenericBugCanBeFiled(self):
"""Test that we can use a Bug object to file a bug report."""
- self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker')
+ self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
bug = reporting.Bug('title', 'summary', 'marker')
- reporting.Reporter._find_issue_by_marker(mox.IgnoreArg()).AndReturn(
+ reporting.Reporter.find_issue_by_marker(mox.IgnoreArg()).AndReturn(
None)
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
@@ -231,7 +231,7 @@
class FindIssueByMarkerTests(mox.MoxTestBase):
- """Tests the _find_issue_by_marker function."""
+ """Tests the find_issue_by_marker function."""
def setUp(self):
super(FindIssueByMarkerTests, self).setUp()
@@ -260,7 +260,7 @@
mox.IgnoreArg())
self.mox.ReplayAll()
- result = reporting.Reporter()._find_issue_by_marker(None)
+ result = reporting.Reporter().find_issue_by_marker(None)
self.assertTrue(result is None)
@@ -340,7 +340,7 @@
def _create_count_label(self, n):
- return '%s%d' % (reporting.Reporter._AUTOFILED_COUNT, n)
+ return '%s%d' % (reporting.Reporter.AUTOFILED_COUNT, n)
def _test_count_label_update(self, labels, remove, expected_count):
@@ -404,7 +404,7 @@
"""Test that autofiled-count increment ignores unusual labels."""
old_count = self._create_count_label(3)
self._test_count_label_update(
- [reporting.Reporter._AUTOFILED_COUNT + 'bogus',
+ [reporting.Reporter.AUTOFILED_COUNT + 'bogus',
self._create_count_label(8) + '-bogus',
old_count],
[old_count], 4)
diff --git a/server/site_tests/platform_InstallTestImage/control b/server/site_tests/platform_InstallTestImage/control
index 4b31f04..9cb9939 100644
--- a/server/site_tests/platform_InstallTestImage/control
+++ b/server/site_tests/platform_InstallTestImage/control
@@ -15,6 +15,7 @@
TEST_CATEGORY = "Install"
TEST_CLASS = "platform"
TEST_TYPE = "server"
+SUITE = "push_to_prod"
DOC = """
This test installs a specified test image onto a servo-connected DUT.
diff --git a/server/site_tests/telemetry_CrosTests/control.LoginStatus b/server/site_tests/telemetry_CrosTests/control.LoginStatus
index ddcdad5..662bd2e 100644
--- a/server/site_tests/telemetry_CrosTests/control.LoginStatus
+++ b/server/site_tests/telemetry_CrosTests/control.LoginStatus
@@ -3,10 +3,11 @@
# found in the LICENSE file.
AUTHOR = "chrome-ui"
-NAME = "Telemetry Login Status Test"
+NAME = "telemetry_CrosTests.LoginStatus"
TIME = "LONG"
TEST_CATEGORY = "Functional"
TEST_TYPE = "server"
+SUITE = "push_to_prod"
DOC = """
This server control file executes the CrOS telemetry test: testLoginStatus.
diff --git a/server/site_utils.py b/server/site_utils.py
index 55b1cc6..7a6ad28 100644
--- a/server/site_utils.py
+++ b/server/site_utils.py
@@ -16,6 +16,7 @@
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
from autotest_lib.server.cros.dynamic_suite import constants
+from autotest_lib.server.cros.dynamic_suite import job_status
_SHERIFF_JS = global_config.global_config.get_config_value(
@@ -272,3 +273,25 @@
host.hostname, labels)
raise error.TestError('Could not lock a device with labels %s' % labels)
+
+
+def get_test_views_from_tko(suite_job_id, tko):
+ """Get test name and result for given suite job ID.
+
+ @param suite_job_id: ID of suite job.
+ @param tko: an instance of TKO as defined in server/frontend.py.
+ @return: A dictionary of test status keyed by test name, e.g.,
+ {'dummy_Fail.Error': 'ERROR', 'dummy_Fail.NAError': 'TEST_NA'}
+ @raise: Exception when there is no test view found.
+
+ """
+ views = tko.run('get_detailed_test_views', afe_job_id=suite_job_id)
+ relevant_views = filter(job_status.view_is_relevant, views)
+ if not relevant_views:
+ raise Exception('Failed to retrieve job results.')
+
+ test_views = {}
+ for view in relevant_views:
+ test_views[view['test_name']] = view['status']
+
+ return test_views