[autotest] emit scheduled tests names and number of them from suite
For a suite, write the number of scheduled tests and names of them into
keyval file. These keyvals can be inserted into tko database when the
suite finishes, which can help goldeneye team debug the missing tests.
BUG=chromium:661703
TEST=unittest, trybot and run tests with the test build.
Change-Id: I3cb194d4d89628943b8052db32ef4a44df51df5e
Reviewed-on: https://chromium-review.googlesource.com/414243
Commit-Ready: Shuqian Zhao <shuqianz@chromium.org>
Tested-by: Shuqian Zhao <shuqianz@chromium.org>
Reviewed-by: Aviv Keshet <akeshet@chromium.org>
diff --git a/server/cros/dynamic_suite/constants.py b/server/cros/dynamic_suite/constants.py
index 044e27c..44d04a5 100644
--- a/server/cros/dynamic_suite/constants.py
+++ b/server/cros/dynamic_suite/constants.py
@@ -17,6 +17,10 @@
# Job keyval indicating the minimum duts required by the suite
SUITE_MIN_DUTS_KEY = 'suite_min_duts'
+# Job keyval indicating the scheduled tests
+SCHEDULED_TEST_COUNT_KEY = 'scheduled_test_count'
+SCHEDULED_TEST_NAMES_KEY = 'scheduled_test_names'
+
# Job attribute and label names
EXPERIMENTAL_PREFIX = 'experimental_'
FWRW_BUILD = 'fwrw_build'
diff --git a/server/cros/dynamic_suite/suite.py b/server/cros/dynamic_suite/suite.py
index 96fe175..c99ffd4 100644
--- a/server/cros/dynamic_suite/suite.py
+++ b/server/cros/dynamic_suite/suite.py
@@ -918,9 +918,18 @@
test.name = constants.EXPERIMENTAL_PREFIX + test.name
tests.append(test)
+ scheduled_test_names = []
for test in tests:
if self._schedule_test(record, test):
- n_scheduled += 1
+ scheduled_test_names.append(test.name)
+
+ # Write the num of scheduled tests and name of them to keyval file.
+ n_scheduled = len(scheduled_test_names)
+ logging.debug('Scheduled %d tests, writing the total to keyval.',
+ n_scheduled)
+ d = {constants.SCHEDULED_TEST_COUNT_KEY: n_scheduled,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(scheduled_test_names)}
+ utils.write_keyval(self._results_dir, d)
except Exception: # pylint: disable=W0703
logging.error(traceback.format_exc())
Status('FAIL', self._tag,
diff --git a/server/cros/dynamic_suite/suite_unittest.py b/server/cros/dynamic_suite/suite_unittest.py
index 7a00992..1b80219 100755
--- a/server/cros/dynamic_suite/suite_unittest.py
+++ b/server/cros/dynamic_suite/suite_unittest.py
@@ -30,6 +30,7 @@
from autotest_lib.server import frontend
from autotest_lib.server.cros import provision
from autotest_lib.server.cros.dynamic_suite import control_file_getter
+from autotest_lib.server.cros.dynamic_suite import constants
from autotest_lib.server.cros.dynamic_suite import job_status
from autotest_lib.server.cros.dynamic_suite import reporting
from autotest_lib.server.cros.dynamic_suite import suite as SuiteBase
@@ -542,9 +543,18 @@
def testScheduleStableTests(self):
"""Should schedule only stable tests with the AFE."""
+ # Since test data_one is experimental, it will be skip.
+ name_list = ['name-data_two', 'name-data_three',
+ 'name-data_four', 'name-data_five', 'name-data_six',
+ 'name-data_seven']
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=False)
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
@@ -555,10 +565,19 @@
def testScheduleStableTestsIgnoreDeps(self):
"""Should schedule only stable tests with the AFE."""
+ # Since test data_one is experimental, it will be skip.
+ name_list = ['name-data_two', 'name-data_three',
+ 'name-data_four', 'name-data_five', 'name-data_six',
+ 'name-data_seven']
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=False,
ignore_deps=True)
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
@@ -570,9 +589,17 @@
def testScheduleUnrunnableTestsTESTNA(self):
"""Tests which fail to schedule should be TEST_NA."""
+ # Since all tests will be fail to schedule, the num of scheduled tests
+ # will be zero.
+ name_list = []
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=True, raises=True)
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
@@ -582,9 +609,17 @@
def testRetryMapAfterScheduling(self):
"""Test job-test and test-job mapping are correctly updated."""
+ name_list = ['name-data_two', 'name-data_three',
+ 'name-data_four', 'name-data_five', 'name-data_six',
+ 'name-data_seven', 'experimental_name-data_one']
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=True)
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
@@ -608,9 +643,17 @@
def testSuiteMaxRetries(self):
"""Test suite max retries."""
+ name_list = ['name-data_two', 'name-data_three',
+ 'name-data_four', 'name-data_five', 'name-data_six',
+ 'name-data_seven', 'experimental_name-data_one']
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=True)
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
@@ -626,10 +669,19 @@
def testSuiteDependencies(self):
"""Should add suite dependencies to tests scheduled."""
+ # Since add_experimental set to False, will skip experimental data_one.
+ name_list = ['name-data_two', 'name-data_three',
+ 'name-data_four', 'name-data_five', 'name-data_six',
+ 'name-data_seven']
+ keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 6,
+ constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
+
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, add_experimental=False,
suite_deps=['extra'])
+ self.mox.StubOutWithMock(utils, 'write_keyval')
+ utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,