[autotest]: Run suites from AFE.
* Updated the AFE to allow for suite runs. When an image is provided
the create_job RPC will redirect to the create_suite_job RPC instead.
* Updated create_suite_job to allow for a control_file to be supplied.
create_suite_job will use this control file rather than what the
devserver stages when it creates the suite job.
* Removed the parameterized job code from create_job as it is not used
and broken.
* The create_job page now includes a 'Pool' text box to specify for
these jobs.
BUG=chromium:358579
TEST=Able to launch suite jobs from run_suite and AFE. And regular tests
kicked off via the AFE still work as well. rpc_interface and
site_rpc_interface unittests, suite_scheduler unittests.
DEPLOY=apache,afe,suite_scheduler
Change-Id: I53312419c32740e78fc07598babeb497c162ba81
Reviewed-on: https://chromium-review.googlesource.com/194349
Reviewed-by: Simran Basi <sbasi@chromium.org>
Commit-Queue: Simran Basi <sbasi@chromium.org>
Tested-by: Simran Basi <sbasi@chromium.org>
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index b3acb0e..271a7d5 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -33,9 +33,10 @@
import datetime
import common
-from autotest_lib.client.common_lib import error, priorities
+from autotest_lib.client.common_lib import priorities
from autotest_lib.frontend.afe import models, model_logic, model_attributes
from autotest_lib.frontend.afe import control_file, rpc_utils
+from autotest_lib.frontend.afe import site_rpc_interface
def get_parameterized_autoupdate_image_url(job):
"""Get the parameterized autoupdate image url from a parameterized job."""
@@ -495,6 +496,30 @@
raise
+def create_job_page_handler(name, priority, control_file, control_type,
+ image=None, hostless=False, **kwargs):
+ """\
+ Create and enqueue a job.
+
+ @param name name of this job
+ @param priority Integer priority of this job. Higher is more important.
+ @param control_file String contents of the control file.
+ @param control_type Type of control file, Client or Server.
+ @param kwargs extra args that will be required by create_suite_job or
+ create_job.
+
+ @returns The created Job id number.
+ """
+ control_file = rpc_utils.encode_ascii(control_file)
+
+ if image and hostless:
+ return site_rpc_interface.create_suite_job(
+ name=name, control_file=control_file, priority=priority,
+ build=image, **kwargs)
+ return create_job(name, priority, control_file, control_type, image=image,
+ hostless=hostless, **kwargs)
+
+
def create_job(name, priority, control_file, control_type,
hosts=(), meta_hosts=(), one_time_hosts=(),
atomic_group_name=None, synch_count=None, is_template=False,
@@ -502,7 +527,7 @@
run_verify=False, email_list='', dependencies=(),
reboot_before=None, reboot_after=None, parse_failed_repair=None,
hostless=False, keyvals=None, drone_set=None, image=None,
- parent_job_id=None, test_retry=0, run_reset=True):
+ parent_job_id=None, test_retry=0, run_reset=True, **kwargs):
"""\
Create and enqueue a job.
@@ -527,7 +552,6 @@
this job will be parsed as part of the job.
@param hostless if true, create a hostless job
@param keyvals dict of keyvals to associate with the job
-
@param hosts List of hosts to run job on.
@param meta_hosts List where each entry is a label name, and for each entry
one host will be chosen from that label to run the job on.
@@ -536,48 +560,14 @@
@param drone_set The name of the drone set to run this test on.
@param image OS image to install before running job.
@param parent_job_id id of a job considered to be parent of created job.
- @param test_retry: Number of times to retry test if the test did not
+ @param test_retry Number of times to retry test if the test did not
complete successfully. (optional, default: 0)
- @param run_reset: Should the host be reset before running the test?
+ @param run_reset Should the host be reset before running the test?
+ @param kwargs extra keyword args. NOT USED.
@returns The created Job id number.
"""
- # Force control files to only contain ascii characters.
- try:
- control_file.encode('ascii')
- except UnicodeDecodeError as e:
- raise error.ControlFileMalformed(str(e))
-
- if image is None:
- return rpc_utils.create_job_common(
- **rpc_utils.get_create_job_common_args(locals()))
-
- # When image is supplied use a known parameterized test already in the
- # database to pass the OS image path from the front end, through the
- # scheduler, and finally to autoserv as the --image parameter.
-
- # The test autoupdate_ParameterizedJob is in afe_autotests and used to
- # instantiate a Test object and from there a ParameterizedJob.
- known_test_obj = models.Test.smart_get('autoupdate_ParameterizedJob')
- known_parameterized_job = models.ParameterizedJob.objects.create(
- test=known_test_obj)
-
- # autoupdate_ParameterizedJob has a single parameter, the image parameter,
- # stored in the table afe_test_parameters. We retrieve and set this
- # instance of the parameter to the OS image path.
- image_parameter = known_test_obj.testparameter_set.get(test=known_test_obj,
- name='image')
- known_parameterized_job.parameterizedjobparameter_set.create(
- test_parameter=image_parameter, parameter_value=image,
- parameter_type='string')
-
- # By passing a parameterized_job to create_job_common the job entry in
- # the afe_jobs table will have the field parameterized_job_id set.
- # The scheduler uses this id in the afe_parameterized_jobs table to
- # match this job to our known test, and then with the
- # afe_parameterized_job_parameters table to get the actual image path.
return rpc_utils.create_job_common(
- parameterized_job=known_parameterized_job.id,
**rpc_utils.get_create_job_common_args(locals()))
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index 7041cd3..182641b 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -811,3 +811,18 @@
host_objects=host_objects,
metahost_objects=metahost_objects,
atomic_group=atomic_group)
+
+
+def encode_ascii(control_file):
+ """Force a control file to only contain ascii characters.
+
+ @param control_file: Control file to encode.
+
+ @returns the control file in an ascii encoding.
+
+ @raises error.ControlFileMalformed: if encoding fails.
+ """
+ try:
+ return control_file.encode('ascii')
+ except UnicodeDecodeError as e:
+ raise error.ControlFileMalformed(str(e))
\ No newline at end of file
diff --git a/frontend/afe/site_rpc_interface.py b/frontend/afe/site_rpc_interface.py
index f69da13..aa7c349 100644
--- a/frontend/afe/site_rpc_interface.py
+++ b/frontend/afe/site_rpc_interface.py
@@ -13,6 +13,7 @@
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import priorities
from autotest_lib.client.common_lib.cros import dev_server
+from autotest_lib.server import utils
from autotest_lib.server.cros.dynamic_suite import constants
from autotest_lib.server.cros.dynamic_suite import control_file_getter
from autotest_lib.server.cros.dynamic_suite import job_status
@@ -43,14 +44,13 @@
return datetime.datetime.now().strftime(job_status.TIME_FMT)
-def get_control_file_contents_by_name(build, board, ds, suite_name):
+def _get_control_file_contents_by_name(build, ds, suite_name):
"""Return control file contents for |suite_name|.
Query the dev server at |ds| for the control file |suite_name|, included
in |build| for |board|.
@param build: unique name by which to refer to the image from now on.
- @param board: the kind of device to run the tests on.
@param ds: a dev_server.DevServer instance to fetch control file with.
@param suite_name: canonicalized suite name, e.g. test_suites/control.bvt.
@raises ControlFileNotFound if a unique suite control file doesn't exist.
@@ -65,7 +65,7 @@
try:
control_file_in = getter.get_control_file_contents_by_name(suite_name)
except error.CrosDynamicSuiteException as e:
- raise type(e)("%s while testing %s for %s." % (e, build, board))
+ raise type(e)("%s while testing %s." % (e, build))
if not control_file_in:
raise error.ControlFileEmpty(
"Fetching %s returned no data." % suite_name)
@@ -78,17 +78,44 @@
return control_file_in
-def create_suite_job(suite_name, board, build, pool, check_hosts=True,
- num=None, file_bugs=False, timeout=24, timeout_mins=None,
- priority=priorities.Priority.DEFAULT,
- suite_args=None, wait_for_results=True):
+def _stage_build_artifacts(build):
+ """
+ Ensure components of |build| necessary for installing images are staged.
+
+ @param build image we want to stage.
+
+ @raises StageBuildFailure: if the dev server throws 500 while staging
+ build.
+
+ @return: dev_server.ImageServer instance to use with this build.
+ @return: timings dictionary containing staging start/end times.
+ """
+ timings = {}
+ # Set synchronous to False to allow other components to be downloaded in
+ # the background.
+ ds = dev_server.ImageServer.resolve(build)
+ timings[constants.DOWNLOAD_STARTED_TIME] = formatted_now()
+ try:
+ ds.stage_artifacts(build, ['test_suites'])
+ except dev_server.DevServerException as e:
+ raise error.StageBuildFailure(
+ "Failed to stage %s: %s" % (build, e))
+ timings[constants.PAYLOAD_FINISHED_TIME] = formatted_now()
+ return (ds, timings)
+
+
+def create_suite_job(name='', board='', build='', pool='', control_file='',
+ check_hosts=True, num=None, file_bugs=False, timeout=24,
+ timeout_mins=None, priority=priorities.Priority.DEFAULT,
+ suite_args=None, wait_for_results=True, **kwargs):
"""
Create a job to run a test suite on the given device with the given image.
When the timeout specified in the control file is reached, the
job is guaranteed to have completed and results will be available.
- @param suite_name: the test suite to run, e.g. 'bvt'.
+ @param name: The test name if control_file is supplied, otherwise the name
+ of the test suite to run, e.g. 'bvt'.
@param board: the kind of device to run the tests on.
@param build: unique name by which to refer to the image from now on.
@param pool: Specify the pool of machines to use for scheduling
@@ -106,6 +133,7 @@
determine which tests to run.
@param wait_for_results: Set to False to run the suite job without waiting
for test jobs to finish. Default is True.
+ @param kwargs: extra keyword args. NOT USED.
@raises ControlFileNotFound: if a unique suite control file doesn't exist.
@raises NoControlFileList: if we can't list the control files at all.
@@ -115,8 +143,6 @@
@return: the job ID of the suite; -1 on error.
"""
- # All suite names are assumed under test_suites/control.XX.
- suite_name = canonicalize_suite_name(suite_name)
if type(num) is not int and num is not None:
raise error.SuiteArgumentException('Ill specified num argument %r. '
'Must be an integer or None.' % num)
@@ -124,26 +150,20 @@
logging.warning("Can't run on 0 hosts; using default.")
num = None
- timings = {}
- # Ensure components of |build| necessary for installing images are staged
- # on the dev server. However set synchronous to False to allow other
- # components to be downloaded in the background.
- ds = dev_server.ImageServer.resolve(build)
- timings[constants.DOWNLOAD_STARTED_TIME] = formatted_now()
- try:
- ds.stage_artifacts(build, ['test_suites'])
- except dev_server.DevServerException as e:
- raise error.StageBuildFailure(
- "Failed to stage %s for %s: %s" % (build, board, e))
- timings[constants.PAYLOAD_FINISHED_TIME] = formatted_now()
+ (ds, timings) = _stage_build_artifacts(build)
- control_file_in = get_control_file_contents_by_name(build, board, ds,
- suite_name)
+ if not control_file:
+ # No control file was supplied so look it up from the build artifacts.
+ suite_name = canonicalize_suite_name(name)
+ control_file = _get_control_file_contents_by_name(build, ds, suite_name)
+ name = '%s-%s' % (build, suite_name)
timeout_mins = timeout_mins or timeout * 60
+ if not board:
+ board = utils.ParseBuildName(build)[0]
- # prepend build and board to the control file
+ # Prepend build and board to the control file.
inject_dict = {'board': board,
'build': build,
'check_hosts': check_hosts,
@@ -158,9 +178,9 @@
'wait_for_results': wait_for_results
}
- control_file = tools.inject_vars(inject_dict, control_file_in)
+ control_file = tools.inject_vars(inject_dict, control_file)
- return _rpc_utils().create_job_common('%s-%s' % (build, suite_name),
+ return _rpc_utils().create_job_common(name,
priority=priority,
timeout_mins=timeout_mins,
max_runtime_mins=timeout*60,
diff --git a/frontend/afe/site_rpc_interface_unittest.py b/frontend/afe/site_rpc_interface_unittest.py
index 85b9ba2..085ae7f 100644
--- a/frontend/afe/site_rpc_interface_unittest.py
+++ b/frontend/afe/site_rpc_interface_unittest.py
@@ -29,8 +29,8 @@
@var _PRIORITY: fake priority with which to reimage.
"""
_NAME = 'name'
- _BOARD = 'board'
- _BUILD = 'build'
+ _BOARD = 'link'
+ _BUILD = 'link-release/R36-5812.0.0'
_PRIORITY = priorities.Priority.DEFAULT
_TIMEOUT = 24
@@ -58,12 +58,15 @@
dev_server.ImageServer.resolve(self._BUILD).AndReturn(self.dev_server)
- def _mockDevServerGetter(self):
+ def _mockDevServerGetter(self, get_control_file=True):
self._setupDevserver()
- self.getter = self.mox.CreateMock(control_file_getter.DevServerGetter)
- self.mox.StubOutWithMock(control_file_getter.DevServerGetter, 'create')
- control_file_getter.DevServerGetter.create(
- mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(self.getter)
+ if get_control_file:
+ self.getter = self.mox.CreateMock(
+ control_file_getter.DevServerGetter)
+ self.mox.StubOutWithMock(control_file_getter.DevServerGetter,
+ 'create')
+ control_file_getter.DevServerGetter.create(
+ mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(self.getter)
def _mockRpcUtils(self, to_return, control_file_substring=''):
@@ -180,11 +183,11 @@
self._SUITE_NAME).AndReturn('f')
self._mockRpcUtils(-1)
self.mox.ReplayAll()
- self.assertEquals(site_rpc_interface.create_suite_job(self._NAME,
- self._BOARD,
- self._BUILD,
- None),
- - 1)
+ self.assertEquals(
+ site_rpc_interface.create_suite_job(name=self._NAME,
+ board=self._BOARD,
+ build=self._BUILD, pool=None),
+ -1)
def testCreateSuiteJobSuccess(self):
@@ -198,11 +201,12 @@
job_id = 5
self._mockRpcUtils(job_id)
self.mox.ReplayAll()
- self.assertEquals(site_rpc_interface.create_suite_job(self._NAME,
- self._BOARD,
- self._BUILD,
- None),
- job_id)
+ self.assertEquals(
+ site_rpc_interface.create_suite_job(name=self._NAME,
+ board=self._BOARD,
+ build=self._BUILD,
+ pool=None),
+ job_id)
def testCreateSuiteJobNoHostCheckSuccess(self):
@@ -216,11 +220,12 @@
job_id = 5
self._mockRpcUtils(job_id)
self.mox.ReplayAll()
- self.assertEquals(site_rpc_interface.create_suite_job(self._NAME,
- self._BOARD,
- self._BUILD,
- None, False),
- job_id)
+ self.assertEquals(
+ site_rpc_interface.create_suite_job(name=self._NAME,
+ board=self._BOARD,
+ build=self._BUILD,
+ pool=None, check_hosts=False),
+ job_id)
def testCreateSuiteIntegerNum(self):
"""Ensures that success results in a successful RPC."""
@@ -233,12 +238,34 @@
job_id = 5
self._mockRpcUtils(job_id, control_file_substring='num=17')
self.mox.ReplayAll()
- self.assertEquals(site_rpc_interface.create_suite_job(self._NAME,
- self._BOARD,
- self._BUILD,
- None, False,
- num=17),
- job_id)
+ self.assertEquals(
+ site_rpc_interface.create_suite_job(name=self._NAME,
+ board=self._BOARD,
+ build=self._BUILD,
+ pool=None,
+ check_hosts=False,
+ num=17),
+ job_id)
+
+
+ def testCreateSuiteJobControlFileSupplied(self):
+ """Ensure we can supply the control file to create_suite_job."""
+ self._mockDevServerGetter(get_control_file=False)
+ self.dev_server.stage_artifacts(self._BUILD,
+ ['test_suites']).AndReturn(True)
+ self.dev_server.url().AndReturn('mox_url')
+ job_id = 5
+ self._mockRpcUtils(job_id)
+ self.mox.ReplayAll()
+ self.assertEquals(
+ site_rpc_interface.create_suite_job(name='%s/%s' % (self._NAME,
+ self._BUILD),
+ board=None,
+ build=self._BUILD,
+ pool=None,
+ control_file='CONTROL FILE'),
+ job_id)
+
if __name__ == '__main__':
diff --git a/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java b/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
index 3cb54d5..a6d6423 100644
--- a/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
+++ b/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
@@ -55,6 +55,7 @@
private RadioChooserDisplay rebootAfter = new RadioChooserDisplay();
private CheckBox parseFailedRepair = new CheckBox();
private CheckBoxImpl hostless = new CheckBoxImpl();
+ private TextBox pool = new TextBox();
private TestSelectorDisplay testSelector = new TestSelectorDisplay();
private CheckBoxPanelDisplay profilersPanel = new CheckBoxPanelDisplay(CHECKBOX_PANEL_COLUMNS);
private CheckBoxImpl runNonProfiledIteration =
@@ -114,6 +115,7 @@
panel.add(rebootAfter, "create_reboot_after");
panel.add(parseFailedRepair, "create_parse_failed_repair");
panel.add(hostless, "create_hostless");
+ panel.add(pool, "create_pool");
panel.add(testSelector, "create_tests");
panel.add(profilerControls, "create_profilers");
panel.add(controlFilePanel, "create_edit_control");
@@ -168,6 +170,10 @@
return hostless;
}
+ public HasText getPool() {
+ return pool;
+ }
+
public HasText getJobName() {
return jobName;
}
diff --git a/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java b/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
index 66621d7..9a1139c 100644
--- a/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
+++ b/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
@@ -75,6 +75,7 @@
public RadioChooser.Display getRebootAfter();
public HasValue<Boolean> getParseFailedRepair();
public ICheckBox getHostless();
+ public HasText getPool();
public HostSelector.Display getHostSelectorDisplay();
public SimplifiedList getDroneSet();
public ITextBox getSynchCountInput();
@@ -613,6 +614,7 @@
display.getEditControlButton().setText(EDIT_CONTROL_STRING);
hostSelector.reset();
dependencies = new JSONArray();
+ display.getPool().setText("");
}
private void submitJob(final boolean isTemplate) {
@@ -667,6 +669,7 @@
args.put("parse_failed_repair",
JSONBoolean.getInstance(display.getParseFailedRepair().getValue()));
args.put("hostless", JSONBoolean.getInstance(display.getHostless().getValue()));
+ args.put("pool", new JSONString(display.getPool().getText()));
if (staticData.getData("drone_sets_enabled").isBoolean().booleanValue()) {
args.put("drone_set", new JSONString(display.getDroneSet().getSelectedName()));
@@ -683,7 +686,7 @@
args.put("image", new JSONString(imageUrlString));
}
- rpcProxy.rpcCall("create_job", args, new JsonRpcCallback() {
+ rpcProxy.rpcCall("create_job_page_handler", args, new JsonRpcCallback() {
@Override
public void onSuccess(JSONValue result) {
int id = (int) result.isNumber().doubleValue();
diff --git a/frontend/client/src/autotest/public/AfeClient.html b/frontend/client/src/autotest/public/AfeClient.html
index bbc0606..194f37f 100644
--- a/frontend/client/src/autotest/public/AfeClient.html
+++ b/frontend/client/src/autotest/public/AfeClient.html
@@ -64,7 +64,7 @@
<span id="view_created"></span><br>
<span class="field-name">Timeout:</span>
<span id="view_timeout"></span> minutes<br>
- <span class="field-name">Image URL/Build:</span>
+ <span class="field-name">Image (Build Name):</span>
<span id="view_image_url"></span><br>
<span class="field-name">Max runtime:</span>
<span id="view_max_runtime"></span> minutes<br>
@@ -127,8 +127,7 @@
<td id="create_kernel_cmdline"></td></tr>
<tr><td class="field-name">Image URL/Build:</td>
<td id="create_image_url"></td>
- <td class="help">Example: "x86-alex-release/R27-3837.0.0" or
- "http://$your_devserver/update/alex-release/R27-3837.0.0"
+ <td class="help">Example: "x86-alex-release/R27-3837.0.0"
</td></tr>
<tr><td class="field-name">Timeout (minutes):</td>
<td id="create_timeout"></td><td></td></tr>
@@ -148,8 +147,10 @@
<td id="create_reboot_after"></td><td></td></tr>
<tr><td class="field-name">Include failed repair results:</td>
<td id="create_parse_failed_repair"></td><td></td></tr>
- <tr><td class="field-name">Hostless:</td>
+ <tr><td class="field-name">Hostless (Suite Job):</td>
<td id="create_hostless"></td><td></td></tr>
+ <tr><td class="field-name">Pool:</td>
+ <td id="create_pool"></td><td></td></tr>
<tr id="create_drone_set_wrapper">
<td class="field-name">Drone set:</td>