[Autotest] merge cleanup and verify
The objective of this CL is to merge cleanup and verify into a single job to
reduce run time of each test. In existing design, by default, a cleanup job is
scheduled after a test is finished, and a verify job is scheduled before a
test is started. By merging these two jobs together, we are seeing the total
run time of these two jobs is reduced from about 47s to 37s, around 10s saving.
That does not include the saving on scheduler to schedule two jobs, which may
take another 5-10s.
The design is to create a new special task, reset, which runs at the beginning
of a job by default. Verify task is changed to not to run by default before a
job starts. Cleanup job will only be run if a job is scheduled to reboot and
any test failed in that job.
BUG=chromium:220679
TEST=tested with run_suite in local machine
DEPLOY=afe,apache,scheduler,change all users' preference on reboot_after to
Never, sql: |update chromeos_autotest_db.afe_users set reboot_after=0|
Change-Id: Ia38baf6b73897b7e09fdf635eadedc752b5eba2f
Reviewed-on: https://gerrit.chromium.org/gerrit/48685
Commit-Queue: Dan Shi <dshi@chromium.org>
Reviewed-by: Dan Shi <dshi@chromium.org>
Tested-by: Dan Shi <dshi@chromium.org>
diff --git a/frontend/afe/doctests/001_rpc_test.txt b/frontend/afe/doctests/001_rpc_test.txt
index f5acb7b..39b7743 100644
--- a/frontend/afe/doctests/001_rpc_test.txt
+++ b/frontend/afe/doctests/001_rpc_test.txt
@@ -156,7 +156,8 @@
... 'test_type': 'Client',
... 'test_class': 'Kernel',
... 'test_time': 'SHORT',
-... 'run_verify': 1,
+... 'run_verify': 0,
+... 'run_reset': 1,
... 'test_category': 'Functional',
... 'path': '/my/path',
... 'test_retry': 0}]
@@ -188,7 +189,7 @@
... 'login': 'showard',
... 'access_level': 1,
... 'reboot_before': 'If dirty',
-... 'reboot_after': 'Always',
+... 'reboot_after': 'Never',
... 'drone_set': None,
... 'show_experimental': False}]
True
@@ -541,10 +542,11 @@
... 'timeout': 24,
... 'max_runtime_mins': 1440,
... 'max_runtime_hrs' : 72,
-... 'run_verify': True,
+... 'run_verify': False,
+... 'run_reset': True,
... 'email_list': '',
... 'reboot_before': 'If dirty',
-... 'reboot_after': 'Always',
+... 'reboot_after': 'Never',
... 'parse_failed_repair': True,
... 'drone_set': drone_set,
... 'parameterized_job': None,
diff --git a/frontend/afe/models.py b/frontend/afe/models.py
index 6f66d74..2a528e2 100644
--- a/frontend/afe/models.py
+++ b/frontend/afe/models.py
@@ -21,7 +21,7 @@
# job options and user preferences
DEFAULT_REBOOT_BEFORE = model_attributes.RebootBefore.IF_DIRTY
-DEFAULT_REBOOT_AFTER = model_attributes.RebootBefore.ALWAYS
+DEFAULT_REBOOT_AFTER = model_attributes.RebootBefore.NEVER
class AclAccessViolation(Exception):
@@ -359,7 +359,7 @@
dirty: true if the host has been used without being rebooted
"""
Status = enum.Enum('Verifying', 'Running', 'Ready', 'Repairing',
- 'Repair Failed', 'Cleaning', 'Pending',
+ 'Repair Failed', 'Cleaning', 'Pending', 'Resetting',
string_values=True)
Protection = host_protections.Protection
@@ -576,6 +576,7 @@
test dependencies.
experimental: If this is set to True production servers will ignore the test
run_verify: Whether or not the scheduler should run the verify stage
+ run_reset: Whether or not the scheduler should run the reset stage
test_retry: Number of times to retry test if the test did not complete
successfully. (optional, default: 0)
"""
@@ -588,7 +589,7 @@
dependencies = dbmodels.CharField(max_length=255, blank=True)
description = dbmodels.TextField(blank=True)
experimental = dbmodels.BooleanField(default=True)
- run_verify = dbmodels.BooleanField(default=True)
+ run_verify = dbmodels.BooleanField(default=False)
test_time = dbmodels.SmallIntegerField(choices=TestTime.choices(),
default=TestTime.MEDIUM)
test_type = dbmodels.SmallIntegerField(
@@ -596,6 +597,7 @@
sync_count = dbmodels.IntegerField(default=1)
path = dbmodels.CharField(max_length=255, unique=True)
test_retry = dbmodels.IntegerField(blank=True, default=0)
+ run_reset = dbmodels.BooleanField(default=True)
dependency_labels = (
dbmodels.ManyToManyField(Label, blank=True,
@@ -1000,6 +1002,7 @@
submitted_on: date of job submission
synch_count: how many hosts should be used per autoserv execution
run_verify: Whether or not to run the verify phase
+ run_reset: Whether or not to run the reset phase
timeout: hours from queuing time until job times out
max_runtime_hrs: DEPRECATED - hours from job starting time until job
times out
@@ -1044,7 +1047,7 @@
created_on = dbmodels.DateTimeField()
synch_count = dbmodels.IntegerField(null=True, default=1)
timeout = dbmodels.IntegerField(default=DEFAULT_TIMEOUT)
- run_verify = dbmodels.BooleanField(default=True)
+ run_verify = dbmodels.BooleanField(default=False)
email_list = dbmodels.CharField(max_length=250, blank=True)
dependency_labels = (
dbmodels.ManyToManyField(Label, blank=True,
@@ -1070,6 +1073,8 @@
test_retry = dbmodels.IntegerField(blank=True, default=0)
+ run_reset = dbmodels.BooleanField(default=True)
+
# custom manager
objects = JobManager()
@@ -1165,7 +1170,8 @@
drone_set=drone_set,
parameterized_job=parameterized_job,
parent_job=options.get('parent_job_id'),
- test_retry=options.get('test_retry'))
+ test_retry=options.get('test_retry'),
+ run_reset=options.get('run_reset'))
job.dependency_labels = options['dependencies']
@@ -1540,7 +1546,8 @@
queue_entry: Host queue entry waiting on this task (or None, if task was not
started in preparation of a job)
"""
- Task = enum.Enum('Verify', 'Cleanup', 'Repair', string_values=True)
+ Task = enum.Enum('Verify', 'Cleanup', 'Repair', 'Reset',
+ string_values=True)
host = dbmodels.ForeignKey(Host, blank=False, null=False)
task = dbmodels.CharField(max_length=64, choices=Task.choices(),
diff --git a/frontend/afe/resources.py b/frontend/afe/resources.py
index 8c9ced3..04171b1 100644
--- a/frontend/afe/resources.py
+++ b/frontend/afe/resources.py
@@ -436,6 +436,7 @@
'dependencies': [],
'machines_per_execution': 1,
'run_verify': bool(_job_fields['run_verify'].default),
+ 'run_reset': bool(_job_fields['run_reset'].default),
'timeout_hrs': _job_fields['timeout'].default,
'maximum_runtime_mins': _job_fields['max_runtime_mins'].default,
'cleanup_before_job':
@@ -477,6 +478,7 @@
in job.dependency_labels.all()],
'machines_per_execution': job.synch_count,
'run_verify': bool(job.run_verify),
+ 'run_reset': bool(job.run_reset),
'timeout_hrs': job.timeout,
'maximum_runtime_mins': job.max_runtime_mins,
'cleanup_before_job':
@@ -686,6 +688,7 @@
max_runtime_mins=execution_info.get('maximum_runtime_mins'),
synch_count=execution_info.get('machines_per_execution'),
run_verify=execution_info.get('run_verify'),
+ run_reset=execution_info.get('run_reset'),
email_list=input_dict.get('email_list', None),
dependencies=execution_info.get('dependencies', ()),
reboot_before=execution_info.get('cleanup_before_job'),
diff --git a/frontend/afe/resources_test.py b/frontend/afe/resources_test.py
index 1f679ee..de7b53a 100755
--- a/frontend/afe/resources_test.py
+++ b/frontend/afe/resources_test.py
@@ -343,9 +343,10 @@
self.assertEquals(info['control_file'], self.CONTROL_FILE_CONTENTS)
self.assertEquals(info['is_server'], False)
self.assertEquals(info['cleanup_before_job'], 'Never')
- self.assertEquals(info['cleanup_after_job'], 'Always')
+ self.assertEquals(info['cleanup_after_job'], 'Never')
self.assertEquals(info['machines_per_execution'], 1)
- self.assertEquals(info['run_verify'], True)
+ self.assertEquals(info['run_verify'], False)
+ self.assertEquals(info['run_reset'], True)
def test_queue_entries(self):
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index d41d376..e910a61 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -419,10 +419,11 @@
meta_hosts=(), one_time_hosts=(),
atomic_group_name=None, synch_count=None,
is_template=False, timeout=None,
- max_runtime_mins=None, run_verify=True,
+ max_runtime_mins=None, run_verify=False,
email_list='', dependencies=(), reboot_before=None,
reboot_after=None, parse_failed_repair=None,
- hostless=False, keyvals=None, drone_set=None):
+ hostless=False, keyvals=None, drone_set=None,
+ run_reset=True):
"""
Creates and enqueues a parameterized job.
@@ -498,11 +499,11 @@
def create_job(name, priority, control_file, control_type,
hosts=(), meta_hosts=(), one_time_hosts=(),
atomic_group_name=None, synch_count=None, is_template=False,
- timeout=None, max_runtime_mins=None, run_verify=True,
+ timeout=None, max_runtime_mins=None, run_verify=False,
email_list='', dependencies=(), reboot_before=None,
reboot_after=None, parse_failed_repair=None, hostless=False,
keyvals=None, drone_set=None, image=None, parent_job_id=None,
- test_retry=0):
+ test_retry=0, run_reset=True):
"""\
Create and enqueue a job.
@@ -536,10 +537,10 @@
@param parent_job_id id of a job considered to be parent of created job.
@param test_retry: Number of times to retry test if the test did not
complete successfully. (optional, default: 0)
+ @param run_reset: Should the host be reset before running the test?
@returns The created Job id number.
"""
-
stats.Counter('create_job').increment()
# Force control files to only contain ascii characters.
try:
@@ -913,7 +914,8 @@
"Gathering": "Gathering log files",
"Template": "Template job for recurring run",
"Waiting": "Waiting for scheduler action",
- "Archiving": "Archiving results"}
+ "Archiving": "Archiving results",
+ "Resetting": "Resetting hosts"}
return result
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index 9c4f245..7d370d1 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -8,7 +8,7 @@
import datetime, os, inspect
import django.http
-from autotest_lib.frontend.afe import models, model_logic, model_attributes
+from autotest_lib.frontend.afe import models, model_logic
from autotest_lib.client.common_lib import control_data
NULL_DATETIME = datetime.datetime.max
@@ -648,7 +648,7 @@
reboot_before=None, reboot_after=None,
parse_failed_repair=None, hostless=False, keyvals=None,
drone_set=None, parameterized_job=None,
- parent_job_id=None, test_retry=0):
+ parent_job_id=None, test_retry=0, run_reset=True):
#pylint: disable-msg=C0111
"""
Common code between creating "standard" jobs and creating parameterized jobs
@@ -753,7 +753,8 @@
drone_set=drone_set,
parameterized_job=parameterized_job,
parent_job_id=parent_job_id,
- test_retry=test_retry)
+ test_retry=test_retry,
+ run_reset=run_reset)
return create_new_job(owner=owner,
options=options,
host_objects=host_objects,
diff --git a/frontend/client/src/autotest/afe/HostDetailView.java b/frontend/client/src/autotest/afe/HostDetailView.java
index be86941..e5fc3d3 100644
--- a/frontend/client/src/autotest/afe/HostDetailView.java
+++ b/frontend/client/src/autotest/afe/HostDetailView.java
@@ -256,7 +256,8 @@
selectionManager.setSelectableRowFilter(this);
jobsTable.setWidgetFactory(selectionManager);
tableDecorator.addTableActionsPanel(this, true);
- tableDecorator.addControl("Show verifies, repairs and cleanups", showSpecialTasks);
+ tableDecorator.addControl("Show verifies, repairs, cleanups and resets",
+ showSpecialTasks);
addWidget(tableDecorator, "view_host_jobs_table");
showSpecialTasks.addClickHandler(new ClickHandler() {
diff --git a/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java b/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
index 4fd2f79..affcaa7 100644
--- a/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
+++ b/frontend/client/src/autotest/afe/create/CreateJobViewDisplay.java
@@ -50,6 +50,7 @@
private TextBox testRetry = new TextBox();
private TextBox emailList = new TextBox();
private CheckBoxImpl skipVerify = new CheckBoxImpl();
+ private CheckBoxImpl skipReset = new CheckBoxImpl();
private RadioChooserDisplay rebootBefore = new RadioChooserDisplay();
private RadioChooserDisplay rebootAfter = new RadioChooserDisplay();
private CheckBox parseFailedRepair = new CheckBox();
@@ -108,6 +109,7 @@
panel.add(emailList, "create_email_list");
panel.add(priorityList, "create_priority");
panel.add(skipVerify, "create_skip_verify");
+ panel.add(skipReset, "create_skip_reset");
panel.add(rebootBefore, "create_reboot_before");
panel.add(rebootAfter, "create_reboot_after");
panel.add(parseFailedRepair, "create_parse_failed_repair");
@@ -218,6 +220,10 @@
return skipVerify;
}
+ public ICheckBox getSkipReset() {
+ return skipReset;
+ }
+
public IButton getSubmitJobButton() {
return submitJobButton;
}
diff --git a/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java b/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
index f136ac4..e59a1fc 100644
--- a/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
+++ b/frontend/client/src/autotest/afe/create/CreateJobViewPresenter.java
@@ -69,6 +69,7 @@
public HasText getTestRetry();
public HasText getEmailList();
public ICheckBox getSkipVerify();
+ public ICheckBox getSkipReset();
public RadioChooser.Display getRebootBefore();
public RadioChooser.Display getRebootAfter();
public HasValue<Boolean> getParseFailedRepair();
@@ -175,6 +176,7 @@
jobObject.get("email_list").isString().stringValue());
display.getSkipVerify().setValue(!jobObject.get("run_verify").isBoolean().booleanValue());
+ display.getSkipReset().setValue(!jobObject.get("run_reset").isBoolean().booleanValue());
rebootBefore.setSelectedChoice(Utils.jsonToString(jobObject.get("reboot_before")));
rebootAfter.setSelectedChoice(Utils.jsonToString(jobObject.get("reboot_after")));
display.getParseFailedRepair().setValue(
@@ -384,6 +386,24 @@
}
}
+ public void handleSkipReset() {
+ boolean shouldSkipReset = false;
+ for (JSONObject test : testSelector.getSelectedTests()) {
+ boolean runReset = test.get("run_reset").isBoolean().booleanValue();
+ if (!runReset) {
+ shouldSkipReset = true;
+ break;
+ }
+ }
+
+ if (shouldSkipReset) {
+ display.getSkipReset().setValue(true);
+ display.getSkipReset().setEnabled(false);
+ } else {
+ display.getSkipReset().setEnabled(true);
+ }
+ }
+
protected int getMaximumRetriesCount() {
int maxRetries = 0;
for (JSONObject test : testSelector.getSelectedTests()) {
@@ -396,6 +416,7 @@
testSelector.setEnabled(true);
profilersPanel.setEnabled(true);
handleSkipVerify();
+ handleSkipReset();
display.getKernel().setEnabled(true);
display.getKernelCmdline().setEnabled(true);
display.getImageUrl().setEnabled(true);
@@ -565,7 +586,8 @@
display.getTestRetry().setText("");
display.getEmailList().setText("");
testSelector.reset();
- display.getSkipVerify().setValue(false);
+ display.getSkipVerify().setValue(true);
+ display.getSkipReset().setValue(false);
profilersPanel.reset();
setInputsEnabled();
controlTypeSelect.setControlType(TestSelector.CLIENT_TYPE);
@@ -624,6 +646,8 @@
args.put("email_list", new JSONString(display.getEmailList().getText()));
args.put("run_verify", JSONBoolean.getInstance(
!display.getSkipVerify().getValue()));
+ args.put("run_reset", JSONBoolean.getInstance(
+ !display.getSkipReset().getValue()));
args.put("is_template", JSONBoolean.getInstance(isTemplate));
args.put("dependencies", getSelectedDependencies());
args.put("reboot_before", new JSONString(rebootBefore.getSelectedChoice()));
diff --git a/frontend/client/src/autotest/public/AfeClient.html b/frontend/client/src/autotest/public/AfeClient.html
index 740b2ae..45d29e5 100644
--- a/frontend/client/src/autotest/public/AfeClient.html
+++ b/frontend/client/src/autotest/public/AfeClient.html
@@ -140,6 +140,8 @@
<td id="create_email_list"></td><td></td></tr>
<tr><td class="field-name">Skip verify:</td>
<td id="create_skip_verify"></td><td></td></tr>
+ <tr><td class="field-name">Skip reset:</td>
+ <td id="create_skip_reset"></td><td></td></tr>
<tr><td class="field-name">Reboot before:</td>
<td id="create_reboot_before"></td><td></td></tr>
<tr><td class="field-name">Reboot after:</td>
diff --git a/frontend/migrations/077_add_run_reset.py b/frontend/migrations/077_add_run_reset.py
new file mode 100644
index 0000000..b71ab9a
--- /dev/null
+++ b/frontend/migrations/077_add_run_reset.py
@@ -0,0 +1,9 @@
+UP_SQL = """
+ALTER TABLE afe_autotests ADD COLUMN run_reset SMALLINT NOT NULL DEFAULT '1';
+ALTER TABLE afe_jobs ADD COLUMN run_reset SMALLINT NOT NULL DEFAULT '1';
+"""
+
+DOWN_SQL = """
+ALTER TABLE afe_autotests DROP COLUMN run_reset;
+ALTER TABLE afe_jobs DROP COLUMN run_reset;
+"""
\ No newline at end of file