two new major features:
(1) added test and job dependencies
-added M2M relationship between tests and labels and between jobs and labels, for tracking the labels on which a test/job depends
-modified test_importer to read the DEPENDENCIES field and create the right M2M relationships
-modified generate_control_file() RPC to compute and return the union of test dependencies. since generate_control_file now returns four pieces of information, i converted its return type from tuple to dict, and changed clients accordingly.
-modified job creation clients (GWT and CLI) to pass this dependency list to the create_job() RPC
-modified the create_job() RPC to check that hosts satisfy job dependencies, and to create M2M relationships
-modified the scheduler to check dependencies when scheduling jobs
-modified JobDetailView to show a job's dependencies
(2) added "only_if_needed" bit to labels; if true, a machine with this label can only be used if the label is requested (either by job dependencies or by the metahost label)
-added boolean field to Labels
-modified CLI label creation/viewing to support this new field
-made create_job() RPC and scheduler check for hosts with such a label that was not requested, and reject such hosts
also did some slight refactoring of other code in create_job() to simplify it while I was changing things there.
a couple notes:
-an only_if_needed label can be used if either the job depends on the label or it's a metahost for that label. we assume that if the user specifically requests the label in a metahost, then it's OK, even if the job doesn't depend on that label.
-one-time-hosts are assumed to satisfy job dependencies.
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2215 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/frontend/afe/doctests/001_rpc_test.txt b/frontend/afe/doctests/001_rpc_test.txt
index 0e869e8..84697b4 100644
--- a/frontend/afe/doctests/001_rpc_test.txt
+++ b/frontend/afe/doctests/001_rpc_test.txt
@@ -51,6 +51,7 @@
... 'name': 'test_label',
... 'platform': 1,
... 'kernel_config': '/my/kernel/config',
+... 'only_if_needed' : False,
... 'invalid': 0}]
True
@@ -63,9 +64,9 @@
4L
>>> data = rpc_interface.get_labels(platform=False)
>>> data == [{'id': 2L, 'name': 'label1', 'platform': 0, 'kernel_config': '',
-... 'invalid': 0},
+... 'only_if_needed': False, 'invalid': 0},
... {'id': 4L, 'name': 'label3', 'platform': 0, 'kernel_config': '',
-... 'invalid': 0}]
+... 'only_if_needed': False, 'invalid': 0}]
True
# delete_label takes an ID or a name as well
@@ -340,13 +341,13 @@
>>> rpc_interface.label_add_hosts(id='my_label', hosts=['my_label_host1', 'my_label_host2'])
# generate a control file
->>> control_file, is_server, is_synch = rpc_interface.generate_control_file(
+>>> cf_info = rpc_interface.generate_control_file(
... tests=['sleeptest', 'my_test'],
... kernel='1.2.3.4',
... label='my_label')
->>> control_file
+>>> cf_info['control_file']
"kernel = '1.2.3.4'\ndef step_init():\n job.next_step([step_test])\n testkernel = job.kernel('1.2.3.4')\n testkernel.config('my_kernel_config')\n testkernel.install()\n testkernel.boot(args='')\n\ndef step_test():\n job.next_step('step0')\n job.next_step('step1')\n\ndef step0():\n job.run_test('testname')\n\ndef step1():\n job.run_test('testname')"
->>> print control_file #doctest: +NORMALIZE_WHITESPACE
+>>> print cf_info['control_file'] #doctest: +NORMALIZE_WHITESPACE
kernel = '1.2.3.4'
def step_init():
job.next_step([step_test])
@@ -362,13 +363,13 @@
job.run_test('testname')
def step1():
job.run_test('testname')
->>> is_server, is_synch
-(False, False)
+>>> cf_info['is_server'], cf_info['is_synchronous'], cf_info['dependencies']
+(False, False, [])
# create a job to run on host1, host2, and any two machines in my_label
>>> rpc_interface.create_job(name='my_job',
... priority='Low',
-... control_file=control_file,
+... control_file=cf_info['control_file'],
... control_type='Client',
... hosts=['host1', 'host2'],
... meta_hosts=['my_label', 'my_label'])
@@ -379,7 +380,7 @@
>>> data = data[0]
>>> data['id'], data['owner'], data['name'], data['priority']
(1L, 'debug_user', 'my_job', 'Low')
->>> data['control_file'] == control_file
+>>> data['control_file'] == cf_info['control_file']
True
>>> data['control_type']
'Client'
@@ -401,7 +402,7 @@
# get_host_queue_entries returns full info about the job within each queue entry
>>> job = data[0]['job']
->>> job == {'control_file': control_file, # refer to the control file we used
+>>> job == {'control_file': cf_info['control_file'], # the control file we used
... 'control_type': 'Client',
... 'created_on': None,
... 'id': 1L,
@@ -493,7 +494,7 @@
# is_synchronous
>>> rpc_interface.create_job(name='my_job',
... priority='Low',
-... control_file=control_file,
+... control_file=cf_info['control_file'],
... control_type='Server',
... is_synchronous=False,
... hosts=['host1'])
diff --git a/frontend/afe/doctests/003_misc_rpc_features.txt b/frontend/afe/doctests/003_misc_rpc_features.txt
index f5e686d..abaac61 100644
--- a/frontend/afe/doctests/003_misc_rpc_features.txt
+++ b/frontend/afe/doctests/003_misc_rpc_features.txt
@@ -10,8 +10,8 @@
3L
# profiler support in control file generation
->>> control_file, is_server, is_synch = rpc_interface.generate_control_file(
+>>> cf_info = rpc_interface.generate_control_file(
... tests=['sleeptest'],
... profilers=['oprofile', 'iostat'])
->>> control_file
+>>> cf_info['control_file']
"def step_init():\n job.next_step('step0')\n job.next_step('step1')\n job.next_step('step2')\n job.next_step('step3')\n job.next_step('step4')\n\ndef step0():\n job.profilers.add('oprofile')\n\ndef step1():\n job.profilers.add('iostat')\n\ndef step2():\n job.run_test('testname')\n\ndef step3():\n job.profilers.delete('oprofile')\n\ndef step4():\n job.profilers.delete('iostat')"
diff --git a/frontend/afe/models.py b/frontend/afe/models.py
index 3451a50..2f79014 100644
--- a/frontend/afe/models.py
+++ b/frontend/afe/models.py
@@ -21,10 +21,13 @@
kernel_config: url/path to kernel config to use for jobs run on this
label
platform: if True, this is a platform label (defaults to False)
+ only_if_needed: if True, a machine with this label can only be used if that
+ that label is requested by the job/test.
"""
name = dbmodels.CharField(maxlength=255, unique=True)
kernel_config = dbmodels.CharField(maxlength=255, blank=True)
platform = dbmodels.BooleanField(default=False)
+ only_if_needed = dbmodels.BooleanField(default=False)
invalid = dbmodels.BooleanField(default=False,
editable=settings.FULL_ADMIN)
@@ -262,6 +265,8 @@
machines.
Optional:
dependencies: What the test requires to run. Comma deliminated list
+ dependency_labels: many-to-many relationship with labels corresponding to
+ test dependencies.
experimental: If this is set to True production servers will ignore the test
run_verify: Whether or not the scheduler should run the verify stage
"""
@@ -286,6 +291,8 @@
synch_type = dbmodels.SmallIntegerField(choices=SynchType.choices(),
default=SynchType.ASYNCHRONOUS)
path = dbmodels.CharField(maxlength=255, unique=True)
+ dependency_labels = dbmodels.ManyToManyField(
+ Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
name_field = 'name'
objects = model_logic.ExtendedManager()
@@ -472,6 +479,24 @@
return all_job_counts
+ def populate_dependencies(self, jobs):
+ job_ids = ','.join(str(job['id']) for job in jobs)
+ cursor = connection.cursor()
+ cursor.execute("""
+ SELECT jobs.id, GROUP_CONCAT(labels.name)
+ FROM jobs
+ INNER JOIN jobs_dependency_labels
+ ON jobs.id = jobs_dependency_labels.job_id
+ INNER JOIN labels ON jobs_dependency_labels.label_id = labels.id
+ WHERE jobs.id IN (%s)
+ GROUP BY jobs.id
+ """ % job_ids)
+ id_to_dependencies = dict((job_id, dependencies)
+ for job_id, dependencies in cursor.fetchall())
+ for job in jobs:
+ job['dependencies'] = id_to_dependencies.get(job['id'], '')
+
+
class Job(dbmodels.Model, model_logic.ModelExtensions):
"""\
owner: username of job owner
@@ -489,6 +514,8 @@
timeout: hours until job times out
email_list: list of people to email on completion delimited by any of:
white space, ',', ':', ';'
+ dependency_labels: many-to-many relationship with labels corresponding to
+ job dependencies
"""
Priority = enum.Enum('Low', 'Medium', 'High', 'Urgent')
ControlType = enum.Enum('Server', 'Client', start_value=1)
@@ -512,6 +539,8 @@
run_verify = dbmodels.BooleanField(default=True)
timeout = dbmodels.IntegerField()
email_list = dbmodels.CharField(maxlength=250, blank=True)
+ dependency_labels = dbmodels.ManyToManyField(
+ Label, blank=True, filter_interface=dbmodels.HORIZONTAL)
# custom manager
@@ -524,7 +553,8 @@
@classmethod
def create(cls, owner, name, priority, control_file, control_type,
- hosts, synch_type, timeout, run_verify, email_list):
+ hosts, synch_type, timeout, run_verify, email_list,
+ dependencies):
"""\
Creates a job by taking some information (the listed args)
and filling in the rest of the necessary information.
@@ -545,6 +575,7 @@
+ ' one host to run on'}
raise model_logic.ValidationError(errors)
job.save()
+ job.dependency_labels = dependencies
return job
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index c471a68..6beb3be 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -37,9 +37,10 @@
# labels
-def add_label(name, kernel_config=None, platform=None):
+def add_label(name, kernel_config=None, platform=None, only_if_needed=None):
return models.Label.add_object(name=name, kernel_config=kernel_config,
- platform=platform).id
+ platform=platform,
+ only_if_needed=only_if_needed).id
def modify_label(id, **data):
@@ -246,10 +247,11 @@
def generate_control_file(tests, kernel=None, label=None, profilers=[]):
"""\
Generates a client-side control file to load a kernel and run a set of
- tests. Returns a tuple (control_file, is_server, is_synchronous):
+ tests. Returns a dict with the following keys:
control_file - the control file text
is_server - is the control file a server-side control file?
is_synchronous - should the control file be run synchronously?
+ dependencies - a list of the names of labels on which the job depends
tests: list of tests to run
kernel: kernel to install in generated control file
@@ -257,21 +259,22 @@
profilers: list of profilers to activate during the job
"""
if not tests:
- return '', False, False
+ return dict(control_file='', is_server=False, is_synchronous=False,
+ dependencies=[])
- is_server, is_synchronous, test_objects, profiler_objects, label = (
+ cf_info, test_objects, profiler_objects, label = (
rpc_utils.prepare_generate_control_file(tests, kernel, label,
profilers))
- cf_text = control_file.generate_control(tests=test_objects, kernel=kernel,
- platform=label,
- profilers=profiler_objects,
- is_server=is_server)
- return cf_text, is_server, is_synchronous
+ cf_info['control_file'] = control_file.generate_control(
+ tests=test_objects, kernel=kernel, platform=label,
+ profilers=profiler_objects, is_server=cf_info['is_server'])
+ return cf_info
def create_job(name, priority, control_file, control_type, timeout=None,
is_synchronous=None, hosts=None, meta_hosts=None,
- run_verify=True, one_time_hosts=None, email_list=''):
+ run_verify=True, one_time_hosts=None, email_list='',
+ dependencies=[]):
"""\
Create and enqueue a job.
@@ -285,6 +288,7 @@
on.
timeout: hours until job times out
email_list: string containing emails to mail when the job is done
+ dependencies: list of label names on which this job depends
"""
if timeout is None:
@@ -299,33 +303,32 @@
"'meta_hosts', or 'one_time_hosts'"
})
- requested_host_counts = {}
+ labels_by_name = dict((label.name, label)
+ for label in models.Label.objects.all())
# convert hostnames & meta hosts to host/label objects
host_objects = []
+ metahost_objects = []
+ metahost_counts = {}
for host in hosts or []:
this_host = models.Host.smart_get(host)
host_objects.append(this_host)
for label in meta_hosts or []:
- this_label = models.Label.smart_get(label)
- host_objects.append(this_label)
- requested_host_counts.setdefault(this_label.name, 0)
- requested_host_counts[this_label.name] += 1
+ this_label = labels_by_name[label]
+ metahost_objects.append(this_label)
+ metahost_counts.setdefault(this_label, 0)
+ metahost_counts[this_label] += 1
for host in one_time_hosts or []:
this_host = models.Host.create_one_time_host(host)
host_objects.append(this_host)
# check that each metahost request has enough hosts under the label
- if meta_hosts:
- labels = models.Label.objects.filter(
- name__in=requested_host_counts.keys())
- for label in labels:
- count = label.host_set.count()
- if requested_host_counts[label.name] > count:
- error = ("You have requested %d %s's, but there are only %d."
- % (requested_host_counts[label.name],
- label.name, count))
- raise model_logic.ValidationError({'arguments' : error})
+ for label, requested_count in metahost_counts.iteritems():
+ available_count = label.host_set.count()
+ if requested_count > available_count:
+ error = ("You have requested %d %s's, but there are only %d."
+ % (requested_count, label.name, available_count))
+ raise model_logic.ValidationError({'meta_hosts' : error})
# default is_synchronous to some appropriate value
ControlType = models.Job.ControlType
@@ -338,15 +341,20 @@
else:
synch_type = models.Test.SynchType.ASYNCHRONOUS
+ rpc_utils.check_job_dependencies(host_objects, dependencies)
+ dependency_labels = [labels_by_name[label_name]
+ for label_name in dependencies]
+
job = models.Job.create(owner=owner, name=name, priority=priority,
control_file=control_file,
control_type=control_type,
synch_type=synch_type,
- hosts=host_objects,
+ hosts=host_objects + metahost_objects,
timeout=timeout,
run_verify=run_verify,
- email_list=email_list.strip())
- job.queue(host_objects)
+ email_list=email_list.strip(),
+ dependencies=dependency_labels)
+ job.queue(host_objects + metahost_objects)
return job.id
@@ -382,8 +390,9 @@
filter_data['extra_args'] = rpc_utils.extra_job_filters(not_yet_run,
running,
finished)
- return rpc_utils.prepare_for_serialization(
- models.Job.list_objects(filter_data))
+ jobs = models.Job.list_objects(filter_data)
+ models.Job.objects.populate_dependencies(jobs)
+ return rpc_utils.prepare_for_serialization(jobs)
def get_num_jobs(not_yet_run=False, running=False, finished=False,
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index ac175f0..fcdc0ba 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -142,4 +142,44 @@
if label:
label = models.Label.smart_get(label)
- return is_server, is_synchronous, test_objects, profiler_objects, label
+ dependencies = set(label.name for label
+ in models.Label.objects.filter(test__in=test_objects))
+
+ return (dict(is_server=is_server, is_synchronous=is_synchronous,
+ dependencies=list(dependencies)),
+ test_objects, profiler_objects, label)
+
+
+def check_job_dependencies(host_objects, job_dependencies):
+ """
+ Check that a set of machines satisfies a job's dependencies.
+ host_objects: list of models.Host objects
+ job_dependencies: list of names of labels
+ """
+ # check that hosts satisfy dependencies
+ host_ids = [host.id for host in host_objects]
+ hosts_in_job = models.Host.objects.filter(id__in=host_ids)
+ ok_hosts = hosts_in_job
+ for index, dependency in enumerate(job_dependencies):
+ ok_hosts &= models.Host.objects.filter_custom_join(
+ '_label%d' % index, labels__name=dependency)
+ failing_hosts = (set(host.hostname for host in host_objects) -
+ set(host.hostname for host in ok_hosts))
+ if failing_hosts:
+ raise model_logic.ValidationError(
+ {'hosts' : 'Host(s) failed to meet job dependencies: ' +
+ ', '.join(failing_hosts)})
+
+ # check for hosts that have only_if_needed labels that aren't requested
+ labels_not_requested = models.Label.objects.filter(only_if_needed=True,
+ host__id__in=host_ids)
+ labels_not_requested = labels_not_requested.exclude(
+ name__in=job_dependencies)
+ errors = []
+ for label in labels_not_requested:
+ hosts_in_label = hosts_in_job.filter(labels=label)
+ errors.append('Cannot use hosts with label "%s" unless requested: %s' %
+ (label.name,
+ ', '.join(host.hostname for host in hosts_in_label)))
+ if errors:
+ raise model_logic.ValidationError({'hosts' : '\n'.join(errors)})
diff --git a/frontend/client/src/autotest/afe/CreateJobView.java b/frontend/client/src/autotest/afe/CreateJobView.java
index 24f31c5..c615362 100644
--- a/frontend/client/src/autotest/afe/CreateJobView.java
+++ b/frontend/client/src/autotest/afe/CreateJobView.java
@@ -170,6 +170,7 @@
protected boolean controlEdited = false;
protected boolean controlReadyForSubmit = false;
+ private JSONArray dependencies = new JSONArray();
public CreateJobView(JobCreateListener listener) {
this.listener = listener;
@@ -316,10 +317,12 @@
rpcProxy.rpcCall("generate_control_file", params, new JsonRpcCallback() {
@Override
public void onSuccess(JSONValue result) {
- JSONArray results = result.isArray();
- String controlFileText = results.get(0).isString().stringValue();
- boolean isServer = results.get(1).isBoolean().booleanValue();
- boolean isSynchronous = results.get(2).isBoolean().booleanValue();
+ JSONObject controlInfo = result.isObject();
+ String controlFileText = controlInfo.get("control_file").isString().stringValue();
+ boolean isServer = controlInfo.get("is_server").isBoolean().booleanValue();
+ boolean isSynchronous =
+ controlInfo.get("is_synchronous").isBoolean().booleanValue();
+ dependencies = controlInfo.get("dependencies").isArray();
controlFile.setText(controlFileText);
controlTypeSelect.setControlType(isServer ? TestSelector.SERVER_TYPE :
TestSelector.CLIENT_TYPE);
@@ -523,6 +526,7 @@
controlFilePanel.setOpen(false);
editControlButton.setText(EDIT_CONTROL_STRING);
hostSelector.reset();
+ dependencies = new JSONArray();
}
protected void submitJob() {
@@ -565,6 +569,7 @@
args.put("meta_hosts", Utils.stringsToJSON(hosts.metaHosts));
args.put("one_time_hosts",
Utils.stringsToJSON(hosts.oneTimeHosts));
+ args.put("dependencies", dependencies);
rpcProxy.rpcCall("create_job", args, new JsonRpcCallback() {
@Override
diff --git a/frontend/client/src/autotest/afe/JobDetailView.java b/frontend/client/src/autotest/afe/JobDetailView.java
index 2adbdbd..c7acbb0 100644
--- a/frontend/client/src/autotest/afe/JobDetailView.java
+++ b/frontend/client/src/autotest/afe/JobDetailView.java
@@ -97,6 +97,7 @@
showField(jobObject, "email_list", "view_email_list");
showField(jobObject, "control_type", "view_control_type");
showField(jobObject, "control_file", "view_control_file");
+ showField(jobObject, "dependencies", "view_dependencies");
String synchType = jobObject.get("synch_type").isString().stringValue();
showText(synchType.toLowerCase(), "view_synch_type");
diff --git a/frontend/client/src/autotest/public/AfeClient.html b/frontend/client/src/autotest/public/AfeClient.html
index 8a50e78..fc4de49 100644
--- a/frontend/client/src/autotest/public/AfeClient.html
+++ b/frontend/client/src/autotest/public/AfeClient.html
@@ -54,6 +54,8 @@
<span id="view_timeout"></span> hours<br>
<span class="field-name">Email List:</span>
<span id="view_email_list"></span><br>
+ <span class="field-name">Dependencies:</span>
+ <span id="view_dependencies"></span><br>
<span class="field-name">Status:</span>
<span id="view_status"></span><br>
diff --git a/frontend/migrations/018_add_label_only_if_needed.py b/frontend/migrations/018_add_label_only_if_needed.py
new file mode 100644
index 0000000..790299a
--- /dev/null
+++ b/frontend/migrations/018_add_label_only_if_needed.py
@@ -0,0 +1,24 @@
+CREATE_MANY2MANY_TABLES = """
+CREATE TABLE `autotests_dependency_labels` (
+ `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+ `test_id` integer NOT NULL REFERENCES `autotests` (`id`),
+ `label_id` integer NOT NULL REFERENCES `labels` (`id`),
+ UNIQUE (`test_id`, `label_id`)
+);
+CREATE TABLE `jobs_dependency_labels` (
+ `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+ `job_id` integer NOT NULL REFERENCES `jobs` (`id`),
+ `label_id` integer NOT NULL REFERENCES `labels` (`id`),
+ UNIQUE (`job_id`, `label_id`)
+);
+"""
+
+def migrate_up(manager):
+ manager.execute('ALTER TABLE labels '
+ 'ADD COLUMN only_if_needed bool NOT NULL')
+ manager.execute_script(CREATE_MANY2MANY_TABLES)
+
+def migrate_down(manager):
+ manager.execute('ALTER TABLE labels DROP COLUMN only_if_needed')
+ manager.execute('DROP TABLE IF EXISTS `autotests_dependency_labels`')
+ manager.execute('DROP TABLE IF EXISTS `jobs_dependency_labels`')