Support for job keyvals
* can be passed as an argument to create_job, stored in AFE DB
* scheduler reads them from the AFE DB and writes them to the job-level keyval file before the job starts
* parser reads them from the keyval file and writes them to the TKO DB in a new table
Since the field name "key" happens to be a MySQL keyword, I went ahead and made db.py support proper quoting of field names. Evetually it'd be really nice to deprecate db.py and use Django models exclusively, but that is a far-off dream.
Still lacking support in the AFE and TKO web clients and CLIs, at least the TKO part will be coming soon
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@4123 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/frontend/afe/models.py b/frontend/afe/models.py
index 07f1b40..41c39c0 100644
--- a/frontend/afe/models.py
+++ b/frontend/afe/models.py
@@ -709,6 +709,11 @@
created_on=datetime.now())
job.dependency_labels = options['dependencies']
+
+ if options['keyvals'] is not None:
+ for key, value in options['keyvals'].iteritems():
+ JobKeyval.objects.create(job=job, key=key, value=value)
+
return job
@@ -755,6 +760,11 @@
return '%s-%s' % (self.id, self.owner)
+ def keyval_dict(self):
+ return dict((keyval.key, keyval.value)
+ for keyval in self.jobkeyval_set.all())
+
+
class Meta:
db_table = 'afe_jobs'
@@ -762,6 +772,18 @@
return u'%s (%s-%s)' % (self.name, self.id, self.owner)
+class JobKeyval(dbmodels.Model, model_logic.ModelExtensions):
+ """Keyvals associated with jobs"""
+ job = dbmodels.ForeignKey(Job)
+ key = dbmodels.CharField(max_length=90)
+ value = dbmodels.CharField(max_length=300)
+
+ objects = model_logic.ExtendedManager()
+
+ class Meta:
+ db_table = 'afe_job_keyvals'
+
+
class IneligibleHostQueue(dbmodels.Model, model_logic.ModelExtensions):
job = dbmodels.ForeignKey(Job)
host = dbmodels.ForeignKey(Host)
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index dd36602..fe56cab 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -402,7 +402,8 @@
atomic_group_name=None, synch_count=None, is_template=False,
timeout=None, max_runtime_hrs=None, run_verify=True,
email_list='', dependencies=(), reboot_before=None,
- reboot_after=None, parse_failed_repair=None, hostless=False):
+ reboot_after=None, parse_failed_repair=None, hostless=False,
+ keyvals=None):
"""\
Create and enqueue a job.
@@ -424,6 +425,7 @@
@param parse_failed_repair if true, results of failed repairs launched by
this job will be parsed as part of the job.
@param hostless if true, create a hostless job
+ @param keyvals dict of keyvals to associate with the job
@param hosts List of hosts to run job on.
@param meta_hosts List where each entry is a label name, and for each entry
@@ -531,7 +533,8 @@
dependencies=dependencies,
reboot_before=reboot_before,
reboot_after=reboot_after,
- parse_failed_repair=parse_failed_repair)
+ parse_failed_repair=parse_failed_repair,
+ keyvals=keyvals)
return rpc_utils.create_new_job(owner=owner,
options=options,
host_objects=host_objects,
@@ -584,10 +587,13 @@
jobs = list(models.Job.query_objects(filter_data))
models.Job.objects.populate_relationships(jobs, models.Label,
'dependencies')
+ models.Job.objects.populate_relationships(jobs, models.JobKeyval, 'keyvals')
for job in jobs:
job_dict = job.get_object_dict()
job_dict['dependencies'] = ','.join(label.name
for label in job.dependencies)
+ job_dict['keyvals'] = dict((keyval.key, keyval.value)
+ for keyval in job.keyvals)
job_dicts.append(job_dict)
return rpc_utils.prepare_for_serialization(job_dicts)
diff --git a/frontend/afe/rpc_interface_unittest.py b/frontend/afe/rpc_interface_unittest.py
index 822ec1f..c84dacf 100755
--- a/frontend/afe/rpc_interface_unittest.py
+++ b/frontend/afe/rpc_interface_unittest.py
@@ -93,6 +93,18 @@
self._check_hostnames(hosts, ['host2'])
+ def test_job_keyvals(self):
+ keyval_dict = {'mykey': 'myvalue'}
+ job_id = rpc_interface.create_job(name='test', priority='Medium',
+ control_file='foo',
+ control_type='Client',
+ hosts=['host1'],
+ keyvals=keyval_dict)
+ jobs = rpc_interface.get_jobs(id=job_id)
+ self.assertEquals(len(jobs), 1)
+ self.assertEquals(jobs[0]['keyvals'], keyval_dict)
+
+
def test_get_jobs_summary(self):
job = self._create_job(hosts=xrange(1, 4))
entries = list(job.hostqueueentry_set.all())
diff --git a/frontend/migrations/047_job_keyvals.py b/frontend/migrations/047_job_keyvals.py
new file mode 100644
index 0000000..d587fd8
--- /dev/null
+++ b/frontend/migrations/047_job_keyvals.py
@@ -0,0 +1,28 @@
+UP_SQL = """
+CREATE TABLE `afe_job_keyvals` (
+ `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+ `job_id` integer NOT NULL,
+ INDEX `afe_job_keyvals_job_id` (`job_id`),
+ FOREIGN KEY (`job_id`) REFERENCES `afe_jobs` (`id`) ON DELETE NO ACTION,
+ `key` varchar(90) NOT NULL,
+ INDEX `afe_job_keyvals_key` (`key`),
+ `value` varchar(300) NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE `tko_job_keyvals` (
+ `id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY,
+ `job_id` int(10) unsigned NOT NULL,
+ INDEX `tko_job_keyvals_job_id` (`job_id`),
+ FOREIGN KEY (`job_id`) REFERENCES `tko_jobs` (`job_idx`)
+ ON DELETE NO ACTION,
+ `key` varchar(90) NOT NULL,
+ INDEX `tko_job_keyvals_key` (`key`),
+ `value` varchar(300) NOT NULL
+) ENGINE=InnoDB;
+"""
+
+
+DOWN_SQL = """
+DROP TABLE afe_job_keyvals;
+DROP TABLE tko_job_keyvals;
+"""
diff --git a/frontend/tko/models.py b/frontend/tko/models.py
index 61b24af..429473d 100644
--- a/frontend/tko/models.py
+++ b/frontend/tko/models.py
@@ -155,6 +155,16 @@
db_table = 'tko_jobs'
+class JobKeyval(dbmodels.Model):
+ job = dbmodels.ForeignKey(Job)
+ key = dbmodels.CharField(max_length=90)
+ value = dbmodels.CharField(blank=True, max_length=300)
+
+
+ class Meta:
+ db_table = 'tko_job_keyvals'
+
+
class Test(dbmodels.Model, model_logic.ModelExtensions,
model_logic.ModelWithAttributes):
test_idx = dbmodels.AutoField(primary_key=True)