two new major features:
(1) added test and job dependencies
-added M2M relationship between tests and labels and between jobs and labels, for tracking the labels on which a test/job depends
-modified test_importer to read the DEPENDENCIES field and create the right M2M relationships
-modified generate_control_file() RPC to compute and return the union of test dependencies. since generate_control_file now returns four pieces of information, i converted its return type from tuple to dict, and changed clients accordingly.
-modified job creation clients (GWT and CLI) to pass this dependency list to the create_job() RPC
-modified the create_job() RPC to check that hosts satisfy job dependencies, and to create M2M relationships
-modified the scheduler to check dependencies when scheduling jobs
-modified JobDetailView to show a job's dependencies
(2) added "only_if_needed" bit to labels; if true, a machine with this label can only be used if the label is requested (either by job dependencies or by the metahost label)
-added boolean field to Labels
-modified CLI label creation/viewing to support this new field
-made create_job() RPC and scheduler check for hosts with such a label that was not requested, and reject such hosts
also did some slight refactoring of other code in create_job() to simplify it while I was changing things there.
a couple notes:
-an only_if_needed label can be used if either the job depends on the label or it's a metahost for that label. we assume that if the user specifically requests the label in a metahost, then it's OK, even if the job doesn't depend on that label.
-one-time-hosts are assumed to satisfy job dependencies.
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2215 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/cli/job.py b/cli/job.py
index 1208789..7a7c30f 100755
--- a/cli/job.py
+++ b/cli/job.py
@@ -321,6 +321,10 @@
self.ctrl_file_data['do_push_packages'] = True
self.ctrl_file_data['use_container'] = options.container
+ # TODO: add support for manually specifying dependencies, when this is
+ # added to the frontend as well
+ self.data['dependencies'] = []
+
return (options, leftover)
@@ -330,20 +334,20 @@
socket.setdefaulttimeout(topic_common.UPLOAD_SOCKET_TIMEOUT)
print 'Uploading Kernel: this may take a while...',
- (ctrl_file, on_server,
- is_synch) = self.execute_rpc(op='generate_control_file',
- item=self.jobname,
- **self.ctrl_file_data)
+ cf_info = self.execute_rpc(op='generate_control_file',
+ item=self.jobname,
+ **self.ctrl_file_data)
if self.ctrl_file_data.has_key('kernel'):
print 'Done'
socket.setdefaulttimeout(topic_common.DEFAULT_SOCKET_TIMEOUT)
- self.data['control_file'] = ctrl_file
- self.data['is_synchronous'] = is_synch
- if on_server:
+ self.data['control_file'] = cf_info['control_file']
+ self.data['is_synchronous'] = cf_info['is_synchronous']
+ if cf_info['is_server']:
self.data['control_type'] = 'Server'
else:
self.data['control_type'] = 'Client'
+ self.data['dependencies'] = cf_info['dependencies']
return super(job_create, self).execute()
diff --git a/cli/job_unittest.py b/cli/job_unittest.py
index 3b2677b..4b3a62b 100755
--- a/cli/job_unittest.py
+++ b/cli/job_unittest.py
@@ -564,15 +564,19 @@
data = {'priority': 'Medium', 'control_file': ctrl_file, 'hosts': ['host0'],
'name': 'test_job0', 'control_type': 'Client',
- 'meta_hosts': [], 'is_synchronous': False}
+ 'meta_hosts': [], 'is_synchronous': False, 'dependencies': []}
def test_execute_create_job(self):
self.run_cmd(argv=['atest', 'job', 'create', '-t', 'sleeptest',
'test_job0', '-m', 'host0'],
- rpcs=[('generate_control_file', {'tests': ['sleeptest'],
- 'use_container': False},
- True, (self.ctrl_file, False, False)),
+ rpcs=[('generate_control_file',
+ {'tests': ['sleeptest'], 'use_container': False},
+ True,
+ {'control_file' : self.ctrl_file,
+ 'is_synchronous' : False,
+ 'is_server' : False,
+ 'dependencies' : []}),
('create_job', self.data, True, 180)],
out_words_ok=['test_job0', 'Created'],
out_words_no=['Uploading', 'Done'])
@@ -592,10 +596,14 @@
data['control_file'] = self.kernel_ctrl_file
self.run_cmd(argv=['atest', 'job', 'create', '-t', 'sleeptest',
'-k', 'kernel', 'test_job0', '-m', 'host0'],
- rpcs=[('generate_control_file', {'tests': ['sleeptest'],
- 'use_container': False, 'kernel': 'kernel',
- 'do_push_packages': True},
- True, (self.kernel_ctrl_file, False, False)),
+ rpcs=[('generate_control_file',
+ {'tests': ['sleeptest'], 'use_container': False,
+ 'kernel': 'kernel', 'do_push_packages': True},
+ True,
+ {'control_file' : self.kernel_ctrl_file,
+ 'is_synchronous' : False,
+ 'is_server' : False,
+ 'dependencies' : []}),
('create_job', data, True, 180)],
out_words_ok=['test_job0', 'Created',
'Uploading', 'Done'])
@@ -608,10 +616,14 @@
self.run_cmd(argv=['atest', 'job', 'create', '-t', 'sleeptest',
'-k', 'kernel', 'test job with spaces',
'-m', 'host0'],
- rpcs=[('generate_control_file', {'tests': ['sleeptest'],
- 'use_container': False, 'kernel': 'kernel',
- 'do_push_packages': True},
- True, (self.kernel_ctrl_file, False, False)),
+ rpcs=[('generate_control_file',
+ {'tests': ['sleeptest'], 'use_container': False,
+ 'kernel': 'kernel', 'do_push_packages': True},
+ True,
+ {'control_file' : self.kernel_ctrl_file,
+ 'is_synchronous' : False,
+ 'is_server' : False,
+ 'dependencies' : []}),
('create_job', data, True, 180)],
# This is actually 8 spaces,
# the tab has been converted by print.
@@ -713,9 +725,13 @@
data = self.data.copy()
self.run_cmd(argv=['atest', 'job', 'create', '-t', 'sleeptest',
'-c', 'test_job0', '-m', 'host0'],
- rpcs=[('generate_control_file', {'tests': ['sleeptest'],
- 'use_container': True}, True, (self.ctrl_file,
- False, False)),
+ rpcs=[('generate_control_file',
+ {'tests': ['sleeptest'], 'use_container': True},
+ True,
+ {'control_file' : self.ctrl_file,
+ 'is_synchronous' : False,
+ 'is_server' : False,
+ 'dependencies' : []}),
('create_job', data, True, 42)],
out_words_ok=['test_job0', 'Created'])
diff --git a/cli/label.py b/cli/label.py
index f8932e7..a4d6ac2 100755
--- a/cli/label.py
+++ b/cli/label.py
@@ -132,9 +132,9 @@
elif not self.all:
results = [label for label in results
if not label['platform']]
- keys = ['name', 'invalid']
+ keys = ['name', 'only_if_needed', 'invalid']
else:
- keys = ['name', 'platform', 'invalid']
+ keys = ['name', 'platform', 'only_if_needed', 'invalid']
super(label_list, self).output(results, keys)
@@ -147,12 +147,17 @@
help='To create this label as a platform',
default=False,
action='store_true')
+ self.parser.add_option('-o', '--only_if_needed',
+ help='To mark the label as "only use if needed',
+ default=False,
+ action='store_true')
def parse(self):
(options, leftover) = super(label_create, self).parse()
self.data_item_key = 'name'
self.data['platform'] = options.platform
+ self.data['only_if_needed'] = options.only_if_needed
return (options, leftover)
diff --git a/cli/label_unittest.py b/cli/label_unittest.py
index d160435..fff879c 100755
--- a/cli/label_unittest.py
+++ b/cli/label_unittest.py
@@ -15,27 +15,32 @@
u'platform': 0,
u'name': u'label0',
u'invalid': 0,
- u'kernel_config': u''},
+ u'kernel_config': u'',
+ u'only_if_needed': 0},
{u'id': 338, # Valid label
u'platform': 0,
u'name': u'label1',
u'invalid': 0,
- u'kernel_config': u''},
+ u'kernel_config': u'',
+ u'only_if_needed': 0},
{u'id': 340, # Invalid label
u'platform': 0,
u'name': u'label2',
u'invalid': 1,
- u'kernel_config': u''},
+ u'kernel_config': u'',
+ u'only_if_needed': 0},
{u'id': 350, # Valid platform
u'platform': 1,
u'name': u'plat0',
u'invalid': 0,
- u'kernel_config': u''},
+ u'kernel_config': u'',
+ u'only_if_needed': 0},
{u'id': 420, # Invalid platform
u'platform': 1,
u'name': u'plat1',
u'invalid': 1,
- u'kernel_config': u''}]
+ u'kernel_config': u'',
+ u'only_if_needed': 0}]
def test_label_list_labels_only(self):
@@ -82,9 +87,13 @@
def test_execute_create_two_labels(self):
self.run_cmd(argv=['atest', 'label', 'create', 'label0', 'label1',
'--ignore_site_file'],
- rpcs=[('add_label', {'name': 'label0', 'platform': False},
+ rpcs=[('add_label',
+ {'name': 'label0', 'platform': False,
+ 'only_if_needed': False},
True, 42),
- ('add_label', {'name': 'label1', 'platform': False},
+ ('add_label',
+ {'name': 'label1', 'platform': False,
+ 'only_if_needed': False},
True, 43)],
out_words_ok=['Created', 'label0', 'label1'])
@@ -92,9 +101,13 @@
def test_execute_create_two_labels_bad(self):
self.run_cmd(argv=['atest', 'label', 'create', 'label0', 'label1',
'--ignore_site_file'],
- rpcs=[('add_label', {'name': 'label0', 'platform': False},
+ rpcs=[('add_label',
+ {'name': 'label0', 'platform': False,
+ 'only_if_needed': False},
True, 3),
- ('add_label', {'name': 'label1', 'platform': False},
+ ('add_label',
+ {'name': 'label1', 'platform': False,
+ 'only_if_needed': False},
False,
'''ValidationError: {'name':
'This value must be unique (label0)'}''')],
diff --git a/cli/topic_common.py b/cli/topic_common.py
index 03e670c..bbec3f6 100755
--- a/cli/topic_common.py
+++ b/cli/topic_common.py
@@ -89,6 +89,7 @@
'created_on': 'Created On',
'synch_type': 'Synch Type',
'control_file': 'Control File',
+ 'only_if_needed': 'Use only if needed',
}
# In the failure, tag that will replace the item.
@@ -116,8 +117,12 @@
return field
-KEYS_CONVERT = {'locked': lambda flag: str(bool(flag)),
+def _int_2_bool_string(value):
+ return str(bool(value))
+
+KEYS_CONVERT = {'locked': _int_2_bool_string,
'invalid': lambda flag: str(bool(not flag)),
+ 'only_if_needed': _int_2_bool_string,
'platform': __convert_platform,
'labels': lambda labels: ', '.join(labels)}