Make the parser read and use the machine platform from host_keyvals, if available. Goodbye machine_transfer! Also made it handle multimachine jobs (use a comma-separated, sorted list of distinct platforms from the individual machines) and atomic group jobs (use the same atomic group override value that was used for machine name, when applicable).
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@3583 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/tko/db.py b/tko/db.py
index ff939b4..ec83355 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -314,8 +314,10 @@
def insert_job(self, tag, job, commit = None):
job.machine_idx = self.lookup_machine(job.machine)
if not job.machine_idx:
- job.machine_idx = self.insert_machine(job,
- commit=commit)
+ job.machine_idx = self.insert_machine(job, commit=commit)
+ else:
+ self.update_machine_information(job, commit=commit)
+
self.insert('jobs', {'tag':tag,
'label': job.label,
'username': job.user,
@@ -375,30 +377,40 @@
def read_machine_map(self):
- self.machine_group = {}
+ if self.machine_group or not self.machine_map:
+ return
for line in open(self.machine_map, 'r').readlines():
(machine, group) = line.split()
self.machine_group[machine] = group
- def insert_machine(self, job, group = None, commit = None):
+ def machine_info_dict(self, job):
hostname = job.machine
- if self.machine_map and not self.machine_group:
- self.read_machine_map()
+ group = job.machine_group
+ owner = job.machine_owner
if not group:
+ self.read_machine_map()
group = self.machine_group.get(hostname, hostname)
- if group == hostname and job.machine_owner:
- group = job.machine_owner + '/' + hostname
+ if group == hostname and owner:
+ group = owner + '/' + hostname
- self.insert('machines',
- { 'hostname' : hostname ,
- 'machine_group' : group ,
- 'owner' : job.machine_owner },
- commit=commit)
+ return {'hostname': hostname, 'machine_group': group, 'owner': owner}
+
+
+ def insert_machine(self, job, commit = None):
+ machine_info = self.machine_info_dict(job)
+ self.insert('machines', machine_info, commit=commit)
return self.get_last_autonumber_value()
+ def update_machine_information(self, job, commit = None):
+ machine_info = self.machine_info_dict(job)
+ self.update('machines', machine_info,
+ where={'hostname': machine_info['hostname']},
+ commit=commit)
+
+
def lookup_machine(self, hostname):
where = { 'hostname' : hostname }
rows = self.select('machine_idx', 'machines', where)
diff --git a/tko/models.py b/tko/models.py
index 8516612..0842e18 100644
--- a/tko/models.py
+++ b/tko/models.py
@@ -6,7 +6,8 @@
class job(object):
def __init__(self, dir, user, label, machine, queued_time, started_time,
- finished_time, machine_owner, aborted_by, aborted_on):
+ finished_time, machine_owner, machine_group, aborted_by,
+ aborted_on):
self.dir = dir
self.tests = []
self.user = user
@@ -16,6 +17,7 @@
self.started_time = started_time
self.finished_time = finished_time
self.machine_owner = machine_owner
+ self.machine_group = machine_group
self.aborted_by = aborted_by
self.aborted_on = aborted_on
@@ -110,7 +112,9 @@
attributes = {}
# grab test+host attributes from the host keyval
- attributes.update(cls.parse_host_keyval(job.dir, job.machine))
+ host_keyval = cls.parse_host_keyval(job.dir, job.machine)
+ attributes.update(dict(("host-%s" % k, v)
+ for k, v in host_keyval.iteritems()))
if existing_instance:
def constructor(*args, **dargs):
@@ -155,8 +159,7 @@
# the keyval is <job_dir>/host_keyvals/<hostname> if it exists
keyval_path = os.path.join(job_dir, "host_keyvals", hostname)
if os.path.isfile(keyval_path):
- keyval = utils.read_keyval(keyval_path)
- return dict(("host-%s" % k, v) for k, v in keyval.iteritems())
+ return utils.read_keyval(keyval_path)
else:
return {}
diff --git a/tko/parsers/version_0.py b/tko/parsers/version_0.py
index 881b8d9..f1be7b4 100644
--- a/tko/parsers/version_0.py
+++ b/tko/parsers/version_0.py
@@ -22,31 +22,59 @@
user = keyval.get("user", None)
label = keyval.get("label", None)
- host_group_name = keyval.get("host_group_name", None)
- machine = keyval.get("hostname", None)
- if not host_group_name and machine and "," in machine:
- try:
- machine = job.find_hostname(dir) # find a unique hostname
- except NoHostnameError:
- pass # just use the comma-separated name
queued_time = tko_utils.get_timestamp(keyval, "job_queued")
started_time = tko_utils.get_timestamp(keyval, "job_started")
finished_time = tko_utils.get_timestamp(keyval, "job_finished")
+ machine = cls.determine_hostname(keyval, dir)
+ machine_group = cls.determine_machine_group(machine, dir)
machine_owner = keyval.get("owner", None)
aborted_by = keyval.get("aborted_by", None)
aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
- tko_utils.dprint("MACHINE NAME: %s" % machine)
- if host_group_name and ((machine and "," in machine) or not machine):
- tko_utils.dprint("Using host_group_name %r instead of "
- "machine name." % host_group_name)
- machine = host_group_name
-
return {"user": user, "label": label, "machine": machine,
"queued_time": queued_time, "started_time": started_time,
"finished_time": finished_time, "machine_owner": machine_owner,
- "aborted_by": aborted_by, "aborted_on": aborted_at}
+ "machine_group": machine_group, "aborted_by": aborted_by,
+ "aborted_on": aborted_at}
+
+
+ @classmethod
+ def determine_hostname(cls, keyval, job_dir):
+ host_group_name = keyval.get("host_group_name", None)
+ machine = keyval.get("hostname", "")
+ is_multimachine = "," in machine
+
+ # determine what hostname to use
+ if host_group_name:
+ if is_multimachine or not machine:
+ tko_utils.dprint("Using host_group_name %r instead of "
+ "machine name." % host_group_name)
+ machine = host_group_name
+ elif is_multimachine:
+ try:
+ machine = job.find_hostname(job_dir) # find a unique hostname
+ except NoHostnameError:
+ pass # just use the comma-separated name
+
+ tko_utils.dprint("MACHINE NAME: %s" % machine)
+ return machine
+
+
+ @classmethod
+ def determine_machine_group(cls, hostname, job_dir):
+ machine_groups = set()
+ for individual_hostname in hostname.split(","):
+ host_keyval = models.test.parse_host_keyval(job_dir,
+ individual_hostname)
+ if not host_keyval:
+ tko_utils.dprint('Unable to parse host keyval for %s'
+ % individual_hostname)
+ elif "platform" in host_keyval:
+ machine_groups.add(host_keyval["platform"])
+ machine_group = ",".join(sorted(machine_groups))
+ tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
+ return machine_group
@staticmethod
diff --git a/tko/parsers/version_0_unittest.py b/tko/parsers/version_0_unittest.py
index 8c076ff..4807bee 100644
--- a/tko/parsers/version_0_unittest.py
+++ b/tko/parsers/version_0_unittest.py
@@ -9,16 +9,6 @@
class test_job_load_from_dir(unittest.TestCase):
- def setUp(self):
- self.god = mock.mock_god()
- self.god.stub_function(models.job, 'read_keyval')
- self.god.stub_function(version_0.job, 'find_hostname')
-
-
- def tearDown(self):
- self.god.unstub_all()
-
-
keyval_return = {'job_queued': 1234567890,
'job_started': 1234567891,
'job_finished': 1234567892,
@@ -27,37 +17,87 @@
'hostname': 'abc123'}
+ def setUp(self):
+ self.god = mock.mock_god()
+ self.god.stub_function(models.job, 'read_keyval')
+ self.god.stub_function(version_0.job, 'find_hostname')
+ self.god.stub_function(models.test, 'parse_host_keyval')
+
+
+ def tearDown(self):
+ self.god.unstub_all()
+
+
+ def _expect_host_keyval(self, hostname, platform=None):
+ return_dict = {}
+ if platform:
+ return_dict['platform'] = platform
+ return_dict['labels'] = platform + ',other_label'
+ (models.test.parse_host_keyval.expect_call('.', hostname)
+ .and_return(return_dict))
+
+
def test_load_from_dir_simple(self):
models.job.read_keyval.expect_call('.').and_return(
dict(self.keyval_return))
+ self._expect_host_keyval('abc123', 'my_platform')
job = version_0.job.load_from_dir('.')
self.assertEqual('janet', job['user'])
self.assertEqual('steeltown', job['label'])
self.assertEqual('abc123', job['machine'])
+ self.assertEqual('my_platform', job['machine_group'])
self.god.check_playback()
- def test_load_from_dir_two_machine(self):
+ def _setup_two_machines(self):
raw_keyval = dict(self.keyval_return)
raw_keyval['hostname'] = 'easyas,abc123'
models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+
+
+ def test_load_from_dir_two_machines(self):
+ self._setup_two_machines()
version_0.job.find_hostname.expect_call('.').and_raises(
- version_0.NoHostnameError('find_hostname stubbed out'))
+ version_0.NoHostnameError('find_hostname stubbed out'))
+ self._expect_host_keyval('easyas', 'platform')
+ self._expect_host_keyval('abc123', 'platform')
+
job = version_0.job.load_from_dir('.')
self.assertEqual('easyas,abc123', job['machine'])
+ self.assertEqual('platform', job['machine_group'])
- models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+ self.god.check_playback()
+
+
+ def test_load_from_dir_two_machines_with_find_hostname(self):
+ self._setup_two_machines()
version_0.job.find_hostname.expect_call('.').and_return('foo')
+ self._expect_host_keyval('foo')
+
job = version_0.job.load_from_dir('.')
self.assertEqual('foo', job['machine'])
self.god.check_playback()
+ def test_load_from_dir_two_machines_different_platforms(self):
+ self._setup_two_machines()
+ version_0.job.find_hostname.expect_call('.').and_raises(
+ version_0.NoHostnameError('find_hostname stubbed out'))
+ self._expect_host_keyval('easyas', 'platformZ')
+ self._expect_host_keyval('abc123', 'platformA')
+
+ job = version_0.job.load_from_dir('.')
+ self.assertEqual('easyas,abc123', job['machine'])
+ self.assertEqual('platformA,platformZ', job['machine_group'])
+
+ self.god.check_playback()
+
def test_load_from_dir_one_machine_group_name(self):
raw_keyval = dict(self.keyval_return)
raw_keyval['host_group_name'] = 'jackson five'
models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+ self._expect_host_keyval('abc123')
job = version_0.job.load_from_dir('.')
self.assertEqual('janet', job['user'])
self.assertEqual('abc123', job['machine'])
@@ -70,6 +110,7 @@
raw_keyval['hostname'] = 'abc123,dancingmachine'
raw_keyval['host_group_name'] = 'jackson five'
models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+ self._expect_host_keyval('jackson five')
job = version_0.job.load_from_dir('.')
self.assertEqual('michael', job['user'])
# The host_group_name is used instead because machine appeared to be
@@ -83,6 +124,7 @@
del raw_keyval['hostname']
raw_keyval['host_group_name'] = 'jackson five'
models.job.read_keyval.expect_call('.').and_return(raw_keyval)
+ self._expect_host_keyval('jackson five')
job = version_0.job.load_from_dir('.')
# The host_group_name is used because there is no machine.
self.assertEqual('jackson five', job['machine'])