Fix a bug in the parser when dealing with test labels. The final
reparse drops the existing test entries and replaces them with new
ones, so that leaves behind a bunch of orphaned labels in the
database and basically kills all the lables you've added. So instead
we add some code to move the existing labels over the new entries that
replace them (making a best effort to match up "new" and "old"
entries).
Add foreign keys to the test_labels_tests table. This also
requires compacting test_labels_tests into an unsigned int(10), since
that's what the tests.test_idx field is.
Risk: Low
Visibility: Fix up the parser's handling of test labels, and add
foreign keys that should've been there from the start (but were
dropped because of MyISAM vs InnoDB issues).
Signed-off-by: John Admanski <jadmanski@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@3022 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/tko/db.py b/tko/db.py
index 8318ce3..ff939b4 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -305,6 +305,7 @@
self.delete('iteration_result', where)
self.delete('iteration_attributes', where)
self.delete('test_attributes', where)
+ self.delete('test_labels_tests', {'test_id': test_idx})
where = {'job_idx' : job_idx}
self.delete('tests', where)
self.delete('jobs', where)
@@ -336,7 +337,8 @@
'reason':test.reason, 'machine_idx':job.machine_idx,
'started_time': test.started_time,
'finished_time':test.finished_time}
- if hasattr(test, "test_idx"):
+ is_update = hasattr(test, "test_idx")
+ if is_update:
test_idx = test.test_idx
self.update('tests', data, {'test_idx': test_idx}, commit=commit)
where = {'test_idx': test_idx}
@@ -366,6 +368,11 @@
'value': value}
self.insert('test_attributes', data, commit=commit)
+ if not is_update:
+ for label_index in test.labels:
+ data = {'test_id': test_idx, 'testlabel_id': label_index}
+ self.insert('test_labels_tests', data, commit=commit)
+
def read_machine_map(self):
self.machine_group = {}
diff --git a/tko/migrations/025_add_test_label_foreign_keys.py b/tko/migrations/025_add_test_label_foreign_keys.py
new file mode 100644
index 0000000..5607222
--- /dev/null
+++ b/tko/migrations/025_add_test_label_foreign_keys.py
@@ -0,0 +1,24 @@
+ADD_FOREIGN_KEYS = """
+ALTER TABLE test_labels_tests MODIFY COLUMN test_id int(10) unsigned NOT NULL;
+
+DELETE FROM test_labels_tests
+ WHERE test_id NOT IN (SELECT test_idx FROM tests);
+
+ALTER TABLE test_labels_tests ADD CONSTRAINT tests_labels_tests_ibfk_1
+ FOREIGN KEY (testlabel_id) REFERENCES test_labels (id);
+
+ALTER TABLE test_labels_tests ADD CONSTRAINT tests_labels_tests_ibfk_2
+ FOREIGN KEY (test_id) REFERENCES tests (test_idx);
+"""
+
+DROP_FOREIGN_KEYS = """
+ALTER TABLE test_labels_tests DROP FOREIGN KEY tests_labels_tests_ibfk_1;
+ALTER TABLE test_labels_tests DROP FOREIGN KEY tests_labels_tests_ibfk_2;
+ALTER TABLE test_labels_tests MODIFY COLUMN test_id int(11) NOT NULL;
+"""
+
+def migrate_up(mgr):
+ mgr.execute_script(ADD_FOREIGN_KEYS)
+
+def migrate_down(mgr):
+ mgr.execute_script(DROP_FOREIGN_KEYS)
diff --git a/tko/models.py b/tko/models.py
index e30f301..8516612 100644
--- a/tko/models.py
+++ b/tko/models.py
@@ -66,7 +66,7 @@
class test(object):
def __init__(self, subdir, testname, status, reason, test_kernel,
machine, started_time, finished_time, iterations,
- attributes):
+ attributes, labels):
self.subdir = subdir
self.testname = testname
self.status = status
@@ -77,6 +77,7 @@
self.finished_time = finished_time
self.iterations = iterations
self.attributes = attributes
+ self.labels = labels
@staticmethod
@@ -119,7 +120,7 @@
constructor = cls
return constructor(subdir, testname, status, reason, test_kernel,
job.machine, started_time, finished_time,
- iterations, attributes)
+ iterations, attributes, [])
@classmethod
@@ -132,7 +133,7 @@
tko_utils.dprint("parsing partial test %s %s" % (subdir, testname))
return cls(subdir, testname, "RUNNING", reason, test_kernel,
- job.machine, started_time, None, [], {})
+ job.machine, started_time, None, [], {}, [])
@staticmethod
diff --git a/tko/parse.py b/tko/parse.py
index 9ae2954..d43ddc5 100755
--- a/tko/parse.py
+++ b/tko/parse.py
@@ -68,14 +68,41 @@
mail.send("", job.user, "", subject, message_header + message)
+def find_old_tests(db, job_idx):
+ """
+ Given a job index, return a list of all the test objects associated with
+ it in the database, including labels, but excluding other "complex"
+ data (attributes, iteration data, kernels).
+ """
+ raw_tests = db.select("test_idx,subdir,test,started_time,finished_time",
+ "tests", {"job_idx": job_idx})
+
+ test_ids = ", ".join(str(raw_test[0]) for raw_test in raw_tests)
+
+ labels = db.select("test_id, testlabel_id", "test_labels_tests",
+ "test_id in (%s)" % test_ids)
+ label_map = {}
+ for test_id, testlabel_id in labels:
+ label_map.setdefault(test_id, []).append(testlabel_id)
+
+ tests = []
+ for raw_test in raw_tests:
+ tests.append(models.test(raw_test[1], raw_test[2], None, None, None,
+ None, raw_test[3], raw_test[4],
+ [], {}, label_map.get(raw_test[0], [])))
+ return tests
+
+
def parse_one(db, jobname, path, reparse, mail_on_failure):
"""
Parse a single job. Optionally send email on failure.
"""
tko_utils.dprint("\nScanning %s (%s)" % (jobname, path))
- if reparse and db.find_job(jobname):
+ old_job_idx = db.find_job(jobname)
+ if reparse and old_job_idx:
tko_utils.dprint("! Deleting old copy of job results to "
"reparse it")
+ old_tests = find_old_tests(db, old_job_idx)
db.delete_job(jobname)
if db.find_job(jobname):
tko_utils.dprint("! Job is already parsed, done")
@@ -100,7 +127,28 @@
status_lines = open(status_log).readlines()
parser.start(job)
tests = parser.end(status_lines)
- job.tests = tests
+
+ # parser.end can return the same object multiple times, so filter out dups
+ job.tests = []
+ already_added = set()
+ for test in tests:
+ if test not in already_added:
+ already_added.add(test)
+ job.tests.append(test)
+
+ # try and port labels over from the old tests, but if old tests stop
+ # matching up with new ones just give up
+ for test, old_test in zip(job.tests, old_tests):
+ tests_are_the_same = (test.testname == old_test.testname and
+ test.subdir == old_test.subdir and
+ test.started_time == old_test.started_time and
+ (test.finished_time == old_test.finished_time or
+ old_test.finished_time is None))
+ if tests_are_the_same:
+ test.labels = old_test.labels
+ else:
+ tko_utils.dprint("! Reparse returned new tests, "
+ "dropping old test labels")
# check for failures
message_lines = [""]