Strip the optional fields (e.g. timestamps) out of the reason fields
when parsing test results. Also, add a test_finished_time field for
storing the timestamp in the database.
Signed-off-by: John Admanski <jadmanski@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@1248 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/tko/db.py b/tko/db.py
index 409d086..15173da 100644
--- a/tko/db.py
+++ b/tko/db.py
@@ -233,7 +233,8 @@
data = {'job_idx':job.index, 'test':test.testname,
'subdir':test.subdir, 'kernel_idx':kver,
'status':self.status_idx[test.status],
- 'reason':test.reason, 'machine_idx':job.machine_idx }
+ 'reason':test.reason, 'machine_idx':job.machine_idx,
+ 'finished_time':test.finished_time}
self.insert('tests', data, commit=commit)
test_idx = self.get_last_autonumber_value()
data = { 'test_idx':test_idx }
diff --git a/tko/migrations/003_add_test_timestamps.py b/tko/migrations/003_add_test_timestamps.py
new file mode 100644
index 0000000..9055f50
--- /dev/null
+++ b/tko/migrations/003_add_test_timestamps.py
@@ -0,0 +1,72 @@
+def migrate_up(manager):
+ manager.execute_script(ADD_COLUMN_SQL)
+ manager.execute_script(ALTER_VIEWS_SQL)
+
+ADD_COLUMN_SQL = """\
+ALTER TABLE tests ADD COLUMN finished_time datetime NULL;
+"""
+
+ALTER_VIEWS_SQL = """\
+ALTER VIEW test_view AS
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word
+FROM tests
+INNER JOIN jobs ON jobs.job_idx = tests.job_idx
+INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
+INNER JOIN kernels ON kernels.kernel_idx = tests.kernel_idx
+INNER JOIN status ON status.status_idx = tests.status;
+
+-- perf_view (to make life easier for people trying to mine performance data)
+ALTER VIEW perf_view AS
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status,
+ tests.reason,
+ tests.machine_idx,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_label,
+ jobs.username AS job_username,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS machine_hostname,
+ machines.machine_group,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel_printable,
+ status.word AS status_word,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
+FROM tests
+INNER JOIN jobs ON jobs.job_idx = tests.job_idx
+INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
+INNER JOIN kernels ON kernels.kernel_idx = tests.kernel_idx
+INNER JOIN status ON status.status_idx = tests.status
+INNER JOIN iteration_result ON iteration_result.test_idx = tests.kernel_idx;
+"""
diff --git a/tko/parse.py b/tko/parse.py
index 08fcb68..254701d 100755
--- a/tko/parse.py
+++ b/tko/parse.py
@@ -1,5 +1,6 @@
#!/usr/bin/python
import os, re, md5, sys, email.Message, smtplib, datetime
+
client_bin = os.path.join(os.path.dirname(__file__), '../client/bin')
sys.path.insert(0, os.path.abspath(client_bin))
from autotest_utils import read_keyval
@@ -63,8 +64,8 @@
sys.stderr.write(str(info) + '\n')
-def keyval_timestamp(keyval, field):
- val = keyval.get(field, None)
+def get_timestamp(mapping, field):
+ val = mapping.get(field, None)
if val is not None:
val = datetime.datetime.fromtimestamp(int(val))
return val
@@ -92,9 +93,9 @@
self.machine = keyval.get('hostname', None)
if self.machine:
assert ',' not in self.machine
- self.queued_time = keyval_timestamp(keyval, 'job_queued')
- self.started_time = keyval_timestamp(keyval, 'job_started')
- self.finished_time = keyval_timestamp(keyval, 'job_finished')
+ self.queued_time = get_timestamp(keyval, 'job_queued')
+ self.started_time = get_timestamp(keyval, 'job_started')
+ self.finished_time = get_timestamp(keyval, 'job_finished')
self.machine_owner = keyval.get('owner', None)
if not self.machine:
@@ -155,21 +156,27 @@
dprint('Found job level start marker. Looking for level 1 groups now')
continue
indent = re.search('^(\t*)', line).group(0).count('\t')
- line = line.strip()
+ line = line.lstrip()
+ line = line.rstrip('\n')
if line.startswith('START\t'):
group_subdir = None
dprint('start line, ignoring')
continue # ignore start lines
reason = None
- if line.startswith('END'):
- elements = line.split(None, 4)[1:]
+ if line.startswith('END '):
+ elements = line.split('\t')
+ elements[0] = elements[0][4:] # remove 'END '
end = True
else:
- elements = line.split(None, 3)
+ elements = line.split('\t')
end = False
- elements.append(None) # in case no reason specified
(status, subdir, testname, reason) = elements[0:4]
- dprint('GROPE_STATUS: ' + str(elements[0:4]))
+ status, subdir, testname = elements[:3]
+ reason = elements[-1]
+ optional_fields = dict(element.split('=', 1)
+ for element in elements[3:-1])
+ dprint('GROPE_STATUS: ' +
+ str([status, subdir, testname, reason]))
if status == 'ALERT':
dprint('job level alert, recording')
alert_pending = reason
@@ -231,7 +238,12 @@
else:
dprint('WARNING: Invalid status code. Ignoring')
continue
- self.tests.append(test(subdir, testname, status, reason, self.kernel, self))
+
+ finished_time = get_timestamp(optional_fields,
+ 'timestamp')
+ self.tests.append(test(subdir, testname, status,
+ reason, self.kernel, self,
+ finished_time))
dprint('')
if reboot_inprogress:
@@ -300,7 +312,8 @@
class test:
- def __init__(self, subdir, testname, status, reason, kernel, job):
+ def __init__(self, subdir, testname, status, reason, kernel, job,
+ finished_time=None):
# NOTE: subdir may be none here for lines that aren't an
# actual test
self.subdir = subdir
@@ -322,6 +335,7 @@
self.iterations = []
self.kernel = kernel
self.machine = job.machine
+ self.finished_time = finished_time
dprint("PARSING TEST %s %s %s" % (subdir, testname, self.keyval))