Clean up the reporting backend, and add CLI functionality
I've pulled out the generic bits into frontend.py. There are way
too many things called "row" and "column" so I've changed the
matrix iterators to "x" and "y" to be less confusing
Removed all the old, specific query reports, in favour of the new, powerful
general one
Signed-off-by: Martin Bligh <mbligh@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@1166 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/tko/compose_query.cgi b/tko/compose_query.cgi
index e286230..976dafe 100755
--- a/tko/compose_query.cgi
+++ b/tko/compose_query.cgi
@@ -53,19 +53,6 @@
"""
-# dictionary used simply for fast lookups
-field_dict = {
- 'kernel': 'kernel_printable',
- 'hostname': 'machine_hostname',
- 'test': 'test',
- 'label': 'job_label',
- 'machine_group': 'machine_group',
- 'reason': 'reason',
- 'tag': 'job_tag',
- 'user': 'job_username',
- 'status': 'status_word',
-}
-
next_field = {
'machine_group': 'hostname',
'hostname': 'tag',
@@ -86,7 +73,7 @@
if not form_field in form:
return field_default
field_input = form[form_field].value.lower()
- if field_input and field_input in field_dict:
+ if field_input and field_input in frontend.test_view_field_dict:
return field_input
return field_default
@@ -98,42 +85,26 @@
form = cgi.FieldStorage()
-row_field = parse_field(form, 'rows', 'kernel')
-column_field = parse_field(form, 'columns', 'machine_group')
+row = parse_field(form, 'rows', 'kernel')
+column = parse_field(form, 'columns', 'machine_group')
condition_field = parse_condition(form, 'condition', '')
cgitb.enable()
db = db.db()
-def get_value(test, field):
- if field == 'kernel':
- return test.kernel_printable
- if field == 'hostname':
- return test.machine_hostname
- if field == 'test':
- return test.testname
- if field == 'label':
- return test.job_label
- if field == 'machine_group':
- return test.machine_group
- if field == 'reason':
- return test.reason
- raise "Unknown field"
-
-
-def construct_link(row_val, column_val):
- next_row = row_field
- next_column = column_field
+def construct_link(x, y):
+ next_row = row
+ next_column = column
condition_list = []
if condition_field != '':
condition_list.append(condition_field)
- if row_val:
- next_row = next_field[row_field]
- condition_list.append("%s='%s'" % (row_field, row_val))
- if column_val:
- next_column = next_field[column_field]
- condition_list.append("%s='%s'" % (column_field, column_val))
+ if y:
+ next_row = next_field[row]
+ condition_list.append("%s='%s'" % (row, y))
+ if x:
+ next_column = next_field[column]
+ condition_list.append("%s='%s'" % (column, x))
next_condition = '&'.join(condition_list)
return 'compose_query.cgi?' + urllib.urlencode({'columns': next_column,
'rows': next_row, 'condition': next_condition})
@@ -142,69 +113,54 @@
def create_select_options(selected_val):
ret = ""
- for option in sorted(field_dict.keys()):
+ for option in sorted(frontend.test_view_field_dict.keys()):
if selected_val == option:
selected = " SELECTED"
else:
selected = ""
- ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % (option,
- selected,
- option)
+ ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % \
+ (option, selected, option)
return ret
-def smart_sort(list, field):
- if field == 'kernel':
- def kernel_encode(kernel):
- return kernel_versions.version_encode(kernel)
- list.sort(key = kernel_encode, reverse = True)
- else:
- list.sort()
-
-
def gen_matrix():
where = None
if condition_field.strip() != '':
where = query_lib.parse_scrub_and_gen_condition(
- condition_field, field_dict)
+ condition_field, frontend.test_view_field_dict)
print "<!-- where clause: %s -->" % (where,)
- ret = frontend.get_matrix_data(db, field_dict[column_field],
- field_dict[row_field], where)
- (data, column_list, row_list, stat_list, job_tags) = ret
+ test_data = frontend.get_matrix_data(db, column, row, where)
- if not row_list:
+ if not test_data.y_values:
msg = "There are no results for this query (yet?)."
return [[display.box(msg)]]
- smart_sort(row_list, row_field)
- smart_sort(column_list, column_field)
-
link = 'compose_query.cgi?columns=%s&rows=%s&condition=%s' % (
- row_field, column_field, condition_field)
+ row, column, condition_field)
header_row = [display.box("<center>(Flip Axis)</center>", link=link)]
- for column in column_list:
- link = construct_link(None, column)
- header_row.append(display.box(column, header=True, link=link))
+ for x in test_data.x_values:
+ link = construct_link(x, None)
+ header_row.append(display.box(x, header=True, link=link))
matrix = [header_row]
- for row in row_list:
- link = construct_link(row, None)
- cur_row = [display.box(row, header=True, link=link)]
- for column in column_list:
+ for y in test_data.y_values:
+ link = construct_link(None, y)
+ cur_row = [display.box(y, header=True, link=link)]
+ for x in test_data.x_values:
try:
- box_data = data[column][row]
+ box_data = test_data.data[x][y].status_count
except:
cur_row.append(display.box(None, None))
continue
- job_tag = job_tags[column][row]
+ job_tag = test_data.data[x][y].job_tag
if job_tag:
link = '/results/%s/' % job_tag
else:
- link = construct_link(row, column)
+ link = construct_link(x, y)
cur_row.append(display.status_precounted_box(db,
box_data,
link))
@@ -219,8 +175,8 @@
print 'Filtered Autotest Results'
print '</title></head><body>'
display.print_main_header()
- print html_header % (create_select_options(column_field),
- create_select_options(row_field),
+ print html_header % (create_select_options(column),
+ create_select_options(row),
condition_field)
display.print_table(gen_matrix())
print '</body></html>'
diff --git a/tko/frontend.py b/tko/frontend.py
index 5f431f3..85f67ec 100755
--- a/tko/frontend.py
+++ b/tko/frontend.py
@@ -1,125 +1,97 @@
#!/usr/bin/python
import os, re, db, sys
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
+client_bin = os.path.abspath(os.path.join(tko, '../client/bin'))
+sys.path.insert(0, client_bin)
+import kernel_versions
+
root_url_file = os.path.join(tko, '.root_url')
if os.path.exists(root_url_file):
html_root = open(root_url_file, 'r').readline().rstrip()
else:
html_root = '/results/'
-def select(db, field, value=None, distinct=False):
- """ returns the relevant index values where the field value matches the
- input value to the function.
- If there is no value passed, then it returns the index values and the
- field values corresponsing to them. """
- fields = { 'kernel': ['printable', 'kernel_idx', 'kernel_idx'],
- 'machine_group': ['machine_group', 'machine_idx', 'machine_idx'],
- 'hostname': ['hostname', 'machine_idx', 'machine_idx'],
- 'label': ['label', 'job_idx', 'job_idx'],
- 'tag': ['tag', 'job_idx', 'job_idx'],
- 'job': ['job_idx', 'job_idx', 'job_idx'],
- 'user': ['username', 'job_idx', 'job_idx'],
- 'test': ['test', 'test', 'test'],
- 'status': ['word', 'status_idx', 'status'],
- 'reason': ['reason', 'test_idx', 'test_idx'] }
- table = { 'kernel': 'kernels',
- 'machine_group': 'machines',
- 'hostname': 'machines',
- 'label': 'jobs',
- 'tag': 'jobs',
- 'job': 'jobs',
- 'user': 'jobs',
- 'test': 'tests',
- 'status': 'status',
- 'reason': 'tests' }
-
- lookup_field, idx_field, field_name_in_main_table = fields[field]
- tablename = table[field]
- # select all the index values that match the given field name.
- sql = ""
- if distinct:
- sql += " distinct "
- if not value:
- sql += " %s , %s " % (lookup_field, idx_field)
- where = " %s is not null " % lookup_field
- else:
- sql += "%s " % idx_field
- if field == 'tag':
- where = " %s LIKE %s " % (lookup_field, value)
- else:
- where = " %s = %s " % (lookup_field, value)
-
- match = db.select(sql, tablename, where)
- # returns the value and its field name
- return match, field_name_in_main_table
+class status_cell:
+ # One cell in the matrix of status data.
+ def __init__(self):
+ # Count is a dictionary: status -> count of tests with status
+ self.status_count = {}
+ self.job_tag = None
+ self.job_tag_count = 0
-def get_axis_data(axis):
- rows = db.select(axis , 'test_view', distinct = True)
- # Need to do a magic sort here if axis == 'kernel_printable'
- return sorted([row[0] for row in rows])
+ def add(self, status, count, job_tags):
+ assert not self.status_count.has_key(status)
+ assert count > 0
+
+ self.job_tag = job_tags
+ self.job_tag_count += count
+ if self.job_tag_count > 1:
+ self.job_tag = None
+
+ self.status_count[status] = count
+
+
+class status_data:
+ def __init__(self, sql_rows, x_field, y_field):
+ data = {}
+ y_values = set()
+
+ # Walk through the query, filing all results by x, y info
+ for (x, y, status, count, job_tags) in sql_rows:
+ if not data.has_key(x):
+ data[x] = {}
+ if not data[x].has_key(y):
+ y_values.add(y)
+ data[x][y] = status_cell()
+ data[x][y].add(status, count, job_tags)
+
+ # 2-d hash of data - [x-value][y-value]
+ self.data = data
+ # List of possible columns (x-values)
+ self.x_values = smart_sort(data.keys(), x_field)
+ # List of rows columns (y-values)
+ self.y_values = smart_sort(list(y_values), y_field)
def get_matrix_data(db, x_axis, y_axis, where = None):
- # Return a 3-d hash of data - [x-value][y-value][status_word]
# Searches on the test_view table - x_axis and y_axis must both be
# column names in that table.
- fields = ('%s, %s, status, COUNT(status_word), ' +
- 'LEFT(GROUP_CONCAT(job_tag), 100)' # limit what's returned
- ) % (x_axis, y_axis)
- group_by = '%s, %s, status' % (x_axis, y_axis)
+ x_field = test_view_field_dict[x_axis]
+ y_field = test_view_field_dict[y_axis]
+ fields = ('%s, %s, status, COUNT(status), ' +
+ 'LEFT(GROUP_CONCAT(job_tag), 100)' # limit what's returned
+ ) % (x_field, y_field)
+ group_by = '%s, %s, status' % (x_field, y_field)
rows = db.select(fields, 'test_view', where=where, group_by=group_by)
- data = {}
- job_tags = {}
- x_set = set()
- y_set = set()
- status_set = set()
- for (x, y, status, count, job_tag) in rows:
- if not data.has_key(x):
- data[x] = {}
- job_tags[x] = {}
- if not data[x].has_key(y):
- data[x][y] = {}
- data[x][y][status] = count
- if job_tags[x].has_key(y) or count != 1:
- job_tags[x][y] = None
- else:
- job_tags[x][y] = job_tag
- x_set.add(x)
- y_set.add(y)
- status_set.add(status)
- return (data, list(x_set), list(y_set), list(status_set), job_tags)
+ return status_data(rows, x_field, y_field)
-class anygroup:
- @classmethod
- def selectunique(klass, db, field):
- """Return unique values for all possible groups within
- the table."""
- rows, field_name_in_main_table = select(db, field, value=None, distinct=True)
- groupnames = sorted([row for row in rows])
+# Dictionary used simply for fast lookups from short reference names for users
+# to fieldnames in test_view
+test_view_field_dict = {
+ 'kernel' : 'kernel_printable',
+ 'hostname' : 'machine_hostname',
+ 'test' : 'test',
+ 'label' : 'job_label',
+ 'machine_group' : 'machine_group',
+ 'reason' : 'reason',
+ 'tag' : 'job_tag',
+ 'user' : 'job_username',
+ 'status' : 'status_word',
+}
- # collapse duplicates where records have the same name but
- # multiple index values
- headers = {}
- for field_name, idx_value in groupnames:
- if headers.has_key(field_name):
- headers[field_name].append(idx_value)
- else:
- headers[field_name] = [idx_value]
- headers = headers.items()
- headers.sort()
- return [klass(db, field_name_in_main_table, groupname) for groupname in headers]
-
-
- def __init__(self, db, idx_name, name):
- self.db = db
- self.name = name[0]
- self.idx_name = idx_name
- self.idx_value = name[1]
+def smart_sort(list, field):
+ if field == 'kernel_printable':
+ def kernel_encode(kernel):
+ return kernel_versions.version_encode(kernel)
+ list.sort(key = kernel_encode, reverse = True)
+ else:
+ list.sort()
+ return list
class group:
@@ -225,7 +197,6 @@
self.url = None
-
def iterations(self):
"""
Caching function for iterations
diff --git a/tko/group_test.cgi b/tko/group_test.cgi
index 82e311b..e69de29 100755
--- a/tko/group_test.cgi
+++ b/tko/group_test.cgi
@@ -1,61 +0,0 @@
-#!/usr/bin/python
-print "Content-type: text/html\n"
-import cgi, cgitb, os, sys, re
-sys.stdout.flush()
-cgitb.enable()
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-import db, display, frontend
-
-db = db.db()
-
-def main():
- display.print_main_header()
-
- form = cgi.FieldStorage()
- kernel_idx = form["kernel"].value
- kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0]
- groups = frontend.group.select(db)
-
- print_kernel_groups_vs_tests(kernel, groups)
-
-
-def print_kernel_groups_vs_tests(kernel, groups):
- # first we have to get a list of all run tests across all machines
- all_tests = set()
- present_groups = []
- for group in groups:
- tests = group.tests({ 'kernel_idx' : kernel.idx })
- if tests:
- present_groups.append(group)
- for test in tests:
- all_tests.add(test.testname)
- all_tests = list(all_tests)
- all_tests.sort()
-
- print '<h1>%s</h1>' % kernel.printable
-
- header_row = [ display.box('Test', header=True) ]
- for group in present_groups:
- group_name = display.group_name(group)
- header_row.append( display.box(group_name, header=True) )
-
- matrix = [header_row]
- for testname in all_tests:
- shortname = re.sub(r'kernel.', r'kernel<br>', testname)
- row = [display.box(shortname)]
- for group in present_groups:
- tests = group.tests({ 'kernel_idx' : kernel.idx ,
- 'test' : testname })
- link = 'machine_kernel_test.cgi?'
- link += 'group=%s&kernel=%s&test=%s' % \
- (group.name, kernel.idx, testname)
- box = display.status_count_box(db, tests, link=link)
- row.append(box)
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-main()
diff --git a/tko/index.html b/tko/index.html
index 8afb843..e69de29 100644
--- a/tko/index.html
+++ b/tko/index.html
@@ -1,71 +0,0 @@
-<html>
-<head>
-</head>
-<body>
-<table border="0">
- <form action="compose_query.cgi" method="get">
-<tr>
- <td>Column: </td>
- <td>
- <SELECT NAME="columns">
- <OPTION VALUE="kernel">kernel
- <OPTION VALUE="hostname">hostname
- <OPTION VALUE="test">test
- <OPTION VALUE="label">label
- <OPTION VALUE="machine_group">machine_group
- <OPTION VALUE="reason">reason
- <OPTION VALUE="tag">tag
- <OPTION VALUE="user">user
- <OPTION VALUE="status">status
- </SELECT>
- </td>
-</tr>
-<tr>
- <td>Row: </td>
- <td>
- <SELECT NAME="rows">
- <OPTION VALUE="kernel">kernel
- <OPTION VALUE="hostname">hostname
- <OPTION VALUE="test" SELECTED>test
- <OPTION VALUE="label">label
- <OPTION VALUE="machine_group">machine_group
- <OPTION VALUE="reason">reason
- <OPTION VALUE="tag">tag
- <OPTION VALUE="user">user
- <OPTION VALUE="status">status
- </SELECT>
- </td>
-</tr>
-<tr>
- <td>Condition: </td>
- <td>
- <input type="text" name="condition" size="80" maxlength="80"><br />
- <input type="hidden" name="title" value="Report">
- </td>
-</tr>
-<tr>
- <td colspan="100%" align="center"><input type="submit" value="Submit">
- </td>
-</tr>
-<tr>
- <td colspan="100%">
-<p>
-Conditions of the form <column>=<value> & <column>=<value> & ... & <column>=<value>
-</p>
-<p>
-Textual values must be quoted.
-</p>
-<p>
-<div>Examples:</div>
-<ul>
-<li>user='johnmacdonald' & test='burnin'</li>
-<li>hostname='bdpk1' & user='yinghan'</li>
-<li>tag~'134-jorlow%' to search for job '134-jorlow'</li>
-</ul>
-</p>
- </td>
-</tr>
-</form>
-</table>
-</body>
-</html>
diff --git a/tko/machine_benchmark.cgi b/tko/machine_benchmark.cgi
index d0b1aef..e69de29 100755
--- a/tko/machine_benchmark.cgi
+++ b/tko/machine_benchmark.cgi
@@ -1,70 +0,0 @@
-#!/usr/bin/python
-print "Content-type: text/html\n"
-import cgi, cgitb, os, sys, re
-sys.stdout.flush()
-cgitb.enable()
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-import db, display, frontend
-
-db = db.db()
-
-benchmark_key = {
-'kernbench' : 'elapsed',
-'dbench' : 'throughput',
-'tbench' : 'throughput',
-}
-
-def main():
-
- display.print_main_header()
-
- rows = db.select('test', 'tests', {}, distinct = True)
- benchmarks = []
- for row in rows:
- benchmark = row[0]
- testname = re.sub(r'\..*', '', benchmark)
- if not benchmark_key.has_key(testname):
- continue
- benchmarks.append(benchmark)
- benchmarks = display.sort_tests(benchmarks)
-
- machine_idx = {}
- benchmark_data = {}
- for benchmark in benchmarks:
- fields = 'machine_idx,machine_hostname,count(status_word)'
- where = { 'subdir': benchmark, 'status_word' : 'GOOD' }
- data = {}
- for (idx, machine, count) in db.select(fields, 'test_view',
- where, group_by = 'machine_hostname'):
- data[machine] = count
- machine_idx[machine] = idx
- benchmark_data[benchmark] = data
-
- print '<h1>Performance</h1>'
-
- header_row = [ display.box('Benchmark', header=True) ]
- header_row += [ display.box(re.sub(r'\.', '<br>', benchmark), header=True) for benchmark in benchmarks ]
-
- matrix = [header_row]
- for machine in machine_idx:
- row = [display.box(machine)]
- for benchmark in benchmarks:
- count = benchmark_data[benchmark].get(machine, None)
- if not count:
- row.append(display.box(None))
- continue
- key = benchmark_key[re.sub(r'\..*', '', benchmark)]
- url = 'machine_test_attribute_graph.cgi'
- url += '?machine=' + str(machine_idx[machine])
- url += '&benchmark=' + benchmark
- url += '&key=' + key
- html = '<a href="%s">%d</a>' % (url, count)
- row.append(display.box(html))
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-main()
diff --git a/tko/machine_kernel.cgi b/tko/machine_kernel.cgi
index dbd52bb..e69de29 100755
--- a/tko/machine_kernel.cgi
+++ b/tko/machine_kernel.cgi
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-print "Content-type: text/html\n"
-import cgi, cgitb, os, sys, re
-sys.stdout.flush()
-cgitb.enable()
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-import db, frontend, display
-client_bin = os.path.abspath(os.path.join(tko, '../client/bin'))
-sys.path.insert(0, client_bin)
-import kernel_versions
-
-db = db.db()
-
-
-def kernel_group_box(kernel, group, box_data):
- machine_idxs = ['%d' % machine.idx for machine in group.machines()]
- link = 'machine_kernel_test.cgi?machine=%s&kernel=%s' % \
- (','.join(machine_idxs), kernel.idx)
- return display.status_precounted_box(db, box_data, link)
-
-
-def kernel_encode(kernel):
- return kernel_versions.version_encode(kernel.printable)
-
-
-def main():
- display.print_main_header()
-
- ret = frontend.get_matrix_data(db, 'machine_group', 'kernel_printable')
- (data, group_list, kernel_list, status_list, job_tags) = ret
-
- groups = frontend.group.select(db)
- group_names = [display.group_name(g) for g in groups]
- headers = ['Version'] + group_names
- header_row = [ display.box(x, header=True) for x in headers ]
-
- kernels = frontend.kernel.select(db)
- kernels.sort(key = kernel_encode, reverse = True)
-
- matrix = [header_row]
- for kernel in kernels:
- link = 'group_test.cgi?kernel=%s' % kernel.idx
- row = [display.box(kernel.printable, link=link)]
- for group in groups:
- try:
- box_data = data[group.name][kernel.printable]
- except:
- row.append(display.box(None, None))
- continue
- row.append(kernel_group_box(kernel, group, box_data))
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-
-main()
diff --git a/tko/machine_kernel_test.cgi b/tko/machine_kernel_test.cgi
index 25f6a38..e69de29 100755
--- a/tko/machine_kernel_test.cgi
+++ b/tko/machine_kernel_test.cgi
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-print "Content-type: text/html\n"
-import cgi, cgitb, os, sys, re
-sys.stdout.flush()
-cgitb.enable()
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-import db, display, frontend
-
-db = db.db()
-
-def main():
- display.print_main_header()
-
- form = cgi.FieldStorage()
-
- kernel_idx = form['kernel'].value
-
- if form.has_key('machine'):
- mlist = form['machine'].value
- machines = []
- for idx in mlist.split(','):
- where = { 'machine_idx' : idx }
- machines.append(frontend.machine.select(db, where)[0])
- elif form.has_key('group'):
- where = { 'machine_group' : form['group'].value }
- machines = frontend.machine.select(db, where)
- mlist = ','.join(['%s'%machine.idx for machine in machines])
- if form.has_key('test'):
- test = form['test'].value
- else:
- test = None
-
- kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0]
- print_kernel_machines_vs_test(machines, kernel, test, mlist)
-
-
-def print_kernel_machines_vs_test(machines, kernel, only_test, mlist):
- # first we have to get a list of all run tests across all machines
- all_tests = []
- results = {} # will be a 2d hash [machine][testname]
- for machine in machines:
- where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx }
- if only_test:
- where['subdir'] = only_test
- tests = frontend.test.select(db, where)
- if not tests:
- continue
- dict = {}
- for test in tests:
- testname = test.testname
- all_tests.append(testname)
- dict[testname] = dict.get(testname, []) + [test]
- results[machine.idx] = dict
- test_list = display.sort_tests(list(set(all_tests)))
-
- print '<h1>%s</h1>' % kernel.printable
-
- header_row = [ display.box('Version', header=True) ]
- for test in [re.sub(r'kernel.', r'kernel<br>', x) for x in test_list]:
- url='machine_kernel_test_jobs.cgi?machine=%s&kernel=%s&test=%s'\
- % (mlist, kernel.idx, test)
- header_row.append( display.box(test, link=url, header=True) )
-
- matrix = [header_row]
- for machine in machines:
- if not results.has_key(machine.idx):
- continue
- if machine.owner:
- hostname = machine.owner + ' ' + machine.hostname
- else:
- hostname = machine.hostname
- url = 'machine_kernel_test_jobs.cgi?machine=%s&kernel=%s' % \
- (machine.idx, kernel.idx)
- row = [display.box(hostname, link = url)]
- for testname in test_list:
- if results[machine.idx].has_key(testname):
- tests = results[machine.idx][testname]
- if len(tests) == 1:
- link = tests[0].url
- else:
- link = 'machine_kernel_test_jobs.cgi?machine=%s&kernel=%s&test=%s' % (machine.idx, kernel.idx, testname)
- else:
- tests = []
- link = None
-
- box = display.status_count_box(db, tests, link = link)
- row.append(box)
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-main()
diff --git a/tko/machine_kernel_test_jobs.cgi b/tko/machine_kernel_test_jobs.cgi
index a8a1625..e69de29 100755
--- a/tko/machine_kernel_test_jobs.cgi
+++ b/tko/machine_kernel_test_jobs.cgi
@@ -1,130 +0,0 @@
-#!/usr/bin/python
-print "Content-type: text/html\n"
-import cgi, cgitb, os, sys, re
-sys.stdout.flush()
-cgitb.enable()
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-import db, display, frontend
-
-db = db.db()
-
-def main():
- display.print_main_header()
- form = cgi.FieldStorage()
-
- kernel_idx = form['kernel'].value
- kernel = frontend.kernel.select(db, {'kernel_idx' : kernel_idx })[0]
-
- if form.has_key('machine'):
- machines = []
- for idx in form['machine'].value.split(','):
- where = { 'machine_idx' : idx }
- machines.append(frontend.machine.select(db, where)[0])
- elif form.has_key('group'):
- where = { 'machine_group' : form['group'].value }
- machines = frontend.machine.select(db, where)
-
- if form.has_key('test'):
- test = form['test'].value
- else:
- test = None
-
- if len(machines) == 1:
- print_jobs_vs_tests(machines[0], kernel, test)
- else:
- print_jobs_vs_machines(machines, kernel, test)
-
-
-def print_jobs_vs_tests(machine, kernel, only_test):
- # first we have to get a list of all run tests
- results = {} # will be a 2d hash on [testname][jobname]
- all_tests = set([])
- all_jobs = set([])
- where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx }
- if only_test:
- where['subdir'] = only_test
- tests = frontend.test.select(db, where)
- for test in tests:
- all_tests.add(test.testname)
- all_jobs.add(test.job.tag)
- if not results.has_key(test.testname):
- results[test.testname] = {}
- results[test.testname][test.job.tag] = test
- test_list = sorted(list(all_tests))
- job_list = sorted(list(all_jobs))
-
- print '<h1>Kernel: %s</h1>\n' % kernel.printable
- print '<h1>Machine: %s</h1>\n' % machine.hostname
-
- header_row = [ display.box('Job', header=True) ]
- for jobname in job_list:
- header_row.append( display.box(jobname, header=True) )
-
- matrix = [header_row]
- for testname in test_list:
- if not results.has_key(testname):
- continue
- row = [display.box(testname)]
- for jobname in job_list:
- test = results[testname].get(jobname, None)
- if test:
- box = display.box(test.status_word,
- color_key = test.status_word,
- link = test.url)
- else:
- box = display.box(None)
- row.append(box)
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-
-def print_jobs_vs_machines(machines, kernel, only_test):
- if not only_test:
- raise "No test specified!"
- results = {} # will be a 2d hash [machine][jobname]
- all_jobs = set([])
- for machine in machines:
- where = { 'kernel_idx':kernel.idx , 'machine_idx':machine.idx,
- 'subdir':only_test }
- tests = frontend.test.select(db, where)
- if tests:
- results[machine] = {}
- for test in tests:
- results[machine][test.job.tag] = test
- all_jobs.add(test.job.tag)
- results[machine.idx] = tests
- job_list = sorted(list(all_jobs))
-
- print '<h1>Kernel: %s</h1>\n' % kernel.printable
- print '<h1>Test: %s</h1>\n' % only_test
-
- header_row = [ display.box('Machine', header=True) ]
- for jobname in job_list:
- header_row.append( display.box(jobname, header=True) )
-
- matrix = [header_row]
- for machine in machines:
- if not results.has_key(machine):
- continue
- tests = results[machine]
- row = [display.box(machine.hostname)]
- for jobname in job_list:
- test = results[machine].get(jobname, None)
- if test:
- box = display.box(test.status_word,
- color_key = test.status_word,
- link = test.url)
- else:
- box = display.box(None)
- row.append(box)
- matrix.append(row)
- matrix.append(header_row)
-
- display.print_table(matrix)
-
-
-main()
diff --git a/tko/query_lib.py b/tko/query_lib.py
index e20a667..9f492b9 100644
--- a/tko/query_lib.py
+++ b/tko/query_lib.py
@@ -88,102 +88,3 @@
raise "Could not parse '%s' (%s)" % (condition, regex)
-
-###
-### Everything past here is depricated.
-###
-
-def generate_sql_condition(condition_list):
- """ generate the sql for the condition list."""
- sql = ''
- value = []
- for field, operator, values in condition_list:
- if len(values) == 1:
- if sql != '':
- sql += " and "
- sql += " %s%s%%s" % (field, operator)
- value.append(values[0][0])
- elif len(values) > 1:
- expression = [" %s %s %%s" % (field, operator) for val in values]
- for val in values:
- value.append(val[0])
- if sql != '':
- sql += " and "
- sql += "(%s)" % " or ".join(expression)
- return sql, value
-
-
-def prune_list(thelist, condition_sql, condition_value):
- """ keep track of which columns do not have any elements."""
- pruned_list = []
- for g in thelist:
- # check for multiple index values in the db.
- sql = "t where "
- expr = [" %s = %%s" % (g.idx_name) for val in g.idx_value]
- sql += " (%s) " % " or ".join(expr)
- value = []
- value.extend(g.idx_value)
- if condition_sql:
- sql += " and "
- sql += condition_sql
- value.extend(condition_value)
- tests = frontend.test.select_sql(db, sql, value)
- if len(tests) > 0:
- pruned_list.append(g)
- return pruned_list
-
-
-def parse_condition(condition):
- """ parse the condition into independent clauses."""
- condition_list = []
- if not condition:
- return condition_list
- attribute_re = r"(\w+)"
- op_re = r"(=|!=)"
- value_re = r"('[^']*')"
- # condition is clause & clause & ..
- clause_re = r"%s\s*%s\s*%s" % (attribute_re, op_re, value_re)
- condition_re = re.compile(r"^\s*%s(\s*&\s*%s)*\s*$" % (clause_re, clause_re))
- if not condition_re.match(condition):
- print "Condition not in the correct format: %s" % condition
- sys.exit(0)
- triples = []
- for clause in [c.strip() for c in condition.split('&')]:
- attribute, op, value = re.match(clause_re, clause).groups()
- triples.append((attribute, op, value))
- for (field_name, operator, value) in triples:
- match, field = frontend.select(db, field_name, value, distinct=True)
- if len(match) > 0:
- condition_list.append((field, operator, match))
- else:
- print "No matching results found for condition %s." % \
- condition
- sys.exit(0)
- return condition_list
-
-
-def get_value(test, field):
- """ get specific field values from the given test object."""
- if field == 'test':
- return test.testname
- elif field == 'kernel_idx':
- return test.kernel_idx
- elif field == 'machine_idx':
- return test.machine_idx
- elif field == 'status':
- return test.status_num
-
-
-def get_tests(condition_sql, condition_value):
- # get all the tests that satify the given condition.
- if condition_sql:
- sql = "t where "
- sql += condition_sql
- value = [str(val) for val in condition_value]
- #print sql , value
- tests = frontend.test.select_sql(db, sql, value)
- else:
- sql = None
- value = None
- tests = frontend.test.select_sql(db, " t ", None)
- return tests
diff --git a/tko/report.py b/tko/report.py
index bd4921f..e69de29 100644
--- a/tko/report.py
+++ b/tko/report.py
@@ -1,256 +0,0 @@
-#!/usr/bin/python
-
-"""
-CLI support to enable user to query the database.
-"""
-
-import sys, os, getopt
-
-tko = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
-sys.path.insert(0, tko)
-
-import query_lib, db, frontend
-
-db = db.db()
-
-help_msg_header = """
-NAME
-report.py - Print the results matching a given condition in the specified format.
-
-SYNOPSIS
-report.py [options]
-
-OPTIONS
-"""
-
-help_msg_trailer = """
-EXAMPLES
-To see every job that has ever been run:
- report.py
-
-To see all the jobs started by johnmacdonald:
- report.py --condition="user='johnmacdonald'"
-
-To see all the jobs started by johnmandonald and on hostname arh22:
- report.py --condition="user='johnmacdonald' & hostname='arh22'"
-
-To see only the test, hostname and user for the reports:
- report.py --columns="test, hostname, user"
-
-You can use both the columns and condition options to generate the kind of report you want.
-"""
-
-condition_desc = """Condition to filter the results with.
- Supported fields are: test, hostname, user, label, machine_group, status, reason, kernel.
- Supported operators are =, != and string values must be quoted within single quotes.
-"""
-
-columns_desc = """Specific columns to display in the results.
- Supported fields are: test, hostname, user, label, machine_group, status, reason, kernel.
-"""
-
-help_desc = """Print command help.
-"""
-
-
-class CliError(Exception):
- pass
-
-
-class InvalidArgsError(CliError):
- def __init__(self, error):
- CliError.__init__(self, 'Unknown arguments: %r\nTry report.py --help' % error)
-
-
-class InvalidColumnValue(CliError):
- def __init__(self, error):
- CliError.__init__(self, 'Unrecognized column value: %r\nTry report.py --help' % error)
-
-
-class cli:
- def __init__(self):
- self.__options = {}
-
-
- def add_option(self, name=None, short_name=None, type=None,
- description=None, value=None):
- """ Adds the options to the cli.
- """
- if not name and not short_name:
- raise Error("No name provided for the option.")
-
- short = False
-
- if not name and short_name:
- short = True
- name = short_name
-
- self.__options[name] = dict(name=name, type=type,
- description=description,
- value=value, short=short)
-
-
- def list_options(self):
- """ Return the options for this cli.
- """
- return self.__options
-
-
- def parse_options(self, args):
- """ Parse the options and the values the cli is invoked with.
- """
- short_opts = ""
- long_opts = []
- for name,val in self.__options.items():
- if val['short']:
- short_opts += val['name']
- if val['type'] != 'bool':
- short_opts += ':'
- else:
- opt = val['name']
- if val['type'] != 'bool':
- opt += '='
- long_opts.append(opt)
-
- opts, args = getopt.getopt(args[1:], short_opts, long_opts)
- return opts, args
-
-
- def usage(self):
- """ Help for the cli.
- """
- msg = help_msg_header
- for opt,value in self.__options.items():
- if value['short']:
- msg += '-'
- else:
- msg += '--'
- msg += '%s \t: %s\n' % (value['name'], value['description'])
-
- msg += help_msg_trailer
- return msg
-
-
-def pretty_print(header, results):
- """ pretty prints the result with all the proper space indentations.
- """
- # add an extra column for the record number.
- header.insert(0, ' # ')
-
- # add the record number to each result.
- for j in xrange(len(results)):
- results[j].insert(0, "[%d]" % (j+1))
-
- # number of columns in the results table.
- size = len(header)
-
- # list containing the max width of each column.
- column_width = [len(col_name) for col_name in header]
-
- # update the column width based on the values in the table.
- for record in results:
- for i in xrange(size):
- column_width[i] = max(column_width[i], len(record[i]))
-
- # Generates the header.
- lines = []
- lines.append(' '.join([header[i].capitalize().ljust(column_width[i])
- for i in xrange(size)]))
- lines.append(' '.join(['-' * c_size for c_size in column_width]))
-
- # Generates the table with the appropriate space indent.
- for record in results:
- lines.append(' '.join([record[i].ljust(column_width[i])
- for i in xrange(size)]))
-
- return '\n'.join(lines)
-
-
-def main(args):
- cli_obj = cli()
-
- # Add all the known and acceptable options.
- cli_obj.add_option(name='condition', type='string',
- description=condition_desc)
- cli_obj.add_option(name='columns', type='string',
- description=columns_desc)
- cli_obj.add_option(name='help', type='bool',
- description=help_desc)
-
- # Parse the options.
- opts,args = cli_obj.parse_options(args)
-
- # unexpected argument.
- if args:
- raise InvalidArgsError(args)
-
- sql = None
- value = None
-
- # by default display these columns
- requested_columns = ['test', 'hostname', 'status', 'reason']
-
- for option, value in opts:
- if option == '--help':
- print cli_obj.usage()
- return
- elif option == '--condition':
- condition_list = query_lib.parse_condition(value.strip('"'))
- sql, value = query_lib.generate_sql_condition(condition_list)
- elif option == '--columns':
- supported_columns = ['test', 'hostname', 'user', 'label',
- 'machine_group', 'status', 'reason', 'kernel']
- requested_columns = [x.strip() for x in value.split(',')]
- for col in requested_columns:
- if col not in supported_columns:
- raise InvalidColumnValue, 'Unknown field %s specified in the columns option' % col
-
- # get the values corresponding to the index fields.
- col_values = {}
- for col in requested_columns:
- if col != 'test' and col != 'status' and col != 'reason':
- # the rest of the columns need the index values.
- col_group = frontend.anygroup.selectunique(db, col)
- col_value, field_name = frontend.select(db, col)
- col_values[col] = list(col_value)
-
- # get all the tests that satisfy the given conditions.
- tests = query_lib.get_tests(sql, value)
-
- # accumulate the fields that are of interest to the user.
- result = []
-
- for test in tests:
- record = []
-
- test_values = {}
- test_values['hostname'] = test.machine_idx
- test_values['user'] = test.job.job_idx
- test_values['label'] = test.job.job_idx
- test_values['machine_group'] = test.machine_idx
- test_values['kernel'] = test.kernel_idx
-
- for col in requested_columns:
- if col == 'test':
- record.append(test.testname)
- elif col == 'status':
- record.append(test.status_word)
- elif col == 'reason':
- record.append(test.reason.strip())
- else:
- column = col_values[col]
- found = False
- for idx_name, idx_value in column:
- if idx_value == test_values[col]:
- record.append(idx_name)
- found = True
- break
- if not found:
- record.append('')
- result.append(record)
-
- # generate the pretty table.
- print pretty_print(requested_columns, result)
-
-
-main(sys.argv)