Add feature to make spreadsheet header fields from combinations of machine labels. The user can create as many different machine-label-based fields as she wishes. For each field, she can enter a list of labels to be included. The field will then group on each combination of those labels.
-added new HeaderField abstract class with two implementations - SimpleHeaderField for normal fields and MachineLabelField for the new machine label fields
-made HeaderSelect capable of creating MachineLabelFields. In single header mode, selecting "Machine labels..." creates one, and deselecting it destroys it. In multiple header mode, each time "Machine labels..." is selected a new machine labels field is created, and deselecting one destroys it.
-made HeaderSelect display text boxes for each MachineLabelField for the user to input the label list.
-created HeaderSelect.addQueryParameters, moved fixed value logic into it (from SpreadsheetView.java), and put logic for machine label header in it.
-made TestGroupDataSource accept raw query parameters, and updated SpreadsheetDataProcessor to pass it through.
-modified SpreadsheetView to use HeaderFields throughout. Eventually other code (such as TableView) should be made to use them.
-added capability for ConditionTestSet to accept raw condition pieces. Eventually it will only work this way and I'll get rid of the field setting logic, since that's been moved to SimpleHeaderField.
-added ExtendedListBox class containing a bunch of utilities for ListBoxes that I've wanted for a long time. Several other parts of the code (DoubleListSelector, some of the graphing stuff) should be changes to use these utilities eventually.
-added ChangeListener support to DoubleListSelector
-made rpc interface accept a new "machine_label_headers" parameters, and added logic to tko_rpc_utils.py to construct SQL for machine label headers
-modified TestView manager to support a join into host labels
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2331 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/new_tko/tko/models.py b/new_tko/tko/models.py
index 7057f10..89aba22 100644
--- a/new_tko/tko/models.py
+++ b/new_tko/tko/models.py
@@ -11,14 +11,21 @@
return self.get_key_on_this_table(field)
- def _get_field_names(self, fields):
- return [self._get_key_unless_is_function(field) for field in fields]
+ def _get_field_names(self, fields, extra_select_fields={}):
+ field_names = []
+ for field in fields:
+ if field in extra_select_fields:
+ field_names.append(field)
+ else:
+ field_names.append(self._get_key_unless_is_function(field))
+ return field_names
def _get_group_query_sql(self, query, group_by, extra_select_fields):
- group_fields = self._get_field_names(group_by)
+ group_fields = self._get_field_names(group_by, extra_select_fields)
- select_fields = list(group_fields)
+ select_fields = [field for field in group_fields
+ if field not in extra_select_fields]
for field_name, field_sql in extra_select_fields.iteritems():
field_sql = self._get_key_unless_is_function(field_sql)
select_fields.append(field_sql + ' AS ' + field_name)
@@ -289,6 +296,14 @@
return query_set.filter(filter_object).distinct()
+ def _get_include_exclude_suffix(self, exclude):
+ if exclude:
+ suffix = '_exclude'
+ else:
+ suffix = '_include'
+ return suffix
+
+
def _add_label_joins(self, query_set, suffix=''):
query_set = self._add_join(query_set, 'test_labels_tests',
join_key='test_id', suffix=suffix,
@@ -306,8 +321,10 @@
return query_set.filter(filter_object)
- def _add_attribute_join(self, query_set, suffix='', join_condition='',
+ def _add_attribute_join(self, query_set, join_condition='', suffix=None,
exclude=False):
+ if suffix is None:
+ suffix = self._get_include_exclude_suffix(exclude)
return self._add_join(query_set, 'test_attributes',
join_condition=join_condition,
suffix=suffix, exclude=exclude)
@@ -320,12 +337,13 @@
return [label['id'] for label in query]
- def get_query_set_with_joins(self, filter_data):
+ def get_query_set_with_joins(self, filter_data, include_host_labels=False):
exclude_labels = filter_data.pop('exclude_labels', [])
query_set = self.get_query_set()
joined = False
# TODO: make this check more thorough if necessary
- if 'test_labels' in filter_data.get('extra_where', ''):
+ extra_where = filter_data.get('extra_where', '')
+ if 'test_labels' in extra_where:
query_set = self._add_label_joins(query_set)
joined = True
@@ -347,19 +365,23 @@
'')
if include_attributes_where:
query_set = self._add_attribute_join(
- query_set, suffix='_include',
- join_condition=include_attributes_where)
+ query_set, join_condition=include_attributes_where)
joined = True
if exclude_attributes_where:
query_set = self._add_attribute_join(
- query_set, suffix='_exclude',
- join_condition=exclude_attributes_where,
+ query_set, join_condition=exclude_attributes_where,
exclude=True)
joined = True
if not joined:
filter_data['no_distinct'] = True
+ if include_host_labels or 'test_attributes_host_labels' in extra_where:
+ query_set = self._add_attribute_join(
+ query_set, suffix='_host_labels',
+ join_condition='test_attributes_host_labels.attribute = '
+ '"host-labels"')
+
return query_set
diff --git a/new_tko/tko/rpc_interface.py b/new_tko/tko/rpc_interface.py
index db2d6ce..a9dcc06 100644
--- a/new_tko/tko/rpc_interface.py
+++ b/new_tko/tko/rpc_interface.py
@@ -17,7 +17,8 @@
def get_group_counts(group_by, header_groups=[], fixed_headers={},
- extra_select_fields={}, **filter_data):
+ machine_label_headers={}, extra_select_fields={},
+ **filter_data):
"""
Queries against TestView grouping by the specified fields and computings
counts for each group.
@@ -27,6 +28,14 @@
* header_groups can be used to get lists of unique combinations of group
fields. It should be a list of tuples of fields from group_by. It's
primarily for use by the spreadsheet view.
+ * fixed_headers can map header fields to lists of values. the header will
+ guaranteed to return exactly those value. this does not work together
+ with header_groups.
+ * machine_label_headers can specify special headers to be constructed from
+ machine labels. It should map arbitrary names to lists of machine labels.
+ a field will be created with the given name containing a comma-separated
+ list indicating which of the given machine labels are on each test. this
+ field can then be grouped on.
Returns a dictionary with two keys:
* header_values contains a list of lists, one for each header group in
@@ -38,11 +47,16 @@
The keys for the extra_select_fields are determined by the "AS" alias of
the field.
"""
- query = models.TestView.query_objects(filter_data)
+ extra_select_fields = dict(extra_select_fields)
+ query = models.TestView.objects.get_query_set_with_joins(
+ filter_data, include_host_labels=bool(machine_label_headers))
+ query = models.TestView.query_objects(filter_data, initial_query=query)
count_alias, count_sql = models.TestView.objects.get_count_sql(query)
extra_select_fields[count_alias] = count_sql
if 'test_idx' not in group_by:
extra_select_fields['test_idx'] = 'test_idx'
+ tko_rpc_utils.add_machine_label_headers(machine_label_headers,
+ extra_select_fields)
group_processor = tko_rpc_utils.GroupDataProcessor(query, group_by,
header_groups,
@@ -61,7 +75,7 @@
def get_status_counts(group_by, header_groups=[], fixed_headers={},
- **filter_data):
+ machine_label_headers={}, **filter_data):
"""
Like get_group_counts, but also computes counts of passed, complete (and
valid), and incomplete tests, stored in keys "pass_count', 'complete_count',
@@ -69,22 +83,27 @@
"""
return get_group_counts(group_by, header_groups=header_groups,
fixed_headers=fixed_headers,
+ machine_label_headers=machine_label_headers,
extra_select_fields=tko_rpc_utils.STATUS_FIELDS,
**filter_data)
def get_latest_tests(group_by, header_groups=[], fixed_headers={},
- **filter_data):
+ machine_label_headers={}, **filter_data):
"""
Similar to get_status_counts, but return only the latest test result per
group. It still returns the same information (i.e. with pass count etc.)
for compatibility.
"""
# find latest test per group
- query = models.TestView.query_objects(filter_data)
+ query = models.TestView.objects.get_query_set_with_joins(
+ filter_data, include_host_labels=bool(machine_label_headers))
+ query = models.TestView.query_objects(filter_data, initial_query=query)
query.exclude(status__in=tko_rpc_utils._INVALID_STATUSES)
extra_fields = {'latest_test_idx' : 'MAX(%s)' %
models.TestView.objects.get_key_on_this_table('test_idx')}
+ tko_rpc_utils.add_machine_label_headers(machine_label_headers,
+ extra_fields)
group_processor = tko_rpc_utils.GroupDataProcessor(query, group_by,
header_groups,
diff --git a/new_tko/tko/tko_rpc_utils.py b/new_tko/tko/tko_rpc_utils.py
index 6abea64..c8da542 100644
--- a/new_tko/tko/tko_rpc_utils.py
+++ b/new_tko/tko/tko_rpc_utils.py
@@ -74,6 +74,32 @@
group_dict[models.TestView.objects._GROUP_COUNT_NAME] = 1
+def _construct_machine_label_header_sql(machine_labels):
+ """
+ Example result for machine_labels=['Index', 'Diskful']:
+ CONCAT_WS(",",
+ IF(FIND_IN_SET("Diskful", test_attributes_host_labels.value),
+ "Diskful", NULL),
+ IF(FIND_IN_SET("Index", test_attributes_host_labels.value),
+ "Index", NULL))
+
+ This would result in field values "Diskful,Index", "Diskful", "Index", NULL.
+ """
+ machine_labels = sorted(machine_labels)
+ if_clauses = []
+ for label in machine_labels:
+ if_clauses.append(
+ 'IF(FIND_IN_SET("%s", test_attributes_host_labels.value), '
+ '"%s", NULL)' % (label, label))
+ return 'CONCAT_WS(",", %s)' % ', '.join(if_clauses)
+
+
+def add_machine_label_headers(machine_label_headers, extra_selects):
+ for field_name, machine_labels in machine_label_headers.iteritems():
+ extra_selects[field_name] = (
+ _construct_machine_label_header_sql(machine_labels))
+
+
class GroupDataProcessor(object):
_MAX_GROUP_RESULTS = 50000