Attached is a large patch for a powerful and flexible new graphing system for new TKO. This system subsumes all the previous kernel graphing scripts under the tko/ directory and is capable of much more. These wiki pages document usage of the new system and give an idea of what it's capable of:
http://autotest.kernel.org/wiki/MetricsPlot
http://autotest.kernel.org/wiki/MachineQualHistograms
Feel free to try it out and please let us know if you run into any trouble.
This system is the work of our summer intern James Ren. Thank you for all your fantastic work, James!
From: James Ren <jamesren@stanford.edu>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2171 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/frontend/client/src/autotest/public/TkoClient.html b/frontend/client/src/autotest/public/TkoClient.html
index e9b888f..babbb41 100644
--- a/frontend/client/src/autotest/public/TkoClient.html
+++ b/frontend/client/src/autotest/public/TkoClient.html
@@ -75,6 +75,8 @@
</div>
<div id="graphing_view" title="Graphing">
+ <span class="field-name">Graph Type:</span>
+ <span id="graphing_type"></span><br /><hr />
<span id="graphing_frontend"></span>
</div>
diff --git a/frontend/client/src/autotest/tko/DBColumnSelector.java b/frontend/client/src/autotest/tko/DBColumnSelector.java
new file mode 100644
index 0000000..5a20d5b
--- /dev/null
+++ b/frontend/client/src/autotest/tko/DBColumnSelector.java
@@ -0,0 +1,80 @@
+package autotest.tko;
+
+import autotest.common.StaticDataRepository;
+import autotest.common.Utils;
+
+import com.google.gwt.json.client.JSONArray;
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.ListBox;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.ArrayList;
+
+public class DBColumnSelector extends Composite {
+
+ public static final String PERF_VIEW = "perf_view";
+ public static final String TEST_VIEW = "test_view";
+
+ private ListBox field = new ListBox();
+ private ArrayList<ChangeListener> listeners = new ArrayList<ChangeListener>();
+
+ public DBColumnSelector(String view) {
+ this(view, false);
+ }
+
+ public DBColumnSelector(String view, boolean canUseSinglePoint) {
+ if (canUseSinglePoint) {
+ field.addItem("(Single Point)", "'data'");
+ }
+
+ StaticDataRepository staticData = StaticDataRepository.getRepository();
+ JSONArray fields = staticData.getData(view).isArray();
+ for (int i = 0; i < fields.size(); i++) {
+ JSONArray perfField = fields.get(i).isArray();
+ String fieldName = Utils.jsonToString(perfField.get(0));
+ String column = Utils.jsonToString(perfField.get(1));
+ field.addItem(fieldName, column);
+ }
+
+ field.addChangeListener(new ChangeListener() {
+ public void onChange(Widget w) {
+ notifyListeners();
+ }
+ });
+
+ initWidget(field);
+ }
+
+ public void addChangeListener(ChangeListener listener) {
+ listeners.add(listener);
+ }
+
+ public String getColumn() {
+ return field.getValue(field.getSelectedIndex());
+ }
+
+ public void setEnabled(boolean enabled) {
+ field.setEnabled(enabled);
+ }
+
+ // Select the value in the drop-down whose column name matches the given parameter
+ public void selectColumn(String column) {
+ for (int i = 0; i < field.getItemCount(); i++) {
+ if (field.getValue(i).equals(column)) {
+ field.setSelectedIndex(i);
+ break;
+ }
+ }
+ }
+
+ public void setSelectedIndex(int index) {
+ field.setSelectedIndex(index);
+ }
+
+ private void notifyListeners() {
+ for (ChangeListener listener : listeners) {
+ listener.onChange(this);
+ }
+ }
+}
diff --git a/frontend/client/src/autotest/tko/ExistingGraphsFrontend.java b/frontend/client/src/autotest/tko/ExistingGraphsFrontend.java
index f979973..6c33207 100644
--- a/frontend/client/src/autotest/tko/ExistingGraphsFrontend.java
+++ b/frontend/client/src/autotest/tko/ExistingGraphsFrontend.java
@@ -1,9 +1,9 @@
package autotest.tko;
import autotest.common.JsonRpcCallback;
-import autotest.common.JsonRpcProxy;
import autotest.common.StaticDataRepository;
import autotest.common.Utils;
+import autotest.common.ui.TabView;
import com.google.gwt.json.client.JSONArray;
import com.google.gwt.json.client.JSONObject;
@@ -13,10 +13,7 @@
import com.google.gwt.user.client.ui.Button;
import com.google.gwt.user.client.ui.CheckBox;
import com.google.gwt.user.client.ui.ClickListener;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.FlexTable;
import com.google.gwt.user.client.ui.FocusListener;
-import com.google.gwt.user.client.ui.HasVerticalAlignment;
import com.google.gwt.user.client.ui.ListBox;
import com.google.gwt.user.client.ui.MultiWordSuggestOracle;
import com.google.gwt.user.client.ui.SuggestBox;
@@ -25,12 +22,14 @@
import com.google.gwt.user.client.ui.TextBox;
import com.google.gwt.user.client.ui.Widget;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
-public class ExistingGraphsFrontend extends Composite {
-
- private JsonRpcProxy rpcProxy = JsonRpcProxy.getProxy();
+public class ExistingGraphsFrontend extends GraphingFrontend {
+
private CheckBox normalize = new CheckBox("Normalize Performance (allows multiple benchmarks" +
" on one graph)");
private MultiWordSuggestOracle oracle = new MultiWordSuggestOracle();
@@ -40,16 +39,11 @@
private TextBox kernel = new TextBox();
private JSONObject hostsAndTests = null;
private Button graphButton = new Button("Graph");
- FlexTable table = new FlexTable();
-
- public ExistingGraphsFrontend() {
+
+ public ExistingGraphsFrontend(final TabView parent) {
normalize.addClickListener(new ClickListener() {
public void onClick(Widget w) {
- benchmark.setMultipleSelect(normalize.isChecked());
- int selectedIndex = benchmark.getSelectedIndex();
- for (int i = 0; i < benchmark.getItemCount(); i++) {
- benchmark.setItemSelected(i, i == selectedIndex);
- }
+ normalizeClicked();
}
});
@@ -72,6 +66,7 @@
graphButton.addClickListener(new ClickListener() {
public void onClick(Widget w) {
+ parent.updateHistory();
showGraph();
}
});
@@ -89,7 +84,8 @@
initWidget(table);
}
-
+
+ @Override
public void refresh() {
setEnabled(false);
rpcProxy.rpcCall("get_hosts_and_tests", new JSONObject(), new JsonRpcCallback() {
@@ -104,15 +100,68 @@
}
});
}
-
- private void addControl(String text, Widget widget) {
- int row = table.getRowCount();
- table.setText(row, 0, text);
- table.setWidget(row, 1, widget);
- table.getFlexCellFormatter().setStylePrimaryName(row, 0, "field-name");
- table.getFlexCellFormatter().setVerticalAlignment(row, 0, HasVerticalAlignment.ALIGN_TOP);
+
+ @Override
+ protected void addToHistory(Map<String, String> args) {
+ args.put("normalize", String.valueOf(normalize.isChecked()));
+ args.put("hostname", hostname.getText());
+
+ // Add the selected benchmarks
+ StringBuilder benchmarks = new StringBuilder();
+ for (int i = 0; i < benchmark.getItemCount(); i++) {
+ if (benchmark.isItemSelected(i)) {
+ benchmarks.append(benchmark.getValue(i));
+ benchmarks.append(",");
+ }
+ }
+
+ args.put("benchmark", benchmarks.toString());
+ args.put("kernel", kernel.getText());
}
-
+
+ @Override
+ protected void handleHistoryArguments(final Map<String, String> args) {
+ setEnabled(false);
+ hostname.setText(args.get("hostname"));
+ normalize.setChecked(Boolean.parseBoolean(args.get("normalize")));
+ normalizeClicked();
+ kernel.setText(args.get("kernel"));
+
+ rpcProxy.rpcCall("get_hosts_and_tests", new JSONObject(), new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ hostsAndTests = result.isObject();
+ refreshTests();
+
+ Set<String> benchmarks =
+ new HashSet<String>(Arrays.asList(args.get("benchmark").split(",")));
+ for (int i = 0; i < benchmark.getItemCount(); i++) {
+ benchmark.setItemSelected(i, benchmarks.contains(benchmark.getValue(i)));
+ }
+ setEnabled(true);
+ }
+ });
+ }
+
+ @Override
+ protected void setDrilldownTrigger() {
+ // No drilldowns
+ }
+
+ @Override
+ protected void addAdditionalEmbeddingParams(JSONObject params) {
+ // No embedding
+ }
+
+ // Change the state of the page based on the status of the "normalize" checkbox
+ private void normalizeClicked() {
+ benchmark.setMultipleSelect(normalize.isChecked());
+ int selectedIndex = benchmark.getSelectedIndex();
+ for (int i = 0; i < benchmark.getItemCount(); i++) {
+ benchmark.setItemSelected(i, i == selectedIndex);
+ }
+ }
+
private void setEnabled(boolean enabled) {
normalize.setEnabled(enabled);
hostname.setEnabled(enabled);
@@ -126,7 +175,7 @@
if (value == null) {
return;
}
-
+
HashSet<String> selectedTests = new HashSet<String>();
for (int i = 0; i < benchmark.getItemCount(); i++) {
if (benchmark.isItemSelected(i)) {
@@ -137,7 +186,7 @@
JSONArray tests = value.isObject().get("tests").isArray();
benchmark.clear();
for (int i = 0; i < tests.size(); i++) {
- String test = tests.get(i).isString().stringValue();
+ String test = Utils.jsonToString(tests.get(i));
benchmark.addItem(test);
if (selectedTests.contains(test)) {
benchmark.setItemSelected(i, true);
@@ -170,7 +219,7 @@
StringBuilder arg = new StringBuilder();
for (int i = 0; i < tests.size(); i++) {
- String test = tests.get(i).isString().stringValue();
+ String test = Utils.jsonToString(tests.get(i));
String key = getKey(test);
if (i != 0) {
arg.append(",");
@@ -189,10 +238,10 @@
url = "/tko/machine_test_attribute_graph.cgi?";
JSONObject hostObject = value.isObject();
- int machine = (int) hostObject.get("id").isNumber().doubleValue();
+ String machine = Utils.jsonToString(hostObject.get("id"));
String benchmarkStr = benchmark.getValue(benchmarkIndex);
- args.put("machine", String.valueOf(machine));
+ args.put("machine", machine);
args.put("benchmark", benchmarkStr);
args.put("key", getKey(benchmarkStr));
}
@@ -202,6 +251,6 @@
private String getKey(String benchmark) {
JSONObject benchmarkKey =
StaticDataRepository.getRepository().getData("benchmark_key").isObject();
- return benchmarkKey.get(benchmark.replaceAll("\\..*", "")).isString().stringValue();
+ return Utils.jsonToString(benchmarkKey.get(benchmark.replaceAll("\\..*", "")));
}
}
diff --git a/frontend/client/src/autotest/tko/FilterSelector.java b/frontend/client/src/autotest/tko/FilterSelector.java
new file mode 100644
index 0000000..77f25b8
--- /dev/null
+++ b/frontend/client/src/autotest/tko/FilterSelector.java
@@ -0,0 +1,224 @@
+package autotest.tko;
+
+import autotest.common.ui.SimpleHyperlink;
+import autotest.tko.FilterStringViewer.EditListener;
+
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.FlexTable;
+import com.google.gwt.user.client.ui.HasHorizontalAlignment;
+import com.google.gwt.user.client.ui.HorizontalPanel;
+import com.google.gwt.user.client.ui.Panel;
+import com.google.gwt.user.client.ui.RadioButton;
+import com.google.gwt.user.client.ui.TextBox;
+import com.google.gwt.user.client.ui.VerticalPanel;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class FilterSelector extends Composite {
+
+ public class Filter extends Composite {
+
+ private DBColumnSelector dbColumnSelector = new DBColumnSelector(dbView);
+ private TextBox condition = new TextBox();
+ private FlexTable flexTable = new FlexTable();
+ private SimpleHyperlink deleteLink = new SimpleHyperlink("[X]");
+
+ private Filter() {
+ ChangeListener listener = new ChangeListener() {
+ public void onChange(Widget w) {
+ buildFilterString();
+ }
+ };
+ dbColumnSelector.addChangeListener(listener);
+ condition.addChangeListener(listener);
+
+ deleteLink.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ if (enabled) {
+ deleteFilter(Filter.this);
+ buildFilterString();
+ }
+ }
+ });
+
+ flexTable.setWidget(0, 0, dbColumnSelector);
+ flexTable.setWidget(0, 1, condition);
+ flexTable.setWidget(0, 2, deleteLink);
+
+ initWidget(flexTable);
+ }
+ }
+
+ private FlexTable table = new FlexTable();
+ private Panel filtersPanel = new VerticalPanel();
+ private List<Filter> filters = new ArrayList<Filter>();
+ private RadioButton all;
+ private RadioButton any;
+ private SimpleHyperlink addLink = new SimpleHyperlink("[Add Filter]");
+ private FilterStringViewer viewer = new FilterStringViewer();
+ private boolean enabled = true;
+ private String dbView;
+ private static int filterSelectorId;
+
+ public FilterSelector(String dbView) {
+ this.dbView = dbView;
+ int id = filterSelectorId;
+ filterSelectorId++;
+
+ all = new RadioButton("booleanOp" + id, "all of");
+ any = new RadioButton("booleanOp" + id, "any of");
+
+ ClickListener booleanOpListener = new ClickListener() {
+ public void onClick(Widget w) {
+ buildFilterString();
+ }
+ };
+ all.addClickListener(booleanOpListener);
+ any.addClickListener(booleanOpListener);
+ all.setChecked(true);
+
+ addLink.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ if (enabled) {
+ addFilter();
+ }
+ }
+ });
+
+ viewer.addEditListener(new EditListener() {
+ public void onEdit() {
+ setEnabled(false);
+ }
+
+ public void onRevert() {
+ setEnabled(true);
+ buildFilterString();
+ }
+ });
+
+ Panel booleanOpPanel = new HorizontalPanel();
+ booleanOpPanel.add(all);
+ booleanOpPanel.add(any);
+ table.setWidget(0, 0, booleanOpPanel);
+ table.setWidget(1, 0, filtersPanel);
+ table.getFlexCellFormatter().setColSpan(1, 0, 2);
+ table.setWidget(2, 1, addLink);
+ table.getFlexCellFormatter().setHorizontalAlignment(
+ 2, 1, HasHorizontalAlignment.ALIGN_RIGHT);
+ table.setWidget(3, 0, viewer);
+ table.getFlexCellFormatter().setColSpan(3, 0, 2);
+ table.setStylePrimaryName("box");
+
+ addFilter();
+
+ initWidget(table);
+ }
+
+ public String getFilterString() {
+ return viewer.getText();
+ }
+
+ public void reset() {
+ filtersPanel.clear();
+ filters.clear();
+ addFilter();
+ }
+
+ protected void addToHistory(Map<String, String> args, String prefix) {
+ // Get all the filters/conditions
+ for (int index = 0; index < filters.size(); index++) {
+ args.put(prefix + "[" + index + "][db]",
+ filters.get(index).dbColumnSelector.getColumn());
+ args.put(prefix + "[" + index + "][condition]", filters.get(index).condition.getText());
+ }
+
+ // Get whether the filter should be "all" or "any"
+ args.put(prefix + "_all", Boolean.toString(all.isChecked()));
+
+ viewer.addToHistory(args, prefix);
+ }
+
+ protected void handleHistoryArguments(Map<String, String> args, String prefix) {
+ int index = 0;
+ String db, condition;
+
+ // Restore all the filters/conditions
+ while ((db = args.get(prefix + "[" + index + "][db]")) != null) {
+ condition = args.get(prefix + "[" + index + "][condition]");
+ Filter filter;
+ if (index == 0) {
+ filter = filters.get(0);
+ } else {
+ filter = addFilter();
+ }
+ filter.dbColumnSelector.selectColumn(db);
+ filter.condition.setText(condition);
+ index++;
+ }
+
+ // Restore the "all" or "any" selection
+ boolean allChecked = Boolean.parseBoolean(args.get(prefix + "_all"));
+ if (allChecked) {
+ all.setChecked(true);
+ } else {
+ any.setChecked(true);
+ }
+
+ buildFilterString();
+ viewer.handleHistoryArguments(args, prefix);
+ }
+
+ private Filter addFilter() {
+ Filter nextFilter = new Filter();
+ filters.add(nextFilter);
+ filtersPanel.add(nextFilter);
+ return nextFilter;
+ }
+
+ private void deleteFilter(Filter filter) {
+ // If there's only one filter, clear it
+ if (filters.size() == 1) {
+ reset();
+ return;
+ }
+
+ filters.remove(filter);
+ filtersPanel.remove(filter);
+ }
+
+ private void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ all.setEnabled(enabled);
+ any.setEnabled(enabled);
+ for (Filter filter : filters) {
+ filter.condition.setEnabled(enabled);
+ filter.dbColumnSelector.setEnabled(enabled);
+ }
+ }
+
+ private void buildFilterString() {
+ StringBuilder filterString = new StringBuilder();
+
+ for (Filter filter : filters) {
+ if (!filter.condition.getText().equals("")) {
+ if (filterString.length() != 0) {
+ if (all.isChecked()) {
+ filterString.append(" AND ");
+ } else {
+ filterString.append(" OR ");
+ }
+ }
+ filterString.append(filter.dbColumnSelector.getColumn());
+ filterString.append(" ");
+ filterString.append(filter.condition.getText());
+ }
+ }
+
+ viewer.setText(filterString.toString());
+ }
+}
diff --git a/frontend/client/src/autotest/tko/FilterStringViewer.java b/frontend/client/src/autotest/tko/FilterStringViewer.java
new file mode 100644
index 0000000..0e65dc7
--- /dev/null
+++ b/frontend/client/src/autotest/tko/FilterStringViewer.java
@@ -0,0 +1,155 @@
+package autotest.tko;
+
+import autotest.common.ui.SimpleHyperlink;
+
+import com.google.gwt.user.client.DOM;
+import com.google.gwt.user.client.Window;
+import com.google.gwt.user.client.ui.Button;
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.DisclosureEvent;
+import com.google.gwt.user.client.ui.DisclosureHandler;
+import com.google.gwt.user.client.ui.DisclosurePanel;
+import com.google.gwt.user.client.ui.HorizontalPanel;
+import com.google.gwt.user.client.ui.Panel;
+import com.google.gwt.user.client.ui.TextArea;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class FilterStringViewer extends Composite {
+
+ public static final String VIEW_FILTER_STRING = "View Filter String";
+ public static final String HIDE_FILTER_STRING = "Hide Filter String";
+ public static final String EDIT_FILTER_STRING = "Edit Filter String";
+ public static final String UNEDIT_FILTER_STRING = "Revert Filter String";
+
+ public static interface EditListener {
+ public void onEdit();
+ public void onRevert();
+ }
+
+ private SimpleHyperlink view = new SimpleHyperlink(VIEW_FILTER_STRING);
+ private Button edit = new Button(EDIT_FILTER_STRING);
+ private TextArea queries = new TextArea();
+ private DisclosurePanel queriesPanel = new DisclosurePanel();
+ private boolean filterEdited = false;
+ private boolean viewerEditable = false;
+ private ArrayList<EditListener> listeners = new ArrayList<EditListener>();
+
+ public FilterStringViewer() {
+ edit.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ changeEditable(true);
+ }
+ });
+
+ queries.setSize("35em", "10em");
+ queries.setReadOnly(true);
+ queries.addChangeListener(new ChangeListener() {
+ public void onChange(Widget w) {
+ filterEdited = true;
+ }
+ });
+
+ Panel viewerHeaderPanel = new HorizontalPanel();
+ viewerHeaderPanel.add(view);
+ viewerHeaderPanel.add(edit);
+
+ queriesPanel.setHeader(viewerHeaderPanel);
+ queriesPanel.add(queries);
+
+ queriesPanel.addEventHandler(new DisclosureHandler() {
+ public void onClose(DisclosureEvent e) {
+ view.setText(VIEW_FILTER_STRING);
+ }
+
+ public void onOpen(DisclosureEvent e) {
+ view.setText(HIDE_FILTER_STRING);
+ }
+ });
+
+ initWidget(queriesPanel);
+ }
+
+ public void setText(String text) {
+ queries.setText(text);
+ }
+
+ public String getText() {
+ return queries.getText();
+ }
+
+ public void addEditListener(EditListener listener) {
+ listeners.add(listener);
+ }
+
+ protected void addToHistory(Map<String, String> args, String prefix) {
+ args.put(prefix + "_viewerOpen", String.valueOf(queriesPanel.isOpen()));
+ args.put(prefix + "_viewerEditable", String.valueOf(viewerEditable));
+ if (viewerEditable) {
+ args.put(prefix + "_viewerEdited", String.valueOf(filterEdited));
+ if (filterEdited) {
+ args.put(prefix + "_viewerText", queries.getText());
+ }
+ }
+ }
+
+ protected void handleHistoryArguments(Map<String, String> args, String prefix) {
+ // No _viewerOpen parameter. This is a preconfig without a specified custom filter.
+ if (args.get(prefix + "_viewerOpen") == null) {
+ queriesPanel.setOpen(false);
+ if (viewerEditable) {
+ changeEditable(false);
+ }
+ return;
+ }
+
+ queriesPanel.setOpen(Boolean.parseBoolean(args.get(prefix + "_viewerOpen")));
+ if (viewerEditable) {
+ changeEditable(false);
+ }
+ if (Boolean.parseBoolean(args.get(prefix + "_viewerEditable"))) {
+ changeEditable(false);
+ filterEdited = Boolean.parseBoolean(args.get(prefix + "_viewerEdited"));
+ if (filterEdited) {
+ queries.setText(args.get(prefix + "_viewerText"));
+ }
+ }
+ }
+
+ // Change the viewer's editable state
+ private void changeEditable(boolean clicked) {
+ if (clicked) {
+ DOM.eventCancelBubble(DOM.eventGetCurrentEvent(), true);
+ }
+
+ if (viewerEditable) {
+ // We only want the confirmation on revert from an edited viewer, and only if "revert"
+ // was clicked (not on programmatic revert)
+ if (filterEdited &&
+ clicked &&
+ !Window.confirm("Are you sure you want to revert your changes?")) {
+ return;
+ }
+
+ viewerEditable = false;
+ filterEdited = false;
+ queries.setReadOnly(true);
+ edit.setText(EDIT_FILTER_STRING);
+ for (EditListener listener : listeners) {
+ listener.onRevert();
+ }
+ } else {
+ viewerEditable = true;
+ queries.setReadOnly(false);
+ edit.setText(UNEDIT_FILTER_STRING);
+ queriesPanel.setOpen(true);
+ for (EditListener listener : listeners) {
+ listener.onEdit();
+ }
+ }
+ }
+}
diff --git a/frontend/client/src/autotest/tko/GraphingFrontend.java b/frontend/client/src/autotest/tko/GraphingFrontend.java
new file mode 100644
index 0000000..3881567
--- /dev/null
+++ b/frontend/client/src/autotest/tko/GraphingFrontend.java
@@ -0,0 +1,147 @@
+package autotest.tko;
+
+import autotest.common.CustomHistory;
+import autotest.common.JsonRpcCallback;
+import autotest.common.JsonRpcProxy;
+import autotest.common.Utils;
+import autotest.common.CustomHistory.CustomHistoryListener;
+import autotest.common.ui.SimpleHyperlink;
+import autotest.tko.TableView.TableSwitchListener;
+
+import com.google.gwt.json.client.JSONNumber;
+import com.google.gwt.json.client.JSONObject;
+import com.google.gwt.json.client.JSONString;
+import com.google.gwt.json.client.JSONValue;
+import com.google.gwt.user.client.Window;
+import com.google.gwt.user.client.ui.Button;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.DialogBox;
+import com.google.gwt.user.client.ui.FlexTable;
+import com.google.gwt.user.client.ui.HasVerticalAlignment;
+import com.google.gwt.user.client.ui.TextBox;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.Map;
+
+public abstract class GraphingFrontend extends Composite implements CustomHistoryListener {
+
+ public static final String HISTORY_TOKEN = "embedded_query";
+
+ protected FlexTable table = new FlexTable();
+ protected JsonRpcProxy rpcProxy = JsonRpcProxy.getProxy(JsonRpcProxy.TKO_URL);
+ protected SimpleHyperlink embeddingLink = new SimpleHyperlink("[Link to this graph]");
+ protected TableSwitchListener listener;
+
+ public abstract void refresh();
+ protected abstract void addToHistory(Map<String, String> args);
+ protected abstract void handleHistoryArguments(Map<String, String> args);
+ protected abstract void setDrilldownTrigger();
+ protected abstract void addAdditionalEmbeddingParams(JSONObject params);
+
+ protected static class GraphingDialog extends DialogBox {
+ protected GraphingDialog(String title, Widget contents) {
+ super(false, false);
+
+ FlexTable flex = new FlexTable();
+ flex.setText(0, 0, title);
+ flex.getFlexCellFormatter().setStylePrimaryName(0, 0, "field-name");
+
+ flex.setWidget(1, 0, contents);
+
+ Button ok = new Button("OK");
+ ok.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ hide();
+ }
+ });
+ flex.setWidget(2, 0, ok);
+
+ add(flex);
+ }
+ }
+
+ protected void setListener(TableSwitchListener listener) {
+ this.listener = listener;
+ }
+
+ protected void addControl(String text, Widget control) {
+ int row = table.getRowCount();
+ table.setText(row, 0, text);
+ table.getFlexCellFormatter().setStylePrimaryName(row, 0, "field-name");
+ table.setWidget(row, 1, control);
+ table.getFlexCellFormatter().setColSpan(row, 1, 2);
+ table.getFlexCellFormatter().setWidth(row, 1, "100%");
+ table.getFlexCellFormatter().setVerticalAlignment(row, 0, HasVerticalAlignment.ALIGN_TOP);
+ }
+
+ protected GraphingFrontend() {
+ CustomHistory.addHistoryListener(this);
+ setDrilldownTrigger();
+
+ embeddingLink.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ JSONObject params = new JSONObject();
+ params.put("url_token", new JSONString(CustomHistory.getLastHistoryToken()));
+ addAdditionalEmbeddingParams(params);
+
+ rpcProxy.rpcCall("get_embedding_id", params, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ String id = Utils.jsonToString(result);
+
+ StringBuilder link = new StringBuilder();
+ link.append("<a href=\"http://");
+ link.append(Window.Location.getHost());
+ link.append(Window.Location.getPath());
+ link.append("#");
+ link.append(HISTORY_TOKEN);
+ link.append("=");
+ link.append(id);
+
+ link.append("\"><img border=\"0\" src=\"http://");
+ link.append(Window.Location.getHost());
+
+ String url = JsonRpcProxy.TKO_URL;
+ link.append(url.substring(0, url.lastIndexOf('/', url.length() - 2)));
+
+ link.append("/plot/?id=");
+ link.append(id);
+
+ link.append("&max_age=10");
+
+ link.append("\"></a>");
+
+ TextBox linkBox = new TextBox();
+ linkBox.setText(link.toString());
+ linkBox.setWidth("100%");
+ linkBox.setSelectionRange(0, link.length());
+
+ new GraphingDialog("Paste HTML to embed in website:", linkBox).center();
+ }
+ });
+ }
+ });
+ }
+
+ public void onHistoryChanged(Map<String, String> arguments) {
+ final String idString = arguments.get(HISTORY_TOKEN);
+ if (idString == null) {
+ return;
+ }
+
+ JSONObject args = new JSONObject();
+ args.put("id", new JSONNumber(Integer.parseInt(idString)));
+ rpcProxy.rpcCall("get_embedded_query_url_token", args, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ String token = Utils.jsonToString(result);
+
+ // since this is happening asynchronously, the history may have changed, so ensure
+ // it's set back to what it should be.
+ CustomHistory.newItem(HISTORY_TOKEN + "=" + idString);
+ CustomHistory.simulateHistoryToken(token);
+ }
+ });
+ }
+}
diff --git a/frontend/client/src/autotest/tko/GraphingView.java b/frontend/client/src/autotest/tko/GraphingView.java
index a7b2c6d..0b47597 100644
--- a/frontend/client/src/autotest/tko/GraphingView.java
+++ b/frontend/client/src/autotest/tko/GraphingView.java
@@ -1,27 +1,66 @@
package autotest.tko;
import autotest.common.ui.TabView;
+import autotest.tko.TableView.TableSwitchListener;
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.DeckPanel;
+import com.google.gwt.user.client.ui.ListBox;
import com.google.gwt.user.client.ui.RootPanel;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.Map;
public class GraphingView extends TabView {
-
- private ExistingGraphsFrontend existingGraphsFrontend = new ExistingGraphsFrontend();
+
+ private ListBox frontendSelection = new ListBox();
+ private MetricsPlotFrontend metricsPlotFrontend = new MetricsPlotFrontend(this);
+ private MachineQualHistogramFrontend machineQualHistogramFrontend =
+ new MachineQualHistogramFrontend(this);
+ private ExistingGraphsFrontend existingGraphsFrontend = new ExistingGraphsFrontend(this);
+ private DeckPanel controlPanel = new DeckPanel();
+ private GraphingFrontend frontends[] = {
+ metricsPlotFrontend,
+ machineQualHistogramFrontend,
+ existingGraphsFrontend,
+ };
+
+ public GraphingView(TableSwitchListener listener) {
+ metricsPlotFrontend.setListener(listener);
+ machineQualHistogramFrontend.setListener(listener);
+ }
@Override
public void initialize() {
- RootPanel.get("graphing_frontend").add(existingGraphsFrontend);
+ frontendSelection.addItem("Metrics Plot");
+ frontendSelection.addItem("Machine Qualification Histogram");
+ frontendSelection.addItem("Existing Graphs");
+
+ frontendSelection.addChangeListener(new ChangeListener() {
+ public void onChange(Widget w) {
+ showSelectedView();
+ updateHistory();
+ }
+ });
+
+ controlPanel.add(metricsPlotFrontend);
+ controlPanel.add(machineQualHistogramFrontend);
+ controlPanel.add(existingGraphsFrontend);
+ controlPanel.showWidget(0);
+
+ RootPanel.get("graphing_type").add(frontendSelection);
+ RootPanel.get("graphing_frontend").add(controlPanel);
}
-
+
@Override
public String getElementId() {
return "graphing_view";
}
-
+
@Override
public void refresh() {
super.refresh();
- existingGraphsFrontend.refresh();
+ frontends[frontendSelection.getSelectedIndex()].refresh();
}
@Override
@@ -29,4 +68,31 @@
super.display();
CommonPanel.getPanel().setConditionVisible(false);
}
+
+ @Override
+ protected Map<String, String> getHistoryArguments() {
+ Map<String, String> args = super.getHistoryArguments();
+ args.put("view", frontendSelection.getValue(frontendSelection.getSelectedIndex()));
+ frontends[frontendSelection.getSelectedIndex()].addToHistory(args);
+ return args;
+ }
+
+ @Override
+ public void handleHistoryArguments(Map<String, String> arguments) {
+ super.handleHistoryArguments(arguments);
+ for (int i = 0; i < frontendSelection.getItemCount(); i++) {
+ if (frontendSelection.getValue(i).equals(arguments.get("view"))) {
+ frontendSelection.setSelectedIndex(i);
+ frontends[i].handleHistoryArguments(arguments);
+ showSelectedView();
+ break;
+ }
+ }
+ }
+
+ private void showSelectedView() {
+ int index = frontendSelection.getSelectedIndex();
+ controlPanel.showWidget(index);
+ frontends[index].refresh();
+ }
}
diff --git a/frontend/client/src/autotest/tko/MachineQualHistogramFrontend.java b/frontend/client/src/autotest/tko/MachineQualHistogramFrontend.java
new file mode 100644
index 0000000..bf74a09
--- /dev/null
+++ b/frontend/client/src/autotest/tko/MachineQualHistogramFrontend.java
@@ -0,0 +1,219 @@
+package autotest.tko;
+
+import autotest.common.JsonRpcCallback;
+import autotest.common.Utils;
+import autotest.common.ui.NotifyManager;
+import autotest.common.ui.TabView;
+import autotest.tko.TableView.TableViewConfig;
+
+import com.google.gwt.json.client.JSONNumber;
+import com.google.gwt.json.client.JSONObject;
+import com.google.gwt.json.client.JSONString;
+import com.google.gwt.json.client.JSONValue;
+import com.google.gwt.user.client.ui.Button;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.HTML;
+import com.google.gwt.user.client.ui.HasHorizontalAlignment;
+import com.google.gwt.user.client.ui.TextBox;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.Map;
+
+public class MachineQualHistogramFrontend extends GraphingFrontend {
+
+ @Override
+ protected void addToHistory(Map<String, String> args) {
+ globalFilters.addToHistory(args, "globalFilter");
+ testFilters.addToHistory(args, "testFilter");
+ args.put("interval", interval.getText());
+ }
+
+ @Override
+ protected void handleHistoryArguments(Map<String, String> args) {
+ setVisible(false);
+ globalFilters.reset();
+ testFilters.reset();
+ globalFilters.handleHistoryArguments(args, "globalFilter");
+ testFilters.handleHistoryArguments(args, "testFilter");
+ interval.setText(args.get("interval"));
+ setVisible(true);
+ }
+
+ private PreconfigSelector preconfig = new PreconfigSelector("qual", this);
+ private FilterSelector globalFilters =
+ new FilterSelector(DBColumnSelector.TEST_VIEW);
+ private FilterSelector testFilters =
+ new FilterSelector(DBColumnSelector.TEST_VIEW);
+ private TextBox interval = new TextBox();
+ private Button graphButton = new Button("Graph");
+ private HTML graph = new HTML();
+
+ public MachineQualHistogramFrontend(final TabView parent) {
+ interval.setText("10");
+
+ graphButton.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ parent.updateHistory();
+ graph.setVisible(false);
+ embeddingLink.setVisible(false);
+
+ JSONObject params = buildParams();
+ if (params == null) {
+ return;
+ }
+
+ rpcProxy.rpcCall("create_qual_histogram", params, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ graph.setHTML(Utils.jsonToString(result));
+ graph.setVisible(true);
+ embeddingLink.setVisible(true);
+ }
+ });
+ }
+ });
+
+ addControl("Preconfigured:", preconfig);
+ addControl("Global filters:", globalFilters);
+ addControl("Test set filters:", testFilters);
+ addControl("Interval:", interval);
+
+ table.setWidget(table.getRowCount(), 1, graphButton);
+ table.setWidget(table.getRowCount(), 0, graph);
+ table.getFlexCellFormatter().setColSpan(table.getRowCount() - 1, 0, 3);
+
+ table.setWidget(table.getRowCount(), 2, embeddingLink);
+ table.getFlexCellFormatter().setHorizontalAlignment(
+ table.getRowCount() - 1, 2, HasHorizontalAlignment.ALIGN_RIGHT);
+
+ graph.setVisible(false);
+ embeddingLink.setVisible(false);
+
+ initWidget(table);
+ }
+
+ @Override
+ public void refresh() {
+ // nothing to refresh
+ }
+
+ @Override
+ protected native void setDrilldownTrigger() /*-{
+ var instance = this;
+ $wnd.showQualDrilldown = function(filterString) {
+ instance.@autotest.tko.MachineQualHistogramFrontend::showDrilldown(Ljava/lang/String;)(filterString);
+ }
+ $wnd.showQualNADialog = function(hosts) {
+ instance.@autotest.tko.MachineQualHistogramFrontend::showNADialog(Ljava/lang/String;)(hosts);
+ }
+ $wnd.showQualEmptyDialog = function() {
+ instance.@autotest.tko.MachineQualHistogramFrontend::showEmptyDialog()();
+ }
+ }-*/;
+
+ @Override
+ protected void addAdditionalEmbeddingParams(JSONObject params) {
+ params.put("graph_type", new JSONString("qual"));
+ params.put("params", buildParams());
+ }
+
+ @SuppressWarnings("unused")
+ private void showDrilldown(final String filterString) {
+ CommonPanel.getPanel().setCondition(new TestSet() {
+ public String getCondition() {
+ return filterString;
+ }
+
+ public boolean isSingleTest() {
+ return false;
+ }
+ });
+ listener.onSwitchToTable(TableViewConfig.PASS_RATE);
+ }
+
+ @SuppressWarnings("unused")
+ private void showNADialog(String hosts) {
+ new GraphingDialog("Did not run any of the selected tests:", new HTML(hosts)).center();
+ }
+
+ @SuppressWarnings("unused")
+ private void showEmptyDialog() {
+ new GraphingDialog("No hosts in this pass rate range", new HTML()).center();
+ }
+
+ private JSONString buildQuery() {
+ String gFilterString = globalFilters.getFilterString();
+ String tFilterString = testFilters.getFilterString();
+ boolean hasGFilter = !gFilterString.equals("");
+ boolean hasTFilter = !tFilterString.equals("");
+
+ StringBuilder sql = new StringBuilder();
+
+ sql.append("SELECT hostname, COUNT(DISTINCT ");
+ if (hasTFilter) {
+ sql.append("IF(");
+ sql.append(tFilterString);
+ sql.append(", test_idx, NULL)");
+ } else {
+ sql.append("test_idx");
+ }
+ sql.append(") 'total', COUNT(DISTINCT IF(");
+ if (hasTFilter) {
+ sql.append("(");
+ sql.append(tFilterString);
+ sql.append(") AND ");
+ }
+
+ sql.append("status = 'GOOD', test_idx, NULL)) 'good' FROM test_view_outer_joins");
+ if (hasGFilter) {
+ sql.append(" WHERE ");
+ sql.append(gFilterString);
+ }
+ sql.append(" GROUP BY hostname");
+ return new JSONString(sql.toString());
+ }
+
+ private JSONString buildFilterString() {
+ StringBuilder filterString = new StringBuilder();
+ String gFilterString = globalFilters.getFilterString();
+ String tFilterString = testFilters.getFilterString();
+ boolean hasGFilter = !gFilterString.equals("");
+ boolean hasTFilter = !tFilterString.equals("");
+ if (hasGFilter) {
+ filterString.append("(");
+ filterString.append(gFilterString);
+ filterString.append(")");
+ if (hasTFilter) {
+ filterString.append(" AND ");
+ }
+ }
+ if (hasTFilter) {
+ filterString.append("(");
+ filterString.append(tFilterString);
+ filterString.append(")");
+ }
+ return new JSONString(filterString.toString());
+ }
+
+ private JSONObject buildParams() {
+ if (interval.getText().equals("")) {
+ NotifyManager.getInstance().showError("You must enter an interval");
+ return null;
+ }
+
+ int intervalValue;
+ try {
+ intervalValue = Integer.parseInt(interval.getText());
+ } catch (NumberFormatException e) {
+ NotifyManager.getInstance().showError("Interval must be an integer");
+ return null;
+ }
+
+ JSONObject params = new JSONObject();
+ params.put("query", buildQuery());
+ params.put("filter_string", buildFilterString());
+ params.put("interval", new JSONNumber(intervalValue));
+
+ return params;
+ }
+}
diff --git a/frontend/client/src/autotest/tko/MetricsPlotFrontend.java b/frontend/client/src/autotest/tko/MetricsPlotFrontend.java
new file mode 100644
index 0000000..3dbb93b
--- /dev/null
+++ b/frontend/client/src/autotest/tko/MetricsPlotFrontend.java
@@ -0,0 +1,431 @@
+package autotest.tko;
+
+import autotest.common.JsonRpcCallback;
+import autotest.common.Utils;
+import autotest.common.ui.NotifyManager;
+import autotest.common.ui.SimpleHyperlink;
+import autotest.common.ui.TabView;
+import autotest.tko.SeriesSelector.Series;
+
+import com.google.gwt.json.client.JSONArray;
+import com.google.gwt.json.client.JSONObject;
+import com.google.gwt.json.client.JSONString;
+import com.google.gwt.json.client.JSONValue;
+import com.google.gwt.user.client.ui.Button;
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.FlexTable;
+import com.google.gwt.user.client.ui.HTML;
+import com.google.gwt.user.client.ui.HasHorizontalAlignment;
+import com.google.gwt.user.client.ui.HorizontalPanel;
+import com.google.gwt.user.client.ui.ListBox;
+import com.google.gwt.user.client.ui.Panel;
+import com.google.gwt.user.client.ui.RadioButton;
+import com.google.gwt.user.client.ui.TextBox;
+import com.google.gwt.user.client.ui.VerticalPanel;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class MetricsPlotFrontend extends GraphingFrontend {
+
+ public static final String NORMALIZE_SINGLE = "single";
+ public static final String NORMALIZE_FIRST = "first";
+ public static final String NORMALIZE_SERIES_PREFIX = "series__";
+ public static final String NORMALIZE_X_PREFIX = "x__";
+
+ private PreconfigSelector preconfig = new PreconfigSelector("metrics", this);
+ private ListBox plotSelector = new ListBox();
+ private DBColumnSelector xAxis = new DBColumnSelector(DBColumnSelector.PERF_VIEW, true);
+ private FilterSelector globalFilter =
+ new FilterSelector(DBColumnSelector.PERF_VIEW);
+ private RadioButton noNormalizeMultiple =
+ new RadioButton("normalize", "No normalization (multiple subplots)");
+ private RadioButton noNormalizeSingle =
+ new RadioButton("normalize", "No normalization (single plot)");
+ private RadioButton normalizeFirst = new RadioButton("normalize", "First data point");
+ private RadioButton normalizeSeries = new RadioButton("normalize", "Specified series:");
+ private ListBox normalizeSeriesSelect = new ListBox();
+ private RadioButton normalizeX = new RadioButton("normalize", "Specified X-axis value:");
+ private TextBox normalizeXSelect = new TextBox();
+ private Button graphButton = new Button("Graph");
+ private HTML graph = new HTML();
+
+ private SeriesSelector seriesSelector = new SeriesSelector(new ChangeListener() {
+ public void onChange(Widget w) {
+ refreshSeries();
+ }
+ });
+
+ public MetricsPlotFrontend(final TabView parent) {
+ noNormalizeSingle.setChecked(true);
+ noNormalizeMultiple.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeXSelect.setEnabled(false);
+ checkInvertible();
+ }
+ });
+ noNormalizeSingle.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeXSelect.setEnabled(false);
+ checkInvertible();
+ }
+ });
+ normalizeFirst.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeXSelect.setEnabled(false);
+ checkInvertible();
+ }
+ });
+ normalizeSeries.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ normalizeSeriesSelect.setEnabled(true);
+ normalizeXSelect.setEnabled(false);
+ checkInvertible();
+ refreshSeries();
+ }
+ });
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeX.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeXSelect.setEnabled(true);
+ checkInvertible();
+ }
+ });
+ normalizeXSelect.setEnabled(false);
+
+ plotSelector.addItem("Line");
+ plotSelector.addItem("Bar");
+ plotSelector.addChangeListener(new ChangeListener() {
+ public void onChange(Widget sender) {
+ checkNormalizeInput();
+ }
+ });
+
+ graphButton.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ parent.updateHistory();
+ graph.setVisible(false);
+ embeddingLink.setVisible(false);
+
+ JSONObject params = buildParams();
+ if (params == null) {
+ return;
+ }
+
+ rpcProxy.rpcCall("create_metrics_plot", params, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ graph.setHTML(Utils.jsonToString(result));
+ graph.setVisible(true);
+ embeddingLink.setVisible(true);
+ }
+ });
+ }
+ });
+
+ Panel normalizePanel = new VerticalPanel();
+ normalizePanel.add(noNormalizeMultiple);
+ normalizePanel.add(noNormalizeSingle);
+ Panel seriesPanel = new HorizontalPanel();
+ seriesPanel.add(normalizeSeries);
+ seriesPanel.add(normalizeSeriesSelect);
+ normalizePanel.add(seriesPanel);
+ normalizePanel.add(normalizeFirst);
+ Panel baselinePanel = new HorizontalPanel();
+ baselinePanel.add(normalizeX);
+ baselinePanel.add(normalizeXSelect);
+ normalizePanel.add(baselinePanel);
+
+ addControl("Preconfigured:", preconfig);
+ addControl("Plot:", plotSelector);
+ addControl("X-axis values:", xAxis);
+ addControl("Global filters:", globalFilter);
+ addControl("Series:", seriesSelector);
+ addControl("Normalize to:", normalizePanel);
+
+ table.setWidget(table.getRowCount(), 1, graphButton);
+ table.setWidget(table.getRowCount(), 0, graph);
+ table.getFlexCellFormatter().setColSpan(table.getRowCount() - 1, 0, 3);
+
+ table.setWidget(table.getRowCount(), 2, embeddingLink);
+ table.getFlexCellFormatter().setHorizontalAlignment(
+ table.getRowCount() - 1, 2, HasHorizontalAlignment.ALIGN_RIGHT);
+
+ graph.setVisible(false);
+ embeddingLink.setVisible(false);
+
+ initWidget(table);
+ }
+
+ @Override
+ public void refresh() {
+ // Nothing to refresh
+ }
+
+ @Override
+ protected void addToHistory(Map<String, String> args) {
+ String plot = plotSelector.getValue(plotSelector.getSelectedIndex());
+ args.put("plot", plot);
+ args.put("xAxis", xAxis.getColumn());
+ globalFilter.addToHistory(args, "globalFilter");
+ seriesSelector.addToHistory(args);
+ if (plot.equals("Line") && noNormalizeSingle.isChecked()) {
+ args.put("normalize", "single");
+ } else if (normalizeFirst.isChecked()) {
+ args.put("normalize", "first");
+ } else if (normalizeSeries.isChecked()) {
+ String series =
+ normalizeSeriesSelect.getValue(normalizeSeriesSelect.getSelectedIndex());
+ args.put("normalize", "series__" + series);
+ } else if (normalizeX.isChecked()) {
+ String baseline = normalizeXSelect.getText();
+ args.put("normalize", "x__" + baseline);
+ }
+ }
+
+ @Override
+ protected void handleHistoryArguments(Map<String, String> args) {
+ setVisible(false);
+ graph.setVisible(false);
+ embeddingLink.setVisible(false);
+ globalFilter.reset();
+ seriesSelector.reset();
+ for (int i = 0; i < plotSelector.getItemCount(); i++) {
+ if (plotSelector.getValue(i).equals(args.get("plot"))) {
+ plotSelector.setSelectedIndex(i);
+ break;
+ }
+ }
+
+ xAxis.selectColumn(args.get("xAxis"));
+ globalFilter.handleHistoryArguments(args, "globalFilter");
+ seriesSelector.handleHistoryArguments(args);
+
+ refreshSeries();
+ noNormalizeMultiple.setChecked(true);
+ normalizeSeriesSelect.setEnabled(false);
+ normalizeXSelect.setEnabled(false);
+ String normalizeString = args.get("normalize");
+ if (normalizeString != null) {
+ if (normalizeString.equals("single")) {
+ noNormalizeSingle.setChecked(true);
+ } else if (normalizeString.equals("first")) {
+ normalizeFirst.setChecked(true);
+ } else if (normalizeString.startsWith(NORMALIZE_SERIES_PREFIX)) {
+ normalizeSeries.setChecked(true);
+ String series = normalizeString.substring(NORMALIZE_SERIES_PREFIX.length());
+ for (int i = 0; i < normalizeSeriesSelect.getItemCount(); i++) {
+ if (normalizeSeriesSelect.getValue(i).equals(series)) {
+ normalizeSeriesSelect.setSelectedIndex(i);
+ break;
+ }
+ }
+ normalizeSeriesSelect.setEnabled(true);
+ } else if (normalizeString.startsWith(NORMALIZE_X_PREFIX)) {
+ normalizeX.setChecked(true);
+ normalizeXSelect.setText(normalizeString.substring(NORMALIZE_X_PREFIX.length()));
+ normalizeXSelect.setEnabled(true);
+ }
+ }
+ checkNormalizeInput();
+ checkInvertible();
+
+ setVisible(true);
+ }
+
+ @Override
+ protected void addAdditionalEmbeddingParams(JSONObject params) {
+ params.put("graph_type", new JSONString("metrics"));
+ params.put("params", buildParams());
+ }
+
+ private void refreshSeries() {
+ int selectedIndex = normalizeSeriesSelect.getSelectedIndex();
+ String selectedValue = null;
+ if (selectedIndex != -1) {
+ selectedValue = normalizeSeriesSelect.getValue(selectedIndex);
+ }
+ normalizeSeriesSelect.clear();
+ for (Series selector : seriesSelector.getAllSeries()) {
+ normalizeSeriesSelect.addItem(selector.getName());
+ if (selector.getName().equals(selectedValue)) {
+ normalizeSeriesSelect.setSelectedIndex(normalizeSeriesSelect.getItemCount() - 1);
+ }
+ }
+ }
+
+ @Override
+ protected native void setDrilldownTrigger() /*-{
+ var instance = this;
+ $wnd.showMetricsDrilldown = function(query, series, param) {
+ instance.@autotest.tko.MetricsPlotFrontend::showDrilldown(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)(query, series, param);
+ }
+ }-*/;
+
+ @SuppressWarnings("unused")
+ private void showDrilldown(String query, final String series, final String param) {
+ JSONObject params = new JSONObject();
+ params.put("query", new JSONString(query));
+ params.put("param", new JSONString(param));
+ rpcProxy.rpcCall("execute_query_with_param", params, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ JSONArray data = result.isArray();
+
+ String title = series + " for " + param;
+ FlexTable contents = new FlexTable();
+ final GraphingDialog drill = new GraphingDialog(title, contents);
+
+ for (int i = 0; i < data.size(); i++) {
+ final JSONArray row = data.get(i).isArray();
+ SimpleHyperlink link = new SimpleHyperlink(Utils.jsonToString(row.get(1)));
+ link.addClickListener(new ClickListener() {
+ public void onClick(Widget sender) {
+ drill.hide();
+ listener.onSelectTest((int) row.get(0).isNumber().doubleValue());
+ }
+ });
+ contents.setWidget(i, 0, link);
+ }
+
+ drill.center();
+ }
+ });
+ }
+
+ // Disable "No Normalization (multiple)" for bar charts
+ private void checkNormalizeInput() {
+ if (plotSelector.getValue(plotSelector.getSelectedIndex()).equals("Line")) {
+ noNormalizeMultiple.setEnabled(true);
+ } else {
+ noNormalizeMultiple.setEnabled(false);
+ if (noNormalizeMultiple.isChecked()) {
+ noNormalizeSingle.setChecked(true);
+ }
+ }
+ }
+
+ private JSONObject buildQueries() {
+ ArrayList<Series> seriesList = seriesSelector.getAllSeries();
+ JSONObject queries = new JSONObject();
+ StringBuilder sql = new StringBuilder();
+
+ sql.append("SELECT ");
+ sql.append(xAxis.getColumn());
+ for (Series series : seriesList) {
+ DBColumnSelector valueSelector = series.getDBColumnSelector();
+
+ StringBuilder ifClause = new StringBuilder();
+ String seriesFilter = series.getFilterString();
+ if (!seriesFilter.equals("")) {
+ ifClause.append("IF(");
+ ifClause.append(seriesFilter);
+ ifClause.append(", ");
+ }
+ ifClause.append(valueSelector.getColumn());
+ if (!seriesFilter.equals("")) {
+ ifClause.append(", NULL)");
+ }
+
+ sql.append(", ");
+ sql.append(series.getAggregation());
+ sql.append(ifClause);
+ sql.append(") '");
+ sql.append(series.getName());
+ sql.append("'");
+ if (series.wantErrorBars()) {
+ sql.append(", STDDEV(");
+ sql.append(ifClause);
+ sql.append(") 'errors-");
+ sql.append(series.getName());
+ sql.append("'");
+ }
+ }
+
+ sql.append(" FROM perf_view_2");
+
+ String xFilterString = globalFilter.getFilterString();
+ if (xFilterString.equals("")) {
+ NotifyManager.getInstance().showError("You must enter a global filter");
+ return null;
+ }
+
+ sql.append(" WHERE ");
+ sql.append(xFilterString);
+
+ sql.append(" GROUP BY ");
+ sql.append(xAxis.getColumn());
+ queries.put("__main__", new JSONString(sql.toString()));
+
+ for (Series series : seriesList) {
+ sql = new StringBuilder();
+ DBColumnSelector valueSelector = series.getDBColumnSelector();
+
+ sql.append("SELECT test_idx, ");
+ sql.append(valueSelector.getColumn());
+ sql.append(" FROM perf_view_2 WHERE ");
+
+ String seriesFilter = series.getFilterString();
+ if (!xFilterString.equals("") || !seriesFilter.equals("")) {
+ sql.append(xFilterString.replace("%", "%%"));
+ if (!xFilterString.equals("") && !seriesFilter.equals("")) {
+ sql.append(" AND ");
+ }
+ sql.append(seriesFilter.replace("%", "%%"));
+ sql.append(" AND ");
+ }
+
+ sql.append(xAxis.getColumn());
+ sql.append(" = %s ORDER BY ");
+ sql.append(valueSelector.getColumn());
+ queries.put("__" + series.getName() + "__", new JSONString(sql.toString()));
+ }
+
+ return queries;
+ }
+
+ // Disable the "Invert y-axis" checkboxes if inversion doesn't make sense
+ private void checkInvertible() {
+ boolean invertible = (
+ noNormalizeMultiple.isChecked() ||
+ normalizeFirst.isChecked() ||
+ normalizeX.isChecked());
+ seriesSelector.setInvertible(invertible);
+ }
+
+ private JSONObject buildParams() {
+ JSONObject queries = buildQueries();
+ if (queries == null) {
+ return null;
+ }
+
+ JSONObject params = new JSONObject();
+
+ params.put("queries", queries);
+ String plot = plotSelector.getValue(plotSelector.getSelectedIndex());
+ params.put("plot", new JSONString(plot));
+
+ if (plot.equals("Line") && noNormalizeSingle.isChecked()) {
+ params.put("normalize", new JSONString(NORMALIZE_SINGLE));
+ } else if (normalizeFirst.isChecked()) {
+ params.put("normalize", new JSONString(NORMALIZE_FIRST));
+ } else if (normalizeSeries.isChecked()) {
+ String series =
+ normalizeSeriesSelect.getValue(normalizeSeriesSelect.getSelectedIndex());
+ params.put("normalize", new JSONString(NORMALIZE_SERIES_PREFIX + series));
+ } else if (normalizeX.isChecked()) {
+ String baseline = normalizeXSelect.getText();
+ params.put("normalize", new JSONString(NORMALIZE_X_PREFIX + baseline));
+ }
+
+ params.put("invert", Utils.stringsToJSON(seriesSelector.getInverted()));
+
+ return params;
+ }
+}
diff --git a/frontend/client/src/autotest/tko/PreconfigSelector.java b/frontend/client/src/autotest/tko/PreconfigSelector.java
new file mode 100644
index 0000000..bcb0b44
--- /dev/null
+++ b/frontend/client/src/autotest/tko/PreconfigSelector.java
@@ -0,0 +1,65 @@
+package autotest.tko;
+
+import autotest.common.JsonRpcCallback;
+import autotest.common.JsonRpcProxy;
+import autotest.common.StaticDataRepository;
+import autotest.common.Utils;
+
+import com.google.gwt.json.client.JSONObject;
+import com.google.gwt.json.client.JSONString;
+import com.google.gwt.json.client.JSONValue;
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.ListBox;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class PreconfigSelector extends Composite {
+
+ public static final String NO_PRECONFIG = "----------";
+
+ private ListBox selector = new ListBox();
+ private JsonRpcProxy rpcProxy = JsonRpcProxy.getProxy();
+
+ public PreconfigSelector(final String preconfigType, final GraphingFrontend parent) {
+ selector.addItem(NO_PRECONFIG);
+ StaticDataRepository staticData = StaticDataRepository.getRepository();
+ JSONObject preconfigs = staticData.getData("preconfigs").isObject();
+ Set<String> keys = preconfigs.get(preconfigType).isObject().keySet();
+ for (String key : keys) {
+ selector.addItem(key);
+ }
+
+ selector.addChangeListener(new ChangeListener() {
+ public void onChange(Widget sender) {
+ String name = selector.getValue(selector.getSelectedIndex());
+
+ if (name.equals(NO_PRECONFIG)) {
+ return;
+ }
+
+ selector.setSelectedIndex(0);
+
+ JSONObject params = new JSONObject();
+ params.put("name", new JSONString(name));
+ params.put("type", new JSONString(preconfigType));
+ rpcProxy.rpcCall("get_preconfig", params, new JsonRpcCallback() {
+ @Override
+ public void onSuccess(JSONValue result) {
+ JSONObject config = result.isObject();
+ Map<String, String> map = new HashMap<String, String>();
+ for (String key : config.keySet()) {
+ map.put(key, Utils.jsonToString(config.get(key)));
+ }
+ parent.handleHistoryArguments(map);
+ }
+ });
+ }
+ });
+
+ initWidget(selector);
+ }
+}
diff --git a/frontend/client/src/autotest/tko/SeriesSelector.java b/frontend/client/src/autotest/tko/SeriesSelector.java
new file mode 100644
index 0000000..9751fbd
--- /dev/null
+++ b/frontend/client/src/autotest/tko/SeriesSelector.java
@@ -0,0 +1,262 @@
+package autotest.tko;
+
+import autotest.common.Utils;
+import autotest.common.ui.SimpleHyperlink;
+
+import com.google.gwt.user.client.ui.ChangeListener;
+import com.google.gwt.user.client.ui.CheckBox;
+import com.google.gwt.user.client.ui.ClickListener;
+import com.google.gwt.user.client.ui.Composite;
+import com.google.gwt.user.client.ui.FlexTable;
+import com.google.gwt.user.client.ui.HasHorizontalAlignment;
+import com.google.gwt.user.client.ui.HasVerticalAlignment;
+import com.google.gwt.user.client.ui.HorizontalPanel;
+import com.google.gwt.user.client.ui.ListBox;
+import com.google.gwt.user.client.ui.Panel;
+import com.google.gwt.user.client.ui.TextBox;
+import com.google.gwt.user.client.ui.Widget;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SeriesSelector extends Composite {
+
+ private FlexTable table = new FlexTable();
+ private ArrayList<Series> series = new ArrayList<Series>();
+ private SimpleHyperlink addLink = new SimpleHyperlink("[Add Series]");
+ private boolean enabled = true;
+ private boolean invertible = false;
+ private final ChangeListener listener;
+
+ public class Series extends Composite {
+
+ private FlexTable seriesTable = new FlexTable();
+ private TextBox name = new TextBox();
+ private CheckBox invert = new CheckBox("Invert y-axis");
+ private DBColumnSelector values = new DBColumnSelector(DBColumnSelector.PERF_VIEW);
+ private ListBox aggregation = new ListBox();
+ private CheckBox errorBars = new CheckBox();
+ private FilterSelector filter = new FilterSelector(DBColumnSelector.PERF_VIEW);
+ private SimpleHyperlink deleteLink = new SimpleHyperlink("Delete Series");
+ private int row;
+
+ private Series(int aRow) {
+ this.row = aRow;
+ deleteLink.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ if (enabled) {
+ deleteSeries(row);
+ }
+ }
+ });
+
+ name.addChangeListener(listener);
+
+ aggregation.addItem("AVG", "AVG(");
+ aggregation.addItem("COUNT (DISTINCT)", "COUNT(DISTINCT ");
+ aggregation.addItem("MIN", "MIN(");
+ aggregation.addItem("MAX", "MAX(");
+ aggregation.addChangeListener(new ChangeListener() {
+ public void onChange(Widget w) {
+ int index = aggregation.getSelectedIndex();
+ if (index == -1) {
+ return;
+ }
+
+ if (aggregation.getValue(index).equals("AVG(")) {
+ errorBars.setEnabled(true);
+ } else {
+ errorBars.setEnabled(false);
+ }
+ }
+ });
+
+ errorBars.setText("error bars");
+
+ Panel aggregationPanel = new HorizontalPanel();
+ aggregationPanel.add(aggregation);
+ aggregationPanel.add(errorBars);
+
+ addControl("Name:", name);
+ addControl("Values:", values);
+ addControl("Aggregation:", aggregationPanel);
+ addControl("Filters:", filter);
+ seriesTable.getFlexCellFormatter().setVerticalAlignment(
+ seriesTable.getRowCount() - 1, 0, HasVerticalAlignment.ALIGN_TOP);
+
+ seriesTable.setWidget(seriesTable.getRowCount() - 1, 2, deleteLink);
+ seriesTable.getFlexCellFormatter().setHorizontalAlignment(
+ seriesTable.getRowCount() - 1, 2, HasHorizontalAlignment.ALIGN_RIGHT);
+ seriesTable.getFlexCellFormatter().setVerticalAlignment(
+ seriesTable.getRowCount() - 1, 2, HasVerticalAlignment.ALIGN_BOTTOM);
+
+ seriesTable.setWidget(0, 2, invert);
+ seriesTable.getFlexCellFormatter().setHorizontalAlignment(
+ 0, 2, HasHorizontalAlignment.ALIGN_RIGHT);
+
+ initWidget(seriesTable);
+ }
+
+ private void addControl(String text, Widget control) {
+ int nextRow = seriesTable.getRowCount();
+ seriesTable.setText(nextRow, 0, text);
+ seriesTable.getFlexCellFormatter().setStylePrimaryName(nextRow, 0, "field-name");
+ seriesTable.setWidget(nextRow, 1, control);
+ }
+
+ public String getAggregation() {
+ return aggregation.getValue(aggregation.getSelectedIndex());
+ }
+
+ public DBColumnSelector getDBColumnSelector() {
+ return values;
+ }
+
+ public boolean wantErrorBars() {
+ int index = aggregation.getSelectedIndex();
+ return (index != -1 &&
+ aggregation.getValue(index).equals("AVG(") &&
+ errorBars.isChecked());
+ }
+
+ public String getName() {
+ return name.getText();
+ }
+
+ public String getFilterString() {
+ return filter.getFilterString();
+ }
+ }
+
+ public SeriesSelector(ChangeListener listener) {
+ this.listener = listener;
+
+ addLink.addClickListener(new ClickListener() {
+ public void onClick(Widget w) {
+ if (enabled) {
+ addSeries();
+ }
+ }
+ });
+ table.setWidget(0, 0, addLink);
+ table.setText(0, 1, "");
+
+ addSeries();
+
+ initWidget(table);
+ }
+
+ public ArrayList<Series> getAllSeries() {
+ return series;
+ }
+
+ public void reset() {
+ for (int i = 0; i < series.size(); i++) {
+ table.removeRow(0);
+ }
+ series.clear();
+ addSeries();
+ }
+
+ public List<String> getInverted() {
+ List<String> inverted = new ArrayList<String>();
+ for (Series s : series) {
+ if (s.invert.isChecked()) {
+ inverted.add(s.getName());
+ }
+ }
+ return inverted;
+ }
+
+ public void setInvertible(boolean invertible) {
+ for (Series s : series) {
+ s.invert.setEnabled(invertible);
+ if (!invertible) {
+ s.invert.setChecked(false);
+ }
+ }
+ this.invertible = invertible;
+ }
+
+ protected void addToHistory(Map<String, String> args) {
+ for (int index = 0; index < series.size(); index++) {
+ Series s = series.get(index);
+ args.put("name[" + index + "]", s.getName());
+ args.put("values[" + index + "]", s.getDBColumnSelector().getColumn());
+ args.put("aggregation[" + index + "]",
+ s.aggregation.getItemText(s.aggregation.getSelectedIndex()));
+ args.put("errorBars[" + index + "]", String.valueOf(s.wantErrorBars()));
+ s.filter.addToHistory(args, "seriesFilters[" + index + "]");
+ }
+ List<String> inverted = getInverted();
+ if (!inverted.isEmpty()) {
+ args.put("inverted", Utils.joinStrings(",", inverted));
+ System.out.println(args.get("inverted"));
+ }
+ }
+
+ protected void handleHistoryArguments(Map<String, String> args) {
+ int index = 0;
+
+ String invertedString = args.get("inverted");
+ Set<String> inverted = null;
+ if (invertedString != null) {
+ inverted = new HashSet<String>();
+ for (String s : invertedString.split(",")) {
+ inverted.add(s);
+ }
+ }
+
+ String name;
+ while ((name = args.get("name[" + index + "]")) != null) {
+ Series s;
+ if (index == 0) {
+ s = (Series) table.getWidget(0, 0);
+ } else {
+ s = addSeries();
+ }
+ s.name.setText(name);
+ s.getDBColumnSelector().selectColumn(args.get("values[" + index + "]"));
+ String aggregation = args.get("aggregation[" + index + "]");
+ for (int i = 0; i < s.aggregation.getItemCount(); i++) {
+ if (s.aggregation.getItemText(i).equals(aggregation)) {
+ s.aggregation.setSelectedIndex(i);
+ break;
+ }
+ }
+ s.errorBars.setChecked(Boolean.parseBoolean(args.get("errorBars[" + index + "]")));
+ s.filter.handleHistoryArguments(args, "seriesFilters[" + index + "]");
+ s.invert.setChecked(inverted != null && inverted.contains(name));
+
+ index++;
+ }
+ }
+
+ private Series addSeries() {
+ int row = table.getRowCount() - 1;
+ Series nextSeries = new Series(row);
+ nextSeries.invert.setEnabled(invertible);
+ series.add(nextSeries);
+ table.insertRow(row);
+ table.setWidget(row, 0, nextSeries);
+ table.getFlexCellFormatter().setColSpan(row, 0, 2);
+ table.getFlexCellFormatter().setStylePrimaryName(row, 0, "box");
+ return nextSeries;
+ }
+
+ private void deleteSeries(int row) {
+ if (series.size() == 1) {
+ reset();
+ return;
+ }
+
+ series.remove(row);
+ table.removeRow(row);
+ for (int i = row; i < series.size(); i++) {
+ series.get(i).row--;
+ }
+ }
+}
diff --git a/frontend/client/src/autotest/tko/SpreadsheetView.java b/frontend/client/src/autotest/tko/SpreadsheetView.java
index ddbc80e..1232892 100644
--- a/frontend/client/src/autotest/tko/SpreadsheetView.java
+++ b/frontend/client/src/autotest/tko/SpreadsheetView.java
@@ -16,6 +16,8 @@
import autotest.tko.Spreadsheet.Header;
import autotest.tko.Spreadsheet.HeaderImpl;
import autotest.tko.Spreadsheet.SpreadsheetListener;
+import autotest.tko.TableView.TableSwitchListener;
+import autotest.tko.TableView.TableViewConfig;
import autotest.tko.TkoUtils.FieldInfo;
import com.google.gwt.json.client.JSONArray;
@@ -49,7 +51,7 @@
private static JsonRpcProxy rpcProxy = JsonRpcProxy.getProxy();
private static JsonRpcProxy afeRpcProxy = JsonRpcProxy.getProxy(JsonRpcProxy.AFE_URL);
- private SpreadsheetViewListener listener;
+ private TableSwitchListener listener;
protected Header currentRowFields;
protected Header currentColumnFields;
protected Map<String,String[]> drilldownMap = new HashMap<String,String[]>();
@@ -69,11 +71,7 @@
private boolean currentShowIncomplete;
private boolean notYetQueried = true;
- public static interface SpreadsheetViewListener extends TestSelectionListener {
- public void onSwitchToTable(boolean isTriageView);
- }
-
- public SpreadsheetView(SpreadsheetViewListener listener) {
+ public SpreadsheetView(TableSwitchListener listener) {
this.listener = listener;
commonPanel.addListener(this);
}
@@ -456,7 +454,13 @@
private void switchToTable(final TestSet tests, boolean isTriageView) {
commonPanel.setCondition(tests);
- listener.onSwitchToTable(isTriageView);
+ TableViewConfig config;
+ if (isTriageView) {
+ config = TableViewConfig.TRIAGE;
+ } else {
+ config = TableViewConfig.DEFAULT;
+ }
+ listener.onSwitchToTable(config);
}
public ContextMenu getActionMenu() {
diff --git a/frontend/client/src/autotest/tko/TableView.java b/frontend/client/src/autotest/tko/TableView.java
index 9675dab..e5d084a 100644
--- a/frontend/client/src/autotest/tko/TableView.java
+++ b/frontend/client/src/autotest/tko/TableView.java
@@ -52,11 +52,16 @@
{"test_idx", "test_name", "job_tag", "hostname", "status"};
private static final String[] TRIAGE_GROUP_COLUMNS =
{"test_name", "status", "reason"};
+ private static final String[] PASS_RATE_GROUP_COLUMNS =
+ {"hostname"};
private static final SortSpec[] TRIAGE_SORT_SPECS = {
new SortSpec("test_name", SortDirection.ASCENDING),
new SortSpec("status", SortDirection.ASCENDING),
new SortSpec("reason", SortDirection.ASCENDING),
};
+ private static final SortSpec[] PASS_RATE_SORT_SPECS = {
+ new SortSpec("test_name", SortDirection.ASCENDING)
+ };
private static final String COUNT_NAME = "Count in group";
private static final String STATUS_COUNTS_NAME = "Test pass rate";
@@ -80,6 +85,14 @@
private List<SortSpec> tableSorts = new ArrayList<SortSpec>();
private Map<String, String> fieldNames = new HashMap<String, String>();
+ public enum TableViewConfig {
+ DEFAULT, PASS_RATE, TRIAGE
+ }
+
+ public static interface TableSwitchListener extends TestSelectionListener {
+ public void onSwitchToTable(TableViewConfig config);
+ }
+
public TableView(TestSelectionListener listener) {
this.listener = listener;
commonPanel.addListener(this);
@@ -141,6 +154,17 @@
// need to copy it so we can mutate it
tableSorts = new ArrayList<SortSpec>(Arrays.asList(TRIAGE_SORT_SPECS));
}
+
+ public void setupPassRate() {
+ // easier if we ensure it's deselected and then select it
+ selectColumns(PASS_RATE_GROUP_COLUMNS);
+ statusGroupCheckbox.setChecked(true);
+ groupCheckbox.setChecked(false);
+ updateCheckboxes();
+
+ // need to copy it so we can mutate it
+ tableSorts = new ArrayList<SortSpec>(Arrays.asList(PASS_RATE_SORT_SPECS));
+ }
private void createTable() {
int numColumns = fields.size();
diff --git a/frontend/client/src/autotest/tko/TkoClient.java b/frontend/client/src/autotest/tko/TkoClient.java
index 01791eb..c3a44dc 100644
--- a/frontend/client/src/autotest/tko/TkoClient.java
+++ b/frontend/client/src/autotest/tko/TkoClient.java
@@ -5,12 +5,13 @@
import autotest.common.StaticDataRepository;
import autotest.common.ui.CustomTabPanel;
import autotest.common.ui.NotifyManager;
-import autotest.tko.SpreadsheetView.SpreadsheetViewListener;
+import autotest.tko.TableView.TableSwitchListener;
+import autotest.tko.TableView.TableViewConfig;
import com.google.gwt.core.client.EntryPoint;
import com.google.gwt.user.client.ui.RootPanel;
-public class TkoClient implements EntryPoint, SpreadsheetViewListener {
+public class TkoClient implements EntryPoint, TableSwitchListener {
private CommonPanel commonPanel;
private SpreadsheetView spreadsheetView;
private TableView tableView;
@@ -37,7 +38,7 @@
commonPanel = CommonPanel.getPanel();
spreadsheetView = new SpreadsheetView(this);
tableView = new TableView(this);
- graphingView = new GraphingView();
+ graphingView = new GraphingView(this);
detailView = new TestDetailView();
mainTabPanel.getCommonAreaPanel().add(commonPanel);
@@ -57,12 +58,18 @@
tabsRoot.removeStyleName("hidden");
}
- public void onSwitchToTable(boolean isTriageView) {
+ public void onSwitchToTable(TableViewConfig config) {
tableView.ensureInitialized();
- if (isTriageView) {
- tableView.setupJobTriage();
- } else {
- tableView.setupDefaultView();
+ switch (config) {
+ case TRIAGE:
+ tableView.setupJobTriage();
+ break;
+ case PASS_RATE:
+ tableView.setupPassRate();
+ break;
+ default:
+ tableView.setupDefaultView();
+ break;
}
tableView.doQuery();
mainTabPanel.selectTabView(tableView);
diff --git a/global_config.ini b/global_config.ini
index 7d65c3c..35cd7c6 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -10,6 +10,7 @@
query_timeout: 3600
min_retry_delay: 20
max_retry_delay: 60
+graph_cache_creation_timeout_minutes: 10
[AUTOTEST_WEB]
host: localhost
diff --git a/new_tko/tko/graphing_utils.py b/new_tko/tko/graphing_utils.py
new file mode 100644
index 0000000..57ffb6a
--- /dev/null
+++ b/new_tko/tko/graphing_utils.py
@@ -0,0 +1,688 @@
+import base64, os, tempfile, operator, pickle, datetime, django.db
+from math import sqrt
+
+os.environ['HOME'] = tempfile.gettempdir()
+
+import matplotlib
+matplotlib.use('Agg')
+
+import matplotlib.figure, matplotlib.backends.backend_agg
+import StringIO, colorsys, PIL.Image, PIL.ImageChops
+from autotest_lib.frontend.afe import readonly_connection
+from autotest_lib.frontend.afe.model_logic import ValidationError
+from autotest_lib.client.common_lib import global_config
+from new_tko.tko import models, tko_rpc_utils
+
+
+class NoDataError(Exception):
+ """\
+ Exception to raise if the graphing query returned an empty resultset.
+ """
+
+
+def _colors(n):
+ """\
+ Returns a generator function for creating n colors. The return value is a
+ tuple representing the RGB of the color.
+ """
+
+ incr = 1.0 / n
+ hue = 0.0
+ for i in xrange(n):
+ yield colorsys.hsv_to_rgb(hue, 1.0, 1.0)
+ hue += incr
+
+
+def _resort(kernel_labels, list_to_sort):
+ """\
+ Resorts a list, using a list of kernel strings as the keys. Returns the
+ resorted list.
+ """
+
+ labels = [tko_rpc_utils.KernelString(label) for label in kernel_labels]
+ resorted_pairs = sorted(zip(labels, list_to_sort))
+
+ # We only want the resorted list; we are not interested in the kernel
+ # strings.
+ resorted_list = [pair[1] for pair in resorted_pairs]
+ return resorted_list
+
+
+_tmpl = """\
+<html><head></head><body>
+<img src="data:image/png;base64,%s" usemap="#%s"
+ border="0" alt="graph">
+<map name="%s">%s</map>
+</body></html>"""
+
+_area = """\
+<area shape="rect" coords="%i,%i,%i,%i" title="%s"
+href="#"
+onclick="%s(%s); return false;">"""
+
+
+def _create_figure(height_inches):
+ """\
+ Creates an instance of matplotlib.figure.Figure, given the height in inches.
+ Returns the figure and the height in pixels.
+ """
+
+ dpi = 100;
+ fig = matplotlib.figure.Figure(figsize=(10, 2 + height_inches),
+ dpi=dpi, facecolor='white')
+ fig.subplots_adjust(bottom=2.0/height_inches)
+ return (fig, fig.get_figheight() * dpi)
+
+
+def _create_line(plots, labels, queries, invert, single):
+ """\
+ Given all the data for the metrics, create a line plot.
+
+ plots: dictionary containing the plot data.
+ x: list of x-values for the plot
+ y: list of corresponding y-values
+ errors: errors for each data point, or None if no error information
+ available
+ label: plot title
+ labels: x-tick labels
+ queries: dictionary containing the relevant drilldown queries for series
+ invert: list of series that should have an inverted y-axis
+ single: True if this should be a single plot, False for multiple subplots
+ """
+
+ area_data = []
+ lines = []
+ if single:
+ h = 6
+ else:
+ h = 4 * len(plots)
+ fig, height = _create_figure(h)
+ plot_index = 1
+
+ if single:
+ sub = fig.add_subplot(1,1,1)
+
+ # Plot all the data
+ for plot, color in zip(plots, _colors(len(plots))):
+ needs_invert = (plot['label'] in invert)
+
+ # Add a new subplot, if user wants multiple subplots
+ # Also handle axis inversion for subplots here
+ if not single:
+ sub = fig.add_subplot(len(plots), 1, plot_index)
+ sub.set_title(plot['label'])
+ if needs_invert:
+ sub.set_ylim(1,0)
+ elif needs_invert:
+ plot['y'] = [-y for y in plot['y']]
+
+ # Plot the series
+ sub.set_xticks(range(0, len(labels)))
+ sub.set_xlim(-1, len(labels))
+ if single:
+ lines += sub.plot(plot['x'], plot['y'], label=plot['label'],
+ marker='o', markersize=4)
+ color = lines[-1].get_color()
+ else:
+ lines += sub.plot(plot['x'], plot['y'], 'bs-', label=plot['label'])
+ color = 'r'
+ if plot['errors']:
+ sub.errorbar(plot['x'], plot['y'], linestyle='None',
+ yerr=plot['errors'], color=color)
+ sub.set_xticklabels([])
+
+ plot_index += 1
+
+ # Construct the information for the drilldowns
+ for line in lines:
+
+ # Get the pixel coordinates of each point on the figure
+ x = line.get_xdata()
+ y = line.get_ydata()
+ label = line.get_label()
+ icoords = line.get_transform().transform(zip(x,y))
+
+ # Get the appropriate drilldown query
+ drill = "'%s'" % (queries['__' + label + '__'].replace("'", "\\'"))
+
+ # Set the title attributes (hover-over tool-tips)
+ x_labels = [labels[x_val] for x_val in x]
+ titles = ['%s - %s: %f' % (label, x_label, y_val)
+ for x_label, y_val in zip(x_labels, y)]
+
+ # Get the appropriate parameters for the drilldown query
+ params = [[drill, "'%s'" % (line.get_label()), "'%s'" % x_label]
+ for x_label in x_labels]
+
+ area_data += [(ix - 5, height - iy - 5, ix + 5, height - iy + 5,
+ title, 'showMetricsDrilldown', ','.join(param))
+ for (ix, iy), title, param
+ in zip(icoords, titles, params)]
+
+ sub.set_xticklabels(labels, rotation=90, size='x-small')
+
+ # Show the legend if there are not multiple subplots
+ if single:
+ prop = matplotlib.font_manager.FontProperties(size='xx-small')
+ legend = fig.legend(lines, [plot['label'] for plot in plots],
+ prop=prop, handlelen=0.03, numpoints=3)
+ # workaround for matplotlib not keeping all line markers in the legend
+ lines = legend.get_lines()
+ for line in lines:
+ line.set_marker('o')
+
+ return (fig, area_data)
+
+
+def _get_adjusted_bar(x, width, index, num_plots):
+ """\
+ Adjust the list 'x' to take the multiple series into account. Each series
+ should be shifted to the right by the width of a bar.
+ """
+ adjust = width * (index - 0.5 * num_plots - 1)
+ return [x_val + adjust for x_val in x]
+
+
+def _create_bar(plots, labels, queries, invert):
+ """\
+ Given all the data for the metrics, create a line plot.
+
+ plots: dictionary containing the plot data.
+ x: list of x-values for the plot
+ y: list of corresponding y-values
+ errors: errors for each data point, or None if no error information
+ available
+ label: plot title
+ labels: x-tick labels
+ queries: dictionary containing the relevant drilldown queries for series
+ invert: list of series that should have an inverted y-axis
+ """
+
+ area_data = []
+ bars = []
+ fig, height = _create_figure(6)
+
+ # Set up the plot
+ sub = fig.add_subplot(1,1,1)
+ sub.set_xticks(range(0, len(labels)))
+ sub.set_xlim(-1, len(labels))
+ sub.set_xticklabels(labels, rotation=90, size=8)
+ sub.axhline(linewidth=2, color='black')
+
+ # width here is the width for each bar in the plot. Matplotlib default is
+ # 0.8.
+ width = 0.8 / len(plots)
+ plot_index = 1
+
+ # Plot the data
+ for plot, color in zip(plots, _colors(len(plots))):
+ # Invert the y-axis if needed
+ if plot['label'] in invert:
+ plot['y'] = [-y for y in plot['y']]
+
+ adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index, len(plots))
+ bars.append(sub.bar(adjusted_x, plot['y'],
+ width=width, yerr=plot['errors'], facecolor=color,
+ label=plot['label'])[0])
+ plot_index += 1
+
+ # Construct the information for the drilldowns
+ plot_index = 1
+ for plot in plots:
+ adjusted_x = _get_adjusted_bar(plot['x'], width, plot_index, len(plots))
+
+ # Let matplotlib plot the data, so that we can get the data-to-image
+ # coordinate transforms
+ line = sub.plot(adjusted_x, plot['y'], linestyle='None')[0]
+ ulcoords = line.get_transform().transform(zip(adjusted_x, plot['y']))
+ brcoords = line.get_transform().transform(
+ [(x + width, 0) for x in adjusted_x])
+
+ # Get the drilldown query
+ key = '__' + plot['label'] + '__'
+ drill = "'%s'" % (queries[key].replace("'", "\\'"))
+
+ # Set the title attributes
+ x_labels = [labels[x] for x in plot['x']]
+ titles = ['%s - %s: %f' % (plot['label'], label, y)
+ for label, y in zip(x_labels, plot['y'])]
+ params = [[drill, "'%s'" % plot['label'], "'%s'" % x_label]
+ for x_label in x_labels]
+ area_data += [(ulx, height - uly, brx, height - bry,
+ title, 'showMetricsDrilldown', ','.join(param))
+ for (ulx, uly), (brx, bry), title, param
+ in zip(ulcoords, brcoords, titles, params)]
+ plot_index += 1
+
+ fig.legend(bars, [plot['label'] for plot in plots])
+ return (fig, area_data)
+
+
+def _normalize(data_values, data_errors, base_values, base_errors):
+ """\
+ Normalize the data against a baseline.
+
+ data_values: y-values for the to-be-normalized data
+ data_errors: standard deviations for the to-be-normalized data
+ base_values: list of values normalize against
+ base_errors: list of standard deviations for those base values
+ """
+
+ values = [100 * (value - base) / base
+ for value, base in zip(data_values, base_values)]
+
+ # Based on error for f(x,y) = 100 * (x - y) / y
+ if data_errors:
+ if not base_errors:
+ base_errors = [0] * len(data_errors)
+ errors = [sqrt(error**2 * (100 / base_value)**2
+ + base_error**2 * (100 * data / base_value**2)**2
+ + error * base_error * (100 / base_value**2)**2)
+ for data, error, base_value, base_error
+ in zip(data_values, data_errors, base_values, base_errors)]
+ else:
+ errors = None
+
+ return (values, errors)
+
+
+def _create_png(fig):
+ """\
+ Given the matplotlib figure, generate the PNG data for it.
+ """
+
+ # Draw the image
+ canvas = matplotlib.backends.backend_agg.FigureCanvasAgg(fig)
+ canvas.draw()
+ size = canvas.get_renderer().get_canvas_width_height()
+ buf = canvas.tostring_rgb()
+ im = PIL.Image.fromstring('RGB', size, buf, 'raw', 'RGB', 0, 1)
+ bg = PIL.Image.new(im.mode, im.size, fig.get_facecolor())
+
+ # Crop the image to remove surrounding whitespace
+ diff = PIL.ImageChops.difference(im, bg)
+ bbox = diff.getbbox()
+ im = im.crop(bbox)
+
+ imdata = StringIO.StringIO()
+ im.save(imdata, format='PNG')
+
+ return imdata.getvalue(), bbox
+
+
+def _create_image_html(fig, area_data, name):
+ """\
+ Given the figure and drilldown data, construct the HTML that will render the
+ graph as a PNG image, and attach the image map to that image.
+
+ fig: figure containing the drawn plot(s)
+ area_data: list of parameters for each area of the image map. See the
+ definition of the template string '_area'
+ name: name to give the image map in the HTML
+ """
+
+ png, bbox = _create_png(fig)
+
+ # Construct the list of image map areas
+ areas = [_area % (data[0] - bbox[0], data[1] - bbox[1],
+ data[2] - bbox[0], data[3] - bbox[1],
+ data[4], data[5], data[6])
+ for data in area_data]
+
+ return _tmpl % (base64.b64encode(png), name,
+ name, '\n'.join(areas))
+
+
+def _create_metrics_plot_helper(queries, plot, invert, normalize=None,
+ extra_text=None):
+ """\
+ Create a metrics plot of the given data.
+
+ queries: dictionary containing the main query and the drilldown queries
+ plot: 'Line' or 'Bar', depending on the plot type the user wants
+ invert: list of series that should be plotted on an inverted y-axis
+ normalize: None - do not normalize
+ 'first' - normalize against the first data point
+ 'x__%s' - normalize against the x-axis value %s
+ 'series__%s' - normalize against the series %s
+ extra_text: text to show at the uppper-left of the graph
+ """
+
+ if normalize is None:
+ normalize = ''
+ query = queries['__main__']
+ cursor = readonly_connection.connection.cursor()
+ cursor.execute(query)
+
+ if not cursor.rowcount:
+ raise NoDataError('query did not return any data')
+
+ if plot == 'Line':
+ line = True
+ elif plot == 'Bar':
+ line = False
+ else:
+ raise ValidationError({
+ 'Plot' : 'Plot must be either Line or Bar'
+ })
+ plots = []
+ labels = [str(row[0]) for row in cursor.fetchall()]
+ needs_resort = (cursor.description[0][0] == 'kernel')
+
+ # Collect all the data for the plot
+ col = 1
+ while col < len(cursor.description):
+ y = [row[col] for row in cursor.fetchall()]
+ label = cursor.description[col][0]
+ col += 1
+ if (col < len(cursor.description) and
+ 'errors-' + label == cursor.description[col][0]):
+ errors = [row[col] for row in cursor.fetchall()]
+ col += 1
+ else:
+ errors = None
+ if needs_resort:
+ y = _resort(labels, y)
+ if errors:
+ errors = _resort(labels, errors)
+
+ x = [enum[0] for enum in enumerate(y) if enum[1] is not None]
+ y = [y[i] for i in x]
+ if errors:
+ errors = [error for error in errors if error is not None]
+ plots.append({
+ 'label': label,
+ 'x': x,
+ 'y': y,
+ 'errors': errors
+ })
+
+ if needs_resort:
+ labels = _resort(labels, labels)
+
+ # Normalize the data if necessary
+ if normalize == 'first' or normalize.startswith('x__'):
+ if normalize != 'first':
+ baseline = normalize[3:]
+ try:
+ baseline_index = labels.index(baseline)
+ except ValueError:
+ raise ValidationError({
+ 'Normalize' : 'Invalid baseline %s' % baseline
+ })
+ for plot in plots:
+ if normalize == 'first':
+ plot_index = 0
+ else:
+ try:
+ plot_index = plot['x'].index(baseline_index)
+ # if the value is not found, then we cannot normalize
+ except ValueError:
+ raise ValidationError({
+ 'Normalize' : ('%s does not have a value for %s'
+ % (plot['label'], normalize[3:]))
+ })
+ base_values = [plot['y'][plot_index]] * len(plot['y'])
+ if plot['errors']:
+ base_errors = [plot['errors'][plot_index]] * len(plot['errors'])
+ plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
+ base_values,
+ None or base_errors)
+
+ elif normalize.startswith('series__'):
+ series = normalize[8:]
+ series_index = [plot['label'] for plot in plots].index(series)
+ plot = plots[series_index]
+ base_x = plot['x']
+ base_values = plot['y']
+ base_errors = plot['errors']
+ del plots[series_index]
+ for plot in plots:
+ # Remove all points in the to-be-normalized data that do not
+ # have a corresponding baseline value
+ to_remove = []
+ for index, data in enumerate(plot['x']):
+ if not data in base_x:
+ to_remove.append(index)
+ to_remove.reverse()
+ for index in to_remove:
+ del plot['x'][index]
+ del plot['y'][index]
+ if plot['errors']:
+ del plot['errors'][index]
+
+ plot['y'], plot['errors'] = _normalize(plot['y'], plot['errors'],
+ base_values, base_errors)
+
+ # Call the appropriate function to draw the line or bar plot
+ params = [plots, labels, queries, invert]
+ if line:
+ func = _create_line
+ params.append(normalize)
+ else:
+ func = _create_bar
+ fig, area_data = func(*params)
+
+ if extra_text:
+ text_y = .95 - .0075 * len(plots)
+ fig.text(.1, text_y, extra_text, size='xx-small')
+
+ return (fig, area_data, 'metrics_drilldown')
+
+def create_metrics_plot(queries, plot, invert, normalize, extra_text=None):
+ """\
+ Wrapper for _create_metrics_plot_helper
+ """
+
+ fig, area_data, name = _create_metrics_plot_helper(queries, plot,
+ invert, normalize,
+ extra_text)
+ return _create_image_html(fig, area_data, name)
+
+
+def _get_hostnames_in_bucket(hist_data, bucket):
+ """\
+ Get all the hostnames that constitute a particular bucket in the histogram.
+
+ hist_data: list containing tuples of (hostname, pass_rate)
+ bucket: tuple containing the (low, high) values of the target bucket
+ """
+
+ return [data[0] for data in hist_data
+ if data[1] >= bucket[0] and data[1] < bucket[1]]
+
+
+def _create_qual_histogram_helper(query, filter_string, interval, extra_text=None):
+ """\
+ Create a machine qualification histogram of the given data.
+
+ query: the main query to retrieve the pass rate information
+ filter_string: filter to apply to the common global filter to show the Table
+ View drilldown of a histogram bucket
+ interval: interval for each bucket. E.g., 10 means that buckets should be
+ 0-10%, 10%-20%, ...
+ extra_text: text to show at the upper-left of the graph
+ """
+ cursor = readonly_connection.connection.cursor()
+ cursor.execute(query)
+
+ if not cursor.rowcount:
+ raise NoDataError('query did not return any data')
+
+ # Lists to store the plot data.
+ # hist_data store tuples of (hostname, pass_rate) for machines that have
+ # pass rates between 0 and 100%, exclusive.
+ # no_tests is a list of machines that have run none of the selected tests
+ # no_pass is a list of machines with 0% pass rate
+ # perfect is a list of machines with a 100% pass rate
+ hist_data = []
+ no_tests = []
+ no_pass = []
+ perfect = []
+
+ # Construct the lists of data to plot
+ for hostname, total, good in cursor.fetchall():
+ if total != 0:
+ if good == 0:
+ no_pass.append(hostname)
+ elif good == total:
+ perfect.append(hostname)
+ else:
+ percentage = 100.0 * good / total
+ hist_data.append((hostname, percentage))
+ else:
+ no_tests.append(hostname)
+
+ bins = range(0, 100, interval)
+ if bins[-1] != 100:
+ bins.append(bins[-1] + interval)
+
+ fig, height = _create_figure(6)
+ sub = fig.add_subplot(1,1,1)
+
+ # Plot the data and get all the bars plotted
+ _,_, bars = sub.hist([data[1] for data in hist_data],
+ bins=bins, align='left')
+ bars += sub.bar([-interval], len(no_pass),
+ width=interval, align='center')
+ bars += sub.bar([bins[-1]], len(perfect),
+ width=interval, align='center')
+ bars += sub.bar([-3 * interval], len(no_tests),
+ width=interval, align='center')
+
+ buckets = [(bin, min(bin + interval, 100)) for bin in bins[:-1]]
+ sub.set_xlim(-4 * interval, bins[-1] + interval)
+ sub.set_xticks([-3 * interval, -interval] + bins + [100 + interval])
+ sub.set_xticklabels(['N/A', '0%'] +
+ ['%d%% - <%d%%' % bucket for bucket in buckets] +
+ ['100%'], rotation=90, size='small')
+
+ # Find the coordinates on the image for each bar
+ x = []
+ y = []
+ for bar in bars:
+ x.append(bar.get_x())
+ y.append(bar.get_height())
+ f = sub.plot(x, y, linestyle='None')[0]
+ ulcoords = f.get_transform().transform(zip(x, y))
+ brcoords = f.get_transform().transform(
+ [(x_val + interval, 0) for x_val in x])
+
+ filter_string_base = filter_string.replace("'", "\\'").replace('%', '%%')
+
+ # Set the title attributes
+ titles = ['%d%% - <%d%%: %d machines' % (bucket[0], bucket[1], y_val)
+ for bucket, y_val in zip(buckets, y)]
+ titles.append('0%%: %d machines' % len(no_pass))
+ titles.append('100%%: %d machines' % len(perfect))
+ titles.append('N/A: %d machines' % len(no_tests))
+
+ # Get the hostnames for each bucket in the histogram
+ names_list = [_get_hostnames_in_bucket(hist_data, bucket)
+ for bucket in buckets]
+ names_list += [no_pass, perfect]
+ if filter_string_base:
+ filter_string_base += ' AND '
+
+ # Construct the list of JavaScript functions to be called when the user
+ # clicks on the bar.
+ funcs = []
+ params = []
+ for names in names_list:
+ if names:
+ s_string = ','.join(["\\'%s\\'"] * len(names))
+ filter_tmpl = '%shostname IN (%s)' % (filter_string_base, s_string)
+ filter_string = filter_tmpl % tuple(names)
+ funcs.append('showQualDrilldown')
+ params.append("'%s'" % filter_string)
+ else:
+ funcs.append('showQualEmptyDialog')
+ params.append([])
+ funcs.append('showQualNADialog')
+ params.append("<html>%s</html>" % ('<br />'.join(no_tests)))
+
+ area_data = [(ulx, height - uly, brx, height - bry,
+ title, func, param)
+ for (ulx, uly), (brx, bry), title, func, param
+ in zip(ulcoords, brcoords, titles, funcs, params)]
+
+ if extra_text:
+ fig.text(.1, .95, extra_text, size='xx-small')
+
+ return (fig, area_data, 'qual_drilldown')
+
+
+def create_qual_histogram(query, filter_string, interval, extra_text=None):
+ """\
+ Wrapper for _create_qual_histogram_helper
+ """
+
+ fig, area_data, name = _create_qual_histogram_helper(query, filter_string,
+ interval, extra_text)
+ return _create_image_html(fig, area_data, name)
+
+
+def create_embedded_plot(model, update_time):
+ """\
+ Given an EmbeddedGraphingQuery object, generate the PNG image for it.
+
+ model: EmbeddedGraphingQuery object
+ update_time: 'Last updated' time
+ """
+
+ if model.graph_type == 'metrics':
+ func = _create_metrics_plot_helper
+ elif model.graph_type == 'qual':
+ func = _create_qual_histogram_helper
+
+ params = pickle.loads(model.params)
+ params['extra_text'] = 'Last updated: %s' % update_time
+ fig, _, _ = func(**params)
+ img, _ = _create_png(fig)
+
+ return img
+
+
+_cache_timeout = global_config.global_config.get_config_value(
+ 'TKO', 'graph_cache_creation_timeout_minutes')
+
+
+def handle_plot_request(id, max_age):
+ """\
+ Given the embedding id of a graph, generate a PNG of the embedded graph
+ associated with that id.
+
+ id: id of the embedded graph
+ max_age: maximum age, in minutes, that a cached version should be held
+ """
+ model = models.EmbeddedGraphingQuery.objects.get(id=id)
+
+ # Check if the cached image needs to be updated
+ now = datetime.datetime.now()
+ update_time = model.last_updated + datetime.timedelta(minutes=int(max_age))
+ if now > update_time:
+ cursor = django.db.connection.cursor()
+
+ # We want this query to update the refresh_time only once, even if
+ # multiple threads are running it at the same time. That is, only the
+ # first thread will win the race, and it will be the one to update the
+ # cached image; all other threads will show that they updated 0 rows
+ query = """
+ UPDATE embedded_graphing_queries
+ SET refresh_time = NOW()
+ WHERE id = %s AND (
+ refresh_time IS NULL OR
+ refresh_time + INTERVAL %s MINUTE < NOW()
+ )
+ """
+ cursor.execute(query, (id, _cache_timeout))
+
+ # Only refresh the cached image if we were successful in updating the
+ # refresh time
+ if cursor.rowcount:
+ model.cached_png = create_embedded_plot(model, now.ctime())
+ model.last_updated = now
+ model.refresh_time = None
+ model.save()
+
+ return model.cached_png
diff --git a/new_tko/tko/models.py b/new_tko/tko/models.py
index c893f8b..5722e0b 100644
--- a/new_tko/tko/models.py
+++ b/new_tko/tko/models.py
@@ -197,6 +197,23 @@
db_table = 'saved_queries'
+class EmbeddedGraphingQuery(dbmodels.Model, model_logic.ModelExtensions):
+ url_token = dbmodels.TextField(null=False, blank=False)
+ graph_type = dbmodels.CharField(maxlength=16, null=False, blank=False)
+ params = dbmodels.TextField(null=False, blank=False)
+ last_updated = dbmodels.DateTimeField(null=False, blank=False,
+ editable=False)
+ # refresh_time shows the time at which a thread is updating the cached
+ # image, or NULL if no one is updating the image. This is used so that only
+ # one thread is updating the cached image at a time (see
+ # graphing_utils.handle_plot_request)
+ refresh_time = dbmodels.DateTimeField(editable=False)
+ cached_png = dbmodels.TextField(editable=False)
+
+ class Meta:
+ db_table = 'embedded_graphing_queries'
+
+
# views
class TestViewManager(TempManager):
diff --git a/new_tko/tko/preconfigs/metrics/kernel_compare b/new_tko/tko/preconfigs/metrics/kernel_compare
new file mode 100644
index 0000000..7035abb
--- /dev/null
+++ b/new_tko/tko/preconfigs/metrics/kernel_compare
@@ -0,0 +1,23 @@
+plot: Bar
+xAxis: test_name
+globalFilter[0][db]: test_name
+globalFilter[0][condition]: IN ('dbench', 'tbench')
+globalFilter[1][db]: iteration_key
+globalFilter[1][condition]: = 'throughput'
+globalFilter[2][db]: hostname
+globalFilter[2][condition]: = 'my_host'
+globalFilter_all: true
+name[0]: my kernel 1
+values[0]: iteration_value
+aggregation[0]: AVG
+errorBars[0]: true
+seriesFilters[0][0][db]: kernel
+seriesFilters[0][0][condition]: LIKE 'my_kernel_1.%'
+seriesFilters[0]_all: true
+name[1]: my kernel 2
+values[1]: iteration_value
+aggregation[1]: AVG
+errorBars[1]: true
+seriesFilters[1][0][db]: kernel
+seriesFilters[1][0][condition]: LIKE 'my_kernel_2.%'
+seriesFilters[1]_all: true
diff --git a/new_tko/tko/preconfigs/metrics/perf b/new_tko/tko/preconfigs/metrics/perf
new file mode 100644
index 0000000..afbae87
--- /dev/null
+++ b/new_tko/tko/preconfigs/metrics/perf
@@ -0,0 +1,51 @@
+plot: Line
+xAxis: kernel
+globalFilter[0][db]: hostname
+globalFilter[0][condition]: = 'my_host'
+globalFilter_all: true
+name[0]: kernbench (elapsed)
+values[0]: iteration_value
+aggregation[0]: AVG
+errorBars[0]: true
+seriesFilters[0][0][db]: iteration_key
+seriesFilters[0][0][condition]: = 'elapsed'
+seriesFilters[0][1][db]: test_name
+seriesFilters[0][1][condition]: = 'kernbench'
+seriesFilters[0]_all: true
+name[1]: dbench (throughput)
+values[1]: iteration_value
+aggregation[1]: AVG
+errorBars[1]: true
+seriesFilters[1][0][db]: iteration_key
+seriesFilters[1][0][condition]: = 'throughput'
+seriesFilters[1][1][db]: test_name
+seriesFilters[1][1][condition]: = 'dbench'
+seriesFilters[1]_all: true
+name[2]: tbench (throughput)
+values[2]: iteration_value
+aggregation[2]: AVG
+errorBars[2]: true
+seriesFilters[2][0][db]: iteration_key
+seriesFilters[2][0][condition]: = 'throughput'
+seriesFilters[2][1][db]: test_name
+seriesFilters[2][1][condition]: = 'tbench'
+seriesFilters[2]_all: true
+name[3]: unixbench (score)
+values[3]: iteration_value
+aggregation[3]: AVG
+errorBars[3]: true
+seriesFilters[3][0][db]: iteration_key
+seriesFilters[3][0][condition]: = 'score'
+seriesFilters[3][1][db]: test_name
+seriesFilters[3][1][condition]: = 'unixbench'
+seriesFilters[3]_all: true
+name[4]: iozone (32768-4096-fwrite)
+values[4]: iteration_value
+aggregation[4]: AVG
+errorBars[4]: true
+seriesFilters[4][0][db]: iteration_key
+seriesFilters[4][0][condition]: = '32768-4096-fwrite'
+seriesFilters[4][1][db]: test_name
+seriesFilters[4][1][condition]: = 'iozone'
+seriesFilters[4]_all: true
+inverted: kernbench (elapsed)
diff --git a/new_tko/tko/preconfigs/qual/pre b/new_tko/tko/preconfigs/qual/pre
new file mode 100644
index 0000000..d302346
--- /dev/null
+++ b/new_tko/tko/preconfigs/qual/pre
@@ -0,0 +1,9 @@
+globalFilter[0][db]: hostname
+globalFilter[0][condition]: LIKE 'my_host%'
+globalFilter[1][db]: hostname
+globalFilter[1][condition]: LIKE 'my_other_host%'
+globalFilter_all: false
+testFilter[0][db]: test_name
+testFilter[0][condition]: = 'my_test_name'
+testFilter_all: true
+interval: 10
diff --git a/new_tko/tko/rpc_interface.py b/new_tko/tko/rpc_interface.py
index 0e4822d..64dbf9c 100644
--- a/new_tko/tko/rpc_interface.py
+++ b/new_tko/tko/rpc_interface.py
@@ -1,8 +1,9 @@
-import re, datetime
+import os, pickle, datetime
from django.db import models as dbmodels
from autotest_lib.frontend import thread_local
from autotest_lib.frontend.afe import rpc_utils, model_logic
-from autotest_lib.new_tko.tko import models, tko_rpc_utils
+from autotest_lib.frontend.afe import readonly_connection
+from new_tko.tko import models, tko_rpc_utils, graphing_utils
# table/spreadsheet view support
@@ -143,6 +144,100 @@
return rpc_utils.prepare_for_serialization(host_info)
+def create_metrics_plot(queries, plot, invert, normalize=None):
+ return graphing_utils.create_metrics_plot(queries, plot, invert, normalize)
+
+
+def create_qual_histogram(query, filter_string, interval):
+ return graphing_utils.create_qual_histogram(query, filter_string, interval)
+
+
+def execute_query_with_param(query, param):
+ cursor = readonly_connection.connection.cursor()
+ cursor.execute(query, param)
+ return cursor.fetchall()
+
+
+_preconfigs = {}
+is_init = False
+
+
+def _get_preconfig_path(suffix):
+ """\
+ Get the absolute path to a prefix directory or file.
+
+ suffix: list of suffixes after the 'preconfigs' directory to navigate to.
+ E.g., ['metrics', 'abc'] gives the path to
+ <tko>/preconfigs/metrics/abc
+ """
+
+ rel_path = os.path.join(os.path.dirname(__file__), 'preconfigs', *suffix)
+ return os.path.abspath(rel_path)
+
+
+def _init_preconfigs():
+ """\
+ Read the names of all the preconfigs from disk and store them in the
+ _preconfigs dictionary.
+ """
+
+ global is_init
+ if not is_init:
+ # Read the data
+ _preconfigs['metrics'] = dict.fromkeys(
+ os.listdir(_get_preconfig_path(['metrics'])))
+ _preconfigs['qual'] = dict.fromkeys(
+ os.listdir(_get_preconfig_path(['qual'])))
+ is_init = True
+
+def _read_preconfig(name, type):
+ """\
+ Populate the _preconfigs dictionary entry for the preconfig described by the
+ given parameters.
+
+ name: specific name of the preconfig
+ type: 'metrics' or 'qual'
+ """
+
+ _preconfigs[type][name] = {}
+ path = _get_preconfig_path([type, name])
+ config = open(path)
+ for line in config.readlines():
+ line.rstrip('\n')
+ parts = line.split(':')
+ _preconfigs[type][name][parts[0]] = parts[1].strip()
+ config.close()
+
+
+def get_preconfig(name, type):
+ _init_preconfigs()
+ if _preconfigs[type][name] is None:
+ _read_preconfig(name, type)
+ return _preconfigs[type][name]
+
+
+def get_embedding_id(url_token, graph_type, params):
+ try:
+ model = models.EmbeddedGraphingQuery.objects.get(url_token=url_token)
+ except models.EmbeddedGraphingQuery.DoesNotExist:
+ params_str = pickle.dumps(params)
+ now = datetime.datetime.now()
+ model = models.EmbeddedGraphingQuery(url_token=url_token,
+ graph_type=graph_type,
+ params=params_str,
+ last_updated=now)
+ model.cached_png = graphing_utils.create_embedded_plot(model,
+ now.ctime())
+ model.save()
+
+ return model.id
+
+
+def get_embedded_query_url_token(id):
+ model = models.EmbeddedGraphingQuery.objects.get(id=id)
+ return model.url_token
+
+
# test label management
def add_test_label(name, description=None):
@@ -219,14 +314,6 @@
# other
-_benchmark_key = {
- 'kernbench' : 'elapsed',
- 'dbench' : 'throughput',
- 'tbench' : 'throughput',
- 'unixbench' : 'score',
- 'iozone' : '32768-4096-fwrite'
-}
-
def get_static_data():
result = {}
@@ -242,10 +329,52 @@
extra_fields = [(field_name.capitalize(), field_sql)
for field_sql, field_name
in models.TestView.extra_fields.iteritems()]
+ _init_preconfigs()
+
+ benchmark_key = {
+ 'kernbench' : 'elapsed',
+ 'dbench' : 'throughput',
+ 'tbench' : 'throughput',
+ 'unixbench' : 'score',
+ 'iozone' : '32768-4096-fwrite'
+ }
+
+ perf_view = [
+ ['Test Index', 'test_idx'],
+ ['Job Index', 'job_idx'],
+ ['Test Name', 'test_name'],
+ ['Subdirectory', 'subdir'],
+ ['Kernel Index', 'kernel_idx'],
+ ['Status Index', 'status_idx'],
+ ['Reason', 'reason'],
+ ['Host Index', 'machine_idx'],
+ ['Test Started Time', 'test_started_time'],
+ ['Test Finished Time', 'test_finished_time'],
+ ['Job Tag', 'job_tag'],
+ ['Job Name', 'job_name'],
+ ['Owner', 'job_owner'],
+ ['Job Queued Time', 'job_queued_time'],
+ ['Job Started Time', 'job_started_time'],
+ ['Job Finished Time', 'job_finished_time'],
+ ['Hostname', 'hostname'],
+ ['Platform', 'platform'],
+ ['Machine Owner', 'machine_owner'],
+ ['Kernel Hash', 'kernel_hash'],
+ ['Kernel Base', 'kernel_base'],
+ ['Kernel', 'kernel'],
+ ['Status', 'status'],
+ ['Iteration Number', 'iteration'],
+ ['Performance Keyval (Key)', 'iteration_key'],
+ ['Performance Keyval (Value)', 'iteration_value'],
+ ]
result['group_fields'] = sorted(group_fields)
result['all_fields'] = sorted(model_fields + extra_fields)
result['test_labels'] = get_test_labels(sort_by=['name'])
result['user_login'] = thread_local.get_user()
- result['benchmark_key'] = _benchmark_key
+ result['benchmark_key'] = benchmark_key
+ result['perf_view'] = perf_view
+ result['test_view'] = model_fields
+ result['preconfigs'] = _preconfigs
+
return result
diff --git a/new_tko/tko/urls.py b/new_tko/tko/urls.py
index 1214cae..5fdd659 100644
--- a/new_tko/tko/urls.py
+++ b/new_tko/tko/urls.py
@@ -2,7 +2,8 @@
from django.conf import settings
import os
-pattern_list = [(r'^(?:|noauth/)rpc/', 'new_tko.tko.views.handle_rpc')]
+pattern_list = [(r'^(?:|noauth/)rpc/', 'new_tko.tko.views.handle_rpc'),
+ (r'^(?:|noauth/)plot/', 'new_tko.tko.views.handle_plot')]
debug_pattern_list = [
(r'^model_doc/', 'new_tko.tko.views.model_documentation'),
diff --git a/new_tko/tko/views.py b/new_tko/tko/views.py
index 7985ef5..3f38893 100644
--- a/new_tko/tko/views.py
+++ b/new_tko/tko/views.py
@@ -1,4 +1,5 @@
-from new_tko.tko import rpc_interface
+import django.http
+from new_tko.tko import rpc_interface, graphing_utils
from autotest_lib.frontend.afe import rpc_handler
rpc_handler_obj = rpc_handler.RpcHandler((rpc_interface,),
@@ -7,3 +8,10 @@
def handle_rpc(request):
return rpc_handler_obj.handle_rpc_request(request)
+
+
+def handle_plot(request):
+ id = request.GET['id']
+ max_age = request.GET['max_age']
+ return django.http.HttpResponse(
+ graphing_utils.handle_plot_request(id, max_age), mimetype='image/png')
diff --git a/tko/migrations/015_support_graphing_interface.py b/tko/migrations/015_support_graphing_interface.py
new file mode 100644
index 0000000..402a7e4
--- /dev/null
+++ b/tko/migrations/015_support_graphing_interface.py
@@ -0,0 +1,101 @@
+def migrate_up(manager):
+ manager.execute(CREATE_QUERIES_TABLE)
+ manager.execute(CREATE_TEST_VIEW_OUTER_JOINS)
+ manager.execute(CREATE_PERF_VIEW_2)
+
+def migrate_down(manager):
+ manager.execute(DROP_QUERIES_TABLE)
+ manager.execute(DROP_TEST_VIEW_OUTER_JOINS)
+ manager.execute(DROP_PERF_VIEW_2)
+
+
+CREATE_QUERIES_TABLE = """\
+CREATE TABLE embedded_graphing_queries (
+ id INT NOT NULL AUTO_INCREMENT,
+ url_token TEXT NOT NULL,
+ graph_type VARCHAR(16) NOT NULL,
+ params TEXT NOT NULL,
+ last_accessed DATETIME NOT NULL,
+ PRIMARY KEY(id),
+ INDEX (url_token(128)))
+"""
+
+DROP_QUERIES_TABLE = """\
+DROP TABLE IF EXISTS embedded_graphing_queries
+"""
+
+CREATE_TEST_VIEW_OUTER_JOINS = """\
+CREATE VIEW test_view_outer_joins AS
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test AS test_name,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status AS status_idx,
+ tests.reason,
+ tests.machine_idx,
+ tests.started_time AS test_started_time,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_name,
+ jobs.username AS job_owner,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS hostname,
+ machines.machine_group AS platform,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel,
+ status.word AS status
+FROM tests
+LEFT OUTER JOIN jobs ON jobs.job_idx = tests.job_idx
+LEFT OUTER JOIN machines ON machines.machine_idx = jobs.machine_idx
+LEFT OUTER JOIN kernels ON kernels.kernel_idx = tests.kernel_idx
+LEFT OUTER JOIN status ON status.status_idx = tests.status;
+"""
+
+DROP_TEST_VIEW_OUTER_JOINS = """\
+DROP VIEW IF EXISTS test_view_outer_joins
+"""
+
+CREATE_PERF_VIEW_2 = """\
+CREATE VIEW perf_view_2 AS
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test AS test_name,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status AS status_idx,
+ tests.reason,
+ tests.machine_idx,
+ tests.started_time AS test_started_time,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_name,
+ jobs.username AS job_owner,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS hostname,
+ machines.machine_group AS platform,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel,
+ status.word AS status,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
+FROM tests
+INNER JOIN jobs ON jobs.job_idx = tests.job_idx
+INNER JOIN machines ON machines.machine_idx = jobs.machine_idx
+INNER JOIN kernels ON kernels.kernel_idx = tests.kernel_idx
+INNER JOIN status ON status.status_idx = tests.status
+INNER JOIN iteration_result ON iteration_result.test_idx = tests.test_idx;
+"""
+
+DROP_PERF_VIEW_2 = """\
+DROP VIEW IF EXISTS perf_view_2
+"""
diff --git a/tko/migrations/016_modify_perf_view_2.py b/tko/migrations/016_modify_perf_view_2.py
new file mode 100644
index 0000000..f94df0f
--- /dev/null
+++ b/tko/migrations/016_modify_perf_view_2.py
@@ -0,0 +1,45 @@
+prev_migration = __import__('015_support_graphing_interface')
+
+def migrate_up(manager):
+ manager.execute(prev_migration.DROP_PERF_VIEW_2)
+ manager.execute(CREATE_NEW_PERF_VIEW_2)
+
+def migrate_down(manager):
+ manager.execute(prev_migration.DROP_PERF_VIEW_2)
+ manager.execute(prev_migration.CREATE_PERF_VIEW_2)
+
+CREATE_NEW_PERF_VIEW_2 = """\
+CREATE VIEW perf_view_2 AS
+SELECT tests.test_idx,
+ tests.job_idx,
+ tests.test AS test_name,
+ tests.subdir,
+ tests.kernel_idx,
+ tests.status AS status_idx,
+ tests.reason,
+ tests.machine_idx,
+ tests.started_time AS test_started_time,
+ tests.finished_time AS test_finished_time,
+ jobs.tag AS job_tag,
+ jobs.label AS job_name,
+ jobs.username AS job_owner,
+ jobs.queued_time AS job_queued_time,
+ jobs.started_time AS job_started_time,
+ jobs.finished_time AS job_finished_time,
+ machines.hostname AS hostname,
+ machines.machine_group AS platform,
+ machines.owner AS machine_owner,
+ kernels.kernel_hash,
+ kernels.base AS kernel_base,
+ kernels.printable AS kernel,
+ status.word AS status,
+ iteration_result.iteration,
+ iteration_result.attribute AS iteration_key,
+ iteration_result.value AS iteration_value
+FROM tests
+LEFT OUTER JOIN jobs ON jobs.job_idx = tests.job_idx
+LEFT OUTER JOIN machines ON machines.machine_idx = jobs.machine_idx
+LEFT OUTER JOIN kernels ON kernels.kernel_idx = tests.kernel_idx
+LEFT OUTER JOIN status ON status.status_idx = tests.status
+LEFT OUTER JOIN iteration_result ON iteration_result.test_idx = tests.test_idx;
+"""
diff --git a/tko/migrations/017_add_embedded_graph_caching.py b/tko/migrations/017_add_embedded_graph_caching.py
new file mode 100644
index 0000000..bee7471
--- /dev/null
+++ b/tko/migrations/017_add_embedded_graph_caching.py
@@ -0,0 +1,33 @@
+def migrate_up(manager):
+ manager.execute_script(ADD_COLUMNS)
+
+def migrate_down(manager):
+ manager.execute_script(DROP_COLUMNS)
+
+ADD_COLUMNS = """\
+DELETE FROM embedded_graphing_queries;
+
+ALTER TABLE embedded_graphing_queries
+DROP COLUMN last_accessed;
+
+ALTER TABLE embedded_graphing_queries
+ADD COLUMN (
+ last_updated DATETIME NOT NULL,
+ refresh_time DATETIME DEFAULT NULL,
+ cached_png MEDIUMBLOB
+);
+"""
+
+DROP_COLUMNS = """\
+ALTER TABLE embedded_graphing_queries
+DROP COLUMN last_updated;
+
+ALTER TABLE embedded_graphing_queries
+DROP COLUMN cached_png;
+
+ALTER TABLE embedded_graphing_queries
+DROP COLUMN refresh_time;
+
+ALTER TABLE embedded_graphing_queries
+ADD COLUMN (last_accessed DATETIME NOT NULL);
+"""