Upload chromeOS Octane benchmark perf results to Chrome's new perf dashboard.
This CL contains 3 related changes:
1) modify croschart_defaults.json to use the new test name/perf key names
for the Telemetry Octane benchmark that's running on chromeOS.
2) modify extract_perf.py to also extract the chrome version number
associated with each perf result (which is available in the results
database).
3) modify generate_perf_graphs.py to actually upload the perf data for
a single platform/perf test (lumpy/Telemetry Octane benchmark) to Chrome's
live perf dashboard.
Once we verify things are working well on the live dashboard for this
single platform/test, we'll open it up to more platforms/tests. In a future
CL, we'll cache results that can't be uploaded to the live dashboard
(e.g., if the live dashboard is temporarily down), so that they can be
uploaded on a later invocation of the script.
BUG=chromium:220573
TEST=Verified that extract_perf.py and generate_perf_graphs.py will properly
upload only results for Telemetry Octane benchmark on lumpy, to a local instance
of Chrome's new perf dashboard (sending the perf data to the URL
"http://localhost:8080/add_point"). The CL now uses the actual URL
of the live dashboard (which can be viewed externally).
Change-Id: I98f28d2813ba4ccd52d342db1eebddcf3eba5a85
Reviewed-on: https://gerrit.chromium.org/gerrit/49257
Reviewed-by: Richard Barnette <jrbarnette@chromium.org>
Reviewed-by: Scott Zawalski <scottz@chromium.org>
Commit-Queue: Dennis Jeffrey <dennisjeffrey@chromium.org>
Tested-by: Dennis Jeffrey <dennisjeffrey@chromium.org>
diff --git a/frontend/perf-dashboard/croschart_defaults.json b/frontend/perf-dashboard/croschart_defaults.json
index 7f2c91e..ced4c91 100644
--- a/frontend/perf-dashboard/croschart_defaults.json
+++ b/frontend/perf-dashboard/croschart_defaults.json
@@ -686,21 +686,6 @@
]
},
{
- "test_name": "telemetry_OctaneBenchmark",
- "master": "ChromiumPerf",
- "graphs": [
- {
- "graph_name": "overall_score",
- "units": "score",
- "better_direction": "higher",
- "info_url": "../../../../../scripts/doc/default.html",
- "keys": [
- "score*"
- ]
- }
- ]
- },
- {
"test_name": "platform_GesturesRegressionTest",
"master": "chromeos.touchpad",
"graphs": [
@@ -714,5 +699,40 @@
]
}
]
+ },
+ {
+ "test_name": "telemetry_Benchmarks.octane",
+ "master": "ChromiumPerf",
+ "graphs": [
+ {
+ "graph_name": "overall_score",
+ "units": "score",
+ "better_direction": "higher",
+ "info_url": "../../../../../scripts/doc/default.html",
+ "keys": [
+ "TELEMETRY--score*"
+ ]
+ },
+ {
+ "graph_name": "individual_scores",
+ "units": "score",
+ "better_direction": "higher",
+ "info_url": "../../../../../scripts/doc/default.html",
+ "keys": [
+ "TELEMETRY--CodeLoad*",
+ "TELEMETRY--Crypto*",
+ "TELEMETRY--DeltaBlue*",
+ "TELEMETRY--EarleyBoyer*",
+ "TELEMETRY--Gameboy*",
+ "TELEMETRY--Mandreel*",
+ "TELEMETRY--NavierStokes*",
+ "TELEMETRY--PdfJS*",
+ "TELEMETRY--RayTrace*",
+ "TELEMETRY--RegExp*",
+ "TELEMETRY--Richards*",
+ "TELEMETRY--Splay*"
+ ]
+ }
+ ]
}
]
diff --git a/frontend/perf-dashboard/extract_perf.py b/frontend/perf-dashboard/extract_perf.py
index 93b62ba..b5afb88 100644
--- a/frontend/perf-dashboard/extract_perf.py
+++ b/frontend/perf-dashboard/extract_perf.py
@@ -109,7 +109,8 @@
@param output_dir: The output directory in which results are being written.
"""
- result_out = [job_id, result_dict['job_name'], result_dict['platform']]
+ result_out = [job_id, result_dict['job_name'], result_dict['platform'],
+ result_dict['chrome_version']]
perf_items = []
for perf_key in result_dict['perf_keys']:
for perf_val in result_dict['perf_keys'][perf_key]:
@@ -174,6 +175,18 @@
if 'platform' not in result:
return False
+ # Get the Chrome version number associated with this job ID.
+ query = ('SELECT DISTINCT value FROM tko_test_attributes '
+ 'INNER JOIN tko_perf_view_2 USING (test_idx) '
+ 'INNER JOIN tko_jobs USING (job_idx) '
+ 'WHERE afe_job_id=%s AND attribute="CHROME_VERSION"')
+ cursor.execute(query, job_id)
+ cursor_results = cursor.fetchall()
+ assert len(cursor_results) <= 1, \
+ 'Expected the Chrome version number to be unique for afe_job_id ' \
+ '%s, but got multiple instead: %s' % (job_id, cursor_results)
+ result['chrome_version'] = cursor_results[0][0] if cursor_results else ''
+
write_perf_info_to_disk(job_id, result, test_dir, output_dir)
return True
diff --git a/frontend/perf-dashboard/generate_perf_graphs.py b/frontend/perf-dashboard/generate_perf_graphs.py
index 63a1add..95383b8 100644
--- a/frontend/perf-dashboard/generate_perf_graphs.py
+++ b/frontend/perf-dashboard/generate_perf_graphs.py
@@ -31,6 +31,7 @@
import simplejson
import sys
import urllib
+import urllib2
_SETTINGS = 'autotest_lib.frontend.settings'
os.environ['DJANGO_SETTINGS_MODULE'] = _SETTINGS
@@ -45,6 +46,9 @@
_COMPLETED_ID_FILE_NAME = 'job_id_complete.txt'
_REV_NUM_FILE_NAME = 'rev_num.txt'
_WILDCARD = '*'
+_TELEMETRY_PERF_KEY_IDENTIFIER = 'TELEMETRY'
+_TELEMETRY_PERF_KEY_DELIMITER = '--'
+_NEW_DASH_UPLOAD_URL = 'http://chromeperf.appspot.com/add_point'
# Values that can be configured through options.
# TODO(dennisjeffrey): Infer the tip-of-tree milestone dynamically once this
@@ -238,41 +242,90 @@
"""Uploads a set of perf values to Chrome's perf dashboard.
@param data_point_info: A dictionary containing information about perf
- data points to plot: key names and values, revision number, chromeOS
- version number.
+ data points to plot: key names and values, chrome(OS) version numbers.
@param platform: The string name of the associated platform.
@param test_name: The string name of the associated test.
@param master_name: The string name of the "buildbot master" to use
(a concept that exists in Chrome's perf dashboard).
- This function is currently a no-op. It will be completed as soon as we're
- ready to start sending actual data to Chrome's perf dashboard.
"""
+
+ # Start slow - only upload results for one bot, one test right now.
+ # TODO(dennisjeffrey): Expand this to include other platforms/tests once
+ # we've proven ourselves with a single bot/test. Longer-term, the
+ # check below will be completely removed as soon as we're ready to upload
+ # the full suite of perf results to the new dashboard. The check is only
+ # in place temporarily to allow a subset of results to be uploaded until
+ # we're ready to upload everything.
+ if platform != 'lumpy' or test_name != 'telemetry_Benchmarks.octane':
+ return
+
+ # Generate a warning and return if any expected values in |data_point_info|
+ # are missing.
+ for expected_val in ['chrome_ver', 'traces', 'ver']:
+ if (expected_val not in data_point_info or
+ not data_point_info[expected_val]):
+ logging.warning('Did not upload data point for test "%s", '
+ 'platform "%s": missing value for "%s"',
+ test_name, platform, expected_val)
+ return
+
traces = data_point_info['traces']
for perf_key in traces:
perf_val = traces[perf_key][0]
perf_err = traces[perf_key][1]
+ units = None
+ if perf_key.startswith(_TELEMETRY_PERF_KEY_IDENTIFIER):
+ # The perf key is associated with a Telemetry test, and has a
+ # specially-formatted perf key that encodes a graph_name,
+ # trace_name, and units. Example Telemetry perf key:
+ # "TELEMETRY--DeltaBlue--DeltaBlue--score__bigger_is_better_"
+ graph_name, trace_name, units = (
+ perf_key.split(_TELEMETRY_PERF_KEY_DELIMITER)[1:])
+ # The Telemetry test name is the name of the tag that has been
+ # appended to |test_name|. For example, autotest name
+ # "telemetry_Benchmarks.octane" corresponds to Telemetry test name
+ # "octane" on chrome's new perf dashboard.
+ test_name = test_name[test_name.find('.') + 1:]
+
new_dash_entry = {
'master': master_name,
- 'bot': platform,
- 'test': '%s/%s' % (test_name, perf_key),
- 'revision': data_point_info['rev'] + 1, # Don't allow 0.
+ 'bot': 'cros-' + platform, # Prefix to make clear it's chromeOS.
+ 'test': '%s/%s/%s' % (test_name, graph_name, trace_name),
'value': perf_val,
'error': perf_err,
'supplemental_columns': {
- 'a_cros_build': data_point_info['ver']
+ 'r_cros_version': data_point_info['ver'],
+ 'r_chrome_version': data_point_info['chrome_ver'],
}
}
+ if units:
+ new_dash_entry['units'] = units
json_string = simplejson.dumps([new_dash_entry], indent=2)
params = urllib.urlencode({'data': json_string})
- # TODO(dennisjeffrey): Upload "params" to Chrome's perf dashboard,
- # once ready.
+ fp = None
+ try:
+ fp = urllib2.urlopen(_NEW_DASH_UPLOAD_URL, params)
+ errors = fp.read().strip()
+ if errors:
+ raise urllib2.URLError(errors)
+ except urllib2.URLError, e:
+ # TODO(dennisjeffrey): If the live dashboard is currently down,
+ # cache results and retry them later when the live dashboard is
+ # back up. For now we skip the current upload if the live
+ # dashboard is down.
+ logging.exception('Error uploading to new dashboard, skipping '
+ 'upload attempt: %s', e)
+ return
+ finally:
+ if fp:
+ fp.close()
def output_graph_data_for_entry(test_name, master_name, graph_name, job_name,
- platform, units, better_direction, url,
- perf_keys, chart_keys, options,
+ platform, chrome_ver, units, better_direction,
+ url, perf_keys, chart_keys, options,
summary_id_to_rev_num, output_data_dir):
"""Outputs data for a perf test result into appropriate graph data files.
@@ -284,6 +337,8 @@
test result.
@param platform: The string name of the platform associated with this test
result.
+ @param chrome_ver: The string Chrome version number associated with this
+ test result.
@param units: The string name of the units displayed on this graph.
@param better_direction: A String representing whether better perf results
are those that are "higher" or "lower".
@@ -355,14 +410,13 @@
entry = {}
entry['traces'] = {}
entry['ver'] = build_num
+ entry['chrome_ver'] = chrome_ver
key_to_vals = {}
for perf_key in perf_keys:
if any([chart_key_matches_actual_key(c, perf_key[0])
for c in chart_keys]):
- # Replace dashes with underscores so different lines show
- # up as different colors in the graphs.
- key = perf_key[0].replace('-', '_')
+ key = perf_key[0]
if key not in key_to_vals:
key_to_vals[key] = []
# There are some cases where results for
@@ -386,9 +440,20 @@
summary_id_to_rev_num[summary_id] = rev + 1
entry['rev'] = rev
- upload_to_chrome_dashboard(entry, platform, test_name,
- master_name)
+ # Upload data point to the new performance dashboard (only
+ # for the tip-of-tree branch).
+ if release == options.tot_milestone:
+ upload_to_chrome_dashboard(entry, platform, test_name,
+ master_name)
+ # For each perf key, replace dashes with underscores so
+ # different lines show up as different colors in the graphs.
+ for orig_key in entry['traces'].keys():
+ new_key = orig_key.replace('-', '_')
+ entry['traces'][new_key] = entry['traces'].pop(orig_key)
+
+ # Output data point to be displayed on the current (deprecated)
+ # dashboard.
with open(summary_file, 'a') as f:
f.write(simplejson.dumps(entry) + '\n')
@@ -431,7 +496,8 @@
job_id = info[0]
job_name = info[1]
platform = info[2]
- perf_keys = info[3]
+ chrome_ver = info[3]
+ perf_keys = info[4]
# Skip this job ID if it's already been processed.
if job_id in completed_ids:
@@ -457,9 +523,9 @@
if store_entry:
output_graph_data_for_entry(
test_name, master_name, graph_name, job_name,
- platform, units, better_direction, url, perf_keys,
- chart_keys, options, summary_id_to_rev_num,
- output_data_dir)
+ platform, chrome_ver, units, better_direction, url,
+ perf_keys, chart_keys, options,
+ summary_id_to_rev_num, output_data_dir)
# Mark this job ID as having been processed.
with open(os.path.join(output_data_dir,