Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Uploads performance data to the performance dashboard. |
| 6 | |
| 7 | Performance tests may output data that needs to be displayed on the performance |
| 8 | dashboard. The autotest TKO parser invokes this module with each test |
| 9 | associated with a job. If a test has performance data associated with it, it |
| 10 | is uploaded to the performance dashboard. The performance dashboard is owned |
| 11 | by Chrome team and is available here: https://chromeperf.appspot.com/. Users |
| 12 | must be logged in with an @google.com account to view chromeOS perf data there. |
| 13 | |
| 14 | """ |
| 15 | |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 16 | import httplib, json, os, re, urllib, urllib2 |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 17 | |
| 18 | import common |
Hsinyu Chao | e0b08e6 | 2015-08-11 10:50:37 +0000 | [diff] [blame] | 19 | from autotest_lib.client.cros import constants |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 20 | from autotest_lib.tko import utils as tko_utils |
| 21 | |
| 22 | _ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 23 | _PRESENTATION_CONFIG_FILE = os.path.join( |
| 24 | _ROOT_DIR, 'perf_dashboard_config.json') |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 25 | _PRESENTATION_SHADOW_CONFIG_FILE = os.path.join( |
| 26 | _ROOT_DIR, 'perf_dashboard_shadow_config.json') |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 27 | _DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point' |
| 28 | |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 29 | # Format for Chrome and Chrome OS version strings. |
| 30 | VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$' |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 31 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 32 | class PerfUploadingError(Exception): |
| 33 | """Exception raised in perf_uploader""" |
| 34 | pass |
| 35 | |
| 36 | |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 37 | def _parse_config_file(config_file): |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 38 | """Parses a presentation config file and stores the info into a dict. |
| 39 | |
| 40 | The config file contains information about how to present the perf data |
| 41 | on the perf dashboard. This is required if the default presentation |
| 42 | settings aren't desired for certain tests. |
| 43 | |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 44 | @param config_file: Path to the configuration file to be parsed. |
| 45 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 46 | @returns A dictionary mapping each unique autotest name to a dictionary |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 47 | of presentation config information. |
| 48 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 49 | @raises PerfUploadingError if config data or master name for the test |
| 50 | is missing from the config file. |
| 51 | |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 52 | """ |
| 53 | json_obj = [] |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 54 | if os.path.exists(config_file): |
| 55 | with open(config_file, 'r') as fp: |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 56 | json_obj = json.load(fp) |
| 57 | config_dict = {} |
| 58 | for entry in json_obj: |
| 59 | config_dict[entry['autotest_name']] = entry |
| 60 | return config_dict |
| 61 | |
| 62 | |
| 63 | def _gather_presentation_info(config_data, test_name): |
| 64 | """Gathers presentation info from config data for the given test name. |
| 65 | |
| 66 | @param config_data: A dictionary of dashboard presentation info for all |
| 67 | tests, as returned by _parse_config_file(). Info is keyed by autotest |
| 68 | name. |
| 69 | @param test_name: The name of an autotest. |
| 70 | |
| 71 | @return A dictionary containing presentation information extracted from |
| 72 | |config_data| for the given autotest name. |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 73 | |
| 74 | @raises PerfUploadingError if some required data is missing. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 75 | """ |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 76 | if not test_name in config_data: |
| 77 | raise PerfUploadingError( |
| 78 | 'No config data is specified for test %s in %s.' % |
| 79 | (test_name, _PRESENTATION_CONFIG_FILE)) |
| 80 | |
| 81 | presentation_dict = config_data[test_name] |
| 82 | try: |
| 83 | master_name = presentation_dict['master_name'] |
| 84 | except KeyError: |
| 85 | raise PerfUploadingError( |
| 86 | 'No master name is specified for test %s in %s.' % |
| 87 | (test_name, _PRESENTATION_CONFIG_FILE)) |
| 88 | if 'dashboard_test_name' in presentation_dict: |
| 89 | test_name = presentation_dict['dashboard_test_name'] |
Fang Deng | 7f24f0b | 2013-11-12 11:22:16 -0800 | [diff] [blame] | 90 | return {'master_name': master_name, 'test_name': test_name} |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 91 | |
| 92 | |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 93 | def _format_for_upload(platform_name, cros_version, chrome_version, |
Kris Rambish | eb04ae4 | 2015-02-04 16:28:44 -0800 | [diff] [blame] | 94 | hardware_id, variant_name, hardware_hostname, |
Keith Haddow | 7a5a7bd | 2016-02-05 20:24:12 -0800 | [diff] [blame] | 95 | perf_data, presentation_info, jobname): |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 96 | """Formats perf data suitable to upload to the perf dashboard. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 97 | |
| 98 | The perf dashboard expects perf data to be uploaded as a |
| 99 | specially-formatted JSON string. In particular, the JSON object must be a |
| 100 | dictionary with key "data", and value being a list of dictionaries where |
| 101 | each dictionary contains all the information associated with a single |
| 102 | measured perf value: master name, bot name, test name, perf value, error |
| 103 | value, units, and build version numbers. |
| 104 | |
| 105 | @param platform_name: The string name of the platform. |
| 106 | @param cros_version: The string chromeOS version number. |
| 107 | @param chrome_version: The string chrome version number. |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 108 | @param hardware_id: String that identifies the type of hardware the test was |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 109 | executed on. |
Kris Rambish | eb04ae4 | 2015-02-04 16:28:44 -0800 | [diff] [blame] | 110 | @param variant_name: String that identifies the variant name of the board. |
“Keith | 390e85c | 2015-01-09 14:28:07 -0800 | [diff] [blame] | 111 | @param hardware_hostname: String that identifies the name of the device the |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 112 | test was executed on. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 113 | @param perf_data: A dictionary of measured perf data as computed by |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 114 | _compute_avg_stddev(). |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 115 | @param presentation_info: A dictionary of dashboard presentation info for |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 116 | the given test, as identified by _gather_presentation_info(). |
Keith Haddow | 7a5a7bd | 2016-02-05 20:24:12 -0800 | [diff] [blame] | 117 | @param jobname: A string uniquely identifying the test run, this enables |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 118 | linking back from a test result to the logs of the test run. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 119 | |
| 120 | @return A dictionary containing the formatted information ready to upload |
| 121 | to the performance dashboard. |
| 122 | |
| 123 | """ |
Kris Rambish | eb04ae4 | 2015-02-04 16:28:44 -0800 | [diff] [blame] | 124 | if variant_name: |
| 125 | platform_name += '-' + variant_name |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 126 | |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 127 | perf_values = perf_data |
| 128 | # Client side case - server side comes with its own charts data section. |
| 129 | if 'charts' not in perf_values: |
| 130 | perf_values = { |
| 131 | 'format_version': '1.0', |
| 132 | 'benchmark_name': presentation_info['test_name'], |
| 133 | 'charts': perf_data, |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 134 | } |
| 135 | |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 136 | dash_entry = { |
| 137 | 'master': presentation_info['master_name'], |
| 138 | 'bot': 'cros-' + platform_name, # Prefix to clarify it's ChromeOS. |
| 139 | 'point_id': _get_id_from_version(chrome_version, cros_version), |
| 140 | 'versions': { |
| 141 | 'cros_version': cros_version, |
| 142 | 'chrome_version': chrome_version, |
| 143 | }, |
| 144 | 'supplemental': { |
| 145 | 'default_rev': 'chrome_version', |
| 146 | 'hardware_identifier': hardware_id, |
| 147 | 'hardware_hostname': hardware_hostname, |
| 148 | 'variant_name': variant_name, |
| 149 | 'jobname': jobname, |
| 150 | }, |
| 151 | 'chart_data': perf_values, |
| 152 | } |
| 153 | return {'data': json.dumps(dash_entry)} |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 154 | |
| 155 | |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 156 | def _get_version_numbers(test_attributes): |
| 157 | """Gets the version numbers from the test attributes and validates them. |
| 158 | |
| 159 | @param test_attributes: The attributes property (which is a dict) of an |
| 160 | autotest tko.models.test object. |
| 161 | |
| 162 | @return A pair of strings (Chrome OS version, Chrome version). |
| 163 | |
| 164 | @raises PerfUploadingError if a version isn't formatted as expected. |
| 165 | """ |
| 166 | chrome_version = test_attributes.get('CHROME_VERSION', '') |
| 167 | cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '') |
| 168 | # Prefix the ChromeOS version number with the Chrome milestone. |
| 169 | cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version |
| 170 | if not re.match(VERSION_REGEXP, cros_version): |
| 171 | raise PerfUploadingError('CrOS version "%s" does not match expected ' |
| 172 | 'format.' % cros_version) |
| 173 | if not re.match(VERSION_REGEXP, chrome_version): |
| 174 | raise PerfUploadingError('Chrome version "%s" does not match expected ' |
| 175 | 'format.' % chrome_version) |
| 176 | return (cros_version, chrome_version) |
| 177 | |
| 178 | |
| 179 | def _get_id_from_version(chrome_version, cros_version): |
| 180 | """Computes the point ID to use, from Chrome and ChromeOS version numbers. |
| 181 | |
| 182 | For ChromeOS row data, data values are associated with both a Chrome |
| 183 | version number and a ChromeOS version number (unlike for Chrome row data |
| 184 | that is associated with a single revision number). This function takes |
| 185 | both version numbers as input, then computes a single, unique integer ID |
| 186 | from them, which serves as a 'fake' revision number that can uniquely |
| 187 | identify each ChromeOS data point, and which will allow ChromeOS data points |
| 188 | to be sorted by Chrome version number, with ties broken by ChromeOS version |
| 189 | number. |
| 190 | |
| 191 | To compute the integer ID, we take the portions of each version number that |
| 192 | serve as the shortest unambiguous names for each (as described here: |
| 193 | http://www.chromium.org/developers/version-numbers). We then force each |
| 194 | component of each portion to be a fixed width (padded by zeros if needed), |
| 195 | concatenate all digits together (with those coming from the Chrome version |
| 196 | number first), and convert the entire string of digits into an integer. |
| 197 | We ensure that the total number of digits does not exceed that which is |
| 198 | allowed by AppEngine NDB for an integer (64-bit signed value). |
| 199 | |
| 200 | For example: |
| 201 | Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2) |
| 202 | ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0) |
| 203 | concatenated together with padding for fixed-width columns: |
| 204 | ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000' |
| 205 | Final integer ID: 14520020390600000 |
| 206 | |
| 207 | @param chrome_ver: The Chrome version number as a string. |
| 208 | @param cros_ver: The ChromeOS version number as a string. |
| 209 | |
| 210 | @return A unique integer ID associated with the two given version numbers. |
| 211 | |
| 212 | """ |
| 213 | |
| 214 | # Number of digits to use from each part of the version string for Chrome |
| 215 | # and Chrome OS versions when building a point ID out of these two versions. |
| 216 | chrome_version_col_widths = [0, 0, 5, 3] |
| 217 | cros_version_col_widths = [0, 5, 3, 2] |
| 218 | |
| 219 | def get_digits_from_version(version_num, column_widths): |
| 220 | if re.match(VERSION_REGEXP, version_num): |
| 221 | computed_string = '' |
| 222 | version_parts = version_num.split('.') |
| 223 | for i, version_part in enumerate(version_parts): |
| 224 | if column_widths[i]: |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 225 | computed_string += version_part.zfill(column_widths[i]) |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 226 | return computed_string |
| 227 | else: |
| 228 | return None |
| 229 | |
| 230 | chrome_digits = get_digits_from_version( |
| 231 | chrome_version, chrome_version_col_widths) |
| 232 | cros_digits = get_digits_from_version( |
| 233 | cros_version, cros_version_col_widths) |
| 234 | if not chrome_digits or not cros_digits: |
| 235 | return None |
| 236 | result_digits = chrome_digits + cros_digits |
| 237 | max_digits = sum(chrome_version_col_widths + cros_version_col_widths) |
| 238 | if len(result_digits) > max_digits: |
| 239 | return None |
| 240 | return int(result_digits) |
| 241 | |
| 242 | |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 243 | def _send_to_dashboard(data_obj): |
| 244 | """Sends formatted perf data to the perf dashboard. |
| 245 | |
| 246 | @param data_obj: A formatted data object as returned by |
| 247 | _format_for_upload(). |
| 248 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 249 | @raises PerfUploadingError if an exception was raised when uploading. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 250 | |
| 251 | """ |
| 252 | encoded = urllib.urlencode(data_obj) |
| 253 | req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded) |
| 254 | try: |
| 255 | urllib2.urlopen(req) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 256 | except urllib2.HTTPError as e: |
| 257 | raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % ( |
| 258 | e.code, e.msg, data_obj['data'])) |
| 259 | except urllib2.URLError as e: |
| 260 | raise PerfUploadingError( |
| 261 | 'URLError: %s for JSON %s\n' % |
| 262 | (str(e.reason), data_obj['data'])) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 263 | except httplib.HTTPException: |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 264 | raise PerfUploadingError( |
| 265 | 'HTTPException for JSON %s\n' % data_obj['data']) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 266 | |
| 267 | |
Keith Haddow | 7a5a7bd | 2016-02-05 20:24:12 -0800 | [diff] [blame] | 268 | def upload_test(job, test, jobname): |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 269 | """Uploads any perf data associated with a test to the perf dashboard. |
| 270 | |
| 271 | @param job: An autotest tko.models.job object that is associated with the |
| 272 | given |test|. |
| 273 | @param test: An autotest tko.models.test object that may or may not be |
| 274 | associated with measured perf data. |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 275 | @param jobname: A string uniquely identifying the test run, this enables |
| 276 | linking back from a test result to the logs of the test run. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 277 | |
| 278 | """ |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 279 | |
| 280 | # Format the perf data for the upload, then upload it. |
| 281 | test_name = test.testname |
| 282 | platform_name = job.machine_group |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 283 | hardware_id = test.attributes.get('hwid', '') |
“Keith | 390e85c | 2015-01-09 14:28:07 -0800 | [diff] [blame] | 284 | hardware_hostname = test.machine |
Kris Rambish | eb04ae4 | 2015-02-04 16:28:44 -0800 | [diff] [blame] | 285 | variant_name = test.attributes.get(constants.VARIANT_KEY, None) |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 286 | config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE) |
Dan Shi | 463dfe2 | 2016-01-26 13:48:21 -0800 | [diff] [blame] | 287 | try: |
| 288 | shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE) |
| 289 | config_data.update(shadow_config_data) |
| 290 | except ValueError as e: |
| 291 | tko_utils.dprint('Failed to parse config file %s: %s.' % |
| 292 | (_PRESENTATION_SHADOW_CONFIG_FILE, e)) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 293 | try: |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 294 | cros_version, chrome_version = _get_version_numbers(test.attributes) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 295 | presentation_info = _gather_presentation_info(config_data, test_name) |
| 296 | formatted_data = _format_for_upload( |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 297 | platform_name, cros_version, chrome_version, hardware_id, |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 298 | variant_name, hardware_hostname, test.perf_values, |
| 299 | presentation_info, jobname) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 300 | _send_to_dashboard(formatted_data) |
| 301 | except PerfUploadingError as e: |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 302 | tko_utils.dprint('Error when uploading perf data to the perf ' |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 303 | 'dashboard for test %s: %s' % (test_name, e)) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 304 | else: |
| 305 | tko_utils.dprint('Successfully uploaded perf data to the perf ' |
| 306 | 'dashboard for test %s.' % test_name) |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame^] | 307 | |