Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
| 5 | """Uploads performance data to the performance dashboard. |
| 6 | |
| 7 | Performance tests may output data that needs to be displayed on the performance |
| 8 | dashboard. The autotest TKO parser invokes this module with each test |
| 9 | associated with a job. If a test has performance data associated with it, it |
| 10 | is uploaded to the performance dashboard. The performance dashboard is owned |
| 11 | by Chrome team and is available here: https://chromeperf.appspot.com/. Users |
| 12 | must be logged in with an @google.com account to view chromeOS perf data there. |
| 13 | |
| 14 | """ |
| 15 | |
Prathmesh Prabhu | 97fe9cb | 2017-02-03 17:17:35 -0800 | [diff] [blame] | 16 | import httplib |
| 17 | import json |
| 18 | import os |
| 19 | import re |
| 20 | import urllib |
| 21 | import urllib2 |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 22 | |
| 23 | import common |
| 24 | from autotest_lib.tko import utils as tko_utils |
| 25 | |
| 26 | _ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 27 | _PRESENTATION_CONFIG_FILE = os.path.join( |
| 28 | _ROOT_DIR, 'perf_dashboard_config.json') |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 29 | _PRESENTATION_SHADOW_CONFIG_FILE = os.path.join( |
| 30 | _ROOT_DIR, 'perf_dashboard_shadow_config.json') |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 31 | _DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point' |
| 32 | |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 33 | # Format for Chrome and Chrome OS version strings. |
| 34 | VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$' |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 35 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 36 | class PerfUploadingError(Exception): |
| 37 | """Exception raised in perf_uploader""" |
| 38 | pass |
| 39 | |
| 40 | |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 41 | def _parse_config_file(config_file): |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 42 | """Parses a presentation config file and stores the info into a dict. |
| 43 | |
| 44 | The config file contains information about how to present the perf data |
| 45 | on the perf dashboard. This is required if the default presentation |
| 46 | settings aren't desired for certain tests. |
| 47 | |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 48 | @param config_file: Path to the configuration file to be parsed. |
| 49 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 50 | @returns A dictionary mapping each unique autotest name to a dictionary |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 51 | of presentation config information. |
| 52 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 53 | @raises PerfUploadingError if config data or master name for the test |
| 54 | is missing from the config file. |
| 55 | |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 56 | """ |
| 57 | json_obj = [] |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 58 | if os.path.exists(config_file): |
| 59 | with open(config_file, 'r') as fp: |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 60 | json_obj = json.load(fp) |
| 61 | config_dict = {} |
| 62 | for entry in json_obj: |
Po-Hsien Wang | 9156575 | 2019-06-13 14:01:02 -0700 | [diff] [blame] | 63 | if 'autotest_regex' in entry: |
| 64 | config_dict[entry['autotest_regex']] = entry |
| 65 | else: |
Po-Hsien Wang | b7d6b02 | 2019-06-28 15:07:37 -0700 | [diff] [blame] | 66 | config_dict['^' + re.escape(entry['autotest_name']) + '$'] = entry |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 67 | return config_dict |
| 68 | |
| 69 | |
| 70 | def _gather_presentation_info(config_data, test_name): |
| 71 | """Gathers presentation info from config data for the given test name. |
| 72 | |
| 73 | @param config_data: A dictionary of dashboard presentation info for all |
| 74 | tests, as returned by _parse_config_file(). Info is keyed by autotest |
| 75 | name. |
| 76 | @param test_name: The name of an autotest. |
| 77 | |
| 78 | @return A dictionary containing presentation information extracted from |
| 79 | |config_data| for the given autotest name. |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 80 | |
| 81 | @raises PerfUploadingError if some required data is missing. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 82 | """ |
Po-Hsien Wang | ccaa059 | 2018-11-12 20:57:20 -0800 | [diff] [blame] | 83 | presentation_dict = None |
| 84 | for regex in config_data: |
| 85 | match = re.match(regex, test_name) |
| 86 | if match: |
Po-Hsien Wang | 9156575 | 2019-06-13 14:01:02 -0700 | [diff] [blame] | 87 | if presentation_dict: |
| 88 | raise PerfUploadingError('Duplicate config data refer to the ' |
| 89 | 'same test %s' % test_name) |
Po-Hsien Wang | ccaa059 | 2018-11-12 20:57:20 -0800 | [diff] [blame] | 90 | presentation_dict = config_data[regex] |
Po-Hsien Wang | ccaa059 | 2018-11-12 20:57:20 -0800 | [diff] [blame] | 91 | |
| 92 | if not presentation_dict: |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 93 | raise PerfUploadingError( |
| 94 | 'No config data is specified for test %s in %s.' % |
| 95 | (test_name, _PRESENTATION_CONFIG_FILE)) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 96 | try: |
| 97 | master_name = presentation_dict['master_name'] |
| 98 | except KeyError: |
| 99 | raise PerfUploadingError( |
| 100 | 'No master name is specified for test %s in %s.' % |
| 101 | (test_name, _PRESENTATION_CONFIG_FILE)) |
| 102 | if 'dashboard_test_name' in presentation_dict: |
| 103 | test_name = presentation_dict['dashboard_test_name'] |
Fang Deng | 7f24f0b | 2013-11-12 11:22:16 -0800 | [diff] [blame] | 104 | return {'master_name': master_name, 'test_name': test_name} |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 105 | |
| 106 | |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 107 | def _format_for_upload(platform_name, cros_version, chrome_version, |
harpreet | 93f7d23 | 2019-07-15 15:59:15 -0700 | [diff] [blame^] | 108 | hardware_id, hardware_hostname, perf_data, |
| 109 | presentation_info, jobname): |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 110 | """Formats perf data suitable to upload to the perf dashboard. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 111 | |
| 112 | The perf dashboard expects perf data to be uploaded as a |
| 113 | specially-formatted JSON string. In particular, the JSON object must be a |
| 114 | dictionary with key "data", and value being a list of dictionaries where |
| 115 | each dictionary contains all the information associated with a single |
| 116 | measured perf value: master name, bot name, test name, perf value, error |
| 117 | value, units, and build version numbers. |
| 118 | |
| 119 | @param platform_name: The string name of the platform. |
| 120 | @param cros_version: The string chromeOS version number. |
| 121 | @param chrome_version: The string chrome version number. |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 122 | @param hardware_id: String that identifies the type of hardware the test was |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 123 | executed on. |
“Keith | 390e85c | 2015-01-09 14:28:07 -0800 | [diff] [blame] | 124 | @param hardware_hostname: String that identifies the name of the device the |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 125 | test was executed on. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 126 | @param perf_data: A dictionary of measured perf data as computed by |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 127 | _compute_avg_stddev(). |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 128 | @param presentation_info: A dictionary of dashboard presentation info for |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 129 | the given test, as identified by _gather_presentation_info(). |
Keith Haddow | 7a5a7bd | 2016-02-05 20:24:12 -0800 | [diff] [blame] | 130 | @param jobname: A string uniquely identifying the test run, this enables |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 131 | linking back from a test result to the logs of the test run. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 132 | |
| 133 | @return A dictionary containing the formatted information ready to upload |
| 134 | to the performance dashboard. |
| 135 | |
| 136 | """ |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 137 | perf_values = perf_data |
| 138 | # Client side case - server side comes with its own charts data section. |
| 139 | if 'charts' not in perf_values: |
| 140 | perf_values = { |
| 141 | 'format_version': '1.0', |
| 142 | 'benchmark_name': presentation_info['test_name'], |
| 143 | 'charts': perf_data, |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 144 | } |
| 145 | |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 146 | dash_entry = { |
| 147 | 'master': presentation_info['master_name'], |
| 148 | 'bot': 'cros-' + platform_name, # Prefix to clarify it's ChromeOS. |
| 149 | 'point_id': _get_id_from_version(chrome_version, cros_version), |
| 150 | 'versions': { |
| 151 | 'cros_version': cros_version, |
| 152 | 'chrome_version': chrome_version, |
| 153 | }, |
| 154 | 'supplemental': { |
Keith Haddow | 52e6552 | 2016-04-01 11:05:38 -0700 | [diff] [blame] | 155 | 'default_rev': 'r_cros_version', |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 156 | 'hardware_identifier': hardware_id, |
| 157 | 'hardware_hostname': hardware_hostname, |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 158 | 'jobname': jobname, |
| 159 | }, |
| 160 | 'chart_data': perf_values, |
| 161 | } |
| 162 | return {'data': json.dumps(dash_entry)} |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 163 | |
| 164 | |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 165 | def _get_version_numbers(test_attributes): |
| 166 | """Gets the version numbers from the test attributes and validates them. |
| 167 | |
| 168 | @param test_attributes: The attributes property (which is a dict) of an |
| 169 | autotest tko.models.test object. |
| 170 | |
| 171 | @return A pair of strings (Chrome OS version, Chrome version). |
| 172 | |
| 173 | @raises PerfUploadingError if a version isn't formatted as expected. |
| 174 | """ |
| 175 | chrome_version = test_attributes.get('CHROME_VERSION', '') |
| 176 | cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '') |
Keith Haddow | 60ddbb3 | 2016-05-23 10:34:54 -0700 | [diff] [blame] | 177 | cros_milestone = test_attributes.get('CHROMEOS_RELEASE_CHROME_MILESTONE') |
| 178 | # Use the release milestone as the milestone if present, othewise prefix the |
| 179 | # cros version with the with the Chrome browser milestone. |
| 180 | if cros_milestone: |
| 181 | cros_version = "%s.%s" % (cros_milestone, cros_version) |
| 182 | else: |
| 183 | cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 184 | if not re.match(VERSION_REGEXP, cros_version): |
| 185 | raise PerfUploadingError('CrOS version "%s" does not match expected ' |
| 186 | 'format.' % cros_version) |
| 187 | if not re.match(VERSION_REGEXP, chrome_version): |
| 188 | raise PerfUploadingError('Chrome version "%s" does not match expected ' |
| 189 | 'format.' % chrome_version) |
| 190 | return (cros_version, chrome_version) |
| 191 | |
| 192 | |
| 193 | def _get_id_from_version(chrome_version, cros_version): |
| 194 | """Computes the point ID to use, from Chrome and ChromeOS version numbers. |
| 195 | |
| 196 | For ChromeOS row data, data values are associated with both a Chrome |
| 197 | version number and a ChromeOS version number (unlike for Chrome row data |
| 198 | that is associated with a single revision number). This function takes |
| 199 | both version numbers as input, then computes a single, unique integer ID |
| 200 | from them, which serves as a 'fake' revision number that can uniquely |
| 201 | identify each ChromeOS data point, and which will allow ChromeOS data points |
| 202 | to be sorted by Chrome version number, with ties broken by ChromeOS version |
| 203 | number. |
| 204 | |
| 205 | To compute the integer ID, we take the portions of each version number that |
| 206 | serve as the shortest unambiguous names for each (as described here: |
| 207 | http://www.chromium.org/developers/version-numbers). We then force each |
| 208 | component of each portion to be a fixed width (padded by zeros if needed), |
| 209 | concatenate all digits together (with those coming from the Chrome version |
| 210 | number first), and convert the entire string of digits into an integer. |
| 211 | We ensure that the total number of digits does not exceed that which is |
| 212 | allowed by AppEngine NDB for an integer (64-bit signed value). |
| 213 | |
| 214 | For example: |
| 215 | Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2) |
| 216 | ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0) |
| 217 | concatenated together with padding for fixed-width columns: |
| 218 | ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000' |
| 219 | Final integer ID: 14520020390600000 |
| 220 | |
| 221 | @param chrome_ver: The Chrome version number as a string. |
| 222 | @param cros_ver: The ChromeOS version number as a string. |
| 223 | |
| 224 | @return A unique integer ID associated with the two given version numbers. |
| 225 | |
| 226 | """ |
| 227 | |
| 228 | # Number of digits to use from each part of the version string for Chrome |
| 229 | # and Chrome OS versions when building a point ID out of these two versions. |
| 230 | chrome_version_col_widths = [0, 0, 5, 3] |
| 231 | cros_version_col_widths = [0, 5, 3, 2] |
| 232 | |
| 233 | def get_digits_from_version(version_num, column_widths): |
| 234 | if re.match(VERSION_REGEXP, version_num): |
| 235 | computed_string = '' |
| 236 | version_parts = version_num.split('.') |
| 237 | for i, version_part in enumerate(version_parts): |
| 238 | if column_widths[i]: |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 239 | computed_string += version_part.zfill(column_widths[i]) |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 240 | return computed_string |
| 241 | else: |
| 242 | return None |
| 243 | |
| 244 | chrome_digits = get_digits_from_version( |
| 245 | chrome_version, chrome_version_col_widths) |
| 246 | cros_digits = get_digits_from_version( |
| 247 | cros_version, cros_version_col_widths) |
| 248 | if not chrome_digits or not cros_digits: |
| 249 | return None |
| 250 | result_digits = chrome_digits + cros_digits |
| 251 | max_digits = sum(chrome_version_col_widths + cros_version_col_widths) |
| 252 | if len(result_digits) > max_digits: |
| 253 | return None |
| 254 | return int(result_digits) |
| 255 | |
| 256 | |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 257 | def _send_to_dashboard(data_obj): |
| 258 | """Sends formatted perf data to the perf dashboard. |
| 259 | |
| 260 | @param data_obj: A formatted data object as returned by |
| 261 | _format_for_upload(). |
| 262 | |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 263 | @raises PerfUploadingError if an exception was raised when uploading. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 264 | |
| 265 | """ |
| 266 | encoded = urllib.urlencode(data_obj) |
| 267 | req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded) |
| 268 | try: |
| 269 | urllib2.urlopen(req) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 270 | except urllib2.HTTPError as e: |
| 271 | raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % ( |
| 272 | e.code, e.msg, data_obj['data'])) |
| 273 | except urllib2.URLError as e: |
| 274 | raise PerfUploadingError( |
| 275 | 'URLError: %s for JSON %s\n' % |
| 276 | (str(e.reason), data_obj['data'])) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 277 | except httplib.HTTPException: |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 278 | raise PerfUploadingError( |
| 279 | 'HTTPException for JSON %s\n' % data_obj['data']) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 280 | |
| 281 | |
Keith Haddow | 7a5a7bd | 2016-02-05 20:24:12 -0800 | [diff] [blame] | 282 | def upload_test(job, test, jobname): |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 283 | """Uploads any perf data associated with a test to the perf dashboard. |
| 284 | |
| 285 | @param job: An autotest tko.models.job object that is associated with the |
| 286 | given |test|. |
| 287 | @param test: An autotest tko.models.test object that may or may not be |
| 288 | associated with measured perf data. |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 289 | @param jobname: A string uniquely identifying the test run, this enables |
| 290 | linking back from a test result to the logs of the test run. |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 291 | |
| 292 | """ |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 293 | |
| 294 | # Format the perf data for the upload, then upload it. |
| 295 | test_name = test.testname |
| 296 | platform_name = job.machine_group |
Chung-yih Wang | d8ede14 | 2016-12-09 13:46:22 +0800 | [diff] [blame] | 297 | # Append the platform name with '.arc' if the suffix of the control |
| 298 | # filename is '.arc'. |
| 299 | if job.label and re.match('.*\.arc$', job.label): |
| 300 | platform_name += '.arc' |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 301 | hardware_id = test.attributes.get('hwid', '') |
“Keith | 390e85c | 2015-01-09 14:28:07 -0800 | [diff] [blame] | 302 | hardware_hostname = test.machine |
Dan Shi | b9fec88 | 2016-01-05 15:18:30 -0800 | [diff] [blame] | 303 | config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE) |
Dan Shi | 463dfe2 | 2016-01-26 13:48:21 -0800 | [diff] [blame] | 304 | try: |
| 305 | shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE) |
| 306 | config_data.update(shadow_config_data) |
| 307 | except ValueError as e: |
| 308 | tko_utils.dprint('Failed to parse config file %s: %s.' % |
| 309 | (_PRESENTATION_SHADOW_CONFIG_FILE, e)) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 310 | try: |
Quinten Yearsley | 5a66aea | 2015-04-14 12:40:25 -0700 | [diff] [blame] | 311 | cros_version, chrome_version = _get_version_numbers(test.attributes) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 312 | presentation_info = _gather_presentation_info(config_data, test_name) |
| 313 | formatted_data = _format_for_upload( |
Keith Haddow | ac0b164 | 2014-11-20 10:26:44 -0800 | [diff] [blame] | 314 | platform_name, cros_version, chrome_version, hardware_id, |
harpreet | 93f7d23 | 2019-07-15 15:59:15 -0700 | [diff] [blame^] | 315 | hardware_hostname, test.perf_values, presentation_info, jobname) |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 316 | _send_to_dashboard(formatted_data) |
| 317 | except PerfUploadingError as e: |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 318 | tko_utils.dprint('Error when uploading perf data to the perf ' |
Fang Deng | 947502e | 2014-05-07 11:59:07 -0700 | [diff] [blame] | 319 | 'dashboard for test %s: %s' % (test_name, e)) |
Dennis Jeffrey | f9bef6c | 2013-08-05 11:01:27 -0700 | [diff] [blame] | 320 | else: |
| 321 | tko_utils.dprint('Successfully uploaded perf data to the perf ' |
| 322 | 'dashboard for test %s.' % test_name) |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 323 | |