tko/perf_upload/perf_uploader.py

# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Uploads performance data to the performance dashboard.

Performance tests may output data that needs to be displayed on the performance
dashboard.  The autotest TKO parser invokes this module with each test
associated with a job.  If a test has performance data associated with it, it
is uploaded to the performance dashboard.  The performance dashboard is owned
by Chrome team and is available here: https://chromeperf.appspot.com/.  Users
must be logged in with an @google.com account to view chromeOS perf data there.

"""

import httplib, json, math, os, re, urllib, urllib2

import common
from autotest_lib.client.cros import constants
from autotest_lib.tko import utils as tko_utils

_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
_PRESENTATION_CONFIG_FILE = os.path.join(
        _ROOT_DIR, 'perf_dashboard_config.json')
_PRESENTATION_SHADOW_CONFIG_FILE = os.path.join(
        _ROOT_DIR, 'perf_dashboard_shadow_config.json')
_DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'

# Format for Chrome and Chrome OS version strings.
VERSION_REGEXP = r'^(\d+)\.(\d+)\.(\d+)\.(\d+)$'

class PerfUploadingError(Exception):
    """Exception raised in perf_uploader"""
    pass


def _aggregate_iterations(perf_values):
    """Aggregate same measurements from multiple iterations.

    Each perf measurement may exist multiple times across multiple iterations
    of a test.  Here, the results for each unique measured perf metric are
    aggregated across multiple iterations.

    @param perf_values: A list of tko.models.perf_value_iteration objects.

    @return A dictionary mapping each unique measured perf value (keyed by
        tuple of its description and graph name) to information about that
        perf value (in particular, the value is a list of values
        for each iteration).

    """
    perf_data = {}
    for perf_iteration in perf_values:
        for perf_dict in perf_iteration.perf_measurements:
            key = (perf_dict['description'], perf_dict['graph'])
            if key not in perf_data:
                perf_data[key] = {
                    'units': perf_dict['units'],
                    'higher_is_better': perf_dict['higher_is_better'],
                    'graph': perf_dict['graph'],
                    'value': [perf_dict['value']],   # Note: a list of values.
                    'stddev': perf_dict['stddev']
                }
            else:
                perf_data[key]['value'].append(perf_dict['value'])
                # Note: the stddev will be recomputed later when the results
                # from each of the multiple iterations are averaged together.
    return perf_data


def _mean_and_stddev(data, precision=4):
    """Computes mean and standard deviation from a list of numbers.

    Assumes that the list contains at least 2 numbers.

    @param data: A list of numeric values.
    @param precision: The integer number of decimal places to which to
        round the results.

    @return A 2-tuple (mean, standard_deviation), in which each value is
        rounded to |precision| decimal places.

    """
    n = len(data)
    mean = float(sum(data)) / n
    # Divide by n-1 to compute "sample standard deviation".
    variance = sum([(elem - mean) ** 2 for elem in data]) / (n - 1)
    return round(mean, precision), round(math.sqrt(variance), precision)


def _compute_avg_stddev(perf_data):
    """Compute average and standard deviations as needed for perf measurements.

    For any perf measurement that exists in multiple iterations (has more than
    one measured value), compute the average and standard deviation for it and
    then store the updated information in the dictionary.

    @param perf_data: A dictionary of measured perf data as computed by
        _aggregate_iterations(), except each value is now a single value, not a
        list of values.

    """
    for perf_dict in perf_data.itervalues():
        if len(perf_dict['value']) > 1:
            perf_dict['value'], perf_dict['stddev'] = (
                    _mean_and_stddev(map(float, perf_dict['value'])))
        else:
            perf_dict['value'] = perf_dict['value'][0]  # Take out of list.


def _parse_config_file(config_file):
    """Parses a presentation config file and stores the info into a dict.

    The config file contains information about how to present the perf data
    on the perf dashboard.  This is required if the default presentation
    settings aren't desired for certain tests.

    @param config_file: Path to the configuration file to be parsed.

    @returns A dictionary mapping each unique autotest name to a dictionary
        of presentation config information.

    @raises PerfUploadingError if config data or master name for the test
        is missing from the config file.

    """
    json_obj = []
    if os.path.exists(config_file):
        with open(config_file, 'r') as fp:
            json_obj = json.load(fp)
    config_dict = {}
    for entry in json_obj:
        config_dict[entry['autotest_name']] = entry
    return config_dict


def _gather_presentation_info(config_data, test_name):
    """Gathers presentation info from config data for the given test name.

    @param config_data: A dictionary of dashboard presentation info for all
        tests, as returned by _parse_config_file().  Info is keyed by autotest
        name.
    @param test_name: The name of an autotest.

    @return A dictionary containing presentation information extracted from
        |config_data| for the given autotest name.

    @raises PerfUploadingError if some required data is missing.
    """
    if not test_name in config_data:
        raise PerfUploadingError(
                'No config data is specified for test %s in %s.' %
                (test_name, _PRESENTATION_CONFIG_FILE))

    presentation_dict = config_data[test_name]
    try:
        master_name = presentation_dict['master_name']
    except KeyError:
        raise PerfUploadingError(
                'No master name is specified for test %s in %s.' %
                (test_name, _PRESENTATION_CONFIG_FILE))
    if 'dashboard_test_name' in presentation_dict:
        test_name = presentation_dict['dashboard_test_name']
    return {'master_name': master_name, 'test_name': test_name}


def _format_for_upload(platform_name, cros_version, chrome_version,
                       hardware_id, variant_name, hardware_hostname,
                       perf_data, presentation_info):
    """Formats perf data suitably to upload to the perf dashboard.

    The perf dashboard expects perf data to be uploaded as a
    specially-formatted JSON string.  In particular, the JSON object must be a
    dictionary with key "data", and value being a list of dictionaries where
    each dictionary contains all the information associated with a single
    measured perf value: master name, bot name, test name, perf value, error
    value, units, and build version numbers.

    @param platform_name: The string name of the platform.
    @param cros_version: The string chromeOS version number.
    @param chrome_version: The string chrome version number.
    @param hardware_id: String that identifies the type of hardware the test was
        executed on.
    @param variant_name: String that identifies the variant name of the board.
    @param hardware_hostname: String that identifies the name of the device the
        test was executed on.
    @param perf_data: A dictionary of measured perf data as computed by
        _compute_avg_stddev().
    @param presentation_info: A dictionary of dashboard presentation info for
        the given test, as identified by _gather_presentation_info().

    @return A dictionary containing the formatted information ready to upload
        to the performance dashboard.

    """
    dash_entries = []
    if variant_name:
        platform_name += '-' + variant_name
    for (desc, graph), data in perf_data.iteritems():
        # Each perf metric is named by a path that encodes the test name,
        # a graph name (if specified), and a description.  This must be defined
        # according to rules set by the Chrome team, as implemented in:
        # chromium/tools/build/scripts/slave/results_dashboard.py.
        if desc.endswith('_ref'):
            desc = 'ref'
        desc = desc.replace('_by_url', '')
        desc = desc.replace('/', '_')
        if data['graph']:
            test_path = '%s/%s/%s' % (presentation_info['test_name'],
                                      data['graph'], desc)
        else:
            test_path = '%s/%s' % (presentation_info['test_name'], desc)

        new_dash_entry = {
            'master': presentation_info['master_name'],
            'bot': 'cros-' + platform_name,  # Prefix to clarify it's chromeOS.
            'test': test_path,
            'value': data['value'],
            'error': data['stddev'],
            'units': data['units'],
            'higher_is_better': data['higher_is_better'],
            'revision': _get_id_from_version(chrome_version, cros_version),
            'supplemental_columns': {
                'r_cros_version': cros_version,
                'r_chrome_version': chrome_version,
                'a_default_rev': 'r_chrome_version',
                'a_hardware_identifier': hardware_id,
                'a_hardware_hostname': hardware_hostname,
            }
        }

        dash_entries.append(new_dash_entry)

    json_string = json.dumps(dash_entries)
    return {'data': json_string}


def _get_version_numbers(test_attributes):
    """Gets the version numbers from the test attributes and validates them.

    @param test_attributes: The attributes property (which is a dict) of an
        autotest tko.models.test object.

    @return A pair of strings (Chrome OS version, Chrome version).

    @raises PerfUploadingError if a version isn't formatted as expected.
    """
    chrome_version = test_attributes.get('CHROME_VERSION', '')
    cros_version = test_attributes.get('CHROMEOS_RELEASE_VERSION', '')
    # Prefix the ChromeOS version number with the Chrome milestone.
    cros_version = chrome_version[:chrome_version.find('.') + 1] + cros_version
    if not re.match(VERSION_REGEXP, cros_version):
        raise PerfUploadingError('CrOS version "%s" does not match expected '
                                 'format.' % cros_version)
    if not re.match(VERSION_REGEXP, chrome_version):
        raise PerfUploadingError('Chrome version "%s" does not match expected '
                                 'format.' % chrome_version)
    return (cros_version, chrome_version)


def _get_id_from_version(chrome_version, cros_version):
    """Computes the point ID to use, from Chrome and ChromeOS version numbers.

    For ChromeOS row data, data values are associated with both a Chrome
    version number and a ChromeOS version number (unlike for Chrome row data
    that is associated with a single revision number).  This function takes
    both version numbers as input, then computes a single, unique integer ID
    from them, which serves as a 'fake' revision number that can uniquely
    identify each ChromeOS data point, and which will allow ChromeOS data points
    to be sorted by Chrome version number, with ties broken by ChromeOS version
    number.

    To compute the integer ID, we take the portions of each version number that
    serve as the shortest unambiguous names for each (as described here:
    http://www.chromium.org/developers/version-numbers).  We then force each
    component of each portion to be a fixed width (padded by zeros if needed),
    concatenate all digits together (with those coming from the Chrome version
    number first), and convert the entire string of digits into an integer.
    We ensure that the total number of digits does not exceed that which is
    allowed by AppEngine NDB for an integer (64-bit signed value).

    For example:
      Chrome version: 27.0.1452.2 (shortest unambiguous name: 1452.2)
      ChromeOS version: 27.3906.0.0 (shortest unambiguous name: 3906.0.0)
      concatenated together with padding for fixed-width columns:
          ('01452' + '002') + ('03906' + '000' + '00') = '014520020390600000'
      Final integer ID: 14520020390600000

    @param chrome_ver: The Chrome version number as a string.
    @param cros_ver: The ChromeOS version number as a string.

    @return A unique integer ID associated with the two given version numbers.

    """

    # Number of digits to use from each part of the version string for Chrome
    # and Chrome OS versions when building a point ID out of these two versions.
    chrome_version_col_widths = [0, 0, 5, 3]
    cros_version_col_widths = [0, 5, 3, 2]

    def get_digits_from_version(version_num, column_widths):
        if re.match(VERSION_REGEXP, version_num):
            computed_string = ''
            version_parts = version_num.split('.')
            for i, version_part in enumerate(version_parts):
                if column_widths[i]:
                    computed_string += version_part.zfill(column_widths[i])
            return computed_string
        else:
            return None

    chrome_digits = get_digits_from_version(
            chrome_version, chrome_version_col_widths)
    cros_digits = get_digits_from_version(
            cros_version, cros_version_col_widths)
    if not chrome_digits or not cros_digits:
        return None
    result_digits = chrome_digits + cros_digits
    max_digits = sum(chrome_version_col_widths + cros_version_col_widths)
    if len(result_digits) > max_digits:
        return None
    return int(result_digits)


def _send_to_dashboard(data_obj):
    """Sends formatted perf data to the perf dashboard.

    @param data_obj: A formatted data object as returned by
        _format_for_upload().

    @raises PerfUploadingError if an exception was raised when uploading.

    """
    encoded = urllib.urlencode(data_obj)
    req = urllib2.Request(_DASHBOARD_UPLOAD_URL, encoded)
    try:
        urllib2.urlopen(req)
    except urllib2.HTTPError as e:
        raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' % (
                e.code, e.msg, data_obj['data']))
    except urllib2.URLError as e:
        raise PerfUploadingError(
                'URLError: %s for JSON %s\n' %
                (str(e.reason), data_obj['data']))
    except httplib.HTTPException:
        raise PerfUploadingError(
                'HTTPException for JSON %s\n' % data_obj['data'])


def upload_test(job, test):
    """Uploads any perf data associated with a test to the perf dashboard.

    @param job: An autotest tko.models.job object that is associated with the
        given |test|.
    @param test: An autotest tko.models.test object that may or may not be
        associated with measured perf data.

    """
    if not test.perf_values:
        return

    # Aggregate values from multiple iterations together.
    perf_data = _aggregate_iterations(test.perf_values)

    # Compute averages and standard deviations as needed for measured perf
    # values that exist in multiple iterations.  Ultimately, we only upload a
    # single measurement (with standard deviation) for every unique measured
    # perf metric.
    _compute_avg_stddev(perf_data)

    # Format the perf data for the upload, then upload it.
    test_name = test.testname
    platform_name = job.machine_group
    hardware_id = test.attributes.get('hwid', '')
    hardware_hostname = test.machine
    variant_name = test.attributes.get(constants.VARIANT_KEY, None)
    config_data = _parse_config_file(_PRESENTATION_CONFIG_FILE)
    shadow_config_data = _parse_config_file(_PRESENTATION_SHADOW_CONFIG_FILE)
    config_data.update(shadow_config_data)
    try:
        cros_version, chrome_version = _get_version_numbers(test.attributes)
        presentation_info = _gather_presentation_info(config_data, test_name)
        formatted_data = _format_for_upload(
                platform_name, cros_version, chrome_version, hardware_id,
                variant_name, hardware_hostname, perf_data, presentation_info)
        _send_to_dashboard(formatted_data)
    except PerfUploadingError as e:
        tko_utils.dprint('Error when uploading perf data to the perf '
                         'dashboard for test %s: %s' % (test_name, e))
    else:
        tko_utils.dprint('Successfully uploaded perf data to the perf '
                         'dashboard for test %s.' % test_name)