gm/rebaseline_server/compare_configs.py

#!/usr/bin/python

"""
Copyright 2014 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.

Compare GM results for two configs, across all builders.
"""

# System-level imports
import argparse
import fnmatch
import json
import logging
import os
import re
import sys
import time

# Imports from within Skia
#
# TODO(epoger): Once we move the create_filepath_url() function out of
# download_actuals into a shared utility module, we won't need to import
# download_actuals anymore.
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory.  That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
if GM_DIRECTORY not in sys.path:
  sys.path.append(GM_DIRECTORY)
import download_actuals
import gm_json
import imagediffdb
import imagepair
import imagepairset
import results


class ConfigComparisons(results.BaseComparisons):
  """Loads results from two different configurations into an ImagePairSet.

  Loads actual and expected results from all builders, except for those skipped
  by BaseComparisons._ignore_builder().
  """

  def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
               diff_base_url=None):
    """
    Args:
      configs: (string, string) tuple; pair of configs to compare
      actuals_root: root directory containing all actual-results.json files
      generated_images_root: directory within which to create all pixel diffs;
          if this directory does not yet exist, it will be created
      diff_base_url: base URL within which the client should look for diff
          images; if not specified, defaults to a "file:///" URL representation
          of generated_images_root
    """
    time_start = int(time.time())
    self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
    self._diff_base_url = (
        diff_base_url or
        download_actuals.create_filepath_url(generated_images_root))
    self._actuals_root = actuals_root
    self._load_config_pairs(configs)
    self._timestamp = int(time.time())
    logging.info('Results complete; took %d seconds.' %
                 (self._timestamp - time_start))

  def _load_config_pairs(self, configs):
    """Loads the results of all tests, across all builders (based on the
    files within self._actuals_root), compares them across two configs,
    and stores the summary in self._results.

    Args:
      configs: tuple of strings; pair of configs to compare
    """
    logging.info('Reading actual-results JSON files from %s...' %
                 self._actuals_root)
    actual_builder_dicts = ConfigComparisons._read_dicts_from_root(
        self._actuals_root)
    configA, configB = configs
    logging.info('Comparing configs %s and %s...' % (configA, configB))

    all_image_pairs = imagepairset.ImagePairSet(
        descriptions=configs,
        diff_base_url=self._diff_base_url)
    failing_image_pairs = imagepairset.ImagePairSet(
        descriptions=configs,
        diff_base_url=self._diff_base_url)

    all_image_pairs.ensure_extra_column_values_in_summary(
        column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
            results.KEY__RESULT_TYPE__FAILED,
            results.KEY__RESULT_TYPE__NOCOMPARISON,
            results.KEY__RESULT_TYPE__SUCCEEDED,
        ])
    failing_image_pairs.ensure_extra_column_values_in_summary(
        column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
            results.KEY__RESULT_TYPE__FAILED,
            results.KEY__RESULT_TYPE__NOCOMPARISON,
        ])

    builders = sorted(actual_builder_dicts.keys())
    num_builders = len(builders)
    builder_num = 0
    for builder in builders:
      builder_num += 1
      logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
                   (builder_num, num_builders, builder))
      actual_results_for_this_builder = (
          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
      for result_type in sorted(actual_results_for_this_builder.keys()):
        results_of_this_type = actual_results_for_this_builder[result_type]
        if not results_of_this_type:
          continue

        tests_found = set()
        for image_name in sorted(results_of_this_type.keys()):
          (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
          tests_found.add(test)

        for test in tests_found:
          # Get image_relative_url (or None) for each of configA, configB
          image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
          configA_image_relative_url = ConfigComparisons._create_relative_url(
              hashtype_and_digest=results_of_this_type.get(image_name_A),
              test_name=test)
          image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
          configB_image_relative_url = ConfigComparisons._create_relative_url(
              hashtype_and_digest=results_of_this_type.get(image_name_B),
              test_name=test)

          # If we have images for at least one of these two configs,
          # add them to our list.
          if configA_image_relative_url or configB_image_relative_url:
            if configA_image_relative_url == configB_image_relative_url:
              result_type = results.KEY__RESULT_TYPE__SUCCEEDED
            elif not configA_image_relative_url:
              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
            elif not configB_image_relative_url:
              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
            else:
              result_type = results.KEY__RESULT_TYPE__FAILED

            extra_columns_dict = {
                results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
                results.KEY__EXTRACOLUMN__BUILDER: builder,
                results.KEY__EXTRACOLUMN__TEST: test,
                # TODO(epoger): Right now, the client UI crashes if it receives
                # results that do not include a 'config' column.
                # Until we fix that, keep the client happy.
                results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
            }

            try:
              image_pair = imagepair.ImagePair(
                  image_diff_db=self._image_diff_db,
                  base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
                  imageA_relative_url=configA_image_relative_url,
                  imageB_relative_url=configB_image_relative_url,
                  extra_columns=extra_columns_dict)
              all_image_pairs.add_image_pair(image_pair)
              if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
                failing_image_pairs.add_image_pair(image_pair)
            except (KeyError, TypeError):
              logging.exception(
                  'got exception while creating ImagePair for image_name '
                  '"%s", builder "%s"' % (image_name, builder))

    self._results = {
      results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
      results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
    }


def main():
  logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                      datefmt='%m/%d/%Y %H:%M:%S',
                      level=logging.INFO)
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--actuals', default=results.DEFAULT_ACTUALS_DIR,
      help='Directory containing all actual-result JSON files; defaults to '
      '\'%(default)s\' .')
  parser.add_argument(
      'config', nargs=2,
      help='Two configurations to compare (8888, gpu, etc.).')
  parser.add_argument(
      '--outfile', required=True,
      help='File to write result summary into, in JSON format.')
  parser.add_argument(
      '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
      help='Which result types to include. Defaults to \'%(default)s\'; '
      'must be one of ' +
      str([results.KEY__HEADER__RESULTS_FAILURES,
           results.KEY__HEADER__RESULTS_ALL]))
  parser.add_argument(
      '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
      help='Directory within which to download images and generate diffs; '
      'defaults to \'%(default)s\' .')
  args = parser.parse_args()
  results_obj = ConfigComparisons(configs=args.config,
                                  actuals_root=args.actuals,
                                  generated_images_root=args.workdir)
  gm_json.WriteToFile(
      results_obj.get_packaged_results_of_type(results_type=args.results),
      args.outfile)


if __name__ == '__main__':
  main()