gm/rebaseline_server/results.py - platform/external/skqp - Gitiles

 #!/usr/bin/python

 """
 Copyright 2013 Google Inc.

 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.

 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
 """

 # System-level imports
 import fnmatch
 import os
 import re

 # Imports from within Skia
 import fix_pythonpath  # must do this first
 import gm_json
 import imagepairset

 # Keys used to link an image to a particular GM test.
 # NOTE: Keep these in sync with static/constants.js
 VALUE__HEADER__SCHEMA_VERSION = 3
 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
 KEY__EXTRACOLUMNS__BUILDER = 'builder'
 KEY__EXTRACOLUMNS__CONFIG = 'config'
 KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType'
 KEY__EXTRACOLUMNS__TEST = 'test'
 KEY__HEADER__DATAHASH = 'dataHash'
 KEY__HEADER__IS_EDITABLE = 'isEditable'
 KEY__HEADER__IS_EXPORTED = 'isExported'
 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
 KEY__HEADER__RESULTS_ALL = 'all'
 KEY__HEADER__RESULTS_FAILURES = 'failures'
 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
 KEY__HEADER__TIME_UPDATED = 'timeUpdated'
 KEY__HEADER__TYPE = 'type'
 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED

 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
 IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)

 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 DEFAULT_ACTUALS_DIR = '.gm-actuals'
 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
     PARENT_DIRECTORY, '.generated-images')

 # Define the default set of builders we will process expectations/actuals for.
 # This allows us to ignore builders for which we don't maintain expectations
 # (trybots, Valgrind, ASAN, TSAN), and avoid problems like
 # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
 # produces error when trying to add baselines for ASAN/TSAN builders')
 DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
 DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
     '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']


 class BaseComparisons(object):
   """Base class for generating summary of comparisons between two image sets.
   """

   def get_results_of_type(self, results_type):
     """Return results of some/all tests (depending on 'results_type' parameter).

     Args:
       results_type: string describing which types of results to include; must
           be one of the RESULTS_* constants

     Results are returned in a dictionary as output by ImagePairSet.as_dict().
     """
     return self._results[results_type]

   def get_packaged_results_of_type(self, results_type, reload_seconds=None,
                                    is_editable=False, is_exported=True):
     """Package the results of some/all tests as a complete response_dict.

     Args:
       results_type: string indicating which set of results to return;
           must be one of the RESULTS_* constants
       reload_seconds: if specified, note that new results may be available once
           these results are reload_seconds old
       is_editable: whether clients are allowed to submit new baselines
       is_exported: whether these results are being made available to other
           network hosts
     """
     response_dict = self._results[results_type]
     time_updated = self.get_timestamp()
     response_dict[imagepairset.KEY__ROOT__HEADER] = {
         KEY__HEADER__SCHEMA_VERSION: (
             VALUE__HEADER__SCHEMA_VERSION),

         # Timestamps:
         # 1. when this data was last updated
         # 2. when the caller should check back for new data (if ever)
         KEY__HEADER__TIME_UPDATED: time_updated,
         KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
             (time_updated+reload_seconds) if reload_seconds else None),

         # The type we passed to get_results_of_type()
         KEY__HEADER__TYPE: results_type,

         # Hash of dataset, which the client must return with any edits--
         # this ensures that the edits were made to a particular dataset.
         KEY__HEADER__DATAHASH: str(hash(repr(
             response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))),

         # Whether the server will accept edits back.
         KEY__HEADER__IS_EDITABLE: is_editable,

         # Whether the service is accessible from other hosts.
         KEY__HEADER__IS_EXPORTED: is_exported,
     }
     return response_dict

   def get_timestamp(self):
     """Return the time at which this object was created, in seconds past epoch
     (UTC).
     """
     return self._timestamp

   _match_builders_pattern_list = [
       re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
   _skip_builders_pattern_list = [
       re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]

   def set_match_builders_pattern_list(self, pattern_list):
     """Override the default set of builders we should process.

     The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .

     Note that skip_builders_pattern_list overrides this; regardless of whether a
     builder is in the "match" list, if it's in the "skip" list, we will skip it.

     Args:
       pattern_list: list of regex patterns; process builders that match any
           entry within this list
     """
     if pattern_list == None:
       pattern_list = []
     self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]

   def set_skip_builders_pattern_list(self, pattern_list):
     """Override the default set of builders we should skip while processing.

     The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .

     This overrides match_builders_pattern_list; regardless of whether a
     builder is in the "match" list, if it's in the "skip" list, we will skip it.

     Args:
       pattern_list: list of regex patterns; skip builders that match any
           entry within this list
     """
     if pattern_list == None:
       pattern_list = []
     self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]

   def _ignore_builder(self, builder):
     """Returns True if we should skip processing this builder.

     Args:
       builder: name of this builder, as a string

     Returns:
       True if we should ignore expectations and actuals for this builder.
     """
     for pattern in self._skip_builders_pattern_list:
       if pattern.match(builder):
         return True
     for pattern in self._match_builders_pattern_list:
       if pattern.match(builder):
         return False
     return True

   def _read_builder_dicts_from_root(self, root, pattern='*.json'):
     """Read all JSON dictionaries within a directory tree.

     Skips any dictionaries belonging to a builder we have chosen to ignore.

     Args:
       root: path to root of directory tree
       pattern: which files to read within root (fnmatch-style pattern)

     Returns:
       A meta-dictionary containing all the JSON dictionaries found within
       the directory tree, keyed by builder name (the basename of the directory
       where each JSON dictionary was found).

     Raises:
       IOError if root does not refer to an existing directory
     """
     # I considered making this call _read_dicts_from_root(), but I decided
     # it was better to prune out the ignored builders within the os.walk().
     if not os.path.isdir(root):
       raise IOError('no directory found at path %s' % root)
     meta_dict = {}
     for dirpath, dirnames, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
         if self._ignore_builder(builder):
           continue
         full_path = os.path.join(dirpath, matching_filename)
         meta_dict[builder] = gm_json.LoadFromFile(full_path)
     return meta_dict

   def _read_dicts_from_root(self, root, pattern='*.json'):
     """Read all JSON dictionaries within a directory tree.

     Args:
       root: path to root of directory tree
       pattern: which files to read within root (fnmatch-style pattern)

     Returns:
       A meta-dictionary containing all the JSON dictionaries found within
       the directory tree, keyed by the pathname (relative to root) of each JSON
       dictionary.

     Raises:
       IOError if root does not refer to an existing directory
     """
     if not os.path.isdir(root):
       raise IOError('no directory found at path %s' % root)
     meta_dict = {}
     for abs_dirpath, dirnames, filenames in os.walk(root):
       rel_dirpath = os.path.relpath(abs_dirpath, root)
       for matching_filename in fnmatch.filter(filenames, pattern):
         abs_path = os.path.join(abs_dirpath, matching_filename)
         rel_path = os.path.join(rel_dirpath, matching_filename)
         meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
     return meta_dict

   @staticmethod
   def _read_noncomment_lines(path):
     """Return a list of all noncomment lines within a file.

     (A "noncomment" line is one that does not start with a '#'.)

     Args:
       path: path to file
     """
     lines = []
     with open(path, 'r') as fh:
       for line in fh:
         if not line.startswith('#'):
           lines.append(line.strip())
     return lines

   @staticmethod
   def _create_relative_url(hashtype_and_digest, test_name):
     """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.

     If we don't have a record of this image, returns None.

     Args:
       hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
           don't have a record of this image
       test_name: string; name of the GM test that created this image
     """
     if not hashtype_and_digest:
       return None
     return gm_json.CreateGmRelativeUrl(
         test_name=test_name,
         hash_type=hashtype_and_digest[0],
         hash_digest=hashtype_and_digest[1])

   @staticmethod
   def combine_subdicts(input_dict):
     """ Flatten out a dictionary structure by one level.

     Input:
       {
         KEY_A1 : {
           KEY_B1 : VALUE_B1,
         },
         KEY_A2 : {
           KEY_B2 : VALUE_B2,
         }
       }

     Output:
       {
         KEY_B1 : VALUE_B1,
         KEY_B2 : VALUE_B2,
       }

     If this would result in any repeated keys, it will raise an Exception.
     """
     output_dict = {}
     for key, subdict in input_dict.iteritems():
       for subdict_key, subdict_value in subdict.iteritems():
         if subdict_key in output_dict:
           raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
         output_dict[subdict_key] = subdict_value
     return output_dict

   @staticmethod
   def get_multilevel(input_dict, *keys):
     """ Returns input_dict[key1][key2][...], or None if any key is not found.
     """
     for key in keys:
       if input_dict == None:
         return None
       input_dict = input_dict.get(key, None)
     return input_dict
	#!/usr/bin/python

	"""
	Copyright 2013 Google Inc.

	Use of this source code is governed by a BSD-style license that can be
	found in the LICENSE file.

	Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
	"""

	# System-level imports
	import fnmatch
	import os
	import re

	# Imports from within Skia
	import fix_pythonpath # must do this first
	import gm_json
	import imagepairset

	# Keys used to link an image to a particular GM test.
	# NOTE: Keep these in sync with static/constants.js
	VALUE__HEADER__SCHEMA_VERSION = 3
	KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
	KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
	KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
	KEY__EXTRACOLUMNS__BUILDER = 'builder'
	KEY__EXTRACOLUMNS__CONFIG = 'config'
	KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType'
	KEY__EXTRACOLUMNS__TEST = 'test'
	KEY__HEADER__DATAHASH = 'dataHash'
	KEY__HEADER__IS_EDITABLE = 'isEditable'
	KEY__HEADER__IS_EXPORTED = 'isExported'
	KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
	KEY__HEADER__RESULTS_ALL = 'all'
	KEY__HEADER__RESULTS_FAILURES = 'failures'
	KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
	KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
	KEY__HEADER__TIME_UPDATED = 'timeUpdated'
	KEY__HEADER__TYPE = 'type'
	KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
	KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
	KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
	KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED

	IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
	IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config)

	PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
	DEFAULT_ACTUALS_DIR = '.gm-actuals'
	DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
	PARENT_DIRECTORY, '.generated-images')

	# Define the default set of builders we will process expectations/actuals for.
	# This allows us to ignore builders for which we don't maintain expectations
	# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
	# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
	# produces error when trying to add baselines for ASAN/TSAN builders')
	DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
	DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
	'.-Trybot', '.Valgrind.', '.TSAN.', '.ASAN.*']


	class BaseComparisons(object):
	"""Base class for generating summary of comparisons between two image sets.
	"""

	def get_results_of_type(self, results_type):
	"""Return results of some/all tests (depending on 'results_type' parameter).

	Args:
	results_type: string describing which types of results to include; must
	be one of the RESULTS_* constants

	Results are returned in a dictionary as output by ImagePairSet.as_dict().
	"""
	return self._results[results_type]

	def get_packaged_results_of_type(self, results_type, reload_seconds=None,
	is_editable=False, is_exported=True):
	"""Package the results of some/all tests as a complete response_dict.

	Args:
	results_type: string indicating which set of results to return;
	must be one of the RESULTS_* constants
	reload_seconds: if specified, note that new results may be available once
	these results are reload_seconds old
	is_editable: whether clients are allowed to submit new baselines
	is_exported: whether these results are being made available to other
	network hosts
	"""
	response_dict = self._results[results_type]
	time_updated = self.get_timestamp()
	response_dict[imagepairset.KEY__ROOT__HEADER] = {
	KEY__HEADER__SCHEMA_VERSION: (
	VALUE__HEADER__SCHEMA_VERSION),

	# Timestamps:
	# 1. when this data was last updated
	# 2. when the caller should check back for new data (if ever)
	KEY__HEADER__TIME_UPDATED: time_updated,
	KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
	(time_updated+reload_seconds) if reload_seconds else None),

	# The type we passed to get_results_of_type()
	KEY__HEADER__TYPE: results_type,

	# Hash of dataset, which the client must return with any edits--
	# this ensures that the edits were made to a particular dataset.
	KEY__HEADER__DATAHASH: str(hash(repr(
	response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))),

	# Whether the server will accept edits back.
	KEY__HEADER__IS_EDITABLE: is_editable,

	# Whether the service is accessible from other hosts.
	KEY__HEADER__IS_EXPORTED: is_exported,
	}
	return response_dict

	def get_timestamp(self):
	"""Return the time at which this object was created, in seconds past epoch
	(UTC).
	"""
	return self._timestamp

	_match_builders_pattern_list = [
	re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
	_skip_builders_pattern_list = [
	re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]

	def set_match_builders_pattern_list(self, pattern_list):
	"""Override the default set of builders we should process.

	The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .

	Note that skip_builders_pattern_list overrides this; regardless of whether a
	builder is in the "match" list, if it's in the "skip" list, we will skip it.

	Args:
	pattern_list: list of regex patterns; process builders that match any
	entry within this list
	"""
	if pattern_list == None:
	pattern_list = []
	self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]

	def set_skip_builders_pattern_list(self, pattern_list):
	"""Override the default set of builders we should skip while processing.

	The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .

	This overrides match_builders_pattern_list; regardless of whether a
	builder is in the "match" list, if it's in the "skip" list, we will skip it.

	Args:
	pattern_list: list of regex patterns; skip builders that match any
	entry within this list
	"""
	if pattern_list == None:
	pattern_list = []
	self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]

	def _ignore_builder(self, builder):
	"""Returns True if we should skip processing this builder.

	Args:
	builder: name of this builder, as a string

	Returns:
	True if we should ignore expectations and actuals for this builder.
	"""
	for pattern in self._skip_builders_pattern_list:
	if pattern.match(builder):
	return True
	for pattern in self._match_builders_pattern_list:
	if pattern.match(builder):
	return False
	return True

	def _read_builder_dicts_from_root(self, root, pattern='*.json'):
	"""Read all JSON dictionaries within a directory tree.

	Skips any dictionaries belonging to a builder we have chosen to ignore.

	Args:
	root: path to root of directory tree
	pattern: which files to read within root (fnmatch-style pattern)

	Returns:
	A meta-dictionary containing all the JSON dictionaries found within
	the directory tree, keyed by builder name (the basename of the directory
	where each JSON dictionary was found).

	Raises:
	IOError if root does not refer to an existing directory
	"""
	# I considered making this call _read_dicts_from_root(), but I decided
	# it was better to prune out the ignored builders within the os.walk().
	if not os.path.isdir(root):
	raise IOError('no directory found at path %s' % root)
	meta_dict = {}
	for dirpath, dirnames, filenames in os.walk(root):
	for matching_filename in fnmatch.filter(filenames, pattern):
	builder = os.path.basename(dirpath)
	if self._ignore_builder(builder):
	continue
	full_path = os.path.join(dirpath, matching_filename)
	meta_dict[builder] = gm_json.LoadFromFile(full_path)
	return meta_dict

	def _read_dicts_from_root(self, root, pattern='*.json'):
	"""Read all JSON dictionaries within a directory tree.

	Args:
	root: path to root of directory tree
	pattern: which files to read within root (fnmatch-style pattern)

	Returns:
	A meta-dictionary containing all the JSON dictionaries found within
	the directory tree, keyed by the pathname (relative to root) of each JSON
	dictionary.

	Raises:
	IOError if root does not refer to an existing directory
	"""
	if not os.path.isdir(root):
	raise IOError('no directory found at path %s' % root)
	meta_dict = {}
	for abs_dirpath, dirnames, filenames in os.walk(root):
	rel_dirpath = os.path.relpath(abs_dirpath, root)
	for matching_filename in fnmatch.filter(filenames, pattern):
	abs_path = os.path.join(abs_dirpath, matching_filename)
	rel_path = os.path.join(rel_dirpath, matching_filename)
	meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
	return meta_dict

	@staticmethod
	def _read_noncomment_lines(path):
	"""Return a list of all noncomment lines within a file.

	(A "noncomment" line is one that does not start with a '#'.)

	Args:
	path: path to file
	"""
	lines = []
	with open(path, 'r') as fh:
	for line in fh:
	if not line.startswith('#'):
	lines.append(line.strip())
	return lines

	@staticmethod
	def _create_relative_url(hashtype_and_digest, test_name):
	"""Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.

	If we don't have a record of this image, returns None.

	Args:
	hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
	don't have a record of this image
	test_name: string; name of the GM test that created this image
	"""
	if not hashtype_and_digest:
	return None
	return gm_json.CreateGmRelativeUrl(
	test_name=test_name,
	hash_type=hashtype_and_digest[0],
	hash_digest=hashtype_and_digest[1])

	@staticmethod
	def combine_subdicts(input_dict):
	""" Flatten out a dictionary structure by one level.

	Input:
	{
	KEY_A1 : {
	KEY_B1 : VALUE_B1,
	},
	KEY_A2 : {
	KEY_B2 : VALUE_B2,
	}
	}

	Output:
	{
	KEY_B1 : VALUE_B1,
	KEY_B2 : VALUE_B2,
	}

	If this would result in any repeated keys, it will raise an Exception.
	"""
	output_dict = {}
	for key, subdict in input_dict.iteritems():
	for subdict_key, subdict_value in subdict.iteritems():
	if subdict_key in output_dict:
	raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
	output_dict[subdict_key] = subdict_value
	return output_dict

	@staticmethod
	def get_multilevel(input_dict, *keys):
	""" Returns input_dict[key1][key2][...], or None if any key is not found.
	"""
	for key in keys:
	if input_dict == None:
	return None
	input_dict = input_dict.get(key, None)
	return input_dict