Create HTTP-based GM results viewer.
For now, it only allows VIEWING results... next, it will allow the user to
rebaseline GM results via the web interface.
R=borenet@google.com
Review URL: https://codereview.chromium.org/24274003
git-svn-id: http://skia.googlecode.com/svn/trunk@11500 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
new file mode 100755
index 0000000..0c50d26
--- /dev/null
+++ b/gm/rebaseline_server/results.py
@@ -0,0 +1,178 @@
+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory. That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+ sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+ """ Loads actual and expected results from all builders, supplying combined
+ reports as requested. """
+
+ def __init__(self, actuals_root, expected_root):
+ """
+ params:
+ actuals_root: root directory containing all actual-results.json files
+ expected_root: root directory containing all expected-results.json files
+ """
+ self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+ self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+ self._all_results = self._Combine()
+
+ def GetAll(self):
+ """Return results of all tests, as a list in this form:
+
+ [
+ {
+ "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+ "test": "bigmatrix",
+ "config": "8888",
+ "resultType": "failed",
+ "expectedHashType": "bitmap-64bitMD5",
+ "expectedHashDigest": "10894408024079689926",
+ "actualHashType": "bitmap-64bitMD5",
+ "actualHashDigest": "2409857384569",
+ },
+ ...
+ ]
+ """
+ return self._all_results
+
+ @staticmethod
+ def _GetDictsFromRoot(root, pattern='*.json'):
+ """Read all JSON dictionaries within a directory tree, returning them within
+ a meta-dictionary (keyed by the builder name for each dictionary).
+
+ params:
+ root: path to root of directory tree
+ pattern: which files to read within root (fnmatch-style pattern)
+ """
+ meta_dict = {}
+ for dirpath, dirnames, filenames in os.walk(root):
+ for matching_filename in fnmatch.filter(filenames, pattern):
+ builder = os.path.basename(dirpath)
+ if builder.endswith('-Trybot'):
+ continue
+ fullpath = os.path.join(dirpath, matching_filename)
+ meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+ return meta_dict
+
+ def _Combine(self):
+ """Returns a list of all tests, across all builders, based on the
+ contents of self._actual_builder_dicts and self._expected_builder_dicts .
+ Returns the list in the same form needed for GetAllResults().
+ """
+ all_tests = []
+ for builder in sorted(self._actual_builder_dicts.keys()):
+ actual_results_for_this_builder = (
+ self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+ for result_type in sorted(actual_results_for_this_builder.keys()):
+ results_of_this_type = actual_results_for_this_builder[result_type]
+ if not results_of_this_type:
+ continue
+ for image_name in sorted(results_of_this_type.keys()):
+ actual_image = results_of_this_type[image_name]
+ try:
+ # TODO(epoger): assumes a single allowed digest per test
+ expected_image = (
+ self._expected_builder_dicts
+ [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
+ [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
+ [0])
+ except (KeyError, TypeError):
+ # There are several cases in which we would expect to find
+ # no expectations for a given test:
+ #
+ # 1. result_type == NOCOMPARISON
+ # There are no expectations for this test yet!
+ #
+ # 2. ignore-tests.txt
+ # If a test has been listed in ignore-tests.txt, then its status
+ # may show as FAILUREIGNORED even if it doesn't have any
+ # expectations yet.
+ #
+ # 3. alternate rendering mode failures (e.g. serialized)
+ # In cases like
+ # https://code.google.com/p/skia/issues/detail?id=1684
+ # ('tileimagefilter GM test failing in serialized render mode'),
+ # the gm-actuals will list a failure for the alternate
+ # rendering mode even though we don't have explicit expectations
+ # for the test (the implicit expectation is that it must
+ # render the same in all rendering modes).
+ #
+ # Don't log types 1 or 2, because they are common.
+ # Log other types, because they are rare and we should know about
+ # them, but don't throw an exception, because we need to keep our
+ # tools working in the meanwhile!
+ if result_type not in [
+ gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
+ gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
+ print 'WARNING: No expectations found for test: %s' % {
+ 'builder': builder,
+ 'image_name': image_name,
+ 'result_type': result_type,
+ }
+ expected_image = [None, None]
+
+ # If this test was recently rebaselined, it will remain in
+ # the "failed" set of actuals until all the bots have
+ # cycled (although the expectations have indeed been set
+ # from the most recent actuals). Treat these as successes
+ # instead of failures.
+ #
+ # TODO(epoger): Do we need to do something similar in
+ # other cases, such as when we have recently marked a test
+ # as ignoreFailure but it still shows up in the "failed"
+ # category? Maybe we should not rely on the result_type
+ # categories recorded within the gm_actuals AT ALL, and
+ # instead evaluate the result_type ourselves based on what
+ # we see in expectations vs actual checksum?
+ if expected_image == actual_image:
+ updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
+ else:
+ updated_result_type = result_type
+
+ # TODO(epoger): For now, don't include succeeded results.
+ # There are so many of them that they make the client too slow.
+ if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+ continue
+
+ (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+ all_tests.append({
+ "builder": builder,
+ "test": test,
+ "config": config,
+ "resultType": updated_result_type,
+ "actualHashType": actual_image[0],
+ "actualHashDigest": str(actual_image[1]),
+ "expectedHashType": expected_image[0],
+ "expectedHashDigest": str(expected_image[1]),
+ })
+ return all_tests
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
new file mode 100755
index 0000000..5b81d8c
--- /dev/null
+++ b/gm/rebaseline_server/server.py
@@ -0,0 +1,228 @@
+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+HTTP server for our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import argparse
+import BaseHTTPServer
+import json
+import os
+import posixpath
+import re
+import shutil
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'tools' directory, so that we can import svn.py within
+# that directory.
+# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
+if TOOLS_DIRECTORY not in sys.path:
+ sys.path.append(TOOLS_DIRECTORY)
+import svn
+
+# Imports from local dir
+import results
+
+ACTUALS_SVN_REPO = 'http://skia-autogen.googlecode.com/svn/gm-actual'
+PATHSPLIT_RE = re.compile('/([^/]+)/(.+)')
+TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+# A simple dictionary of file name extensions to MIME types. The empty string
+# entry is used as the default when no extension was given or if the extension
+# has no entry in this dictionary.
+MIME_TYPE_MAP = {'': 'application/octet-stream',
+ 'html': 'text/html',
+ 'css': 'text/css',
+ 'png': 'image/png',
+ 'js': 'application/javascript',
+ 'json': 'application/json'
+ }
+
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_PORT = 8888
+
+_SERVER = None # This gets filled in by main()
+
+class Server(object):
+ """ HTTP server for our HTML rebaseline viewer.
+
+ params:
+ actuals_dir: directory under which we will check out the latest actual
+ GM results
+ expectations_dir: directory under which to find GM expectations (they
+ must already be in that directory)
+ port: which TCP port to listen on for HTTP requests
+ export: whether to allow HTTP clients on other hosts to access this server
+ """
+ def __init__(self,
+ actuals_dir=DEFAULT_ACTUALS_DIR,
+ expectations_dir=DEFAULT_EXPECTATIONS_DIR,
+ port=DEFAULT_PORT, export=False):
+ self._actuals_dir = actuals_dir
+ self._expectations_dir = expectations_dir
+ self._port = port
+ self._export = export
+
+ def fetch_results(self):
+ """ Create self.results, based on the expectations in
+ self._expectations_dir and the latest actuals from skia-autogen.
+
+ TODO(epoger): Add a new --browseonly mode setting. In that mode,
+ the gm-actuals and expectations will automatically be updated every few
+ minutes. See discussion in https://codereview.chromium.org/24274003/ .
+ """
+ print 'Checking out latest actual GM results from %s into %s ...' % (
+ ACTUALS_SVN_REPO, self._actuals_dir)
+ actuals_repo = svn.Svn(self._actuals_dir)
+ if not os.path.isdir(self._actuals_dir):
+ os.makedirs(self._actuals_dir)
+ actuals_repo.Checkout(ACTUALS_SVN_REPO, '.')
+ else:
+ actuals_repo.Update('.')
+ print 'Parsing results from actuals in %s and expectations in %s ...' % (
+ self._actuals_dir, self._expectations_dir)
+ self.results = results.Results(
+ actuals_root=self._actuals_dir,
+ expected_root=self._expectations_dir)
+
+ def run(self):
+ self.fetch_results()
+ if self._export:
+ server_address = ('', self._port)
+ print ('WARNING: Running in "export" mode. Users on other machines will '
+ 'be able to modify your GM expectations!')
+ else:
+ server_address = ('127.0.0.1', self._port)
+ http_server = BaseHTTPServer.HTTPServer(server_address, HTTPRequestHandler)
+ print 'Ready for requests on http://%s:%d' % (
+ http_server.server_name, http_server.server_port)
+ http_server.serve_forever()
+
+
+class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+ """ HTTP request handlers for various types of queries this server knows
+ how to handle (static HTML and Javascript, expected/actual results, etc.)
+ """
+ def do_GET(self):
+ """ Handles all GET requests, forwarding them to the appropriate
+ do_GET_* dispatcher. """
+ if self.path == '' or self.path == '/' or self.path == '/index.html' :
+ self.redirect_to('/static/view.html')
+ return
+ if self.path == '/favicon.ico' :
+ self.redirect_to('/static/favicon.ico')
+ return
+
+ # All requests must be of this form:
+ # /dispatcher/remainder
+ # where "dispatcher" indicates which do_GET_* dispatcher to run
+ # and "remainder" is the remaining path sent to that dispatcher.
+ normpath = posixpath.normpath(self.path)
+ (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
+ dispatchers = {
+ 'results': self.do_GET_results,
+ 'static': self.do_GET_static,
+ }
+ dispatcher = dispatchers[dispatcher_name]
+ dispatcher(remainder)
+
+ def do_GET_results(self, result_type):
+ """ Handle a GET request for GM results.
+ For now, we ignore the remaining path info, because we only know how to
+ return all results.
+
+ TODO(epoger): Unless we start making use of result_type, remove that
+ parameter."""
+ print 'do_GET_results: sending results of type "%s"' % result_type
+ response_dict = _SERVER.results.GetAll()
+ if response_dict:
+ self.send_json_dict(response_dict)
+ else:
+ self.send_error(404)
+
+ def do_GET_static(self, path):
+ """ Handle a GET request for a file under the 'static' directory. """
+ print 'do_GET_static: sending file "%s"' % path
+ self.send_file(posixpath.join('static', path))
+
+ def redirect_to(self, url):
+ """ Redirect the HTTP client to a different url. """
+ self.send_response(301)
+ self.send_header('Location', url)
+ self.end_headers()
+
+ def send_file(self, path):
+ """ Send the contents of the file at this path, with a mimetype based
+ on the filename extension. """
+ # Grab the extension if there is one
+ extension = os.path.splitext(path)[1]
+ if len(extension) >= 1:
+ extension = extension[1:]
+
+ # Determine the MIME type of the file from its extension
+ mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP[''])
+
+ # Open the file and send it over HTTP
+ if os.path.isfile(path):
+ with open(path, 'rb') as sending_file:
+ self.send_response(200)
+ self.send_header('Content-type', mime_type)
+ self.end_headers()
+ self.wfile.write(sending_file.read())
+ else:
+ self.send_error(404)
+
+ def send_json_dict(self, json_dict):
+ """ Send the contents of this dictionary in JSON format, with a JSON
+ mimetype. """
+ self.send_response(200)
+ self.send_header('Content-type', 'application/json')
+ self.end_headers()
+ json.dump(json_dict, self.wfile)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--actuals-dir',
+ help=('Directory into which we will check out the latest '
+ 'actual GM results. If this directory does not '
+ 'exist, it will be created. Defaults to %(default)s'),
+ default=DEFAULT_ACTUALS_DIR)
+ parser.add_argument('--expectations-dir',
+ help=('Directory under which to find GM expectations; '
+ 'defaults to %(default)s'),
+ default=DEFAULT_EXPECTATIONS_DIR)
+ parser.add_argument('--export', action='store_true',
+ help=('Instead of only allowing access from HTTP clients '
+ 'on localhost, allow HTTP clients on other hosts '
+ 'to access this server. WARNING: doing so will '
+ 'allow users on other hosts to modify your '
+ 'GM expectations!'))
+ parser.add_argument('--port',
+ help=('Which TCP port to listen on for HTTP requests; '
+ 'defaults to %(default)s'),
+ default=DEFAULT_PORT)
+ args = parser.parse_args()
+ global _SERVER
+ _SERVER = Server(expectations_dir=args.expectations_dir,
+ port=args.port, export=args.export)
+ _SERVER.run()
+
+if __name__ == '__main__':
+ main()
diff --git a/gm/rebaseline_server/static/loader.js b/gm/rebaseline_server/static/loader.js
new file mode 100644
index 0000000..68da73a
--- /dev/null
+++ b/gm/rebaseline_server/static/loader.js
@@ -0,0 +1,20 @@
+/*
+ * Loader:
+ * Reads GM result reports written out by results_loader.py, and imports
+ * their data into $scope.results .
+ */
+var Loader = angular.module(
+ 'Loader',
+ []
+);
+Loader.controller(
+ 'Loader.Controller',
+ function($scope, $http) {
+ $http.get("/results/all").then(
+ function(response) {
+ $scope.results = response.data;
+ $scope.sortColumn = 'test';
+ }
+ );
+ }
+);
diff --git a/gm/rebaseline_server/static/view.html b/gm/rebaseline_server/static/view.html
new file mode 100644
index 0000000..89ef538
--- /dev/null
+++ b/gm/rebaseline_server/static/view.html
@@ -0,0 +1,101 @@
+<!DOCTYPE html>
+
+<html ng-app="Loader">
+
+<head>
+ <title>Current GM Results</title>
+ <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.1.5/angular.js"></script>
+ <script src="loader.js"></script>
+</head>
+
+<body>
+ <div ng-controller="Loader.Controller">
+
+ <!-- TODO(epoger): Add a warning banner if the server is running in
+ --export mode
+ -->
+
+ Settings:
+ <ul>
+ <!-- TODO(epoger): Now that we get multiple result types in a single
+ fetch, modify the UI: add a column showing resultType, and allow
+ the user to sort/filter on that column just like all the
+ others. -->
+ <li>show results of type
+ <select ng-model="showResultsOfType"
+ ng-init="showResultsOfType='failed'">
+ <option>failed</option>
+ <option>failure-ignored</option>
+ <!--
+ <option>no-comparison</option>
+
+ TODO(epoger): For now, I have disabled viewing the
+ no-comparison results because there are so many of them, and
+ the browser takes forever to download all the images. Maybe
+ we should use some sort of lazy-loading technique
+ (e.g. http://www.appelsiini.net/projects/lazyload ), so that
+ the images are only loaded as they become viewable...
+ -->
+ <!--
+ <option>succeeded</option>
+
+ TODO(epoger): See results.py: for now, I have disabled
+ returning succeeded tests as part of the JSON, because it
+ makes the returned JSON too big (and slows down the client).
+ -->
+ </select>
+ </li>
+ <li>image size
+ <input type="text" ng-model="imageSize" ng-init="imageSize=100"
+ maxlength="4"/>
+ </li>
+ </ul>
+
+ <p>
+ <!-- TODO(epoger): Show some sort of "loading" message, instead of
+ an empty table, while the data is loading. Otherwise, if there are
+ a lot of failures and it takes a long time to load them, the user
+ might think there are NO failures and leave the page! -->
+ <table border="1">
+ <tr>
+ <th ng:click="sortColumn='builder'">Builder</th>
+ <th ng:click="sortColumn='test'">Test</th>
+ <th ng:click="sortColumn='config'">Config</th>
+ <th ng:click="sortColumn='expectedHashDigest'">Expected Image</th>
+ <th ng:click="sortColumn='actualHashDigest'">Actual Image</th>
+ <!-- TODO(epoger): Add more columns, such as...
+ pixel diff
+ notes/bugs
+ ignoreFailure boolean
+ -->
+ </tr>
+ <!-- TODO(epoger): improve the column sorting, as per
+ http://jsfiddle.net/vojtajina/js64b/14/ -->
+ <tr ng-repeat="result in results | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
+ <td>{{result.builder}}</td>
+ <td>{{result.test}}</td>
+ <td>{{result.config}}</td>
+ <td>
+ <a target="_blank" href="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.expectedHashType}}/{{result.test}}/{{result.expectedHashDigest}}.png">
+ <img width="{{imageSize}}" src="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.expectedHashType}}/{{result.test}}/{{result.expectedHashDigest}}.png"/>
+ </a>
+ </td>
+ <td>
+ <a target="_blank" href="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.actualHashType}}/{{result.test}}/{{result.actualHashDigest}}.png">
+ <img width="{{imageSize}}" src="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.actualHashType}}/{{result.test}}/{{result.actualHashDigest}}.png"/>
+ </a>
+ </td>
+ </tr>
+ </table>
+ </div>
+
+ <!-- TODO(epoger): Can we get the base URLs (commondatastorage and
+ issues list) from
+ http://skia.googlecode.com/svn/buildbot/site_config/global_variables.json
+ ? I tried importing the
+ http://skia.googlecode.com/svn/buildbot/skia_tools.js script and using
+ that to do so, but I got Access-Control-Allow-Origin errors.
+ -->
+
+</body>
+</html>
diff --git a/tools/svn.py b/tools/svn.py
index 9b60413..72d8ad8 100644
--- a/tools/svn.py
+++ b/tools/svn.py
@@ -85,6 +85,15 @@
"""
return self._RunCommand([SVN, 'checkout', url, path])
+ def Update(self, path):
+ """Update the working copy.
+ Returns stdout as a single string.
+
+ @param path path (within self._directory) within which to run
+ "svn update"
+ """
+ return self._RunCommand([SVN, 'update', path])
+
def ListSubdirs(self, url):
"""Returns a list of all subdirectories (not files) within a given SVN
url.