Add tool that downloads gm baseline images from the bots.
Review URL: http://codereview.appspot.com/5503082
git-svn-id: http://skia.googlecode.com/svn/trunk@2937 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/tools/download-baselines.py b/tools/download-baselines.py
new file mode 100644
index 0000000..fa0e0b3
--- /dev/null
+++ b/tools/download-baselines.py
@@ -0,0 +1,141 @@
+'''
+Downloads the actual gm results most recently generated by the Skia buildbots,
+and adds any new ones to SVN control.
+
+This tool makes it much easier to check in new baselines, via the following
+steps:
+
+cd .../trunk
+svn update
+# make sure there are no files awaiting svn commit
+tools/download-baselines.py gm/base-macmini-lion-fixed # or other gm/ subdir
+# upload CL for review
+# validate that the new images look right
+# commit CL
+
+
+Copyright 2011 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+# common Python modules
+import re
+import sys
+import urllib2
+
+# modules declared within this same directory
+import svn
+
+# Where to download recently generated baseline images for each baseline type.
+#
+# For now this only works for our Mac buildbots; our other buildbots aren't
+# uploading their results to a web server yet.
+#
+# Note also that these will currently work only within the Google corporate
+# network; that will also change soon.
+ACTUALS_BY_BASELINE_SUBDIR = {
+ 'gm/base-macmini':
+ 'http://172.29.92.185/b/build/slave/Skia_Mac_Float_NoDebug/gm/actual',
+ 'gm/base-macmini-fixed':
+ 'http://172.29.92.185/b/build/slave/Skia_Mac_Fixed_NoDebug/gm/actual',
+ 'gm/base-macmini-lion-fixed':
+ 'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Fixed_NoDebug/gm/actual',
+ 'gm/base-macmini-lion-float':
+ 'http://172.29.92.179/b/build/slave/Skia_MacMiniLion_Float_NoDebug/gm/actual',
+}
+
+IMAGE_REGEX = '.+\.png'
+IMAGE_MIMETYPE = 'image/png'
+
+def GetPlatformUrl(baseline_subdir):
+ """Return URL within which the buildbots store generated baseline images,
+ as of multiple svn revisions.
+
+ Raises KeyError if we don't have a URL matching this baseline_subdir.
+
+ @param baseline_subdir indicates which platform we want images for
+ """
+ try:
+ return ACTUALS_BY_BASELINE_SUBDIR[baseline_subdir]
+ except KeyError:
+ raise KeyError(
+ 'unknown baseline_subdir "%s", try one of these instead: %s' % (
+ baseline_subdir, ACTUALS_BY_BASELINE_SUBDIR.keys()))
+
+def GetLatestResultsUrl(baseline_subdir):
+ """Return URL from which we can download the MOST RECENTLY generated
+ images for this baseline type.
+
+ @param baseline_subdir indicates which platform we want images for
+ """
+ base_platform_url = GetPlatformUrl(baseline_subdir)
+ print 'base_platform_url is %s' % base_platform_url
+
+ # Find the most recently generated baseline images within base_platform_url
+ response = urllib2.urlopen(base_platform_url)
+ html = response.read()
+ link_regex = re.compile('<a href="(.*)">')
+ links = link_regex.findall(html)
+ last_link = links[-1]
+ most_recent_result_url = '%s/%s' % (base_platform_url, last_link)
+ print 'most_recent_result_url is %s' % most_recent_result_url
+ return most_recent_result_url
+
+def DownloadMatchingFiles(source_url, filename_regex, dest_dir):
+ """Download all files from source_url that match filename_regex, and save
+ them (with their original filenames) in dest_dir.
+
+ @param source_url
+ @param filename_regex
+ @param dest_dir
+ """
+ while source_url.endswith('/'):
+ source_url = source_url[:-1]
+ response = urllib2.urlopen(source_url)
+ html = response.read()
+ link_regex = re.compile('<a href="(%s)">' % filename_regex)
+ links = link_regex.findall(html)
+ for link in links:
+ DownloadBinaryFile('%s/%s' % (source_url, link),
+ '%s/%s' % (dest_dir, link))
+
+def DownloadBinaryFile(source_url, dest_path):
+ """Download a single file from its source_url and save it to local disk
+ at dest_path.
+
+ @param source_url
+ @param dest_path
+ """
+ print 'DownloadBinaryFile: %s -> %s' % (source_url, dest_path)
+ url_fh = urllib2.urlopen(source_url)
+ local_fh = open(dest_path, 'wb')
+ local_fh.write(url_fh.read())
+ local_fh.close()
+
+def Main(arglist):
+ """Download most recently generated baseline images for a given platform,
+ and add any new ones to SVN control.
+
+ @param arglist sys.argv or equivalent
+ """
+ num_args = len(arglist)
+ if num_args != 2:
+ raise Exception('usage: %s <baseline_subdir>' % __file__)
+
+ baseline_subdir = arglist[1]
+ while baseline_subdir.endswith('/'):
+ baseline_subdir = baseline_subdir[:-1]
+
+ results_url = GetLatestResultsUrl(baseline_subdir)
+ DownloadMatchingFiles(results_url, IMAGE_REGEX, baseline_subdir)
+ svn_handler = svn.Svn(baseline_subdir)
+ new_files = svn_handler.GetNewFiles()
+ if new_files:
+ svn_handler.AddFiles(new_files)
+ svn_handler.SetProperty(new_files, svn.PROPERTY_MIMETYPE,
+ IMAGE_MIMETYPE)
+
+if __name__ == '__main__':
+ Main(sys.argv)
diff --git a/tools/svn.py b/tools/svn.py
new file mode 100644
index 0000000..e878301
--- /dev/null
+++ b/tools/svn.py
@@ -0,0 +1,66 @@
+'''
+Copyright 2011 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+import re
+import subprocess
+
+PROPERTY_MIMETYPE = 'svn:mime-type'
+
+class Svn:
+
+ def __init__(self, directory):
+ """Set up to manipulate SVN control within the given directory.
+
+ @param directory
+ """
+ self._directory = directory
+
+ def _RunCommand(self, args):
+ """Run a command (from self._directory) and return stdout as a single
+ string.
+
+ @param args a list of arguments
+ """
+ proc = subprocess.Popen(args, cwd=self._directory,
+ stdout=subprocess.PIPE)
+ stdout = proc.communicate()[0]
+ returncode = proc.returncode
+ if returncode is not 0:
+ raise Exception('command "%s" failed in dir "%s": returncode=%s' %
+ (args, self._directory, returncode))
+ return stdout
+
+ def GetNewFiles(self):
+ """Return a list of files which are in this directory but NOT under
+ SVN control.
+ """
+ stdout = self._RunCommand(['svn', 'status'])
+ new_regex = re.compile('\? +(.+)')
+ files = new_regex.findall(stdout)
+ return files
+
+ def AddFiles(self, filenames):
+ """Adds these files to SVN control.
+
+ @param filenames files to add to SVN control
+ """
+ args = ['svn', 'add']
+ args.extend(filenames)
+ print '\n\nAddFiles: %s' % args
+ print self._RunCommand(args)
+
+ def SetProperty(self, filenames, property_name, property_value):
+ """Sets a svn property for these files.
+
+ @param filenames files to set property on
+ @param property_name property_name to set for each file
+ @param property_value what to set the property_name to
+ """
+ args = ['svn', 'propset', property_name, property_value]
+ args.extend(filenames)
+ print '\n\nSetProperty: %s' % args
+ print self._RunCommand(args)