tools/download-baselines.py

'''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.

This tool makes it much easier to check in new baselines, via the following
steps:

cd .../trunk
svn update
# make sure there are no files awaiting svn commit
python tools/download-baselines.py gm/base-macmini-lion-fixed  # or other gm/ subdir
# validate that the new images look right (maybe using compare-baselines.py)
# upload CL for review
# validate that the new images look right in the review tool
# commit CL

Launch with --help to see more options.


Copyright 2011 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''

# common Python modules
import fnmatch
import optparse
import os
import re
import shutil
import sys
import tempfile

# modules declared within this same directory
import svn

# Base URL of SVN repository where buildbots store actual gm image results.
SVN_BASE_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'

USAGE_STRING = 'usage: %s [options] <baseline_subdir>'
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
OPTION_ADD_NEW_FILES = '--add-new-files'

def GetLatestResultsSvnUrl(baseline_subdir):
    """Return SVN URL from which we can check out the MOST RECENTLY generated
    images for this baseline type.

    @param baseline_subdir indicates which platform we want images for
    """
    # trim off 'gm/' prefix
    gm_prefix = 'gm%s' % os.sep
    if not baseline_subdir.startswith(gm_prefix):
        raise Exception('baseline_subdir "%s" should start with "%s"' % (
            baseline_subdir, gm_prefix))
    return '%s/%s' % (SVN_BASE_URL, baseline_subdir[len(gm_prefix):])

def CopyMatchingFiles(source_dir, dest_dir, filename_pattern,
                      only_copy_updates=False):
    """Copy all files from source_dir that match filename_pattern, and
    save them (with their original filenames) in dest_dir.

    @param source_dir
    @param dest_dir where to save the copied files
    @param filename_pattern only copy files that match this Unix-style filename
           pattern (e.g., '*.jpg')
    @param only_copy_updates if True, only copy files that are already
           present in dest_dir
    """
    all_filenames = os.listdir(source_dir)
    matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
    for filename in matching_filenames:
        source_path = os.path.join(source_dir, filename)
        dest_path = os.path.join(dest_dir, filename)
        if only_copy_updates and not os.path.isfile(dest_path):
            continue
        shutil.copyfile(source_path, dest_path)

def Main(options, args):
    """Download most recently generated baseline images for a given platform,
    and add any new ones to SVN control.

    @param options
    @param args
    """
    num_args = len(args)
    if num_args != 1:
        RaiseUsageException()

    # Create repo_to_modify to handle the SVN repository we will add files to.
    baseline_subdir = args[0].rstrip(os.sep);
    if not os.path.isdir(baseline_subdir):
        raise Exception('could not find baseline_subdir "%s"' % baseline_subdir)
    repo_to_modify = svn.Svn(baseline_subdir)

    # If there are any locally modified files in that directory, exit
    # (so that we don't risk overwriting the user's previous work).
    new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
    if not options.ignore_local_mods:
        if new_and_modified_files:
            raise Exception('Exiting because there are already new and/or '
                            'modified files in %s.  To continue in spite of '
                            'that, run with %s option.' % (
                                baseline_subdir, OPTION_IGNORE_LOCAL_MODS))

    # Download actual gm images into a separate repo in a temporary directory.
    tempdir = tempfile.mkdtemp()
    download_repo = svn.Svn(tempdir)
    download_repo.Checkout(GetLatestResultsSvnUrl(baseline_subdir), '.')

    # Copy any of those files we are interested in into repo_to_modify,
    # and then delete the temporary directory.
    CopyMatchingFiles(source_dir=tempdir, dest_dir=baseline_subdir,
                      filename_pattern='*.png',
                      only_copy_updates=(not options.add_new_files))
    shutil.rmtree(tempdir)
    download_repo = None

    # Add any new files to SVN control (if we are running with add_new_files).
    new_files = repo_to_modify.GetNewFiles()
    if new_files and options.add_new_files:
        repo_to_modify.AddFiles(new_files)

    # Set the mimetype property on *all* image files in baseline_subdir, even
    # the ones that were already there (in case that property wasn't properly
    # set already).
    repo_to_modify.SetPropertyByFilenamePattern(
        '*.png', svn.PROPERTY_MIMETYPE, 'image/png')
    repo_to_modify.SetPropertyByFilenamePattern(
        '*.pdf', svn.PROPERTY_MIMETYPE, 'application/pdf')

def RaiseUsageException():
    raise Exception(USAGE_STRING %  __file__)

if __name__ == '__main__':
    parser = optparse.OptionParser(USAGE_STRING % '%prog')
    parser.add_option(OPTION_IGNORE_LOCAL_MODS,
                      action='store_true', default=False,
                      help='allow tool to run even if there are already '
                      'local modifications in the baseline_subdir')
    parser.add_option(OPTION_ADD_NEW_FILES,
                      action='store_true', default=False,
                      help='in addition to downloading new versions of '
                      'existing baselines, also download baselines that are '
                      'not under SVN control yet')
    (options, args) = parser.parse_args()
    Main(options, args)