tools/download_baselines.py

'''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.

Launch with --help to see more information.


Copyright 2011 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''

# common Python modules
import fnmatch
import optparse
import os
import re
import shutil
import sys
import tempfile

# modules declared within this same directory
import compare_baselines
import svn

USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...'
HELP_STRING = '''

Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.

If no baseline_subdir is given, then this tool will download the most-recently
generated actual gm results for ALL platforms.

''' + compare_baselines.HOWTO_STRING

# Base URL of SVN repository where buildbots store actual gm image results.
GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'

# GM baseline image URL in regular Skia SVN repository
GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected'

GM_EXPECTED_DIR = 'gm-expected'
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
OPTION_ADD_NEW_FILES = '--add-new-files'

def GetLatestResultsSvnUrl(baseline_subdir):
    """Return SVN URL from which we can check out the MOST RECENTLY generated
    images for this baseline type.

    @param baseline_subdir indicates which platform we want images for
    """
    return '%s/%s' % (GM_ACTUAL_URL, baseline_subdir)

def GetBaselineSvnUrl(baseline_subdir):
    """Return SVN URL from which we can check out the baseline images for this
    baseline type.

    @param baseline_subdir indicates which platform we want baselines for
    """
    return '%s/%s' % (GM_BASELINE_URL, baseline_subdir)

def CopyMatchingFiles(source_dir, dest_dir, filename_pattern,
                      only_copy_updates=False):
    """Copy all files from source_dir that match filename_pattern, and
    save them (with their original filenames) in dest_dir.

    @param source_dir
    @param dest_dir where to save the copied files
    @param filename_pattern only copy files that match this Unix-style filename
           pattern (e.g., '*.jpg')
    @param only_copy_updates if True, only copy files that are already
           present in dest_dir
    """
    all_filenames = os.listdir(source_dir)
    matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
    for filename in matching_filenames:
        source_path = os.path.join(source_dir, filename)
        dest_path = os.path.join(dest_dir, filename)
        if only_copy_updates and not os.path.isfile(dest_path):
            continue
        shutil.copyfile(source_path, dest_path)

def DownloadBaselinesForOnePlatform(baseline_subdir):
    """Download most recently generated baseline images for a single platform,
    and add any new ones to SVN control.

    @param baseline_subdir
    """
    # Create repo_to_modify to handle the SVN repository we will add files to.
    gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk...
    try:
        os.makedirs(gm_dir)
    except:
        pass
    repo_to_modify = svn.Svn(gm_dir)
    repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir)

    # If there are any locally modified files in that directory, exit
    # (so that we don't risk overwriting the user's previous work).
    new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
    if not options.ignore_local_mods:
        if new_and_modified_files:
            raise Exception('Exiting because there are already new and/or '
                            'modified files in %s.  To continue in spite of '
                            'that, run with %s option.' % (
                                baseline_subdir, OPTION_IGNORE_LOCAL_MODS))

    # Download actual gm images into a separate repo in a temporary directory.
    actual_dir = tempfile.mkdtemp()
    actual_repo = svn.Svn(actual_dir)
    print 'Using %s as a temp dir' % actual_dir
    actual_repo.Checkout(GetLatestResultsSvnUrl(baseline_subdir), '.')

    # Copy any of those files we are interested in into repo_to_modify,
    # and then delete the temporary directory.
    CopyMatchingFiles(source_dir=actual_dir,
                      dest_dir=os.path.join(gm_dir, baseline_subdir),
                      filename_pattern='*.png',
                      only_copy_updates=(not options.add_new_files))
    shutil.rmtree(actual_dir)
    actual_repo = None

    # Add any new files to SVN control (if we are running with add_new_files).
    if options.add_new_files:
        new_files = repo_to_modify.GetNewFiles()
        if new_files:
            repo_to_modify.AddFiles(sorted(new_files))

    # Set the mimetype property on any new/modified image files in
    # baseline_subdir.  (We used to set the mimetype property on *all* image
    # files in the directory, even those whose content wasn't changing,
    # but that caused confusion.  See
    # http://code.google.com/p/skia/issues/detail?id=618 .)
    modified_files = repo_to_modify.GetNewAndModifiedFiles()
    repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
                               svn.PROPERTY_MIMETYPE, 'image/png')
    repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
                               svn.PROPERTY_MIMETYPE, 'application/pdf')

def RaiseUsageException():
    raise Exception('%s\nRun with --help for more detail.' % (
        USAGE_STRING % __file__))

def Main(options, args):
    """Allow other scripts to call this script with fake command-line args.
    """
    # If no platforms are specified, do 'em all.
    num_args = len(args)
    if num_args == 0:
        # TODO(epoger): automate the default set of platforms. We want to ensure
        # that the user gets all of the platforms that the bots are running,
        # not just whatever subdirectories he happens to have checked out...
        # See http://code.google.com/p/skia/issues/detail?id=678
        #
        # For now, I generate this list using these Unix commands:
        # svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort >/tmp/baselines
        # svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base | sort >/tmp/actual
        # comm -1 -2 /tmp/baselines /tmp/actual
        args = [
            'base-android-galaxy-nexus',
            'base-android-nexus-7',
            'base-android-nexus-s',
            'base-android-xoom',
            'base-macmini',
            'base-macmini-lion-float',
            'base-shuttle-win7-intel-float',
            'base-shuttle_ubuntu12_ati5770',
            ]

    # Trim all subdir names.
    baseline_subdirs = []
    for arg in args:
        baseline_subdirs.append(arg.rstrip(os.sep))

    # Process the subdirs, one at a time.
    for baseline_subdir in baseline_subdirs:
        DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir)

if __name__ == '__main__':
    parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
    parser.add_option(OPTION_IGNORE_LOCAL_MODS,
                      action='store_true', default=False,
                      help='allow tool to run even if there are already '
                      'local modifications in the baseline_subdir')
    parser.add_option(OPTION_ADD_NEW_FILES,
                      action='store_true', default=False,
                      help='in addition to downloading new versions of '
                      'existing baselines, also download baselines that are '
                      'not under SVN control yet')
    (options, args) = parser.parse_args()
    Main(options, args)