tools/roll_deps.py

#!/usr/bin/python2

# Copyright 2014 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Skia's Chromium DEPS roll script.

This script:
- searches through the last N Skia git commits to find out the hash that is
  associated with the SVN revision number.
- creates a new branch in the Chromium tree, modifies the DEPS file to
  point at the given Skia commit, commits, uploads to Rietveld, and
  deletes the local copy of the branch.
- creates a whitespace-only commit and uploads that to to Rietveld.
- returns the Chromium tree to its previous state.

Usage:
  %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
"""


import optparse
import os
import re
import shutil
import subprocess
from subprocess import check_call
import sys
import tempfile


class DepsRollConfig(object):
    """Contains configuration options for this module.

    Attributes:
        git: (string) The git executable.
        chromium_path: (string) path to a local chromium git repository.
        save_branches: (boolean) iff false, delete temporary branches.
        verbose: (boolean)  iff false, suppress the output from git-cl.
        search_depth: (int) how far back to look for the revision.
        skia_url: (string) Skia's git repository.
        self.skip_cl_upload: (boolean)
        self.cl_bot_list: (list of strings)
    """

    # pylint: disable=I0011,R0903,R0902
    def __init__(self, options=None):
        self.skia_url = 'https://skia.googlesource.com/skia.git'
        self.revision_format = (
            'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')

        if not options:
            options = DepsRollConfig.GetOptionParser()
        # pylint: disable=I0011,E1103
        self.verbose = options.verbose
        self.save_branches = options.save_branches
        self.search_depth = options.search_depth
        self.chromium_path = options.chromium_path
        self.git = options.git_path
        self.skip_cl_upload = options.skip_cl_upload
        # Split and remove empty strigns from the bot list.
        self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
        self.skia_git_checkout_path = options.skia_git_path
        self.default_branch_name = 'autogenerated_deps_roll_branch'

    @staticmethod
    def GetOptionParser():
        # pylint: disable=I0011,C0103
        """Returns an optparse.OptionParser object.

        Returns:
            An optparse.OptionParser object.

        Called by the main() function.
        """
        default_bots_list = [
            'android_clang_dbg',
            'android_dbg',
            'android_rel',
            'cros_daisy',
            'linux',
            'linux_asan',
            'linux_chromeos',
            'linux_chromeos_asan',
            'linux_gpu',
            'linux_heapcheck',
            'linux_layout',
            'linux_layout_rel',
            'mac',
            'mac_asan',
            'mac_gpu',
            'mac_layout',
            'mac_layout_rel',
            'win',
            'win_gpu',
            'win_layout',
            'win_layout_rel',
            ]

        option_parser = optparse.OptionParser(usage=__doc__)
        # Anyone using this script on a regular basis should set the
        # CHROMIUM_CHECKOUT_PATH environment variable.
        option_parser.add_option(
            '-c', '--chromium_path', help='Path to local Chromium Git'
            ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
            ' if that environment variable is set.',
            default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
        option_parser.add_option(
            '-r', '--revision', type='int', default=None,
            help='The Skia SVN revision number, defaults to top of tree.')
        # Anyone using this script on a regular basis should set the
        # SKIA_GIT_CHECKOUT_PATH environment variable.
        option_parser.add_option(
            '', '--skia_git_path',
            help='Path of a pure-git Skia repository checkout.  If empty,'
            ' a temporary will be cloned.  Defaults to SKIA_GIT_CHECKOUT'
            '_PATH, if that environment variable is set.',
            default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
        option_parser.add_option(
            '', '--search_depth', type='int', default=100,
            help='How far back to look for the revision.')
        option_parser.add_option(
            '', '--git_path', help='Git executable, defaults to "git".',
            default='git')
        option_parser.add_option(
            '', '--save_branches', help='Save the temporary branches',
            action='store_true', dest='save_branches', default=False)
        option_parser.add_option(
            '', '--verbose', help='Do not suppress the output from `git cl`.',
            action='store_true', dest='verbose', default=False)
        option_parser.add_option(
            '', '--skip_cl_upload', help='Skip the cl upload step; useful'
            ' for testing or with --save_branches.',
            action='store_true', default=False)

        default_bots_help = (
            'Comma-separated list of bots, defaults to a list of %d bots.'
            '  To skip `git cl try`, set this to an empty string.'
            % len(default_bots_list))
        default_bots = ','.join(default_bots_list)
        option_parser.add_option(
            '', '--bots', help=default_bots_help, default=default_bots)

        return option_parser


def test_git_executable(git_executable):
    """Test the git executable.

    Args:
        git_executable: git executable path.
    Returns:
        True if test is successful.
    """
    with open(os.devnull, 'w') as devnull:
        try:
            subprocess.call([git_executable, '--version'], stdout=devnull)
        except (OSError,):
            return False
    return True


class DepsRollError(Exception):
    """Exceptions specific to this module."""
    pass


def strip_output(*args, **kwargs):
    """Wrap subprocess.check_output and str.strip().

    Pass the given arguments into subprocess.check_output() and return
    the results, after stripping any excess whitespace.

    Args:
        *args: to be passed to subprocess.check_output()
        **kwargs: to be passed to subprocess.check_output()

    Returns:
        The output of the process as a string without leading or
        trailing whitespace.
    Raises:
        OSError or subprocess.CalledProcessError: raised by check_output.
    """
    return str(subprocess.check_output(*args, **kwargs)).strip()


def create_temp_skia_clone(config, depth):
    """Clones Skia in a temp dir.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        depth: (int) how far back to clone the tree.
    Returns:
        temporary directory path if succcessful.
    Raises:
        OSError, subprocess.CalledProcessError on failure.
    """
    git = config.git
    skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
    try:
        check_call(
            [git, 'clone', '-q', '--depth=%d' % depth,
             '--single-branch', config.skia_url, skia_dir])
        return skia_dir
    except (OSError, subprocess.CalledProcessError) as error:
        shutil.rmtree(skia_dir)
        raise error


def find_revision_and_hash(config, revision):
    """Finds revision number and git hash of origin/master in the Skia tree.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int or None) SVN revision number.  If None, use
            tip-of-tree.

    Returns:
        A tuple (revision, hash)
            revision: (int) SVN revision number.
            hash: (string) full Git commit hash.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    git = config.git
    use_temp = False
    skia_dir = None
    depth = 1 if (revision is None) else config.search_depth
    try:
        if config.skia_git_checkout_path:
            skia_dir = config.skia_git_checkout_path
            ## Update origin/master if needed.
            check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir)
        else:
            skia_dir = create_temp_skia_clone(config, depth)
            assert skia_dir
            use_temp = True

        if revision is None:
            message = subprocess.check_output(
                [git, 'log', '-n', '1', '--format=format:%B',
                 'origin/master'], cwd=skia_dir)
            svn_format = (
                'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ')
            search = re.search(svn_format, message)
            if not search:
                raise DepsRollError(
                    'Revision number missing from origin/master.')
            revision = int(search.group(1))
            git_hash = strip_output(
                [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir)
        else:
            revision_regex = config.revision_format % revision
            git_hash = strip_output(
                [git, 'log', '--grep', revision_regex, '--format=format:%H',
                 'origin/master'], cwd=skia_dir)

        if revision < 0  or not git_hash:
            raise DepsRollError('Git hash can not be found.')
        return revision, git_hash
    finally:
        if use_temp:
            shutil.rmtree(skia_dir)


class GitBranchCLUpload(object):
    """Class to manage git branches and git-cl-upload.

    This class allows one to create a new branch in a repository based
    off of origin/master, make changes to the tree inside the
    with-block, upload that new branch to Rietveld, restore the original
    tree state, and delete the local copy of the new branch.

    See roll_deps() for an example of use.

    Constructor Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        message: (string) the commit message, can be multiline.
        set_brach_name: (string or none) if not None, the name of the
            branch to use.  If None, then use a temporary branch that
            will be deleted.

    Attributes:
        issue: a string describing the codereview issue, after __exit__
            has been called, othrwise, None.

    Raises:
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    # pylint: disable=I0011,R0903,R0902

    def __init__(self, config, message, set_branch_name):
        self._message = message
        self._file_list = []
        self._branch_name = set_branch_name
        self._stash = None
        self._original_branch = None
        self._config = config
        self._svn_info = None
        self.issue = None

    def stage_for_commit(self, *paths):
        """Calls `git add ...` on each argument.

        Args:
            *paths: (list of strings) list of filenames to pass to `git add`.
        """
        self._file_list.extend(paths)

    def __enter__(self):
        git = self._config.git
        def branch_exists(branch):
            """Return true iff branch exists."""
            return 0 == subprocess.call(
                [git, 'show-ref', '--quiet', branch])
        def has_diff():
            """Return true iff repository has uncommited changes."""
            return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD']))
        self._stash = has_diff()
        if self._stash:
            check_call([git, 'stash', 'save'])
        try:
            self._original_branch = strip_output(
                [git, 'symbolic-ref', '--short', 'HEAD'])
        except (subprocess.CalledProcessError,):
            self._original_branch = strip_output(
                [git, 'rev-parse', 'HEAD'])

        if not self._branch_name:
            self._branch_name = self._config.default_branch_name

        if branch_exists(self._branch_name):
            check_call([git, 'checkout', '-q', 'master'])
            check_call([git, 'branch', '-q', '-D', self._branch_name])

        check_call(
            [git, 'checkout', '-q', '-b',
             self._branch_name, 'origin/master'])

        svn_info = subprocess.check_output(['git', 'svn', 'info'])
        svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info)
        assert svn_info_search
        self._svn_info = svn_info_search.group(1)

    def __exit__(self, etype, value, traceback):
        # pylint: disable=I0011,R0912
        git = self._config.git
        def quiet_check_call(*args, **kwargs):
            """Call check_call, but pipe output to devnull."""
            with open(os.devnull, 'w') as devnull:
                check_call(*args, stdout=devnull, **kwargs)

        for filename in self._file_list:
            assert os.path.exists(filename)
            check_call([git, 'add', filename])
        check_call([git, 'commit', '-q', '-m', self._message])

        git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com',
                  '--bypass-hooks', '--bypass-watchlists']
        git_try = [git, 'cl', 'try', '--revision', self._svn_info]
        git_try.extend([arg for bot in self._config.cl_bot_list
                        for arg in ('-b', bot)])

        if self._config.skip_cl_upload:
            print ' '.join(git_cl)
            print
            if self._config.cl_bot_list:
                print ' '.join(git_try)
                print
            self.issue = ''
        else:
            if self._config.verbose:
                check_call(git_cl)
                print
            else:
                quiet_check_call(git_cl)
            self.issue = strip_output([git, 'cl', 'issue'])
            if self._config.cl_bot_list:
                if self._config.verbose:
                    check_call(git_try)
                    print
                else:
                    quiet_check_call(git_try)

        # deal with the aftermath of failed executions of this script.
        if self._config.default_branch_name == self._original_branch:
            self._original_branch = 'master'
        check_call([git, 'checkout', '-q', self._original_branch])

        if self._config.default_branch_name == self._branch_name:
            check_call([git, 'branch', '-q', '-D', self._branch_name])
        if self._stash:
            check_call([git, 'stash', 'pop'])


def change_skia_deps(revision, git_hash, depspath):
    """Update the DEPS file.

    Modify the skia_revision and skia_hash entries in the given DEPS file.

    Args:
        revision: (int) Skia SVN revision.
        git_hash: (string) Skia Git hash.
        depspath: (string) path to DEPS file.
    """
    temp_file = tempfile.NamedTemporaryFile(delete=False,
                                            prefix='skia_DEPS_ROLL_tmp_')
    try:
        deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
        deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')

        deps_regex_rev_repl = '"skia_revision": "%d",' % revision
        deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash

        with open(depspath, 'r') as input_stream:
            for line in input_stream:
                line = deps_regex_rev.sub(deps_regex_rev_repl, line)
                line = deps_regex_hash.sub(deps_regex_hash_repl, line)
                temp_file.write(line)
    finally:
        temp_file.close()
    shutil.move(temp_file.name, depspath)


def branch_name(message):
    """Return the first line of a commit message to be used as a branch name.

    Args:
        message: (string)

    Returns:
        A string derived from message suitable for a branch name.
    """
    return message.lstrip().split('\n')[0].rstrip().replace(' ', '_')


def roll_deps(config, revision, git_hash):
    """Upload changed DEPS and a whitespace change.

    Given the correct git_hash, create two Reitveld issues.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int) Skia SVN revision.
        git_hash: (string) Skia Git hash.

    Returns:
        a tuple containing textual description of the two issues.

    Raises:
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    git = config.git
    cwd = os.getcwd()
    os.chdir(config.chromium_path)
    try:
        check_call([git, 'fetch', '-q', 'origin'])
        master_hash = strip_output(
            [git, 'show-ref', 'origin/master', '--hash'])

        # master_hash[8] gives each whitespace CL a unique name.
        message = ('whitespace change %s\n\nThis CL was created by'
                   ' Skia\'s roll_deps.py script.\n') % master_hash[:8]
        branch = branch_name(message) if config.save_branches else None

        codereview = GitBranchCLUpload(config, message, branch)
        with codereview:
            with open('build/whitespace_file.txt', 'a') as output_stream:
                output_stream.write('\nCONTROL\n')
            codereview.stage_for_commit('build/whitespace_file.txt')
        whitespace_cl = codereview.issue
        if branch:
            whitespace_cl = '%s\n    branch: %s' % (whitespace_cl, branch)
        control_url_match = re.search('https?://[^) ]+', codereview.issue)
        if control_url_match:
            message = ('roll skia DEPS to %d\n\nThis CL was created by'
                       ' Skia\'s roll_deps.py script.\n\ncontrol: %s'
                       % (revision, control_url_match.group(0)))
        else:
            message = ('roll skia DEPS to %d\n\nThis CL was created by'
                       ' Skia\'s roll_deps.py script.') % revision
        branch = branch_name(message) if config.save_branches else None
        codereview = GitBranchCLUpload(config, message, branch)
        with codereview:
            change_skia_deps(revision, git_hash, 'DEPS')
            codereview.stage_for_commit('DEPS')
        deps_cl = codereview.issue
        if branch:
            deps_cl = '%s\n    branch: %s' % (deps_cl, branch)

        return deps_cl, whitespace_cl
    finally:
        os.chdir(cwd)


def find_hash_and_roll_deps(config, revision):
    """Call find_hash_from_revision() and roll_deps().

    The calls to git will be verbose on standard output.  After a
    successful upload of both issues, print links to the new
    codereview issues.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int or None) the Skia SVN revision number or None
            to use the tip of the tree.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    revision, git_hash = find_revision_and_hash(config, revision)

    print 'revision=%r\nhash=%r\n' % (revision, git_hash)

    deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)

    print 'DEPS roll:\n    %s\n' % deps_issue
    print 'Whitespace change:\n    %s\n' % whitespace_issue


def main(args):
    """main function; see module-level docstring and GetOptionParser help.

    Args:
        args: sys.argv[1:]-type argument list.
    """
    option_parser = DepsRollConfig.GetOptionParser()
    options = option_parser.parse_args(args)[0]

    if not options.chromium_path:
        option_parser.error('Must specify chromium_path.')
    if not os.path.isdir(options.chromium_path):
        option_parser.error('chromium_path must be a directory.')
    if not test_git_executable(options.git_path):
        option_parser.error('Invalid git executable.')

    config = DepsRollConfig(options)
    find_hash_and_roll_deps(config, options.revision)


if __name__ == '__main__':
    main(sys.argv[1:])