tools/roll_deps.py

#!/usr/bin/python2

# Copyright 2014 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Skia's Chromium DEPS roll script.

This script:
- searches through the last N Skia git commits to find out the hash that is
  associated with the SVN revision number.
- creates a new branch in the Chromium tree, modifies the DEPS file to
  point at the given Skia commit, commits, uploads to Rietveld, and
  deletes the local copy of the branch.
- creates a whitespace-only commit and uploads that to to Rietveld.
- returns the Chromium tree to its previous state.

To specify the location of the git executable, set the GIT_EXECUTABLE
environment variable.

Usage:
  %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
"""


import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile

import git_utils
import misc_utils


DEFAULT_BOTS_LIST = [
    'android_clang_dbg',
    'android_dbg',
    'android_rel',
    'cros_daisy',
    'linux',
    'linux_asan',
    'linux_chromeos',
    'linux_chromeos_asan',
    'linux_chromium_gn_dbg',
    'linux_gpu',
    'linux_layout',
    'linux_layout_rel',
    'mac',
    'mac_asan',
    'mac_gpu',
    'mac_layout',
    'mac_layout_rel',
    'win',
    'win_gpu',
    'win_layout',
    'win_layout_rel',
]


class DepsRollConfig(object):
    """Contains configuration options for this module.

    Attributes:
        git: (string) The git executable.
        chromium_path: (string) path to a local chromium git repository.
        save_branches: (boolean) iff false, delete temporary branches.
        verbose: (boolean)  iff false, suppress the output from git-cl.
        search_depth: (int) how far back to look for the revision.
        skia_url: (string) Skia's git repository.
        self.skip_cl_upload: (boolean)
        self.cl_bot_list: (list of strings)
    """

    # pylint: disable=I0011,R0903,R0902
    def __init__(self, options=None):
        self.skia_url = 'https://skia.googlesource.com/skia.git'
        self.revision_format = (
            'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')

        self.git = git_utils.git_executable()

        if not options:
            options = DepsRollConfig.GetOptionParser()
        # pylint: disable=I0011,E1103
        self.verbose = options.verbose
        self.vsp = misc_utils.VerboseSubprocess(self.verbose)
        self.save_branches = not options.delete_branches
        self.search_depth = options.search_depth
        self.chromium_path = options.chromium_path
        self.skip_cl_upload = options.skip_cl_upload
        # Split and remove empty strigns from the bot list.
        self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
        self.skia_git_checkout_path = options.skia_git_path
        self.default_branch_name = 'autogenerated_deps_roll_branch'
        self.reviewers_list = ','.join([
            # 'rmistry@google.com',
            # 'reed@google.com',
            # 'bsalomon@google.com',
            # 'robertphillips@google.com',
            ])
        self.cc_list = ','.join([
            # 'skia-team@google.com',
            ])

    @staticmethod
    def GetOptionParser():
        # pylint: disable=I0011,C0103
        """Returns an optparse.OptionParser object.

        Returns:
            An optparse.OptionParser object.

        Called by the main() function.
        """
        option_parser = optparse.OptionParser(usage=__doc__)
        # Anyone using this script on a regular basis should set the
        # CHROMIUM_CHECKOUT_PATH environment variable.
        option_parser.add_option(
            '-c', '--chromium_path', help='Path to local Chromium Git'
            ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
            ' if that environment variable is set.',
            default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
        option_parser.add_option(
            '-r', '--revision', type='int', default=None,
            help='The Skia SVN revision number, defaults to top of tree.')
        option_parser.add_option(
            '-g', '--git_hash', default=None,
            help='A partial Skia Git hash.  Do not set this and revision.')

        # Anyone using this script on a regular basis should set the
        # SKIA_GIT_CHECKOUT_PATH environment variable.
        option_parser.add_option(
            '', '--skia_git_path',
            help='Path of a pure-git Skia repository checkout.  If empty,'
            ' a temporary will be cloned.  Defaults to SKIA_GIT_CHECKOUT'
            '_PATH, if that environment variable is set.',
            default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
        option_parser.add_option(
            '', '--search_depth', type='int', default=100,
            help='How far back to look for the revision.')
        option_parser.add_option(
            '', '--delete_branches', help='Delete the temporary branches',
            action='store_true', dest='delete_branches', default=False)
        option_parser.add_option(
            '', '--verbose', help='Do not suppress the output from `git cl`.',
            action='store_true', dest='verbose', default=False)
        option_parser.add_option(
            '', '--skip_cl_upload', help='Skip the cl upload step; useful'
            ' for testing.',
            action='store_true', default=False)

        default_bots_help = (
            'Comma-separated list of bots, defaults to a list of %d bots.'
            '  To skip `git cl try`, set this to an empty string.'
            % len(DEFAULT_BOTS_LIST))
        default_bots = ','.join(DEFAULT_BOTS_LIST)
        option_parser.add_option(
            '', '--bots', help=default_bots_help, default=default_bots)

        return option_parser


class DepsRollError(Exception):
    """Exceptions specific to this module."""
    pass


def get_svn_revision(config, commit):
    """Works in both git and git-svn. returns a string."""
    svn_format = (
        '(git-svn-id: [^@ ]+@|SVN changes up to revision |'
        'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)')
    svn_revision = misc_utils.ReSearch.search_within_output(
        config.verbose, svn_format, None,
        [config.git, 'log', '-n', '1', '--format=format:%B', commit])
    if not svn_revision:
        raise DepsRollError(
            'Revision number missing from Chromium origin/master.')
    return int(svn_revision)


class SkiaGitCheckout(object):
    """Class to create a temporary skia git checkout, if necessary.
    """
    # pylint: disable=I0011,R0903

    def __init__(self, config, depth):
        self._config = config
        self._depth = depth
        self._use_temp = None
        self._original_cwd = None

    def __enter__(self):
        config = self._config
        git = config.git
        skia_dir = None
        self._original_cwd = os.getcwd()
        if config.skia_git_checkout_path:
            if config.skia_git_checkout_path != os.curdir:
                skia_dir = config.skia_git_checkout_path
                ## Update origin/master if needed.
                if self._config.verbose:
                    print '~~$', 'cd', skia_dir
                os.chdir(skia_dir)
            config.vsp.check_call([git, 'fetch', '-q', 'origin'])
            self._use_temp = None
        else:
            skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
            self._use_temp = skia_dir
            try:
                os.chdir(skia_dir)
                config.vsp.check_call(
                    [git, 'clone', '-q', '--depth=%d' % self._depth,
                     '--single-branch', config.skia_url, '.'])
            except (OSError, subprocess.CalledProcessError) as error:
                shutil.rmtree(skia_dir)
                raise error

    def __exit__(self, etype, value, traceback):
        if self._config.skia_git_checkout_path != os.curdir:
            if self._config.verbose:
                print '~~$', 'cd', self._original_cwd
            os.chdir(self._original_cwd)
        if self._use_temp:
            shutil.rmtree(self._use_temp)


def revision_and_hash(config):
    """Finds revision number and git hash of origin/master in the Skia tree.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.

    Returns:
        A tuple (revision, hash)
            revision: (int) SVN revision number.
            git_hash: (string) full Git commit hash.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    with SkiaGitCheckout(config, 1):
        revision = get_svn_revision(config, 'origin/master')
        git_hash = config.vsp.strip_output(
            [config.git, 'show-ref', 'origin/master', '--hash'])
        if not git_hash:
            raise DepsRollError('Git hash can not be found.')
    return revision, git_hash


def revision_and_hash_from_revision(config, revision):
    """Finds revision number and git hash of a commit in the Skia tree.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int) SVN revision number.

    Returns:
        A tuple (revision, hash)
            revision: (int) SVN revision number.
            git_hash: (string) full Git commit hash.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    with SkiaGitCheckout(config, config.search_depth):
        revision_regex = config.revision_format % revision
        git_hash = config.vsp.strip_output(
            [config.git, 'log', '--grep', revision_regex,
             '--format=format:%H', 'origin/master'])
        if not git_hash:
            raise DepsRollError('Git hash can not be found.')
    return revision, git_hash


def revision_and_hash_from_partial(config, partial_hash):
    """Returns the SVN revision number and full git hash.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        partial_hash: (string) Partial git commit hash.

    Returns:
        A tuple (revision, hash)
            revision: (int) SVN revision number.
            git_hash: (string) full Git commit hash.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """
    with SkiaGitCheckout(config, config.search_depth):
        git_hash = config.vsp.strip_output(
            ['git', 'log', '-n', '1', '--format=format:%H', partial_hash])
        if not git_hash:
            raise DepsRollError('Partial Git hash can not be found.')
        revision = get_svn_revision(config, git_hash)
    return revision, git_hash


def change_skia_deps(revision, git_hash, depspath):
    """Update the DEPS file.

    Modify the skia_revision and skia_hash entries in the given DEPS file.

    Args:
        revision: (int) Skia SVN revision.
        git_hash: (string) Skia Git hash.
        depspath: (string) path to DEPS file.
    """
    temp_file = tempfile.NamedTemporaryFile(delete=False,
                                            prefix='skia_DEPS_ROLL_tmp_')
    try:
        deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
        deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')

        deps_regex_rev_repl = '"skia_revision": "%d",' % revision
        deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash

        with open(depspath, 'r') as input_stream:
            for line in input_stream:
                line = deps_regex_rev.sub(deps_regex_rev_repl, line)
                line = deps_regex_hash.sub(deps_regex_hash_repl, line)
                temp_file.write(line)
    finally:
        temp_file.close()
    shutil.move(temp_file.name, depspath)


def git_cl_uploader(config, message, file_list):
    """Create a commit in the current git branch; upload via git-cl.

    Assumes that you are already on the branch you want to be on.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        message: (string) the commit message, can be multiline.
        file_list: (list of strings) list of filenames to pass to `git add`.

    Returns:
        The output of `git cl issue`, if not config.skip_cl_upload, else ''.
    """

    git, vsp = config.git, config.vsp
    svn_info = str(get_svn_revision(config, 'HEAD'))

    for filename in file_list:
        assert os.path.exists(filename)
        vsp.check_call([git, 'add', filename])

    vsp.check_call([git, 'commit', '-q', '-m', message])

    git_cl = [git, 'cl', 'upload', '-f',
              '--bypass-hooks', '--bypass-watchlists']
    if config.cc_list:
        git_cl.append('--cc=%s' % config.cc_list)
    if config.reviewers_list:
        git_cl.append('--reviewers=%s' % config.reviewers_list)

    git_try = [
        git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info]
    git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)])

    branch_name = git_utils.git_branch_name(vsp.verbose)

    if config.skip_cl_upload:
        space = '   '
        print 'You should call:'
        print '%scd %s' % (space, os.getcwd())
        misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name])
        misc_utils.print_subprocess_args(space, git_cl)
        if config.cl_bot_list:
            misc_utils.print_subprocess_args(space, git_try)
        print
        return ''
    else:
        vsp.check_call(git_cl)
        issue = vsp.strip_output([git, 'cl', 'issue'])
        if config.cl_bot_list:
            vsp.check_call(git_try)
        return issue


def roll_deps(config, revision, git_hash):
    """Upload changed DEPS and a whitespace change.

    Given the correct git_hash, create two Reitveld issues.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int) Skia SVN revision.
        git_hash: (string) Skia Git hash.

    Returns:
        a tuple containing textual description of the two issues.

    Raises:
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """

    git = config.git
    with misc_utils.ChangeDir(config.chromium_path, config.verbose):
        config.vsp.check_call([git, 'fetch', '-q', 'origin'])

        old_revision = misc_utils.ReSearch.search_within_output(
            config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
            [git, 'show', 'origin/master:DEPS'])
        assert old_revision
        if revision == int(old_revision):
            print 'DEPS is up to date!'
            return (None, None)

        master_hash = config.vsp.strip_output(
            [git, 'show-ref', 'origin/master', '--hash'])
        master_revision = get_svn_revision(config, 'origin/master')

        # master_hash[8] gives each whitespace CL a unique name.
        if config.save_branches:
            branch = 'control_%s' % master_hash[:8]
        else:
            branch = None
        message = ('whitespace change %s\n\n'
                   'Chromium base revision: %d / %s\n\n'
                   'This CL was created by Skia\'s roll_deps.py script.\n'
                  ) % (master_hash[:8], master_revision, master_hash[:8])
        with git_utils.ChangeGitBranch(branch, 'origin/master',
                                       config.verbose):
            branch = git_utils.git_branch_name(config.vsp.verbose)

            with open('build/whitespace_file.txt', 'a') as output_stream:
                output_stream.write('\nCONTROL\n')

            whitespace_cl = git_cl_uploader(
                config, message, ['build/whitespace_file.txt'])

            control_url = misc_utils.ReSearch.search_within_string(
                whitespace_cl, '(?P<return>https?://[^) ]+)', '?')
            if config.save_branches:
                whitespace_cl = '%s\n    branch: %s' % (whitespace_cl, branch)

        if config.save_branches:
            branch = 'roll_%d_%s' % (revision, master_hash[:8])
        else:
            branch = None
        message = (
            'roll skia DEPS to %d\n\n'
            'Chromium base revision: %d / %s\n'
            'Old Skia revision: %s\n'
            'New Skia revision: %d\n'
            'Control CL: %s\n\n'
            'This CL was created by Skia\'s roll_deps.py script.\n\n'
            'Bypassing commit queue trybots:\n'
            'NOTRY=true\n'
            % (revision, master_revision, master_hash[:8],
               old_revision, revision, control_url))
        with git_utils.ChangeGitBranch(branch, 'origin/master',
                                       config.verbose):
            branch = git_utils.git_branch_name(config.vsp.verbose)

            change_skia_deps(revision, git_hash, 'DEPS')
            deps_cl = git_cl_uploader(config, message, ['DEPS'])
            if config.save_branches:
                deps_cl = '%s\n    branch: %s' % (deps_cl, branch)

        return deps_cl, whitespace_cl


def find_hash_and_roll_deps(config, revision=None, partial_hash=None):
    """Call find_hash_from_revision() and roll_deps().

    The calls to git will be verbose on standard output.  After a
    successful upload of both issues, print links to the new
    codereview issues.

    Args:
        config: (roll_deps.DepsRollConfig) object containing options.
        revision: (int or None) the Skia SVN revision number or None
            to use the tip of the tree.
        partial_hash: (string or None) a partial pure-git Skia commit
            hash.  Don't pass both partial_hash and revision.

    Raises:
        roll_deps.DepsRollError: if the revision can't be found.
        OSError: failed to execute git or git-cl.
        subprocess.CalledProcessError: git returned unexpected status.
    """

    if revision and partial_hash:
        raise DepsRollError('Pass revision or partial_hash, not both.')

    if partial_hash:
        revision, git_hash = revision_and_hash_from_partial(
            config, partial_hash)
    elif revision:
        revision, git_hash = revision_and_hash_from_revision(config, revision)
    else:
        revision, git_hash = revision_and_hash(config)

    print 'revision=%r\nhash=%r\n' % (revision, git_hash)

    deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)

    if deps_issue and whitespace_issue:
        print 'DEPS roll:\n    %s\n' % deps_issue
        print 'Whitespace change:\n    %s\n' % whitespace_issue
    else:
        print >> sys.stderr, 'No issues created.'


def main(args):
    """main function; see module-level docstring and GetOptionParser help.

    Args:
        args: sys.argv[1:]-type argument list.
    """
    option_parser = DepsRollConfig.GetOptionParser()
    options = option_parser.parse_args(args)[0]

    if not options.chromium_path:
        option_parser.error('Must specify chromium_path.')
    if not os.path.isdir(options.chromium_path):
        option_parser.error('chromium_path must be a directory.')

    if not git_utils.git_executable():
        option_parser.error('Invalid git executable.')

    config = DepsRollConfig(options)
    find_hash_and_roll_deps(config, options.revision, options.git_hash)


if __name__ == '__main__':
    main(sys.argv[1:])