DEPS roll script
This script:
- searches through the last N commits to find out the hash that is
associated with the revision number.
- creates a new branch in the chromium tree, modifies the DEPS
file, commits, and uploads to Rietveld.
- create a whitespace-only commit and uploads that to to Rietveld.
BUG=
R=borenet@google.com
Review URL: https://codereview.chromium.org/123523003
git-svn-id: http://skia.googlecode.com/svn/trunk@12921 2bbb7eff-a529-9590-31e7-b0007b416f81
diff --git a/tools/roll_deps.py b/tools/roll_deps.py
new file mode 100755
index 0000000..42b9174
--- /dev/null
+++ b/tools/roll_deps.py
@@ -0,0 +1,552 @@
+#!/usr/bin/python2
+
+# Copyright 2014 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Skia's Chromium DEPS roll script.
+
+This script:
+- searches through the last N Skia git commits to find out the hash that is
+ associated with the SVN revision number.
+- creates a new branch in the Chromium tree, modifies the DEPS file to
+ point at the given Skia commit, commits, uploads to Rietveld, and
+ deletes the local copy of the branch.
+- creates a whitespace-only commit and uploads that to to Rietveld.
+- returns the Chromium tree to its previous state.
+
+Usage:
+ %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
+"""
+
+
+import optparse
+import os
+import re
+import shutil
+import subprocess
+from subprocess import check_call
+import sys
+import tempfile
+
+
+class DepsRollConfig(object):
+ """Contains configuration options for this module.
+
+ Attributes:
+ git: (string) The git executable.
+ chromium_path: (string) path to a local chromium git repository.
+ save_branches: (boolean) iff false, delete temporary branches.
+ verbose: (boolean) iff false, suppress the output from git-cl.
+ search_depth: (int) how far back to look for the revision.
+ skia_url: (string) Skia's git repository.
+ self.skip_cl_upload: (boolean)
+ self.cl_bot_list: (list of strings)
+ """
+
+ # pylint: disable=I0011,R0903,R0902
+ def __init__(self, options=None):
+ self.skia_url = 'https://skia.googlesource.com/skia.git'
+ self.revision_format = (
+ 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
+
+ if not options:
+ options = DepsRollConfig.GetOptionParser()
+ # pylint: disable=I0011,E1103
+ self.verbose = options.verbose
+ self.save_branches = options.save_branches
+ self.search_depth = options.search_depth
+ self.chromium_path = options.chromium_path
+ self.git = options.git_path
+ self.skip_cl_upload = options.skip_cl_upload
+ # Split and remove empty strigns from the bot list.
+ self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
+ self.skia_git_checkout_path = options.skia_git_path
+ self.default_branch_name = 'autogenerated_deps_roll_branch'
+
+ @staticmethod
+ def GetOptionParser():
+ # pylint: disable=I0011,C0103
+ """Returns an optparse.OptionParser object.
+
+ Returns:
+ An optparse.OptionParser object.
+
+ Called by the main() function.
+ """
+ default_bots_list = [
+ 'android_clang_dbg',
+ 'android_dbg',
+ 'android_rel',
+ 'cros_daisy',
+ 'linux',
+ 'linux_asan',
+ 'linux_chromeos',
+ 'linux_chromeos_asan',
+ 'linux_gpu',
+ 'linux_heapcheck',
+ 'linux_layout',
+ 'linux_layout_rel',
+ 'mac',
+ 'mac_asan',
+ 'mac_gpu',
+ 'mac_layout',
+ 'mac_layout_rel',
+ 'win',
+ 'win_gpu',
+ 'win_layout',
+ 'win_layout_rel',
+ ]
+
+ option_parser = optparse.OptionParser(usage=__doc__)
+ # Anyone using this script on a regular basis should set the
+ # CHROMIUM_CHECKOUT_PATH environment variable.
+ option_parser.add_option(
+ '-c', '--chromium_path', help='Path to local Chromium Git'
+ ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
+ ' if that environment variable is set.',
+ default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
+ option_parser.add_option(
+ '-r', '--revision', type='int', default=None,
+ help='The Skia SVN revision number, defaults to top of tree.')
+ # Anyone using this script on a regular basis should set the
+ # SKIA_GIT_CHECKOUT_PATH environment variable.
+ option_parser.add_option(
+ '', '--skia_git_path',
+ help='Path of a pure-git Skia repository checkout. If empty,'
+ ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT'
+ '_PATH, if that environment variable is set.',
+ default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
+ option_parser.add_option(
+ '', '--search_depth', type='int', default=100,
+ help='How far back to look for the revision.')
+ option_parser.add_option(
+ '', '--git_path', help='Git executable, defaults to "git".',
+ default='git')
+ option_parser.add_option(
+ '', '--save_branches', help='Save the temporary branches',
+ action='store_true', dest='save_branches', default=False)
+ option_parser.add_option(
+ '', '--verbose', help='Do not suppress the output from `git cl`.',
+ action='store_true', dest='verbose', default=False)
+ option_parser.add_option(
+ '', '--skip_cl_upload', help='Skip the cl upload step; useful'
+ ' for testing or with --save_branches.',
+ action='store_true', default=False)
+
+ default_bots_help = (
+ 'Comma-separated list of bots, defaults to a list of %d bots.'
+ ' To skip `git cl try`, set this to an empty string.'
+ % len(default_bots_list))
+ default_bots = ','.join(default_bots_list)
+ option_parser.add_option(
+ '', '--bots', help=default_bots_help, default=default_bots)
+
+ return option_parser
+
+
+def test_git_executable(git_executable):
+ """Test the git executable.
+
+ Args:
+ git_executable: git executable path.
+ Returns:
+ True if test is successful.
+ """
+ with open(os.devnull, 'w') as devnull:
+ try:
+ subprocess.call([git_executable, '--version'], stdout=devnull)
+ except (OSError,):
+ return False
+ return True
+
+
+class DepsRollError(Exception):
+ """Exceptions specific to this module."""
+ pass
+
+
+def strip_output(*args, **kwargs):
+ """Wrap subprocess.check_output and str.strip().
+
+ Pass the given arguments into subprocess.check_output() and return
+ the results, after stripping any excess whitespace.
+
+ Args:
+ *args: to be passed to subprocess.check_output()
+ **kwargs: to be passed to subprocess.check_output()
+
+ Returns:
+ The output of the process as a string without leading or
+ trailing whitespace.
+ Raises:
+ OSError or subprocess.CalledProcessError: raised by check_output.
+ """
+ return str(subprocess.check_output(*args, **kwargs)).strip()
+
+
+def create_temp_skia_clone(config, depth):
+ """Clones Skia in a temp dir.
+
+ Args:
+ config: (roll_deps.DepsRollConfig) object containing options.
+ depth: (int) how far back to clone the tree.
+ Returns:
+ temporary directory path if succcessful.
+ Raises:
+ OSError, subprocess.CalledProcessError on failure.
+ """
+ git = config.git
+ skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
+ try:
+ check_call(
+ [git, 'clone', '-q', '--depth=%d' % depth,
+ '--single-branch', config.skia_url, skia_dir])
+ return skia_dir
+ except (OSError, subprocess.CalledProcessError) as error:
+ shutil.rmtree(skia_dir)
+ raise error
+
+
+def find_revision_and_hash(config, revision):
+ """Finds revision number and git hash of origin/master in the Skia tree.
+
+ Args:
+ config: (roll_deps.DepsRollConfig) object containing options.
+ revision: (int or None) SVN revision number. If None, use
+ tip-of-tree.
+
+ Returns:
+ A tuple (revision, hash)
+ revision: (int) SVN revision number.
+ hash: (string) full Git commit hash.
+
+ Raises:
+ roll_deps.DepsRollError: if the revision can't be found.
+ OSError: failed to execute git or git-cl.
+ subprocess.CalledProcessError: git returned unexpected status.
+ """
+ git = config.git
+ use_temp = False
+ skia_dir = None
+ depth = 1 if (revision is None) else config.search_depth
+ try:
+ if config.skia_git_checkout_path:
+ skia_dir = config.skia_git_checkout_path
+ ## Update origin/master if needed.
+ check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir)
+ else:
+ skia_dir = create_temp_skia_clone(config, depth)
+ assert skia_dir
+ use_temp = True
+
+ if revision is None:
+ message = subprocess.check_output(
+ [git, 'log', '-n', '1', '--format=format:%B',
+ 'origin/master'], cwd=skia_dir)
+ svn_format = (
+ 'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ')
+ search = re.search(svn_format, message)
+ if not search:
+ raise DepsRollError(
+ 'Revision number missing from origin/master.')
+ revision = int(search.group(1))
+ git_hash = strip_output(
+ [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir)
+ else:
+ revision_regex = config.revision_format % revision
+ git_hash = strip_output(
+ [git, 'log', '--grep', revision_regex, '--format=format:%H',
+ 'origin/master'], cwd=skia_dir)
+
+ if revision < 0 or not git_hash:
+ raise DepsRollError('Git hash can not be found.')
+ return revision, git_hash
+ finally:
+ if use_temp:
+ shutil.rmtree(skia_dir)
+
+
+class GitBranchCLUpload(object):
+ """Class to manage git branches and git-cl-upload.
+
+ This class allows one to create a new branch in a repository based
+ off of origin/master, make changes to the tree inside the
+ with-block, upload that new branch to Rietveld, restore the original
+ tree state, and delete the local copy of the new branch.
+
+ See roll_deps() for an example of use.
+
+ Constructor Args:
+ config: (roll_deps.DepsRollConfig) object containing options.
+ message: (string) the commit message, can be multiline.
+ set_brach_name: (string or none) if not None, the name of the
+ branch to use. If None, then use a temporary branch that
+ will be deleted.
+
+ Attributes:
+ issue: a string describing the codereview issue, after __exit__
+ has been called, othrwise, None.
+
+ Raises:
+ OSError: failed to execute git or git-cl.
+ subprocess.CalledProcessError: git returned unexpected status.
+ """
+ # pylint: disable=I0011,R0903,R0902
+
+ def __init__(self, config, message, set_branch_name):
+ self._message = message
+ self._file_list = []
+ self._branch_name = set_branch_name
+ self._stash = None
+ self._original_branch = None
+ self._config = config
+ self._svn_info = None
+ self.issue = None
+
+ def stage_for_commit(self, *paths):
+ """Calls `git add ...` on each argument.
+
+ Args:
+ *paths: (list of strings) list of filenames to pass to `git add`.
+ """
+ self._file_list.extend(paths)
+
+ def __enter__(self):
+ git = self._config.git
+ def branch_exists(branch):
+ """Return true iff branch exists."""
+ return 0 == subprocess.call(
+ [git, 'show-ref', '--quiet', branch])
+ def has_diff():
+ """Return true iff repository has uncommited changes."""
+ return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD']))
+ self._stash = has_diff()
+ if self._stash:
+ check_call([git, 'stash', 'save'])
+ try:
+ self._original_branch = strip_output(
+ [git, 'symbolic-ref', '--short', 'HEAD'])
+ except (subprocess.CalledProcessError,):
+ self._original_branch = strip_output(
+ [git, 'rev-parse', 'HEAD'])
+
+ if not self._branch_name:
+ self._branch_name = self._config.default_branch_name
+
+ if branch_exists(self._branch_name):
+ check_call([git, 'checkout', '-q', 'master'])
+ check_call([git, 'branch', '-q', '-D', self._branch_name])
+
+ check_call(
+ [git, 'checkout', '-q', '-b',
+ self._branch_name, 'origin/master'])
+
+ svn_info = subprocess.check_output(['git', 'svn', 'info'])
+ svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info)
+ assert svn_info_search
+ self._svn_info = svn_info_search.group(1)
+
+ def __exit__(self, etype, value, traceback):
+ # pylint: disable=I0011,R0912
+ git = self._config.git
+ def quiet_check_call(*args, **kwargs):
+ """Call check_call, but pipe output to devnull."""
+ with open(os.devnull, 'w') as devnull:
+ check_call(*args, stdout=devnull, **kwargs)
+
+ for filename in self._file_list:
+ assert os.path.exists(filename)
+ check_call([git, 'add', filename])
+ check_call([git, 'commit', '-q', '-m', self._message])
+
+ git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com',
+ '--bypass-hooks', '--bypass-watchlists']
+ git_try = [git, 'cl', 'try', '--revision', self._svn_info]
+ git_try.extend([arg for bot in self._config.cl_bot_list
+ for arg in ('-b', bot)])
+
+ if self._config.skip_cl_upload:
+ print ' '.join(git_cl)
+ print
+ if self._config.cl_bot_list:
+ print ' '.join(git_try)
+ print
+ self.issue = ''
+ else:
+ if self._config.verbose:
+ check_call(git_cl)
+ print
+ else:
+ quiet_check_call(git_cl)
+ self.issue = strip_output([git, 'cl', 'issue'])
+ if self._config.cl_bot_list:
+ if self._config.verbose:
+ check_call(git_try)
+ print
+ else:
+ quiet_check_call(git_try)
+
+ # deal with the aftermath of failed executions of this script.
+ if self._config.default_branch_name == self._original_branch:
+ self._original_branch = 'master'
+ check_call([git, 'checkout', '-q', self._original_branch])
+
+ if self._config.default_branch_name == self._branch_name:
+ check_call([git, 'branch', '-q', '-D', self._branch_name])
+ if self._stash:
+ check_call([git, 'stash', 'pop'])
+
+
+def change_skia_deps(revision, git_hash, depspath):
+ """Update the DEPS file.
+
+ Modify the skia_revision and skia_hash entries in the given DEPS file.
+
+ Args:
+ revision: (int) Skia SVN revision.
+ git_hash: (string) Skia Git hash.
+ depspath: (string) path to DEPS file.
+ """
+ temp_file = tempfile.NamedTemporaryFile(delete=False,
+ prefix='skia_DEPS_ROLL_tmp_')
+ try:
+ deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
+ deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')
+
+ deps_regex_rev_repl = '"skia_revision": "%d",' % revision
+ deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash
+
+ with open(depspath, 'r') as input_stream:
+ for line in input_stream:
+ line = deps_regex_rev.sub(deps_regex_rev_repl, line)
+ line = deps_regex_hash.sub(deps_regex_hash_repl, line)
+ temp_file.write(line)
+ finally:
+ temp_file.close()
+ shutil.move(temp_file.name, depspath)
+
+
+def branch_name(message):
+ """Return the first line of a commit message to be used as a branch name.
+
+ Args:
+ message: (string)
+
+ Returns:
+ A string derived from message suitable for a branch name.
+ """
+ return message.lstrip().split('\n')[0].rstrip().replace(' ', '_')
+
+
+def roll_deps(config, revision, git_hash):
+ """Upload changed DEPS and a whitespace change.
+
+ Given the correct git_hash, create two Reitveld issues.
+
+ Args:
+ config: (roll_deps.DepsRollConfig) object containing options.
+ revision: (int) Skia SVN revision.
+ git_hash: (string) Skia Git hash.
+
+ Returns:
+ a tuple containing textual description of the two issues.
+
+ Raises:
+ OSError: failed to execute git or git-cl.
+ subprocess.CalledProcessError: git returned unexpected status.
+ """
+ git = config.git
+ cwd = os.getcwd()
+ os.chdir(config.chromium_path)
+ try:
+ check_call([git, 'fetch', '-q', 'origin'])
+ master_hash = strip_output(
+ [git, 'show-ref', 'origin/master', '--hash'])
+
+ # master_hash[8] gives each whitespace CL a unique name.
+ message = ('whitespace change %s\n\nThis CL was created by'
+ ' Skia\'s roll_deps.py script.\n') % master_hash[:8]
+ branch = branch_name(message) if config.save_branches else None
+
+ codereview = GitBranchCLUpload(config, message, branch)
+ with codereview:
+ with open('build/whitespace_file.txt', 'a') as output_stream:
+ output_stream.write('\nCONTROL\n')
+ codereview.stage_for_commit('build/whitespace_file.txt')
+ whitespace_cl = codereview.issue
+ if branch:
+ whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch)
+ control_url_match = re.search('https?://[^) ]+', codereview.issue)
+ if control_url_match:
+ message = ('roll skia DEPS to %d\n\nThis CL was created by'
+ ' Skia\'s roll_deps.py script.\n\ncontrol: %s'
+ % (revision, control_url_match.group(0)))
+ else:
+ message = ('roll skia DEPS to %d\n\nThis CL was created by'
+ ' Skia\'s roll_deps.py script.') % revision
+ branch = branch_name(message) if config.save_branches else None
+ codereview = GitBranchCLUpload(config, message, branch)
+ with codereview:
+ change_skia_deps(revision, git_hash, 'DEPS')
+ codereview.stage_for_commit('DEPS')
+ deps_cl = codereview.issue
+ if branch:
+ deps_cl = '%s\n branch: %s' % (deps_cl, branch)
+
+ return deps_cl, whitespace_cl
+ finally:
+ os.chdir(cwd)
+
+
+def find_hash_and_roll_deps(config, revision):
+ """Call find_hash_from_revision() and roll_deps().
+
+ The calls to git will be verbose on standard output. After a
+ successful upload of both issues, print links to the new
+ codereview issues.
+
+ Args:
+ config: (roll_deps.DepsRollConfig) object containing options.
+ revision: (int or None) the Skia SVN revision number or None
+ to use the tip of the tree.
+
+ Raises:
+ roll_deps.DepsRollError: if the revision can't be found.
+ OSError: failed to execute git or git-cl.
+ subprocess.CalledProcessError: git returned unexpected status.
+ """
+ revision, git_hash = find_revision_and_hash(config, revision)
+
+ print 'revision=%r\nhash=%r\n' % (revision, git_hash)
+
+ deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)
+
+ print 'DEPS roll:\n %s\n' % deps_issue
+ print 'Whitespace change:\n %s\n' % whitespace_issue
+
+
+def main(args):
+ """main function; see module-level docstring and GetOptionParser help.
+
+ Args:
+ args: sys.argv[1:]-type argument list.
+ """
+ option_parser = DepsRollConfig.GetOptionParser()
+ options = option_parser.parse_args(args)[0]
+
+ if not options.chromium_path:
+ option_parser.error('Must specify chromium_path.')
+ if not os.path.isdir(options.chromium_path):
+ option_parser.error('chromium_path must be a directory.')
+ if not test_git_executable(options.git_path):
+ option_parser.error('Invalid git executable.')
+
+ config = DepsRollConfig(options)
+ find_hash_and_roll_deps(config, options.revision)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
+