halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 1 | #!/usr/bin/python2 |
| 2 | |
| 3 | # Copyright 2014 Google Inc. |
| 4 | # |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | """Skia's Chromium DEPS roll script. |
| 9 | |
| 10 | This script: |
| 11 | - searches through the last N Skia git commits to find out the hash that is |
| 12 | associated with the SVN revision number. |
| 13 | - creates a new branch in the Chromium tree, modifies the DEPS file to |
| 14 | point at the given Skia commit, commits, uploads to Rietveld, and |
| 15 | deletes the local copy of the branch. |
| 16 | - creates a whitespace-only commit and uploads that to to Rietveld. |
| 17 | - returns the Chromium tree to its previous state. |
| 18 | |
| 19 | Usage: |
| 20 | %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS] |
| 21 | """ |
| 22 | |
| 23 | |
| 24 | import optparse |
| 25 | import os |
| 26 | import re |
| 27 | import shutil |
| 28 | import subprocess |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 29 | import sys |
| 30 | import tempfile |
| 31 | |
| 32 | |
borenet@google.com | 17bb951 | 2014-01-27 18:08:55 +0000 | [diff] [blame] | 33 | DEFAULT_BOTS_LIST = [ |
| 34 | 'android_clang_dbg', |
| 35 | 'android_dbg', |
| 36 | 'android_rel', |
| 37 | 'cros_daisy', |
| 38 | 'linux', |
| 39 | 'linux_asan', |
| 40 | 'linux_chromeos', |
| 41 | 'linux_chromeos_asan', |
| 42 | 'linux_gpu', |
| 43 | 'linux_layout', |
| 44 | 'linux_layout_rel', |
| 45 | 'mac', |
| 46 | 'mac_asan', |
| 47 | 'mac_gpu', |
| 48 | 'mac_layout', |
| 49 | 'mac_layout_rel', |
| 50 | 'win', |
| 51 | 'win_gpu', |
| 52 | 'win_layout', |
| 53 | 'win_layout_rel', |
| 54 | ] |
| 55 | |
| 56 | |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 57 | class DepsRollConfig(object): |
| 58 | """Contains configuration options for this module. |
| 59 | |
| 60 | Attributes: |
| 61 | git: (string) The git executable. |
| 62 | chromium_path: (string) path to a local chromium git repository. |
| 63 | save_branches: (boolean) iff false, delete temporary branches. |
| 64 | verbose: (boolean) iff false, suppress the output from git-cl. |
| 65 | search_depth: (int) how far back to look for the revision. |
| 66 | skia_url: (string) Skia's git repository. |
| 67 | self.skip_cl_upload: (boolean) |
| 68 | self.cl_bot_list: (list of strings) |
| 69 | """ |
| 70 | |
| 71 | # pylint: disable=I0011,R0903,R0902 |
| 72 | def __init__(self, options=None): |
| 73 | self.skia_url = 'https://skia.googlesource.com/skia.git' |
| 74 | self.revision_format = ( |
| 75 | 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') |
| 76 | |
| 77 | if not options: |
| 78 | options = DepsRollConfig.GetOptionParser() |
| 79 | # pylint: disable=I0011,E1103 |
| 80 | self.verbose = options.verbose |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 81 | self.vsp = VerboseSubprocess(self.verbose) |
| 82 | self.save_branches = not options.delete_branches |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 83 | self.search_depth = options.search_depth |
| 84 | self.chromium_path = options.chromium_path |
| 85 | self.git = options.git_path |
| 86 | self.skip_cl_upload = options.skip_cl_upload |
| 87 | # Split and remove empty strigns from the bot list. |
| 88 | self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] |
| 89 | self.skia_git_checkout_path = options.skia_git_path |
| 90 | self.default_branch_name = 'autogenerated_deps_roll_branch' |
commit-bot@chromium.org | 34f47f9 | 2014-01-21 21:38:49 +0000 | [diff] [blame] | 91 | self.reviewers_list = ','.join([ |
| 92 | # 'rmistry@google.com', |
| 93 | # 'reed@google.com', |
| 94 | # 'bsalomon@google.com', |
| 95 | # 'robertphillips@google.com', |
| 96 | ]) |
| 97 | self.cc_list = ','.join([ |
| 98 | # 'skia-team@google.com', |
| 99 | ]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 100 | |
| 101 | @staticmethod |
| 102 | def GetOptionParser(): |
| 103 | # pylint: disable=I0011,C0103 |
| 104 | """Returns an optparse.OptionParser object. |
| 105 | |
| 106 | Returns: |
| 107 | An optparse.OptionParser object. |
| 108 | |
| 109 | Called by the main() function. |
| 110 | """ |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 111 | option_parser = optparse.OptionParser(usage=__doc__) |
| 112 | # Anyone using this script on a regular basis should set the |
| 113 | # CHROMIUM_CHECKOUT_PATH environment variable. |
| 114 | option_parser.add_option( |
| 115 | '-c', '--chromium_path', help='Path to local Chromium Git' |
| 116 | ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH' |
| 117 | ' if that environment variable is set.', |
| 118 | default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) |
| 119 | option_parser.add_option( |
| 120 | '-r', '--revision', type='int', default=None, |
| 121 | help='The Skia SVN revision number, defaults to top of tree.') |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 122 | option_parser.add_option( |
| 123 | '-g', '--git_hash', default=None, |
| 124 | help='A partial Skia Git hash. Do not set this and revision.') |
| 125 | |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 126 | # Anyone using this script on a regular basis should set the |
| 127 | # SKIA_GIT_CHECKOUT_PATH environment variable. |
| 128 | option_parser.add_option( |
| 129 | '', '--skia_git_path', |
| 130 | help='Path of a pure-git Skia repository checkout. If empty,' |
| 131 | ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' |
| 132 | '_PATH, if that environment variable is set.', |
| 133 | default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) |
| 134 | option_parser.add_option( |
| 135 | '', '--search_depth', type='int', default=100, |
| 136 | help='How far back to look for the revision.') |
| 137 | option_parser.add_option( |
| 138 | '', '--git_path', help='Git executable, defaults to "git".', |
| 139 | default='git') |
| 140 | option_parser.add_option( |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 141 | '', '--delete_branches', help='Delete the temporary branches', |
| 142 | action='store_true', dest='delete_branches', default=False) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 143 | option_parser.add_option( |
| 144 | '', '--verbose', help='Do not suppress the output from `git cl`.', |
| 145 | action='store_true', dest='verbose', default=False) |
| 146 | option_parser.add_option( |
| 147 | '', '--skip_cl_upload', help='Skip the cl upload step; useful' |
| 148 | ' for testing or with --save_branches.', |
| 149 | action='store_true', default=False) |
| 150 | |
| 151 | default_bots_help = ( |
| 152 | 'Comma-separated list of bots, defaults to a list of %d bots.' |
| 153 | ' To skip `git cl try`, set this to an empty string.' |
borenet@google.com | 17bb951 | 2014-01-27 18:08:55 +0000 | [diff] [blame] | 154 | % len(DEFAULT_BOTS_LIST)) |
| 155 | default_bots = ','.join(DEFAULT_BOTS_LIST) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 156 | option_parser.add_option( |
| 157 | '', '--bots', help=default_bots_help, default=default_bots) |
| 158 | |
| 159 | return option_parser |
| 160 | |
| 161 | |
| 162 | def test_git_executable(git_executable): |
| 163 | """Test the git executable. |
| 164 | |
| 165 | Args: |
| 166 | git_executable: git executable path. |
| 167 | Returns: |
| 168 | True if test is successful. |
| 169 | """ |
| 170 | with open(os.devnull, 'w') as devnull: |
| 171 | try: |
| 172 | subprocess.call([git_executable, '--version'], stdout=devnull) |
| 173 | except (OSError,): |
| 174 | return False |
| 175 | return True |
| 176 | |
| 177 | |
| 178 | class DepsRollError(Exception): |
| 179 | """Exceptions specific to this module.""" |
| 180 | pass |
| 181 | |
| 182 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 183 | class VerboseSubprocess(object): |
| 184 | """Call subprocess methods, but print out command before executing. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 185 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 186 | Attributes: |
| 187 | verbose: (boolean) should we print out the command or not. If |
| 188 | not, this is the same as calling the subprocess method |
| 189 | quiet: (boolean) suppress stdout on check_call and call. |
| 190 | prefix: (string) When verbose, what to print before each command. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 191 | """ |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 192 | |
| 193 | def __init__(self, verbose): |
| 194 | self.verbose = verbose |
| 195 | self.quiet = not verbose |
| 196 | self.prefix = '~~$ ' |
| 197 | |
| 198 | @staticmethod |
| 199 | def _fix(string): |
| 200 | """Quote and escape a string if necessary.""" |
| 201 | if ' ' in string or '\n' in string: |
| 202 | string = '"%s"' % string.replace('\n', '\\n') |
| 203 | return string |
| 204 | |
| 205 | @staticmethod |
| 206 | def print_subprocess_args(prefix, *args, **kwargs): |
| 207 | """Print out args in a human-readable manner.""" |
| 208 | if 'cwd' in kwargs: |
| 209 | print '%scd %s' % (prefix, kwargs['cwd']) |
| 210 | print prefix + ' '.join(VerboseSubprocess._fix(arg) for arg in args[0]) |
| 211 | if 'cwd' in kwargs: |
| 212 | print '%scd -' % prefix |
| 213 | |
| 214 | def check_call(self, *args, **kwargs): |
| 215 | """Wrapper for subprocess.check_call(). |
| 216 | |
| 217 | Args: |
| 218 | *args: to be passed to subprocess.check_call() |
| 219 | **kwargs: to be passed to subprocess.check_call() |
| 220 | Returns: |
| 221 | Whatever subprocess.check_call() returns. |
| 222 | Raises: |
| 223 | OSError or subprocess.CalledProcessError: raised by check_call. |
| 224 | """ |
| 225 | if self.verbose: |
| 226 | self.print_subprocess_args(self.prefix, *args, **kwargs) |
| 227 | if self.quiet: |
| 228 | with open(os.devnull, 'w') as devnull: |
| 229 | return subprocess.check_call(*args, stdout=devnull, **kwargs) |
| 230 | else: |
| 231 | return subprocess.check_call(*args, **kwargs) |
| 232 | |
| 233 | def call(self, *args, **kwargs): |
| 234 | """Wrapper for subprocess.check(). |
| 235 | |
| 236 | Args: |
| 237 | *args: to be passed to subprocess.check_call() |
| 238 | **kwargs: to be passed to subprocess.check_call() |
| 239 | Returns: |
| 240 | Whatever subprocess.call() returns. |
| 241 | Raises: |
| 242 | OSError or subprocess.CalledProcessError: raised by call. |
| 243 | """ |
| 244 | if self.verbose: |
| 245 | self.print_subprocess_args(self.prefix, *args, **kwargs) |
| 246 | if self.quiet: |
| 247 | with open(os.devnull, 'w') as devnull: |
| 248 | return subprocess.call(*args, stdout=devnull, **kwargs) |
| 249 | else: |
| 250 | return subprocess.call(*args, **kwargs) |
| 251 | |
| 252 | def check_output(self, *args, **kwargs): |
| 253 | """Wrapper for subprocess.check_output(). |
| 254 | |
| 255 | Args: |
| 256 | *args: to be passed to subprocess.check_output() |
| 257 | **kwargs: to be passed to subprocess.check_output() |
| 258 | Returns: |
| 259 | Whatever subprocess.check_output() returns. |
| 260 | Raises: |
| 261 | OSError or subprocess.CalledProcessError: raised by check_output. |
| 262 | """ |
| 263 | if self.verbose: |
| 264 | self.print_subprocess_args(self.prefix, *args, **kwargs) |
| 265 | return subprocess.check_output(*args, **kwargs) |
| 266 | |
| 267 | def strip_output(self, *args, **kwargs): |
| 268 | """Wrap subprocess.check_output and str.strip(). |
| 269 | |
| 270 | Pass the given arguments into subprocess.check_output() and return |
| 271 | the results, after stripping any excess whitespace. |
| 272 | |
| 273 | Args: |
| 274 | *args: to be passed to subprocess.check_output() |
| 275 | **kwargs: to be passed to subprocess.check_output() |
| 276 | |
| 277 | Returns: |
| 278 | The output of the process as a string without leading or |
| 279 | trailing whitespace. |
| 280 | Raises: |
| 281 | OSError or subprocess.CalledProcessError: raised by check_output. |
| 282 | """ |
| 283 | if self.verbose: |
| 284 | self.print_subprocess_args(self.prefix, *args, **kwargs) |
| 285 | return str(subprocess.check_output(*args, **kwargs)).strip() |
| 286 | |
| 287 | def popen(self, *args, **kwargs): |
| 288 | """Wrapper for subprocess.Popen(). |
| 289 | |
| 290 | Args: |
| 291 | *args: to be passed to subprocess.Popen() |
| 292 | **kwargs: to be passed to subprocess.Popen() |
| 293 | Returns: |
| 294 | The output of subprocess.Popen() |
| 295 | Raises: |
| 296 | OSError or subprocess.CalledProcessError: raised by Popen. |
| 297 | """ |
| 298 | if self.verbose: |
| 299 | self.print_subprocess_args(self.prefix, *args, **kwargs) |
| 300 | return subprocess.Popen(*args, **kwargs) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 301 | |
| 302 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 303 | class ChangeDir(object): |
| 304 | """Use with a with-statement to temporarily change directories.""" |
| 305 | # pylint: disable=I0011,R0903 |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 306 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 307 | def __init__(self, directory, verbose=False): |
| 308 | self._directory = directory |
| 309 | self._verbose = verbose |
| 310 | |
| 311 | def __enter__(self): |
| 312 | if self._verbose: |
| 313 | print '~~$ cd %s' % self._directory |
| 314 | cwd = os.getcwd() |
| 315 | os.chdir(self._directory) |
| 316 | self._directory = cwd |
| 317 | |
| 318 | def __exit__(self, etype, value, traceback): |
| 319 | if self._verbose: |
| 320 | print '~~$ cd %s' % self._directory |
| 321 | os.chdir(self._directory) |
| 322 | |
| 323 | |
| 324 | class ReSearch(object): |
| 325 | """A collection of static methods for regexing things.""" |
| 326 | |
| 327 | @staticmethod |
| 328 | def search_within_stream(input_stream, pattern, default=None): |
| 329 | """Search for regular expression in a file-like object. |
| 330 | |
| 331 | Opens a file for reading and searches line by line for a match to |
| 332 | the regex and returns the parenthesized group named return for the |
| 333 | first match. Does not search across newlines. |
| 334 | |
| 335 | For example: |
| 336 | pattern = '^root(:[^:]*){4}:(?P<return>[^:]*)' |
| 337 | with open('/etc/passwd', 'r') as stream: |
| 338 | return search_within_file(stream, pattern) |
| 339 | should return root's home directory (/root on my system). |
| 340 | |
| 341 | Args: |
| 342 | input_stream: file-like object to be read |
| 343 | pattern: (string) to be passed to re.compile |
| 344 | default: what to return if no match |
| 345 | |
| 346 | Returns: |
| 347 | A string or whatever default is |
| 348 | """ |
| 349 | pattern_object = re.compile(pattern) |
| 350 | for line in input_stream: |
| 351 | match = pattern_object.search(line) |
| 352 | if match: |
| 353 | return match.group('return') |
| 354 | return default |
| 355 | |
| 356 | @staticmethod |
| 357 | def search_within_string(input_string, pattern, default=None): |
| 358 | """Search for regular expression in a string. |
| 359 | |
| 360 | Args: |
| 361 | input_string: (string) to be searched |
| 362 | pattern: (string) to be passed to re.compile |
| 363 | default: what to return if no match |
| 364 | |
| 365 | Returns: |
| 366 | A string or whatever default is |
| 367 | """ |
| 368 | match = re.search(pattern, input_string) |
| 369 | return match.group('return') if match else default |
| 370 | |
| 371 | @staticmethod |
| 372 | def search_within_output(verbose, pattern, default, *args, **kwargs): |
| 373 | """Search for regular expression in a process output. |
| 374 | |
| 375 | Does not search across newlines. |
| 376 | |
| 377 | Args: |
| 378 | verbose: (boolean) shoule we call |
| 379 | VerboseSubprocess.print_subprocess_args? |
| 380 | pattern: (string) to be passed to re.compile |
| 381 | default: what to return if no match |
| 382 | *args: to be passed to subprocess.Popen() |
| 383 | **kwargs: to be passed to subprocess.Popen() |
| 384 | |
| 385 | Returns: |
| 386 | A string or whatever default is |
| 387 | """ |
| 388 | if verbose: |
| 389 | VerboseSubprocess.print_subprocess_args( |
| 390 | '~~$ ', *args, **kwargs) |
| 391 | proc = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs) |
| 392 | return ReSearch.search_within_stream(proc.stdout, pattern, default) |
| 393 | |
| 394 | |
| 395 | def get_svn_revision(config, commit): |
| 396 | """Works in both git and git-svn. returns a string.""" |
| 397 | svn_format = ( |
| 398 | '(git-svn-id: [^@ ]+@|SVN changes up to revision |' |
| 399 | 'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)') |
| 400 | svn_revision = ReSearch.search_within_output( |
| 401 | config.verbose, svn_format, None, |
| 402 | [config.git, 'log', '-n', '1', '--format=format:%B', commit]) |
| 403 | if not svn_revision: |
| 404 | raise DepsRollError( |
| 405 | 'Revision number missing from Chromium origin/master.') |
| 406 | return int(svn_revision) |
| 407 | |
| 408 | |
| 409 | class SkiaGitCheckout(object): |
| 410 | """Class to create a temporary skia git checkout, if necessary. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 411 | """ |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 412 | # pylint: disable=I0011,R0903 |
| 413 | |
| 414 | def __init__(self, config, depth): |
| 415 | self._config = config |
| 416 | self._depth = depth |
| 417 | self._use_temp = None |
| 418 | self._original_cwd = None |
| 419 | |
| 420 | def __enter__(self): |
| 421 | config = self._config |
| 422 | git = config.git |
| 423 | skia_dir = None |
| 424 | self._original_cwd = os.getcwd() |
| 425 | if config.skia_git_checkout_path: |
| 426 | skia_dir = config.skia_git_checkout_path |
| 427 | ## Update origin/master if needed. |
| 428 | if self._config.verbose: |
| 429 | print '~~$', 'cd', skia_dir |
| 430 | os.chdir(skia_dir) |
| 431 | config.vsp.check_call([git, 'fetch', '-q', 'origin']) |
| 432 | self._use_temp = None |
| 433 | else: |
| 434 | skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') |
| 435 | self._use_temp = skia_dir |
| 436 | try: |
| 437 | os.chdir(skia_dir) |
| 438 | config.vsp.check_call( |
| 439 | [git, 'clone', '-q', '--depth=%d' % self._depth, |
| 440 | '--single-branch', config.skia_url, '.']) |
| 441 | except (OSError, subprocess.CalledProcessError) as error: |
| 442 | shutil.rmtree(skia_dir) |
| 443 | raise error |
| 444 | |
| 445 | def __exit__(self, etype, value, traceback): |
| 446 | if self._config.verbose: |
| 447 | print '~~$', 'cd', self._original_cwd |
| 448 | os.chdir(self._original_cwd) |
| 449 | if self._use_temp: |
| 450 | shutil.rmtree(self._use_temp) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 451 | |
| 452 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 453 | def revision_and_hash(config): |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 454 | """Finds revision number and git hash of origin/master in the Skia tree. |
| 455 | |
| 456 | Args: |
| 457 | config: (roll_deps.DepsRollConfig) object containing options. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 458 | |
| 459 | Returns: |
| 460 | A tuple (revision, hash) |
| 461 | revision: (int) SVN revision number. |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 462 | git_hash: (string) full Git commit hash. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 463 | |
| 464 | Raises: |
| 465 | roll_deps.DepsRollError: if the revision can't be found. |
| 466 | OSError: failed to execute git or git-cl. |
| 467 | subprocess.CalledProcessError: git returned unexpected status. |
| 468 | """ |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 469 | with SkiaGitCheckout(config, 1): |
| 470 | revision = get_svn_revision(config, 'origin/master') |
| 471 | git_hash = config.vsp.strip_output( |
| 472 | [config.git, 'show-ref', 'origin/master', '--hash']) |
| 473 | if not git_hash: |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 474 | raise DepsRollError('Git hash can not be found.') |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 475 | return revision, git_hash |
| 476 | |
| 477 | |
| 478 | def revision_and_hash_from_revision(config, revision): |
| 479 | """Finds revision number and git hash of a commit in the Skia tree. |
| 480 | |
| 481 | Args: |
| 482 | config: (roll_deps.DepsRollConfig) object containing options. |
| 483 | revision: (int) SVN revision number. |
| 484 | |
| 485 | Returns: |
| 486 | A tuple (revision, hash) |
| 487 | revision: (int) SVN revision number. |
| 488 | git_hash: (string) full Git commit hash. |
| 489 | |
| 490 | Raises: |
| 491 | roll_deps.DepsRollError: if the revision can't be found. |
| 492 | OSError: failed to execute git or git-cl. |
| 493 | subprocess.CalledProcessError: git returned unexpected status. |
| 494 | """ |
| 495 | with SkiaGitCheckout(config, config.search_depth): |
| 496 | revision_regex = config.revision_format % revision |
| 497 | git_hash = config.vsp.strip_output( |
| 498 | [config.git, 'log', '--grep', revision_regex, |
| 499 | '--format=format:%H', 'origin/master']) |
| 500 | if not git_hash: |
| 501 | raise DepsRollError('Git hash can not be found.') |
| 502 | return revision, git_hash |
| 503 | |
| 504 | |
| 505 | def revision_and_hash_from_partial(config, partial_hash): |
| 506 | """Returns the SVN revision number and full git hash. |
| 507 | |
| 508 | Args: |
| 509 | config: (roll_deps.DepsRollConfig) object containing options. |
| 510 | partial_hash: (string) Partial git commit hash. |
| 511 | |
| 512 | Returns: |
| 513 | A tuple (revision, hash) |
| 514 | revision: (int) SVN revision number. |
| 515 | git_hash: (string) full Git commit hash. |
| 516 | |
| 517 | Raises: |
| 518 | roll_deps.DepsRollError: if the revision can't be found. |
| 519 | OSError: failed to execute git or git-cl. |
| 520 | subprocess.CalledProcessError: git returned unexpected status. |
| 521 | """ |
| 522 | with SkiaGitCheckout(config, config.search_depth): |
| 523 | git_hash = config.vsp.strip_output( |
| 524 | ['git', 'log', '-n', '1', '--format=format:%H', partial_hash]) |
| 525 | if not git_hash: |
| 526 | raise DepsRollError('Partial Git hash can not be found.') |
| 527 | revision = get_svn_revision(config, git_hash) |
| 528 | return revision, git_hash |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 529 | |
| 530 | |
| 531 | class GitBranchCLUpload(object): |
| 532 | """Class to manage git branches and git-cl-upload. |
| 533 | |
| 534 | This class allows one to create a new branch in a repository based |
| 535 | off of origin/master, make changes to the tree inside the |
| 536 | with-block, upload that new branch to Rietveld, restore the original |
| 537 | tree state, and delete the local copy of the new branch. |
| 538 | |
| 539 | See roll_deps() for an example of use. |
| 540 | |
| 541 | Constructor Args: |
| 542 | config: (roll_deps.DepsRollConfig) object containing options. |
| 543 | message: (string) the commit message, can be multiline. |
| 544 | set_brach_name: (string or none) if not None, the name of the |
| 545 | branch to use. If None, then use a temporary branch that |
| 546 | will be deleted. |
| 547 | |
| 548 | Attributes: |
| 549 | issue: a string describing the codereview issue, after __exit__ |
| 550 | has been called, othrwise, None. |
| 551 | |
| 552 | Raises: |
| 553 | OSError: failed to execute git or git-cl. |
| 554 | subprocess.CalledProcessError: git returned unexpected status. |
| 555 | """ |
| 556 | # pylint: disable=I0011,R0903,R0902 |
| 557 | |
| 558 | def __init__(self, config, message, set_branch_name): |
| 559 | self._message = message |
| 560 | self._file_list = [] |
| 561 | self._branch_name = set_branch_name |
| 562 | self._stash = None |
| 563 | self._original_branch = None |
| 564 | self._config = config |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 565 | self.issue = None |
| 566 | |
| 567 | def stage_for_commit(self, *paths): |
| 568 | """Calls `git add ...` on each argument. |
| 569 | |
| 570 | Args: |
| 571 | *paths: (list of strings) list of filenames to pass to `git add`. |
| 572 | """ |
| 573 | self._file_list.extend(paths) |
| 574 | |
| 575 | def __enter__(self): |
| 576 | git = self._config.git |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 577 | vsp = self._config.vsp |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 578 | def branch_exists(branch): |
| 579 | """Return true iff branch exists.""" |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 580 | return 0 == vsp.call([git, 'show-ref', '--quiet', branch]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 581 | def has_diff(): |
| 582 | """Return true iff repository has uncommited changes.""" |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 583 | return bool(vsp.call([git, 'diff', '--quiet', 'HEAD'])) |
| 584 | |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 585 | self._stash = has_diff() |
| 586 | if self._stash: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 587 | vsp.check_call([git, 'stash', 'save']) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 588 | try: |
borenet@google.com | 4d5e136 | 2014-01-10 21:02:25 +0000 | [diff] [blame] | 589 | full_branch = vsp.strip_output([git, 'symbolic-ref', 'HEAD']) |
| 590 | self._original_branch = full_branch.split('/')[-1] |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 591 | except (subprocess.CalledProcessError,): |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 592 | self._original_branch = vsp.strip_output( |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 593 | [git, 'rev-parse', 'HEAD']) |
| 594 | |
| 595 | if not self._branch_name: |
| 596 | self._branch_name = self._config.default_branch_name |
| 597 | |
| 598 | if branch_exists(self._branch_name): |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 599 | vsp.check_call([git, 'checkout', '-q', 'master']) |
borenet@google.com | 4d5e136 | 2014-01-10 21:02:25 +0000 | [diff] [blame] | 600 | vsp.check_call([git, 'branch', '-D', self._branch_name]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 601 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 602 | vsp.check_call( |
| 603 | [git, 'checkout', '-q', '-b', self._branch_name, 'origin/master']) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 604 | |
| 605 | def __exit__(self, etype, value, traceback): |
| 606 | # pylint: disable=I0011,R0912 |
| 607 | git = self._config.git |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 608 | vsp = self._config.vsp |
| 609 | svn_info = str(get_svn_revision(self._config, 'HEAD')) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 610 | |
| 611 | for filename in self._file_list: |
| 612 | assert os.path.exists(filename) |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 613 | vsp.check_call([git, 'add', filename]) |
| 614 | vsp.check_call([git, 'commit', '-q', '-m', self._message]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 615 | |
commit-bot@chromium.org | 34f47f9 | 2014-01-21 21:38:49 +0000 | [diff] [blame] | 616 | git_cl = [git, 'cl', 'upload', '-f', |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 617 | '--bypass-hooks', '--bypass-watchlists'] |
commit-bot@chromium.org | 34f47f9 | 2014-01-21 21:38:49 +0000 | [diff] [blame] | 618 | if self._config.cc_list: |
| 619 | git_cl.append('--cc=%s' % self._config.cc_list) |
| 620 | if self._config.reviewers_list: |
| 621 | git_cl.append('--reviewers=%s' % self._config.reviewers_list) |
| 622 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 623 | git_try = [git, 'cl', 'try', '--revision', svn_info] |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 624 | git_try.extend([arg for bot in self._config.cl_bot_list |
| 625 | for arg in ('-b', bot)]) |
| 626 | |
| 627 | if self._config.skip_cl_upload: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 628 | print 'You should call:' |
| 629 | print ' cd %s' % os.getcwd() |
| 630 | VerboseSubprocess.print_subprocess_args( |
| 631 | ' ', [git, 'checkout', self._branch_name]) |
| 632 | VerboseSubprocess.print_subprocess_args(' ', git_cl) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 633 | if self._config.cl_bot_list: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 634 | VerboseSubprocess.print_subprocess_args(' ', git_try) |
| 635 | print |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 636 | self.issue = '' |
| 637 | else: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 638 | vsp.check_call(git_cl) |
| 639 | self.issue = vsp.strip_output([git, 'cl', 'issue']) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 640 | if self._config.cl_bot_list: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 641 | vsp.check_call(git_try) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 642 | |
| 643 | # deal with the aftermath of failed executions of this script. |
| 644 | if self._config.default_branch_name == self._original_branch: |
| 645 | self._original_branch = 'master' |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 646 | vsp.check_call([git, 'checkout', '-q', self._original_branch]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 647 | |
| 648 | if self._config.default_branch_name == self._branch_name: |
borenet@google.com | 4d5e136 | 2014-01-10 21:02:25 +0000 | [diff] [blame] | 649 | vsp.check_call([git, 'branch', '-D', self._branch_name]) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 650 | if self._stash: |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 651 | vsp.check_call([git, 'stash', 'pop']) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 652 | |
| 653 | |
| 654 | def change_skia_deps(revision, git_hash, depspath): |
| 655 | """Update the DEPS file. |
| 656 | |
| 657 | Modify the skia_revision and skia_hash entries in the given DEPS file. |
| 658 | |
| 659 | Args: |
| 660 | revision: (int) Skia SVN revision. |
| 661 | git_hash: (string) Skia Git hash. |
| 662 | depspath: (string) path to DEPS file. |
| 663 | """ |
| 664 | temp_file = tempfile.NamedTemporaryFile(delete=False, |
| 665 | prefix='skia_DEPS_ROLL_tmp_') |
| 666 | try: |
| 667 | deps_regex_rev = re.compile('"skia_revision": "[0-9]*",') |
| 668 | deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",') |
| 669 | |
| 670 | deps_regex_rev_repl = '"skia_revision": "%d",' % revision |
| 671 | deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash |
| 672 | |
| 673 | with open(depspath, 'r') as input_stream: |
| 674 | for line in input_stream: |
| 675 | line = deps_regex_rev.sub(deps_regex_rev_repl, line) |
| 676 | line = deps_regex_hash.sub(deps_regex_hash_repl, line) |
| 677 | temp_file.write(line) |
| 678 | finally: |
| 679 | temp_file.close() |
| 680 | shutil.move(temp_file.name, depspath) |
| 681 | |
| 682 | |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 683 | def roll_deps(config, revision, git_hash): |
| 684 | """Upload changed DEPS and a whitespace change. |
| 685 | |
| 686 | Given the correct git_hash, create two Reitveld issues. |
| 687 | |
| 688 | Args: |
| 689 | config: (roll_deps.DepsRollConfig) object containing options. |
| 690 | revision: (int) Skia SVN revision. |
| 691 | git_hash: (string) Skia Git hash. |
| 692 | |
| 693 | Returns: |
| 694 | a tuple containing textual description of the two issues. |
| 695 | |
| 696 | Raises: |
| 697 | OSError: failed to execute git or git-cl. |
| 698 | subprocess.CalledProcessError: git returned unexpected status. |
| 699 | """ |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 700 | |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 701 | git = config.git |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 702 | with ChangeDir(config.chromium_path, config.verbose): |
| 703 | config.vsp.check_call([git, 'fetch', '-q', 'origin']) |
| 704 | |
| 705 | old_revision = ReSearch.search_within_output( |
| 706 | config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None, |
| 707 | [git, 'show', 'origin/master:DEPS']) |
| 708 | assert old_revision |
| 709 | if revision == int(old_revision): |
| 710 | print 'DEPS is up to date!' |
| 711 | return None |
| 712 | |
| 713 | master_hash = config.vsp.strip_output( |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 714 | [git, 'show-ref', 'origin/master', '--hash']) |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 715 | master_revision = get_svn_revision(config, 'origin/master') |
| 716 | |
| 717 | branch = None |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 718 | |
| 719 | # master_hash[8] gives each whitespace CL a unique name. |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 720 | message = ('whitespace change %s\n\n' |
| 721 | 'Chromium base revision: %d / %s\n\n' |
| 722 | 'This CL was created by Skia\'s roll_deps.py script.\n' |
| 723 | ) % (master_hash[:8], master_revision, master_hash[:8]) |
| 724 | if config.save_branches: |
| 725 | branch = 'control_%s' % master_hash[:8] |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 726 | |
| 727 | codereview = GitBranchCLUpload(config, message, branch) |
| 728 | with codereview: |
| 729 | with open('build/whitespace_file.txt', 'a') as output_stream: |
| 730 | output_stream.write('\nCONTROL\n') |
| 731 | codereview.stage_for_commit('build/whitespace_file.txt') |
| 732 | whitespace_cl = codereview.issue |
| 733 | if branch: |
| 734 | whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 735 | |
| 736 | control_url = ReSearch.search_within_string( |
| 737 | codereview.issue, '(?P<return>https?://[^) ]+)', '?') |
| 738 | |
| 739 | if config.save_branches: |
| 740 | branch = 'roll_%d_%s' % (revision, master_hash[:8]) |
| 741 | message = ( |
| 742 | 'roll skia DEPS to %d\n\n' |
| 743 | 'Chromium base revision: %d / %s\n' |
| 744 | 'Old Skia revision: %s\n' |
| 745 | 'New Skia revision: %d\n' |
| 746 | 'Control CL: %s\n\n' |
commit-bot@chromium.org | 06cd6c1 | 2014-01-27 22:42:51 +0000 | [diff] [blame^] | 747 | 'This CL was created by Skia\'s roll_deps.py script.\n\n' |
| 748 | 'Bypassing commit queue trybots:\n' |
| 749 | 'NOTRY=true\n' |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 750 | % (revision, master_revision, master_hash[:8], |
| 751 | old_revision, revision, control_url)) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 752 | codereview = GitBranchCLUpload(config, message, branch) |
| 753 | with codereview: |
| 754 | change_skia_deps(revision, git_hash, 'DEPS') |
| 755 | codereview.stage_for_commit('DEPS') |
| 756 | deps_cl = codereview.issue |
| 757 | if branch: |
| 758 | deps_cl = '%s\n branch: %s' % (deps_cl, branch) |
| 759 | |
| 760 | return deps_cl, whitespace_cl |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 761 | |
| 762 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 763 | def find_hash_and_roll_deps(config, revision=None, partial_hash=None): |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 764 | """Call find_hash_from_revision() and roll_deps(). |
| 765 | |
| 766 | The calls to git will be verbose on standard output. After a |
| 767 | successful upload of both issues, print links to the new |
| 768 | codereview issues. |
| 769 | |
| 770 | Args: |
| 771 | config: (roll_deps.DepsRollConfig) object containing options. |
| 772 | revision: (int or None) the Skia SVN revision number or None |
| 773 | to use the tip of the tree. |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 774 | partial_hash: (string or None) a partial pure-git Skia commit |
| 775 | hash. Don't pass both partial_hash and revision. |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 776 | |
| 777 | Raises: |
| 778 | roll_deps.DepsRollError: if the revision can't be found. |
| 779 | OSError: failed to execute git or git-cl. |
| 780 | subprocess.CalledProcessError: git returned unexpected status. |
| 781 | """ |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 782 | |
| 783 | if revision and partial_hash: |
| 784 | raise DepsRollError('Pass revision or partial_hash, not both.') |
| 785 | |
| 786 | if partial_hash: |
| 787 | revision, git_hash = revision_and_hash_from_partial( |
| 788 | config, partial_hash) |
| 789 | elif revision: |
| 790 | revision, git_hash = revision_and_hash_from_revision(config, revision) |
| 791 | else: |
| 792 | revision, git_hash = revision_and_hash(config) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 793 | |
| 794 | print 'revision=%r\nhash=%r\n' % (revision, git_hash) |
| 795 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 796 | roll = roll_deps(config, revision, git_hash) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 797 | |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 798 | if roll: |
| 799 | deps_issue, whitespace_issue = roll |
| 800 | print 'DEPS roll:\n %s\n' % deps_issue |
| 801 | print 'Whitespace change:\n %s\n' % whitespace_issue |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 802 | |
| 803 | |
| 804 | def main(args): |
| 805 | """main function; see module-level docstring and GetOptionParser help. |
| 806 | |
| 807 | Args: |
| 808 | args: sys.argv[1:]-type argument list. |
| 809 | """ |
| 810 | option_parser = DepsRollConfig.GetOptionParser() |
| 811 | options = option_parser.parse_args(args)[0] |
| 812 | |
| 813 | if not options.chromium_path: |
| 814 | option_parser.error('Must specify chromium_path.') |
| 815 | if not os.path.isdir(options.chromium_path): |
| 816 | option_parser.error('chromium_path must be a directory.') |
| 817 | if not test_git_executable(options.git_path): |
| 818 | option_parser.error('Invalid git executable.') |
| 819 | |
| 820 | config = DepsRollConfig(options) |
halcanary@google.com | 8c5d2c1 | 2014-01-08 21:29:34 +0000 | [diff] [blame] | 821 | find_hash_and_roll_deps(config, options.revision, options.git_hash) |
halcanary@google.com | 31fdb92 | 2014-01-06 19:50:22 +0000 | [diff] [blame] | 822 | |
| 823 | |
| 824 | if __name__ == '__main__': |
| 825 | main(sys.argv[1:]) |
| 826 | |