Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2013 the V8 project authors. All rights reserved. |
| 3 | # Redistribution and use in source and binary forms, with or without |
| 4 | # modification, are permitted provided that the following conditions are |
| 5 | # met: |
| 6 | # |
| 7 | # * Redistributions of source code must retain the above copyright |
| 8 | # notice, this list of conditions and the following disclaimer. |
| 9 | # * Redistributions in binary form must reproduce the above |
| 10 | # copyright notice, this list of conditions and the following |
| 11 | # disclaimer in the documentation and/or other materials provided |
| 12 | # with the distribution. |
| 13 | # * Neither the name of Google Inc. nor the names of its |
| 14 | # contributors may be used to endorse or promote products derived |
| 15 | # from this software without specific prior written permission. |
| 16 | # |
| 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 18 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 19 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 20 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 21 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | |
| 29 | import argparse |
| 30 | import datetime |
| 31 | import httplib |
| 32 | import glob |
| 33 | import imp |
| 34 | import json |
| 35 | import os |
| 36 | import re |
| 37 | import shutil |
| 38 | import subprocess |
| 39 | import sys |
| 40 | import textwrap |
| 41 | import time |
| 42 | import urllib |
| 43 | import urllib2 |
| 44 | |
| 45 | from git_recipes import GitRecipesMixin |
| 46 | from git_recipes import GitFailedException |
| 47 | |
| 48 | CHANGELOG_FILE = "ChangeLog" |
| 49 | DAY_IN_SECONDS = 24 * 60 * 60 |
| 50 | PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P<git_rev>[a-fA-F0-9]+)\)$") |
| 51 | PUSH_MSG_NEW_RE = re.compile(r"^Version \d+\.\d+\.\d+$") |
| 52 | VERSION_FILE = os.path.join("include", "v8-version.h") |
Ben Murdoch | 097c5b2 | 2016-05-18 11:27:45 +0100 | [diff] [blame] | 53 | WATCHLISTS_FILE = "WATCHLISTS" |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 54 | |
| 55 | # V8 base directory. |
| 56 | V8_BASE = os.path.dirname( |
| 57 | os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 58 | |
| 59 | |
| 60 | def TextToFile(text, file_name): |
| 61 | with open(file_name, "w") as f: |
| 62 | f.write(text) |
| 63 | |
| 64 | |
| 65 | def AppendToFile(text, file_name): |
| 66 | with open(file_name, "a") as f: |
| 67 | f.write(text) |
| 68 | |
| 69 | |
| 70 | def LinesInFile(file_name): |
| 71 | with open(file_name) as f: |
| 72 | for line in f: |
| 73 | yield line |
| 74 | |
| 75 | |
| 76 | def FileToText(file_name): |
| 77 | with open(file_name) as f: |
| 78 | return f.read() |
| 79 | |
| 80 | |
| 81 | def MSub(rexp, replacement, text): |
| 82 | return re.sub(rexp, replacement, text, flags=re.MULTILINE) |
| 83 | |
| 84 | |
| 85 | def Fill80(line): |
| 86 | # Replace tabs and remove surrounding space. |
| 87 | line = re.sub(r"\t", r" ", line.strip()) |
| 88 | |
| 89 | # Format with 8 characters indentation and line width 80. |
| 90 | return textwrap.fill(line, width=80, initial_indent=" ", |
| 91 | subsequent_indent=" ") |
| 92 | |
| 93 | |
| 94 | def MakeComment(text): |
| 95 | return MSub(r"^( ?)", "#", text) |
| 96 | |
| 97 | |
| 98 | def StripComments(text): |
| 99 | # Use split not splitlines to keep terminal newlines. |
| 100 | return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n"))) |
| 101 | |
| 102 | |
| 103 | def MakeChangeLogBody(commit_messages, auto_format=False): |
| 104 | result = "" |
| 105 | added_titles = set() |
| 106 | for (title, body, author) in commit_messages: |
| 107 | # TODO(machenbach): Better check for reverts. A revert should remove the |
| 108 | # original CL from the actual log entry. |
| 109 | title = title.strip() |
| 110 | if auto_format: |
| 111 | # Only add commits that set the LOG flag correctly. |
| 112 | log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:(?:Y(?:ES)?)|TRUE)" |
| 113 | if not re.search(log_exp, body, flags=re.I | re.M): |
| 114 | continue |
| 115 | # Never include reverts. |
| 116 | if title.startswith("Revert "): |
| 117 | continue |
| 118 | # Don't include duplicates. |
| 119 | if title in added_titles: |
| 120 | continue |
| 121 | |
| 122 | # Add and format the commit's title and bug reference. Move dot to the end. |
| 123 | added_titles.add(title) |
| 124 | raw_title = re.sub(r"(\.|\?|!)$", "", title) |
| 125 | bug_reference = MakeChangeLogBugReference(body) |
| 126 | space = " " if bug_reference else "" |
| 127 | result += "%s\n" % Fill80("%s%s%s." % (raw_title, space, bug_reference)) |
| 128 | |
| 129 | # Append the commit's author for reference if not in auto-format mode. |
| 130 | if not auto_format: |
| 131 | result += "%s\n" % Fill80("(%s)" % author.strip()) |
| 132 | |
| 133 | result += "\n" |
| 134 | return result |
| 135 | |
| 136 | |
| 137 | def MakeChangeLogBugReference(body): |
| 138 | """Grep for "BUG=xxxx" lines in the commit message and convert them to |
| 139 | "(issue xxxx)". |
| 140 | """ |
| 141 | crbugs = [] |
| 142 | v8bugs = [] |
| 143 | |
| 144 | def AddIssues(text): |
| 145 | ref = re.match(r"^BUG[ \t]*=[ \t]*(.+)$", text.strip()) |
| 146 | if not ref: |
| 147 | return |
| 148 | for bug in ref.group(1).split(","): |
| 149 | bug = bug.strip() |
| 150 | match = re.match(r"^v8:(\d+)$", bug) |
| 151 | if match: v8bugs.append(int(match.group(1))) |
| 152 | else: |
| 153 | match = re.match(r"^(?:chromium:)?(\d+)$", bug) |
| 154 | if match: crbugs.append(int(match.group(1))) |
| 155 | |
| 156 | # Add issues to crbugs and v8bugs. |
| 157 | map(AddIssues, body.splitlines()) |
| 158 | |
| 159 | # Filter duplicates, sort, stringify. |
| 160 | crbugs = map(str, sorted(set(crbugs))) |
| 161 | v8bugs = map(str, sorted(set(v8bugs))) |
| 162 | |
| 163 | bug_groups = [] |
| 164 | def FormatIssues(prefix, bugs): |
| 165 | if len(bugs) > 0: |
| 166 | plural = "s" if len(bugs) > 1 else "" |
| 167 | bug_groups.append("%sissue%s %s" % (prefix, plural, ", ".join(bugs))) |
| 168 | |
| 169 | FormatIssues("", v8bugs) |
| 170 | FormatIssues("Chromium ", crbugs) |
| 171 | |
| 172 | if len(bug_groups) > 0: |
| 173 | return "(%s)" % ", ".join(bug_groups) |
| 174 | else: |
| 175 | return "" |
| 176 | |
| 177 | |
| 178 | def SortingKey(version): |
| 179 | """Key for sorting version number strings: '3.11' > '3.2.1.1'""" |
| 180 | version_keys = map(int, version.split(".")) |
| 181 | # Fill up to full version numbers to normalize comparison. |
| 182 | while len(version_keys) < 4: # pragma: no cover |
| 183 | version_keys.append(0) |
| 184 | # Fill digits. |
| 185 | return ".".join(map("{0:04d}".format, version_keys)) |
| 186 | |
| 187 | |
| 188 | # Some commands don't like the pipe, e.g. calling vi from within the script or |
| 189 | # from subscripts like git cl upload. |
| 190 | def Command(cmd, args="", prefix="", pipe=True, cwd=None): |
| 191 | cwd = cwd or os.getcwd() |
| 192 | # TODO(machenbach): Use timeout. |
| 193 | cmd_line = "%s %s %s" % (prefix, cmd, args) |
| 194 | print "Command: %s" % cmd_line |
| 195 | print "in %s" % cwd |
| 196 | sys.stdout.flush() |
| 197 | try: |
| 198 | if pipe: |
| 199 | return subprocess.check_output(cmd_line, shell=True, cwd=cwd) |
| 200 | else: |
| 201 | return subprocess.check_call(cmd_line, shell=True, cwd=cwd) |
| 202 | except subprocess.CalledProcessError: |
| 203 | return None |
| 204 | finally: |
| 205 | sys.stdout.flush() |
| 206 | sys.stderr.flush() |
| 207 | |
| 208 | |
| 209 | def SanitizeVersionTag(tag): |
| 210 | version_without_prefix = re.compile(r"^\d+\.\d+\.\d+(?:\.\d+)?$") |
| 211 | version_with_prefix = re.compile(r"^tags\/\d+\.\d+\.\d+(?:\.\d+)?$") |
| 212 | |
| 213 | if version_without_prefix.match(tag): |
| 214 | return tag |
| 215 | elif version_with_prefix.match(tag): |
| 216 | return tag[len("tags/"):] |
| 217 | else: |
| 218 | return None |
| 219 | |
| 220 | |
| 221 | def NormalizeVersionTags(version_tags): |
| 222 | normalized_version_tags = [] |
| 223 | |
| 224 | # Remove tags/ prefix because of packed refs. |
| 225 | for current_tag in version_tags: |
| 226 | version_tag = SanitizeVersionTag(current_tag) |
| 227 | if version_tag != None: |
| 228 | normalized_version_tags.append(version_tag) |
| 229 | |
| 230 | return normalized_version_tags |
| 231 | |
| 232 | |
| 233 | # Wrapper for side effects. |
| 234 | class SideEffectHandler(object): # pragma: no cover |
| 235 | def Call(self, fun, *args, **kwargs): |
| 236 | return fun(*args, **kwargs) |
| 237 | |
| 238 | def Command(self, cmd, args="", prefix="", pipe=True, cwd=None): |
| 239 | return Command(cmd, args, prefix, pipe, cwd=cwd) |
| 240 | |
| 241 | def ReadLine(self): |
| 242 | return sys.stdin.readline().strip() |
| 243 | |
| 244 | def ReadURL(self, url, params=None): |
| 245 | # pylint: disable=E1121 |
| 246 | url_fh = urllib2.urlopen(url, params, 60) |
| 247 | try: |
| 248 | return url_fh.read() |
| 249 | finally: |
| 250 | url_fh.close() |
| 251 | |
| 252 | def ReadClusterFuzzAPI(self, api_key, **params): |
| 253 | params["api_key"] = api_key.strip() |
| 254 | params = urllib.urlencode(params) |
| 255 | |
| 256 | headers = {"Content-type": "application/x-www-form-urlencoded"} |
| 257 | |
| 258 | conn = httplib.HTTPSConnection("backend-dot-cluster-fuzz.appspot.com") |
| 259 | conn.request("POST", "/_api/", params, headers) |
| 260 | |
| 261 | response = conn.getresponse() |
| 262 | data = response.read() |
| 263 | |
| 264 | try: |
| 265 | return json.loads(data) |
| 266 | except: |
| 267 | print data |
| 268 | print "ERROR: Could not read response. Is your key valid?" |
| 269 | raise |
| 270 | |
| 271 | def Sleep(self, seconds): |
| 272 | time.sleep(seconds) |
| 273 | |
| 274 | def GetDate(self): |
| 275 | return datetime.date.today().strftime("%Y-%m-%d") |
| 276 | |
| 277 | def GetUTCStamp(self): |
| 278 | return time.mktime(datetime.datetime.utcnow().timetuple()) |
| 279 | |
| 280 | DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler() |
| 281 | |
| 282 | |
| 283 | class NoRetryException(Exception): |
| 284 | pass |
| 285 | |
| 286 | |
| 287 | class VCInterface(object): |
| 288 | def InjectStep(self, step): |
| 289 | self.step=step |
| 290 | |
| 291 | def Pull(self): |
| 292 | raise NotImplementedError() |
| 293 | |
| 294 | def Fetch(self): |
| 295 | raise NotImplementedError() |
| 296 | |
| 297 | def GetTags(self): |
| 298 | raise NotImplementedError() |
| 299 | |
| 300 | def GetBranches(self): |
| 301 | raise NotImplementedError() |
| 302 | |
| 303 | def MasterBranch(self): |
| 304 | raise NotImplementedError() |
| 305 | |
| 306 | def CandidateBranch(self): |
| 307 | raise NotImplementedError() |
| 308 | |
| 309 | def RemoteMasterBranch(self): |
| 310 | raise NotImplementedError() |
| 311 | |
| 312 | def RemoteCandidateBranch(self): |
| 313 | raise NotImplementedError() |
| 314 | |
| 315 | def RemoteBranch(self, name): |
| 316 | raise NotImplementedError() |
| 317 | |
| 318 | def CLLand(self): |
| 319 | raise NotImplementedError() |
| 320 | |
| 321 | def Tag(self, tag, remote, message): |
| 322 | """Sets a tag for the current commit. |
| 323 | |
| 324 | Assumptions: The commit already landed and the commit message is unique. |
| 325 | """ |
| 326 | raise NotImplementedError() |
| 327 | |
| 328 | |
| 329 | class GitInterface(VCInterface): |
| 330 | def Pull(self): |
| 331 | self.step.GitPull() |
| 332 | |
| 333 | def Fetch(self): |
| 334 | self.step.Git("fetch") |
| 335 | |
| 336 | def GetTags(self): |
| 337 | return self.step.Git("tag").strip().splitlines() |
| 338 | |
| 339 | def GetBranches(self): |
| 340 | # Get relevant remote branches, e.g. "branch-heads/3.25". |
| 341 | branches = filter( |
| 342 | lambda s: re.match(r"^branch\-heads/\d+\.\d+$", s), |
| 343 | self.step.GitRemotes()) |
| 344 | # Remove 'branch-heads/' prefix. |
| 345 | return map(lambda s: s[13:], branches) |
| 346 | |
| 347 | def MasterBranch(self): |
| 348 | return "master" |
| 349 | |
| 350 | def CandidateBranch(self): |
| 351 | return "candidates" |
| 352 | |
| 353 | def RemoteMasterBranch(self): |
| 354 | return "origin/master" |
| 355 | |
| 356 | def RemoteCandidateBranch(self): |
| 357 | return "origin/candidates" |
| 358 | |
| 359 | def RemoteBranch(self, name): |
| 360 | # Assume that if someone "fully qualified" the ref, they know what they |
| 361 | # want. |
| 362 | if name.startswith('refs/'): |
| 363 | return name |
| 364 | if name in ["candidates", "master"]: |
| 365 | return "refs/remotes/origin/%s" % name |
| 366 | try: |
| 367 | # Check if branch is in heads. |
| 368 | if self.step.Git("show-ref refs/remotes/origin/%s" % name).strip(): |
| 369 | return "refs/remotes/origin/%s" % name |
| 370 | except GitFailedException: |
| 371 | pass |
| 372 | try: |
| 373 | # Check if branch is in branch-heads. |
| 374 | if self.step.Git("show-ref refs/remotes/branch-heads/%s" % name).strip(): |
| 375 | return "refs/remotes/branch-heads/%s" % name |
| 376 | except GitFailedException: |
| 377 | pass |
| 378 | self.Die("Can't find remote of %s" % name) |
| 379 | |
| 380 | def Tag(self, tag, remote, message): |
| 381 | # Wait for the commit to appear. Assumes unique commit message titles (this |
| 382 | # is the case for all automated merge and push commits - also no title is |
| 383 | # the prefix of another title). |
| 384 | commit = None |
Ben Murdoch | da12d29 | 2016-06-02 14:46:10 +0100 | [diff] [blame^] | 385 | for wait_interval in [10, 30, 60, 60, 60, 60, 60]: |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 386 | self.step.Git("fetch") |
| 387 | commit = self.step.GitLog(n=1, format="%H", grep=message, branch=remote) |
| 388 | if commit: |
| 389 | break |
| 390 | print("The commit has not replicated to git. Waiting for %s seconds." % |
| 391 | wait_interval) |
| 392 | self.step._side_effect_handler.Sleep(wait_interval) |
| 393 | else: |
| 394 | self.step.Die("Couldn't determine commit for setting the tag. Maybe the " |
| 395 | "git updater is lagging behind?") |
| 396 | |
| 397 | self.step.Git("tag %s %s" % (tag, commit)) |
| 398 | self.step.Git("push origin %s" % tag) |
| 399 | |
| 400 | def CLLand(self): |
| 401 | self.step.GitCLLand() |
| 402 | |
| 403 | |
| 404 | class Step(GitRecipesMixin): |
| 405 | def __init__(self, text, number, config, state, options, handler): |
| 406 | self._text = text |
| 407 | self._number = number |
| 408 | self._config = config |
| 409 | self._state = state |
| 410 | self._options = options |
| 411 | self._side_effect_handler = handler |
| 412 | self.vc = GitInterface() |
| 413 | self.vc.InjectStep(self) |
| 414 | |
| 415 | # The testing configuration might set a different default cwd. |
| 416 | self.default_cwd = (self._config.get("DEFAULT_CWD") or |
| 417 | os.path.join(self._options.work_dir, "v8")) |
| 418 | |
| 419 | assert self._number >= 0 |
| 420 | assert self._config is not None |
| 421 | assert self._state is not None |
| 422 | assert self._side_effect_handler is not None |
| 423 | |
| 424 | def __getitem__(self, key): |
| 425 | # Convenience method to allow direct [] access on step classes for |
| 426 | # manipulating the backed state dict. |
| 427 | return self._state.get(key) |
| 428 | |
| 429 | def __setitem__(self, key, value): |
| 430 | # Convenience method to allow direct [] access on step classes for |
| 431 | # manipulating the backed state dict. |
| 432 | self._state[key] = value |
| 433 | |
| 434 | def Config(self, key): |
| 435 | return self._config[key] |
| 436 | |
| 437 | def Run(self): |
| 438 | # Restore state. |
| 439 | state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"] |
| 440 | if not self._state and os.path.exists(state_file): |
| 441 | self._state.update(json.loads(FileToText(state_file))) |
| 442 | |
| 443 | print ">>> Step %d: %s" % (self._number, self._text) |
| 444 | try: |
| 445 | return self.RunStep() |
| 446 | finally: |
| 447 | # Persist state. |
| 448 | TextToFile(json.dumps(self._state), state_file) |
| 449 | |
| 450 | def RunStep(self): # pragma: no cover |
| 451 | raise NotImplementedError |
| 452 | |
| 453 | def Retry(self, cb, retry_on=None, wait_plan=None): |
| 454 | """ Retry a function. |
| 455 | Params: |
| 456 | cb: The function to retry. |
| 457 | retry_on: A callback that takes the result of the function and returns |
| 458 | True if the function should be retried. A function throwing an |
| 459 | exception is always retried. |
| 460 | wait_plan: A list of waiting delays between retries in seconds. The |
| 461 | maximum number of retries is len(wait_plan). |
| 462 | """ |
| 463 | retry_on = retry_on or (lambda x: False) |
| 464 | wait_plan = list(wait_plan or []) |
| 465 | wait_plan.reverse() |
| 466 | while True: |
| 467 | got_exception = False |
| 468 | try: |
| 469 | result = cb() |
| 470 | except NoRetryException as e: |
| 471 | raise e |
| 472 | except Exception as e: |
| 473 | got_exception = e |
| 474 | if got_exception or retry_on(result): |
| 475 | if not wait_plan: # pragma: no cover |
| 476 | raise Exception("Retried too often. Giving up. Reason: %s" % |
| 477 | str(got_exception)) |
| 478 | wait_time = wait_plan.pop() |
| 479 | print "Waiting for %f seconds." % wait_time |
| 480 | self._side_effect_handler.Sleep(wait_time) |
| 481 | print "Retrying..." |
| 482 | else: |
| 483 | return result |
| 484 | |
| 485 | def ReadLine(self, default=None): |
| 486 | # Don't prompt in forced mode. |
| 487 | if self._options.force_readline_defaults and default is not None: |
| 488 | print "%s (forced)" % default |
| 489 | return default |
| 490 | else: |
| 491 | return self._side_effect_handler.ReadLine() |
| 492 | |
| 493 | def Command(self, name, args, cwd=None): |
| 494 | cmd = lambda: self._side_effect_handler.Command( |
| 495 | name, args, "", True, cwd=cwd or self.default_cwd) |
| 496 | return self.Retry(cmd, None, [5]) |
| 497 | |
| 498 | def Git(self, args="", prefix="", pipe=True, retry_on=None, cwd=None): |
| 499 | cmd = lambda: self._side_effect_handler.Command( |
| 500 | "git", args, prefix, pipe, cwd=cwd or self.default_cwd) |
| 501 | result = self.Retry(cmd, retry_on, [5, 30]) |
| 502 | if result is None: |
| 503 | raise GitFailedException("'git %s' failed." % args) |
| 504 | return result |
| 505 | |
| 506 | def Editor(self, args): |
| 507 | if self._options.requires_editor: |
| 508 | return self._side_effect_handler.Command( |
| 509 | os.environ["EDITOR"], |
| 510 | args, |
| 511 | pipe=False, |
| 512 | cwd=self.default_cwd) |
| 513 | |
| 514 | def ReadURL(self, url, params=None, retry_on=None, wait_plan=None): |
| 515 | wait_plan = wait_plan or [3, 60, 600] |
| 516 | cmd = lambda: self._side_effect_handler.ReadURL(url, params) |
| 517 | return self.Retry(cmd, retry_on, wait_plan) |
| 518 | |
| 519 | def GetDate(self): |
| 520 | return self._side_effect_handler.GetDate() |
| 521 | |
| 522 | def Die(self, msg=""): |
| 523 | if msg != "": |
| 524 | print "Error: %s" % msg |
| 525 | print "Exiting" |
| 526 | raise Exception(msg) |
| 527 | |
| 528 | def DieNoManualMode(self, msg=""): |
| 529 | if not self._options.manual: # pragma: no cover |
| 530 | msg = msg or "Only available in manual mode." |
| 531 | self.Die(msg) |
| 532 | |
| 533 | def Confirm(self, msg): |
| 534 | print "%s [Y/n] " % msg, |
| 535 | answer = self.ReadLine(default="Y") |
| 536 | return answer == "" or answer == "Y" or answer == "y" |
| 537 | |
| 538 | def DeleteBranch(self, name, cwd=None): |
| 539 | for line in self.GitBranch(cwd=cwd).splitlines(): |
| 540 | if re.match(r"\*?\s*%s$" % re.escape(name), line): |
| 541 | msg = "Branch %s exists, do you want to delete it?" % name |
| 542 | if self.Confirm(msg): |
| 543 | self.GitDeleteBranch(name, cwd=cwd) |
| 544 | print "Branch %s deleted." % name |
| 545 | else: |
| 546 | msg = "Can't continue. Please delete branch %s and try again." % name |
| 547 | self.Die(msg) |
| 548 | |
| 549 | def InitialEnvironmentChecks(self, cwd): |
| 550 | # Cancel if this is not a git checkout. |
| 551 | if not os.path.exists(os.path.join(cwd, ".git")): # pragma: no cover |
| 552 | self.Die("This is not a git checkout, this script won't work for you.") |
| 553 | |
| 554 | # Cancel if EDITOR is unset or not executable. |
| 555 | if (self._options.requires_editor and (not os.environ.get("EDITOR") or |
| 556 | self.Command( |
| 557 | "which", os.environ["EDITOR"]) is None)): # pragma: no cover |
| 558 | self.Die("Please set your EDITOR environment variable, you'll need it.") |
| 559 | |
| 560 | def CommonPrepare(self): |
| 561 | # Check for a clean workdir. |
| 562 | if not self.GitIsWorkdirClean(): # pragma: no cover |
| 563 | self.Die("Workspace is not clean. Please commit or undo your changes.") |
| 564 | |
| 565 | # Checkout master in case the script was left on a work branch. |
| 566 | self.GitCheckout('origin/master') |
| 567 | |
| 568 | # Fetch unfetched revisions. |
| 569 | self.vc.Fetch() |
| 570 | |
| 571 | def PrepareBranch(self): |
| 572 | # Delete the branch that will be created later if it exists already. |
| 573 | self.DeleteBranch(self._config["BRANCHNAME"]) |
| 574 | |
| 575 | def CommonCleanup(self): |
| 576 | self.GitCheckout('origin/master') |
| 577 | self.GitDeleteBranch(self._config["BRANCHNAME"]) |
| 578 | |
| 579 | # Clean up all temporary files. |
| 580 | for f in glob.iglob("%s*" % self._config["PERSISTFILE_BASENAME"]): |
| 581 | if os.path.isfile(f): |
| 582 | os.remove(f) |
| 583 | if os.path.isdir(f): |
| 584 | shutil.rmtree(f) |
| 585 | |
| 586 | def ReadAndPersistVersion(self, prefix=""): |
| 587 | def ReadAndPersist(var_name, def_name): |
| 588 | match = re.match(r"^#define %s\s+(\d*)" % def_name, line) |
| 589 | if match: |
| 590 | value = match.group(1) |
| 591 | self["%s%s" % (prefix, var_name)] = value |
| 592 | for line in LinesInFile(os.path.join(self.default_cwd, VERSION_FILE)): |
| 593 | for (var_name, def_name) in [("major", "V8_MAJOR_VERSION"), |
| 594 | ("minor", "V8_MINOR_VERSION"), |
| 595 | ("build", "V8_BUILD_NUMBER"), |
| 596 | ("patch", "V8_PATCH_LEVEL")]: |
| 597 | ReadAndPersist(var_name, def_name) |
| 598 | |
| 599 | def WaitForLGTM(self): |
| 600 | print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit " |
| 601 | "your change. (If you need to iterate on the patch or double check " |
| 602 | "that it's sane, do so in another shell, but remember to not " |
| 603 | "change the headline of the uploaded CL.") |
| 604 | answer = "" |
| 605 | while answer != "LGTM": |
| 606 | print "> ", |
| 607 | answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM") |
| 608 | if answer != "LGTM": |
| 609 | print "That was not 'LGTM'." |
| 610 | |
| 611 | def WaitForResolvingConflicts(self, patch_file): |
| 612 | print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", " |
| 613 | "or resolve the conflicts, stage *all* touched files with " |
| 614 | "'git add', and type \"RESOLVED<Return>\"") |
| 615 | self.DieNoManualMode() |
| 616 | answer = "" |
| 617 | while answer != "RESOLVED": |
| 618 | if answer == "ABORT": |
| 619 | self.Die("Applying the patch failed.") |
| 620 | if answer != "": |
| 621 | print "That was not 'RESOLVED' or 'ABORT'." |
| 622 | print "> ", |
| 623 | answer = self.ReadLine() |
| 624 | |
| 625 | # Takes a file containing the patch to apply as first argument. |
| 626 | def ApplyPatch(self, patch_file, revert=False): |
| 627 | try: |
| 628 | self.GitApplyPatch(patch_file, revert) |
| 629 | except GitFailedException: |
| 630 | self.WaitForResolvingConflicts(patch_file) |
| 631 | |
| 632 | def GetVersionTag(self, revision): |
| 633 | tag = self.Git("describe --tags %s" % revision).strip() |
| 634 | return SanitizeVersionTag(tag) |
| 635 | |
| 636 | def GetRecentReleases(self, max_age): |
| 637 | # Make sure tags are fetched. |
| 638 | self.Git("fetch origin +refs/tags/*:refs/tags/*") |
| 639 | |
| 640 | # Current timestamp. |
| 641 | time_now = int(self._side_effect_handler.GetUTCStamp()) |
| 642 | |
| 643 | # List every tag from a given period. |
| 644 | revisions = self.Git("rev-list --max-age=%d --tags" % |
| 645 | int(time_now - max_age)).strip() |
| 646 | |
| 647 | # Filter out revisions who's tag is off by one or more commits. |
| 648 | return filter(lambda r: self.GetVersionTag(r), revisions.splitlines()) |
| 649 | |
| 650 | def GetLatestVersion(self): |
| 651 | # Use cached version if available. |
| 652 | if self["latest_version"]: |
| 653 | return self["latest_version"] |
| 654 | |
| 655 | # Make sure tags are fetched. |
| 656 | self.Git("fetch origin +refs/tags/*:refs/tags/*") |
| 657 | |
| 658 | all_tags = self.vc.GetTags() |
| 659 | only_version_tags = NormalizeVersionTags(all_tags) |
| 660 | |
| 661 | version = sorted(only_version_tags, |
| 662 | key=SortingKey, reverse=True)[0] |
| 663 | self["latest_version"] = version |
| 664 | return version |
| 665 | |
| 666 | def GetLatestRelease(self): |
| 667 | """The latest release is the git hash of the latest tagged version. |
| 668 | |
| 669 | This revision should be rolled into chromium. |
| 670 | """ |
| 671 | latest_version = self.GetLatestVersion() |
| 672 | |
| 673 | # The latest release. |
| 674 | latest_hash = self.GitLog(n=1, format="%H", branch=latest_version) |
| 675 | assert latest_hash |
| 676 | return latest_hash |
| 677 | |
| 678 | def GetLatestReleaseBase(self, version=None): |
| 679 | """The latest release base is the latest revision that is covered in the |
| 680 | last change log file. It doesn't include cherry-picked patches. |
| 681 | """ |
| 682 | latest_version = version or self.GetLatestVersion() |
| 683 | |
| 684 | # Strip patch level if it exists. |
| 685 | latest_version = ".".join(latest_version.split(".")[:3]) |
| 686 | |
| 687 | # The latest release base. |
| 688 | latest_hash = self.GitLog(n=1, format="%H", branch=latest_version) |
| 689 | assert latest_hash |
| 690 | |
| 691 | title = self.GitLog(n=1, format="%s", git_hash=latest_hash) |
| 692 | match = PUSH_MSG_GIT_RE.match(title) |
| 693 | if match: |
| 694 | # Legacy: In the old process there's one level of indirection. The |
| 695 | # version is on the candidates branch and points to the real release |
| 696 | # base on master through the commit message. |
| 697 | return match.group("git_rev") |
| 698 | match = PUSH_MSG_NEW_RE.match(title) |
| 699 | if match: |
| 700 | # This is a new-style v8 version branched from master. The commit |
| 701 | # "latest_hash" is the version-file change. Its parent is the release |
| 702 | # base on master. |
| 703 | return self.GitLog(n=1, format="%H", git_hash="%s^" % latest_hash) |
| 704 | |
| 705 | self.Die("Unknown latest release: %s" % latest_hash) |
| 706 | |
| 707 | def ArrayToVersion(self, prefix): |
| 708 | return ".".join([self[prefix + "major"], |
| 709 | self[prefix + "minor"], |
| 710 | self[prefix + "build"], |
| 711 | self[prefix + "patch"]]) |
| 712 | |
| 713 | def StoreVersion(self, version, prefix): |
| 714 | version_parts = version.split(".") |
| 715 | if len(version_parts) == 3: |
| 716 | version_parts.append("0") |
| 717 | major, minor, build, patch = version_parts |
| 718 | self[prefix + "major"] = major |
| 719 | self[prefix + "minor"] = minor |
| 720 | self[prefix + "build"] = build |
| 721 | self[prefix + "patch"] = patch |
| 722 | |
| 723 | def SetVersion(self, version_file, prefix): |
| 724 | output = "" |
| 725 | for line in FileToText(version_file).splitlines(): |
| 726 | if line.startswith("#define V8_MAJOR_VERSION"): |
| 727 | line = re.sub("\d+$", self[prefix + "major"], line) |
| 728 | elif line.startswith("#define V8_MINOR_VERSION"): |
| 729 | line = re.sub("\d+$", self[prefix + "minor"], line) |
| 730 | elif line.startswith("#define V8_BUILD_NUMBER"): |
| 731 | line = re.sub("\d+$", self[prefix + "build"], line) |
| 732 | elif line.startswith("#define V8_PATCH_LEVEL"): |
| 733 | line = re.sub("\d+$", self[prefix + "patch"], line) |
| 734 | elif (self[prefix + "candidate"] and |
| 735 | line.startswith("#define V8_IS_CANDIDATE_VERSION")): |
| 736 | line = re.sub("\d+$", self[prefix + "candidate"], line) |
| 737 | output += "%s\n" % line |
| 738 | TextToFile(output, version_file) |
| 739 | |
| 740 | |
| 741 | class BootstrapStep(Step): |
| 742 | MESSAGE = "Bootstrapping checkout and state." |
| 743 | |
| 744 | def RunStep(self): |
| 745 | # Reserve state entry for json output. |
| 746 | self['json_output'] = {} |
| 747 | |
| 748 | if os.path.realpath(self.default_cwd) == os.path.realpath(V8_BASE): |
| 749 | self.Die("Can't use v8 checkout with calling script as work checkout.") |
| 750 | # Directory containing the working v8 checkout. |
| 751 | if not os.path.exists(self._options.work_dir): |
| 752 | os.makedirs(self._options.work_dir) |
| 753 | if not os.path.exists(self.default_cwd): |
| 754 | self.Command("fetch", "v8", cwd=self._options.work_dir) |
| 755 | |
| 756 | |
| 757 | class UploadStep(Step): |
| 758 | MESSAGE = "Upload for code review." |
| 759 | |
| 760 | def RunStep(self): |
| 761 | if self._options.reviewer: |
| 762 | print "Using account %s for review." % self._options.reviewer |
| 763 | reviewer = self._options.reviewer |
| 764 | else: |
| 765 | print "Please enter the email address of a V8 reviewer for your patch: ", |
| 766 | self.DieNoManualMode("A reviewer must be specified in forced mode.") |
| 767 | reviewer = self.ReadLine() |
| 768 | self.GitUpload(reviewer, self._options.author, self._options.force_upload, |
| 769 | bypass_hooks=self._options.bypass_upload_hooks, |
| 770 | cc=self._options.cc) |
| 771 | |
| 772 | |
| 773 | def MakeStep(step_class=Step, number=0, state=None, config=None, |
| 774 | options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER): |
| 775 | # Allow to pass in empty dictionaries. |
| 776 | state = state if state is not None else {} |
| 777 | config = config if config is not None else {} |
| 778 | |
| 779 | try: |
| 780 | message = step_class.MESSAGE |
| 781 | except AttributeError: |
| 782 | message = step_class.__name__ |
| 783 | |
| 784 | return step_class(message, number=number, config=config, |
| 785 | state=state, options=options, |
| 786 | handler=side_effect_handler) |
| 787 | |
| 788 | |
| 789 | class ScriptsBase(object): |
| 790 | def __init__(self, |
| 791 | config=None, |
| 792 | side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER, |
| 793 | state=None): |
| 794 | self._config = config or self._Config() |
| 795 | self._side_effect_handler = side_effect_handler |
| 796 | self._state = state if state is not None else {} |
| 797 | |
| 798 | def _Description(self): |
| 799 | return None |
| 800 | |
| 801 | def _PrepareOptions(self, parser): |
| 802 | pass |
| 803 | |
| 804 | def _ProcessOptions(self, options): |
| 805 | return True |
| 806 | |
| 807 | def _Steps(self): # pragma: no cover |
| 808 | raise Exception("Not implemented.") |
| 809 | |
| 810 | def _Config(self): |
| 811 | return {} |
| 812 | |
| 813 | def MakeOptions(self, args=None): |
| 814 | parser = argparse.ArgumentParser(description=self._Description()) |
| 815 | parser.add_argument("-a", "--author", default="", |
| 816 | help="The author email used for rietveld.") |
| 817 | parser.add_argument("--dry-run", default=False, action="store_true", |
| 818 | help="Perform only read-only actions.") |
| 819 | parser.add_argument("--json-output", |
| 820 | help="File to write results summary to.") |
| 821 | parser.add_argument("-r", "--reviewer", default="", |
| 822 | help="The account name to be used for reviews.") |
| 823 | parser.add_argument("-s", "--step", |
| 824 | help="Specify the step where to start work. Default: 0.", |
| 825 | default=0, type=int) |
| 826 | parser.add_argument("--work-dir", |
| 827 | help=("Location where to bootstrap a working v8 " |
| 828 | "checkout.")) |
| 829 | self._PrepareOptions(parser) |
| 830 | |
| 831 | if args is None: # pragma: no cover |
| 832 | options = parser.parse_args() |
| 833 | else: |
| 834 | options = parser.parse_args(args) |
| 835 | |
| 836 | # Process common options. |
| 837 | if options.step < 0: # pragma: no cover |
| 838 | print "Bad step number %d" % options.step |
| 839 | parser.print_help() |
| 840 | return None |
| 841 | |
| 842 | # Defaults for options, common to all scripts. |
| 843 | options.manual = getattr(options, "manual", True) |
| 844 | options.force = getattr(options, "force", False) |
| 845 | options.bypass_upload_hooks = False |
| 846 | |
| 847 | # Derived options. |
| 848 | options.requires_editor = not options.force |
| 849 | options.wait_for_lgtm = not options.force |
| 850 | options.force_readline_defaults = not options.manual |
| 851 | options.force_upload = not options.manual |
| 852 | |
| 853 | # Process script specific options. |
| 854 | if not self._ProcessOptions(options): |
| 855 | parser.print_help() |
| 856 | return None |
| 857 | |
| 858 | if not options.work_dir: |
| 859 | options.work_dir = "/tmp/v8-release-scripts-work-dir" |
| 860 | return options |
| 861 | |
| 862 | def RunSteps(self, step_classes, args=None): |
| 863 | options = self.MakeOptions(args) |
| 864 | if not options: |
| 865 | return 1 |
| 866 | |
| 867 | state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"] |
| 868 | if options.step == 0 and os.path.exists(state_file): |
| 869 | os.remove(state_file) |
| 870 | |
| 871 | steps = [] |
| 872 | for (number, step_class) in enumerate([BootstrapStep] + step_classes): |
| 873 | steps.append(MakeStep(step_class, number, self._state, self._config, |
| 874 | options, self._side_effect_handler)) |
| 875 | |
| 876 | try: |
| 877 | for step in steps[options.step:]: |
| 878 | if step.Run(): |
| 879 | return 0 |
| 880 | finally: |
| 881 | if options.json_output: |
| 882 | with open(options.json_output, "w") as f: |
| 883 | json.dump(self._state['json_output'], f) |
| 884 | |
| 885 | return 0 |
| 886 | |
| 887 | def Run(self, args=None): |
| 888 | return self.RunSteps(self._Steps(), args) |