borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | |
| 8 | """ |
| 9 | submit_try: Submit a try request. |
| 10 | |
| 11 | This is a thin wrapper around the try request utilities in depot_tools which |
| 12 | adds some validation and supports both git and svn. |
| 13 | """ |
| 14 | |
| 15 | |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 16 | import httplib |
| 17 | import json |
| 18 | import os |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 19 | import re |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 20 | import subprocess |
borenet@google.com | a74302d | 2013-03-18 18:18:26 +0000 | [diff] [blame] | 21 | import svn |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 22 | import sys |
borenet@google.com | fe7533e | 2013-03-11 20:09:40 +0000 | [diff] [blame] | 23 | |
| 24 | |
borenet@google.com | 4bfbc7f | 2013-03-20 18:00:09 +0000 | [diff] [blame] | 25 | GLOBAL_VARIABLES = json.loads(svn.Cat('http://skia.googlecode.com/svn/' |
| 26 | 'buildbot/site_config/' |
| 27 | 'global_variables.json')) |
borenet@google.com | fe7533e | 2013-03-11 20:09:40 +0000 | [diff] [blame] | 28 | |
| 29 | |
| 30 | def GetGlobalVariable(var_name): |
| 31 | return GLOBAL_VARIABLES[var_name]['value'] |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 32 | |
| 33 | |
| 34 | # Alias which can be used to run a try on every builder. |
| 35 | ALL_BUILDERS = 'all' |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 36 | # Alias which can be used to run a try on all compile builders. |
| 37 | COMPILE_BUILDERS = 'compile' |
| 38 | # Alias which can be used to run a try on all builders that are run in the CQ. |
| 39 | CQ_BUILDERS = 'cq' |
| 40 | # Alias which can be used to specify a regex to choose builders. |
| 41 | REGEX = 'regex' |
| 42 | |
| 43 | ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, CQ_BUILDERS, REGEX] |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 44 | |
| 45 | # Contact information for the build master. |
borenet@google.com | fe7533e | 2013-03-11 20:09:40 +0000 | [diff] [blame] | 46 | SKIA_BUILD_MASTER_HOST = str(GetGlobalVariable('master_host')) |
| 47 | SKIA_BUILD_MASTER_PORT = str(GetGlobalVariable('external_port')) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 48 | |
| 49 | # All try builders have this suffix. |
borenet@google.com | 99da601 | 2013-05-02 12:14:35 +0000 | [diff] [blame] | 50 | TRYBOT_SUFFIX = '-Trybot' |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 51 | |
| 52 | # Location of the codereview.settings file in the Skia repo. |
| 53 | SKIA_URL = 'skia.googlecode.com' |
| 54 | CODEREVIEW_SETTINGS = '/svn/codereview.settings' |
| 55 | |
| 56 | # String for matching the svn url of the try server inside codereview.settings. |
| 57 | TRYSERVER_SVN_URL = 'TRYSERVER_SVN_URL: ' |
| 58 | |
| 59 | # Strings used for matching svn config properties. |
borenet@google.com | a74302d | 2013-03-18 18:18:26 +0000 | [diff] [blame] | 60 | URL_STR = 'URL' |
| 61 | REPO_ROOT_STR = 'Repository Root' |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 62 | |
| 63 | |
| 64 | def FindDepotTools(): |
| 65 | """ Find depot_tools on the local machine and return its location. """ |
| 66 | which_cmd = 'where' if os.name == 'nt' else 'which' |
| 67 | cmd = [which_cmd, 'gcl'] |
| 68 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) |
| 69 | if proc.wait() != 0: |
| 70 | raise Exception('Couldn\'t find depot_tools in PATH!') |
borenet@google.com | 6c55b51 | 2013-01-24 21:38:51 +0000 | [diff] [blame] | 71 | gcl = proc.communicate()[0].split('\n')[0].rstrip() |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 72 | depot_tools_dir = os.path.dirname(gcl) |
| 73 | return depot_tools_dir |
| 74 | |
| 75 | |
| 76 | def GetCheckoutRoot(is_svn=True): |
| 77 | """ Determine where the local checkout is rooted. |
| 78 | |
| 79 | is_svn: boolean; whether we're in an SVN checkout. If False, assume we're in |
| 80 | a git checkout. |
| 81 | """ |
| 82 | if is_svn: |
borenet@google.com | a74302d | 2013-03-18 18:18:26 +0000 | [diff] [blame] | 83 | repo = svn.Svn(os.curdir) |
| 84 | svn_info = repo.GetInfo() |
| 85 | url = svn_info.get(URL_STR, None) |
| 86 | repo_root = svn_info.get(REPO_ROOT_STR, None) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 87 | if not url or not repo_root: |
| 88 | raise Exception('Couldn\'t find checkout root!') |
| 89 | if url == repo_root: |
| 90 | return 'svn' |
| 91 | return url[len(repo_root)+1:] |
| 92 | else: |
| 93 | cmd = ['git', 'rev-parse', '--show-toplevel'] |
| 94 | proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, |
| 95 | stderr=subprocess.STDOUT) |
| 96 | if proc.wait() != 0: |
| 97 | raise Exception('Couldn\'t find checkout root!') |
| 98 | return os.path.basename(proc.communicate()[0]) |
| 99 | |
| 100 | |
| 101 | def GetTryRepo(): |
| 102 | """ Determine the TRYSERVER_SVN_URL from the codereview.settings file in the |
| 103 | Skia repo. """ |
| 104 | connection = httplib.HTTPConnection(SKIA_URL) |
| 105 | connection.request('GET', CODEREVIEW_SETTINGS) |
| 106 | content = connection.getresponse().read() |
| 107 | for line in content.split('\n'): |
| 108 | if line.startswith(TRYSERVER_SVN_URL): |
| 109 | return line[len(TRYSERVER_SVN_URL):].rstrip() |
| 110 | raise Exception('Couldn\'t determine the TRYSERVER_SVN_URL. Make sure it is ' |
| 111 | 'defined in the %s file.' % CODEREVIEW_SETTINGS) |
| 112 | |
| 113 | |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 114 | def RetrieveTrybotList(json_filename): |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 115 | """ Retrieve the list of known trybots from the build master, stripping |
| 116 | TRYBOT_SUFFIX from the name. """ |
| 117 | trybots = [] |
| 118 | connection = httplib.HTTPConnection(SKIA_BUILD_MASTER_HOST, |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 119 | SKIA_BUILD_MASTER_PORT) |
| 120 | connection.request('GET', '/json/%s' % json_filename) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 121 | response = connection.getresponse() |
| 122 | builders = json.load(response) |
| 123 | |
| 124 | for builder in builders: |
| 125 | if builder.endswith(TRYBOT_SUFFIX): |
| 126 | trybots.append(builder[:-len(TRYBOT_SUFFIX)]) |
| 127 | return trybots |
| 128 | |
| 129 | |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 130 | def ValidateArgs(argv, trybots, is_svn=True): |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 131 | """ Parse and validate command-line arguments. If the arguments are valid, |
| 132 | returns a tuple of (<changelist name>, <list of trybots>). |
| 133 | |
| 134 | trybots: A list of the known try builders. |
| 135 | """ |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 136 | |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 137 | class CollectedArgs(object): |
| 138 | def __init__(self, bots, changelist, revision): |
| 139 | self._bots = bots |
| 140 | self._changelist = changelist |
| 141 | self._revision = revision |
| 142 | |
| 143 | @property |
| 144 | def bots(self): |
| 145 | for bot in self._bots: |
| 146 | yield bot |
| 147 | |
| 148 | @property |
| 149 | def changelist(self): |
| 150 | return self._changelist |
| 151 | |
| 152 | @property |
| 153 | def revision(self): |
| 154 | return self._revision |
| 155 | |
| 156 | usage = ( |
| 157 | """submit_try: Submit a try request. |
| 158 | submit_try %s--bot <buildername> [<buildername> ...] |
| 159 | |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 160 | -b, --bot Builder(s) or Alias on which to run the try. Required. |
| 161 | Allowed aliases: %s |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 162 | -h, --help Show this message. |
| 163 | -r <revision#> Revision from which to run the try. |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 164 | -l, --list_bots List the available try builders and aliases and exit. |
| 165 | """ % ('<changelist> ' if is_svn else '', ALL_ALIASES)) |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 166 | |
| 167 | def Error(msg=None): |
| 168 | if msg: |
| 169 | print msg |
| 170 | print usage |
| 171 | sys.exit(1) |
| 172 | |
| 173 | using_bots = None |
| 174 | changelist = None |
| 175 | revision = None |
| 176 | |
| 177 | while argv: |
| 178 | arg = argv.pop(0) |
| 179 | if arg == '-h' or arg == '--help': |
| 180 | Error() |
| 181 | elif arg == '-l' or arg == '--list_bots': |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 182 | format_args = ['\n '.join(trybots)] + ALL_ALIASES |
| 183 | print ( |
| 184 | """ |
| 185 | submit_try: Available builders:\n %s |
| 186 | |
| 187 | Can also use the following aliases to run on groups of builders- |
| 188 | %s: Will run against all trybots. |
| 189 | %s: Will run against all compile trybots. |
| 190 | %s: Will run against the same trybots as the commit queue. |
| 191 | %s: You will be prompted to enter a regex to select builders with. |
| 192 | |
| 193 | """ % tuple(format_args)) |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 194 | sys.exit(0) |
borenet@google.com | 02009c7 | 2013-04-01 17:59:16 +0000 | [diff] [blame] | 195 | elif arg == '-b' or arg == '--bot': |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 196 | if using_bots: |
| 197 | Error('--bot specified multiple times.') |
| 198 | if len(argv) < 1: |
| 199 | Error('You must specify a builder with "--bot".') |
| 200 | using_bots = [] |
| 201 | while argv and not argv[0].startswith('-'): |
epoger@google.com | 50d6862 | 2013-04-03 18:35:35 +0000 | [diff] [blame] | 202 | for bot in argv.pop(0).split(','): |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 203 | if bot in ALL_ALIASES: |
epoger@google.com | 50d6862 | 2013-04-03 18:35:35 +0000 | [diff] [blame] | 204 | if using_bots: |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 205 | Error('Cannot specify "%s" with additional builder names or ' |
| 206 | 'aliases.' % bot) |
| 207 | if bot == ALL_BUILDERS: |
borenet@google.com | 6fb7756 | 2013-07-12 18:11:04 +0000 | [diff] [blame^] | 208 | are_you_sure = raw_input('Running a try on every bot is very ' |
| 209 | 'expensive. You may be able to get ' |
| 210 | 'enough information by running on a ' |
| 211 | 'smaller set of bots. Are you sure you ' |
| 212 | 'want to run your try job on all of the ' |
| 213 | 'trybots? [y,n]: ') |
| 214 | if are_you_sure == 'y': |
| 215 | using_bots = trybots |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 216 | elif bot == COMPILE_BUILDERS: |
borenet@google.com | 6fb7756 | 2013-07-12 18:11:04 +0000 | [diff] [blame^] | 217 | using_bots = [t for t in trybots if t.startswith('Build')] |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 218 | elif bot == CQ_BUILDERS: |
| 219 | using_bots = RetrieveTrybotList(json_filename='cqtrybots') |
| 220 | elif bot == REGEX: |
| 221 | while True: |
| 222 | regex = raw_input("Enter your trybot regex: ") |
| 223 | p = re.compile(regex) |
| 224 | using_bots = [t for t in trybots if p.match(t)] |
| 225 | print '\n\nTrybots that match your regex:\n%s\n\n' % '\n'.join( |
| 226 | using_bots) |
| 227 | if raw_input('Re-enter regex? [y,n]: ') == 'n': |
| 228 | break |
epoger@google.com | 50d6862 | 2013-04-03 18:35:35 +0000 | [diff] [blame] | 229 | break |
| 230 | else: |
| 231 | if not bot in trybots: |
| 232 | Error('Unrecognized builder: %s' % bot) |
| 233 | using_bots.append(bot) |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 234 | elif arg == '-r': |
| 235 | if len(argv) < 1: |
| 236 | Error('You must specify a revision with "-r".') |
| 237 | revision = argv.pop(0) |
| 238 | else: |
| 239 | if changelist or not is_svn: |
| 240 | Error('Unknown argument: %s' % arg) |
| 241 | changelist = arg |
| 242 | if is_svn and not changelist: |
| 243 | Error('You must specify a changelist name.') |
| 244 | if not using_bots: |
| 245 | Error('You must specify one or more builders using --bot.') |
| 246 | return CollectedArgs(bots=using_bots, changelist=changelist, |
| 247 | revision=revision) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 248 | |
| 249 | |
| 250 | def SubmitTryRequest(args, is_svn=True): |
| 251 | """ Submits a try request for the given changelist on the given list of |
| 252 | trybots. |
| 253 | |
| 254 | args: Object whose properties are derived from command-line arguments. If |
| 255 | is_svn is True, it should contain: |
| 256 | - changelist: string; the name of the changelist to try. |
| 257 | - bot: list of strings; the names of the try builders to run. |
| 258 | - revision: optional, int; the revision number from which to run the try. |
| 259 | If is_svn is False, it should contain: |
| 260 | - bot: list of strings; the names of the try builders to run. |
| 261 | - revision: optional, int; the revision number from which to run the try. |
| 262 | is_svn: boolean; are we in an SVN repo? |
| 263 | """ |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 264 | botlist = ','.join(['%s%s' % (bot, TRYBOT_SUFFIX) for bot in args.bots]) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 265 | if is_svn: |
borenet@google.com | 6c55b51 | 2013-01-24 21:38:51 +0000 | [diff] [blame] | 266 | gcl_cmd = 'gcl.bat' if os.name == 'nt' else 'gcl' |
| 267 | try_args = [gcl_cmd, 'try', args.changelist, |
| 268 | '--root', GetCheckoutRoot(is_svn), |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 269 | '--bot', botlist] |
| 270 | if args.revision: |
| 271 | try_args.extend(['-r', args.revision]) |
borenet@google.com | 6c55b51 | 2013-01-24 21:38:51 +0000 | [diff] [blame] | 272 | print ' '.join(try_args) |
| 273 | proc = subprocess.Popen(try_args, stdout=subprocess.PIPE, |
| 274 | stderr=subprocess.STDOUT) |
| 275 | if proc.wait() != 0: |
| 276 | raise Exception('Failed to submit try request: %s' % ( |
| 277 | proc.communicate()[0])) |
| 278 | print proc.communicate()[0] |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 279 | else: |
borenet@google.com | 6c55b51 | 2013-01-24 21:38:51 +0000 | [diff] [blame] | 280 | # First, find depot_tools. This is needed to import trychange. |
| 281 | sys.path.append(FindDepotTools()) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 282 | import trychange |
| 283 | try_args = ['--use_svn', |
| 284 | '--svn_repo', GetTryRepo(), |
| 285 | '--root', GetCheckoutRoot(is_svn), |
| 286 | '--bot', botlist] |
| 287 | if args.revision: |
| 288 | try_args.extend(['-r', args.revision]) |
| 289 | trychange.TryChange(try_args, None, False) |
| 290 | |
| 291 | |
| 292 | def main(): |
| 293 | # Retrieve the list of active try builders from the build master. |
rmistry@google.com | f5c4fc8 | 2013-04-09 11:46:46 +0000 | [diff] [blame] | 294 | trybots = RetrieveTrybotList(json_filename='trybots') |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 295 | |
| 296 | # Determine if we're in an SVN checkout. |
| 297 | is_svn = os.path.isdir('.svn') |
| 298 | |
| 299 | # Parse and validate the command-line arguments. |
borenet@google.com | a5d621f | 2013-01-25 20:55:35 +0000 | [diff] [blame] | 300 | args = ValidateArgs(sys.argv[1:], trybots=trybots, is_svn=is_svn) |
borenet@google.com | 6b5388e | 2013-01-23 20:54:29 +0000 | [diff] [blame] | 301 | |
| 302 | # Submit the try request. |
| 303 | SubmitTryRequest(args, is_svn=is_svn) |
| 304 | |
| 305 | |
| 306 | if __name__ == '__main__': |
epoger@google.com | 50d6862 | 2013-04-03 18:35:35 +0000 | [diff] [blame] | 307 | sys.exit(main()) |