recipe-roller | 955ddaf | 2019-05-20 13:03:55 -0700 | [diff] [blame^] | 1 | #!/bin/sh |
| 2 | # Copyright 2019 The LUCI Authors. All rights reserved. |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 3 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 | # that can be found in the LICENSE file. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 5 | |
recipe-roller | 955ddaf | 2019-05-20 13:03:55 -0700 | [diff] [blame^] | 6 | # We want to run python in unbuffered mode; however shebangs on linux grab the |
| 7 | # entire rest of the shebang line as a single argument, leading to errors like: |
| 8 | # |
| 9 | # /usr/bin/env: 'python -u': No such file or directory |
| 10 | # |
| 11 | # This little shell hack is a triple-quoted noop in python, but in sh it |
| 12 | # evaluates to re-exec'ing this script in unbuffered mode. |
| 13 | # pylint: disable=pointless-string-statement |
| 14 | ''''exec python -u -- "$0" ${1+"$@"} # ''' |
| 15 | # vi: syntax=python |
| 16 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 17 | """Bootstrap script to clone and forward to the recipe engine tool. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 18 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 19 | ******************* |
| 20 | ** DO NOT MODIFY ** |
| 21 | ******************* |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 22 | |
Robert Iannucci | e1c2354 | 2018-12-07 17:42:12 -0800 | [diff] [blame] | 23 | This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/recipes.py. |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 24 | To fix bugs, fix in the googlesource repo then run the autoroller. |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 25 | """ |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 26 | |
recipe-roller | 955ddaf | 2019-05-20 13:03:55 -0700 | [diff] [blame^] | 27 | # pylint: disable=wrong-import-position |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 28 | import argparse |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 29 | import json |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 30 | import logging |
recipe-roller | 42e16b0 | 2017-05-11 04:27:01 -0700 | [diff] [blame] | 31 | import os |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 32 | import subprocess |
| 33 | import sys |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 34 | import urlparse |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 35 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 36 | from collections import namedtuple |
| 37 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 38 | # The dependency entry for the recipe_engine in the client repo's recipes.cfg |
| 39 | # |
| 40 | # url (str) - the url to the engine repo we want to use. |
| 41 | # revision (str) - the git revision for the engine to get. |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 42 | # branch (str) - the branch to fetch for the engine as an absolute ref (e.g. |
| 43 | # refs/heads/master) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 44 | EngineDep = namedtuple('EngineDep', |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 45 | 'url revision branch') |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 46 | |
| 47 | |
| 48 | class MalformedRecipesCfg(Exception): |
| 49 | def __init__(self, msg, path): |
| 50 | super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' |
| 51 | % (msg, path)) |
| 52 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 53 | |
| 54 | def parse(repo_root, recipes_cfg_path): |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 55 | """Parse is a lightweight a recipes.cfg file parser. |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 56 | |
| 57 | Args: |
| 58 | repo_root (str) - native path to the root of the repo we're trying to run |
| 59 | recipes for. |
| 60 | recipes_cfg_path (str) - native path to the recipes.cfg file to process. |
| 61 | |
| 62 | Returns (as tuple): |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 63 | engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the |
| 64 | current repo IS the recipe_engine. |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 65 | recipes_path (str) - native path to where the recipes live inside of the |
| 66 | current repo (i.e. the folder containing `recipes/` and/or |
| 67 | `recipe_modules`) |
| 68 | """ |
| 69 | with open(recipes_cfg_path, 'rU') as fh: |
Robert Iannucci | 654dfee | 2017-03-27 20:52:15 -0700 | [diff] [blame] | 70 | pb = json.load(fh) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 71 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 72 | try: |
| 73 | if pb['api_version'] != 2: |
| 74 | raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], |
| 75 | recipes_cfg_path) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 76 | |
Robert Iannucci | e1c2354 | 2018-12-07 17:42:12 -0800 | [diff] [blame] | 77 | # If we're running ./recipes.py from the recipe_engine repo itself, then |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 78 | # return None to signal that there's no EngineDep. |
Robert Iannucci | f4d4b87 | 2019-02-16 14:10:41 -0800 | [diff] [blame] | 79 | repo_name = pb.get('repo_name') |
| 80 | if not repo_name: |
| 81 | repo_name = pb['project_id'] |
| 82 | if repo_name == 'recipe_engine': |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 83 | return None, pb.get('recipes_path', '') |
| 84 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 85 | engine = pb['deps']['recipe_engine'] |
| 86 | |
| 87 | if 'url' not in engine: |
| 88 | raise MalformedRecipesCfg( |
| 89 | 'Required field "url" in dependency "recipe_engine" not found', |
| 90 | recipes_cfg_path) |
| 91 | |
| 92 | engine.setdefault('revision', '') |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 93 | engine.setdefault('branch', 'refs/heads/master') |
| 94 | recipes_path = pb.get('recipes_path', '') |
| 95 | |
| 96 | # TODO(iannucci): only support absolute refs |
| 97 | if not engine['branch'].startswith('refs/'): |
| 98 | engine['branch'] = 'refs/heads/' + engine['branch'] |
| 99 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 100 | recipes_path = os.path.join( |
| 101 | repo_root, recipes_path.replace('/', os.path.sep)) |
| 102 | return EngineDep(**engine), recipes_path |
| 103 | except KeyError as ex: |
| 104 | raise MalformedRecipesCfg(ex.message, recipes_cfg_path) |
| 105 | |
| 106 | |
recipe-roller | 198498b | 2018-06-22 07:38:45 -0700 | [diff] [blame] | 107 | _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else '' |
| 108 | GIT = 'git' + _BAT |
| 109 | VPYTHON = 'vpython' + _BAT |
recipe-roller | 11078c65 | 2019-04-05 11:34:44 -0700 | [diff] [blame] | 110 | CIPD = 'cipd' + _BAT |
| 111 | REQUIRED_BINARIES = {GIT, VPYTHON, CIPD} |
| 112 | |
| 113 | |
| 114 | def _is_executable(path): |
| 115 | return os.path.isfile(path) and os.access(path, os.X_OK) |
| 116 | |
| 117 | # TODO: Use shutil.which once we switch to Python3. |
| 118 | def _is_on_path(basename): |
| 119 | for path in os.environ['PATH'].split(os.pathsep): |
| 120 | full_path = os.path.join(path, basename) |
| 121 | if _is_executable(full_path): |
| 122 | return True |
| 123 | return False |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 124 | |
| 125 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 126 | def _subprocess_call(argv, **kwargs): |
| 127 | logging.info('Running %r', argv) |
| 128 | return subprocess.call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 129 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 130 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 131 | def _git_check_call(argv, **kwargs): |
| 132 | argv = [GIT]+argv |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 133 | logging.info('Running %r', argv) |
| 134 | subprocess.check_call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 135 | |
| 136 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 137 | def _git_output(argv, **kwargs): |
| 138 | argv = [GIT]+argv |
| 139 | logging.info('Running %r', argv) |
| 140 | return subprocess.check_output(argv, **kwargs) |
| 141 | |
| 142 | |
| 143 | def parse_args(argv): |
| 144 | """This extracts a subset of the arguments that this bootstrap script cares |
| 145 | about. Currently this consists of: |
Robert Iannucci | e1c2354 | 2018-12-07 17:42:12 -0800 | [diff] [blame] | 146 | * an override for the recipe engine in the form of `-O recipe_engine=/path` |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 147 | * the --package option. |
| 148 | """ |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 149 | PREFIX = 'recipe_engine=' |
| 150 | |
Eric Boren | c1e9617 | 2017-04-19 11:12:20 +0000 | [diff] [blame] | 151 | p = argparse.ArgumentParser(add_help=False) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 152 | p.add_argument('-O', '--project-override', action='append') |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 153 | p.add_argument('--package', type=os.path.abspath) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 154 | args, _ = p.parse_known_args(argv) |
| 155 | for override in args.project_override or (): |
| 156 | if override.startswith(PREFIX): |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 157 | return override[len(PREFIX):], args.package |
| 158 | return None, args.package |
| 159 | |
| 160 | |
| 161 | def checkout_engine(engine_path, repo_root, recipes_cfg_path): |
| 162 | dep, recipes_path = parse(repo_root, recipes_cfg_path) |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 163 | if dep is None: |
| 164 | # we're running from the engine repo already! |
| 165 | return os.path.join(repo_root, recipes_path) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 166 | |
| 167 | url = dep.url |
| 168 | |
| 169 | if not engine_path and url.startswith('file://'): |
| 170 | engine_path = urlparse.urlparse(url).path |
| 171 | |
| 172 | if not engine_path: |
| 173 | revision = dep.revision |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 174 | branch = dep.branch |
| 175 | |
| 176 | # Ensure that we have the recipe engine cloned. |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 177 | engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 178 | |
| 179 | with open(os.devnull, 'w') as NUL: |
| 180 | # Note: this logic mirrors the logic in recipe_engine/fetch.py |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 181 | _git_check_call(['init', engine_path], stdout=NUL) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 182 | |
| 183 | try: |
| 184 | _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 185 | cwd=engine_path, stdout=NUL, stderr=NUL) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 186 | except subprocess.CalledProcessError: |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 187 | _git_check_call(['fetch', url, branch], cwd=engine_path, stdout=NUL, |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 188 | stderr=NUL) |
| 189 | |
| 190 | try: |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 191 | _git_check_call(['diff', '--quiet', revision], cwd=engine_path) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 192 | except subprocess.CalledProcessError: |
recipe-roller | 1ce80fb | 2019-01-16 15:11:31 -0800 | [diff] [blame] | 193 | _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 194 | |
recipe-roller | 98b14ff | 2019-03-28 13:03:40 -0700 | [diff] [blame] | 195 | # If the engine has refactored/moved modules we need to clean all .pyc files |
| 196 | # or things will get squirrely. |
| 197 | _git_check_call(['clean', '-qxf'], cwd=engine_path) |
| 198 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 199 | return engine_path |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 200 | |
| 201 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 202 | def main(): |
recipe-roller | 11078c65 | 2019-04-05 11:34:44 -0700 | [diff] [blame] | 203 | for required_binary in REQUIRED_BINARIES: |
| 204 | if not _is_on_path(required_binary): |
| 205 | return 'Required binary is not found on PATH: %s' % required_binary |
| 206 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 207 | if '--verbose' in sys.argv: |
| 208 | logging.getLogger().setLevel(logging.INFO) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 209 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 210 | args = sys.argv[1:] |
| 211 | engine_override, recipes_cfg_path = parse_args(args) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 212 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 213 | if recipes_cfg_path: |
| 214 | # calculate repo_root from recipes_cfg_path |
| 215 | repo_root = os.path.dirname( |
| 216 | os.path.dirname( |
| 217 | os.path.dirname(recipes_cfg_path))) |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 218 | else: |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 219 | # find repo_root with git and calculate recipes_cfg_path |
| 220 | repo_root = (_git_output( |
| 221 | ['rev-parse', '--show-toplevel'], |
| 222 | cwd=os.path.abspath(os.path.dirname(__file__))).strip()) |
| 223 | repo_root = os.path.abspath(repo_root) |
| 224 | recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') |
| 225 | args = ['--package', recipes_cfg_path] + args |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 226 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 227 | engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 228 | |
recipe-roller | 955ddaf | 2019-05-20 13:03:55 -0700 | [diff] [blame^] | 229 | try: |
| 230 | return _subprocess_call([ |
| 231 | VPYTHON, '-u', |
| 232 | os.path.join(engine_path, 'recipe_engine', 'main.py')] + args) |
| 233 | except KeyboardInterrupt: |
| 234 | return 1 |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 235 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 236 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 237 | if __name__ == '__main__': |
| 238 | sys.exit(main()) |