borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 2 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 3 | # Copyright 2017 The LUCI Authors. All rights reserved. |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 4 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 5 | # that can be found in the LICENSE file. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 6 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 7 | """Bootstrap script to clone and forward to the recipe engine tool. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 8 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 9 | ******************* |
| 10 | ** DO NOT MODIFY ** |
| 11 | ******************* |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 12 | |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 13 | This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/doc/recipes.py. |
| 14 | To fix bugs, fix in the googlesource repo then run the autoroller. |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 15 | """ |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 16 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 17 | import argparse |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 18 | import json |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 19 | import logging |
recipe-roller | 42e16b0 | 2017-05-11 04:27:01 -0700 | [diff] [blame] | 20 | import os |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 21 | import random |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 22 | import subprocess |
| 23 | import sys |
| 24 | import time |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 25 | import urlparse |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 26 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 27 | from collections import namedtuple |
| 28 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 29 | from cStringIO import StringIO |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 30 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 31 | # The dependency entry for the recipe_engine in the client repo's recipes.cfg |
| 32 | # |
| 33 | # url (str) - the url to the engine repo we want to use. |
| 34 | # revision (str) - the git revision for the engine to get. |
| 35 | # path_override (str) - the subdirectory in the engine repo we should use to |
| 36 | # find it's recipes.py entrypoint. This is here for completeness, but will |
| 37 | # essentially always be empty. It would be used if the recipes-py repo was |
| 38 | # merged as a subdirectory of some other repo and you depended on that |
| 39 | # subdirectory. |
| 40 | # branch (str) - the branch to fetch for the engine as an absolute ref (e.g. |
| 41 | # refs/heads/master) |
| 42 | # repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon. |
| 43 | EngineDep = namedtuple('EngineDep', |
| 44 | 'url revision path_override branch repo_type') |
| 45 | |
| 46 | |
| 47 | class MalformedRecipesCfg(Exception): |
| 48 | def __init__(self, msg, path): |
| 49 | super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' |
| 50 | % (msg, path)) |
| 51 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 52 | |
| 53 | def parse(repo_root, recipes_cfg_path): |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 54 | """Parse is a lightweight a recipes.cfg file parser. |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 55 | |
| 56 | Args: |
| 57 | repo_root (str) - native path to the root of the repo we're trying to run |
| 58 | recipes for. |
| 59 | recipes_cfg_path (str) - native path to the recipes.cfg file to process. |
| 60 | |
| 61 | Returns (as tuple): |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 62 | engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the |
| 63 | current repo IS the recipe_engine. |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 64 | recipes_path (str) - native path to where the recipes live inside of the |
| 65 | current repo (i.e. the folder containing `recipes/` and/or |
| 66 | `recipe_modules`) |
| 67 | """ |
| 68 | with open(recipes_cfg_path, 'rU') as fh: |
Robert Iannucci | 654dfee | 2017-03-27 20:52:15 -0700 | [diff] [blame] | 69 | pb = json.load(fh) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 70 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 71 | try: |
| 72 | if pb['api_version'] != 2: |
| 73 | raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], |
| 74 | recipes_cfg_path) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 75 | |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 76 | # If we're running ./doc/recipes.py from the recipe_engine repo itself, then |
| 77 | # return None to signal that there's no EngineDep. |
| 78 | if pb['project_id'] == 'recipe_engine': |
| 79 | return None, pb.get('recipes_path', '') |
| 80 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 81 | engine = pb['deps']['recipe_engine'] |
| 82 | |
| 83 | if 'url' not in engine: |
| 84 | raise MalformedRecipesCfg( |
| 85 | 'Required field "url" in dependency "recipe_engine" not found', |
| 86 | recipes_cfg_path) |
| 87 | |
| 88 | engine.setdefault('revision', '') |
| 89 | engine.setdefault('path_override', '') |
| 90 | engine.setdefault('branch', 'refs/heads/master') |
| 91 | recipes_path = pb.get('recipes_path', '') |
| 92 | |
| 93 | # TODO(iannucci): only support absolute refs |
| 94 | if not engine['branch'].startswith('refs/'): |
| 95 | engine['branch'] = 'refs/heads/' + engine['branch'] |
| 96 | |
| 97 | engine.setdefault('repo_type', 'GIT') |
| 98 | if engine['repo_type'] not in ('GIT', 'GITILES'): |
| 99 | raise MalformedRecipesCfg( |
| 100 | 'Unsupported "repo_type" value in dependency "recipe_engine"', |
| 101 | recipes_cfg_path) |
| 102 | |
| 103 | recipes_path = os.path.join( |
| 104 | repo_root, recipes_path.replace('/', os.path.sep)) |
| 105 | return EngineDep(**engine), recipes_path |
| 106 | except KeyError as ex: |
| 107 | raise MalformedRecipesCfg(ex.message, recipes_cfg_path) |
| 108 | |
| 109 | |
recipe-roller | 198498b | 2018-06-22 07:38:45 -0700 | [diff] [blame] | 110 | _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else '' |
| 111 | GIT = 'git' + _BAT |
| 112 | VPYTHON = 'vpython' + _BAT |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 113 | |
| 114 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 115 | def _subprocess_call(argv, **kwargs): |
| 116 | logging.info('Running %r', argv) |
| 117 | return subprocess.call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 118 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 119 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 120 | def _git_check_call(argv, **kwargs): |
| 121 | argv = [GIT]+argv |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 122 | logging.info('Running %r', argv) |
| 123 | subprocess.check_call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 124 | |
| 125 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 126 | def _git_output(argv, **kwargs): |
| 127 | argv = [GIT]+argv |
| 128 | logging.info('Running %r', argv) |
| 129 | return subprocess.check_output(argv, **kwargs) |
| 130 | |
| 131 | |
| 132 | def parse_args(argv): |
| 133 | """This extracts a subset of the arguments that this bootstrap script cares |
| 134 | about. Currently this consists of: |
| 135 | * an override for the recipe engine in the form of `-O recipe_engin=/path` |
| 136 | * the --package option. |
| 137 | """ |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 138 | PREFIX = 'recipe_engine=' |
| 139 | |
Eric Boren | c1e9617 | 2017-04-19 11:12:20 +0000 | [diff] [blame] | 140 | p = argparse.ArgumentParser(add_help=False) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 141 | p.add_argument('-O', '--project-override', action='append') |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 142 | p.add_argument('--package', type=os.path.abspath) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 143 | args, _ = p.parse_known_args(argv) |
| 144 | for override in args.project_override or (): |
| 145 | if override.startswith(PREFIX): |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 146 | return override[len(PREFIX):], args.package |
| 147 | return None, args.package |
| 148 | |
| 149 | |
| 150 | def checkout_engine(engine_path, repo_root, recipes_cfg_path): |
| 151 | dep, recipes_path = parse(repo_root, recipes_cfg_path) |
Eric Boren | 8e0c2c9 | 2017-09-27 13:03:35 -0400 | [diff] [blame] | 152 | if dep is None: |
| 153 | # we're running from the engine repo already! |
| 154 | return os.path.join(repo_root, recipes_path) |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 155 | |
| 156 | url = dep.url |
| 157 | |
| 158 | if not engine_path and url.startswith('file://'): |
| 159 | engine_path = urlparse.urlparse(url).path |
| 160 | |
| 161 | if not engine_path: |
| 162 | revision = dep.revision |
| 163 | subpath = dep.path_override |
| 164 | branch = dep.branch |
| 165 | |
| 166 | # Ensure that we have the recipe engine cloned. |
| 167 | engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') |
| 168 | engine_path = os.path.join(engine, subpath) |
| 169 | |
| 170 | with open(os.devnull, 'w') as NUL: |
| 171 | # Note: this logic mirrors the logic in recipe_engine/fetch.py |
| 172 | _git_check_call(['init', engine], stdout=NUL) |
| 173 | |
| 174 | try: |
| 175 | _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], |
| 176 | cwd=engine, stdout=NUL, stderr=NUL) |
| 177 | except subprocess.CalledProcessError: |
| 178 | _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL, |
| 179 | stderr=NUL) |
| 180 | |
| 181 | try: |
| 182 | _git_check_call(['diff', '--quiet', revision], cwd=engine) |
| 183 | except subprocess.CalledProcessError: |
| 184 | _git_check_call(['reset', '-q', '--hard', revision], cwd=engine) |
| 185 | |
| 186 | return engine_path |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 187 | |
| 188 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 189 | def main(): |
| 190 | if '--verbose' in sys.argv: |
| 191 | logging.getLogger().setLevel(logging.INFO) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 192 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 193 | args = sys.argv[1:] |
| 194 | engine_override, recipes_cfg_path = parse_args(args) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 195 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 196 | if recipes_cfg_path: |
| 197 | # calculate repo_root from recipes_cfg_path |
| 198 | repo_root = os.path.dirname( |
| 199 | os.path.dirname( |
| 200 | os.path.dirname(recipes_cfg_path))) |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 201 | else: |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 202 | # find repo_root with git and calculate recipes_cfg_path |
| 203 | repo_root = (_git_output( |
| 204 | ['rev-parse', '--show-toplevel'], |
| 205 | cwd=os.path.abspath(os.path.dirname(__file__))).strip()) |
| 206 | repo_root = os.path.abspath(repo_root) |
| 207 | recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') |
| 208 | args = ['--package', recipes_cfg_path] + args |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 209 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 210 | engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 211 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 212 | return _subprocess_call([ |
recipe-roller | 198498b | 2018-06-22 07:38:45 -0700 | [diff] [blame] | 213 | VPYTHON, '-u', |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 214 | os.path.join(engine_path, 'recipes.py')] + args) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 215 | |
Robert Iannucci | 3734c7d | 2017-05-09 11:06:48 -0700 | [diff] [blame] | 216 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 217 | if __name__ == '__main__': |
| 218 | sys.exit(main()) |