borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 2 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 3 | # Copyright 2016 The LUCI Authors. All rights reserved. |
| 4 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 5 | # that can be found in the LICENSE file. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 6 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 7 | """Bootstrap script to clone and forward to the recipe engine tool. |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 8 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 9 | *********************************************************************** |
| 10 | ** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. ** |
| 11 | *********************************************************************** |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 12 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 13 | This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py. |
| 14 | To fix bugs, fix in the github repo then copy it back to here and fix the |
| 15 | PER-REPO CONFIGURATION section to look like this one. |
| 16 | """ |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 17 | |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 18 | import os |
| 19 | |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 20 | # IMPORTANT: Do not alter the header or footer line for the |
| 21 | # "PER-REPO CONFIGURATION" section below, or the autoroller will not be able |
| 22 | # to automatically update this file! All lines between the header and footer |
| 23 | # lines will be retained verbatim by the autoroller. |
| 24 | |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 25 | #### PER-REPO CONFIGURATION (editable) #### |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 26 | # The root of the repository relative to the directory of this file. |
| 27 | REPO_ROOT = os.path.join(os.pardir, os.pardir) |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 28 | # The path of the recipes.cfg file relative to the root of the repository. |
| 29 | RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg') |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 30 | #### END PER-REPO CONFIGURATION #### |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 31 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 32 | BOOTSTRAP_VERSION = 1 |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 33 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 34 | import argparse |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 35 | import json |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 36 | import logging |
| 37 | import random |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 38 | import subprocess |
| 39 | import sys |
| 40 | import time |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 41 | import urlparse |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 42 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 43 | from cStringIO import StringIO |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 44 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 45 | |
| 46 | def parse(repo_root, recipes_cfg_path): |
| 47 | """Parse is transitional code which parses a recipes.cfg file as either jsonpb |
| 48 | or as textpb. |
| 49 | |
| 50 | Args: |
| 51 | repo_root (str) - native path to the root of the repo we're trying to run |
| 52 | recipes for. |
| 53 | recipes_cfg_path (str) - native path to the recipes.cfg file to process. |
| 54 | |
| 55 | Returns (as tuple): |
| 56 | engine_url (str) - the url to the engine repo we want to use. |
| 57 | engine_revision (str) - the git revision for the engine to get. |
| 58 | engine_subpath (str) - the subdirectory in the engine repo we should use to |
| 59 | find it's recipes.py entrypoint. This is here for completeness, but will |
| 60 | essentially always be empty. It would be used if the recipes-py repo was |
| 61 | merged as a subdirectory of some other repo and you depended on that |
| 62 | subdirectory. |
| 63 | recipes_path (str) - native path to where the recipes live inside of the |
| 64 | current repo (i.e. the folder containing `recipes/` and/or |
| 65 | `recipe_modules`) |
| 66 | """ |
| 67 | with open(recipes_cfg_path, 'rU') as fh: |
Robert Iannucci | 654dfee | 2017-03-27 20:52:15 -0700 | [diff] [blame] | 68 | pb = json.load(fh) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 69 | |
recipe-roller | 775663f | 2017-04-17 12:55:16 -0700 | [diff] [blame] | 70 | if pb['api_version'] == 1: |
| 71 | # TODO(iannucci): remove when we only support version 2 |
| 72 | engine = next( |
| 73 | (d for d in pb['deps'] if d['project_id'] == 'recipe_engine'), None) |
| 74 | if engine is None: |
| 75 | raise ValueError('could not find recipe_engine dep in %r' |
| 76 | % recipes_cfg_path) |
| 77 | else: |
| 78 | engine = pb['deps']['recipe_engine'] |
Robert Iannucci | 654dfee | 2017-03-27 20:52:15 -0700 | [diff] [blame] | 79 | engine_url = engine['url'] |
| 80 | engine_revision = engine.get('revision', '') |
| 81 | engine_subpath = engine.get('path_override', '') |
| 82 | recipes_path = pb.get('recipes_path', '') |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 83 | |
| 84 | recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep)) |
| 85 | return engine_url, engine_revision, engine_subpath, recipes_path |
| 86 | |
| 87 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 88 | def _subprocess_call(argv, **kwargs): |
| 89 | logging.info('Running %r', argv) |
| 90 | return subprocess.call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 91 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 92 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 93 | def _subprocess_check_call(argv, **kwargs): |
| 94 | logging.info('Running %r', argv) |
| 95 | subprocess.check_call(argv, **kwargs) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 96 | |
| 97 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 98 | def find_engine_override(argv): |
| 99 | """Since the bootstrap process attempts to defer all logic to the recipes-py |
| 100 | repo, we need to be aware if the user is overriding the recipe_engine |
| 101 | dependency. This looks for and returns the overridden recipe_engine path, if |
| 102 | any, or None if the user didn't override it.""" |
| 103 | PREFIX = 'recipe_engine=' |
| 104 | |
recipe-roller | 6bc873f | 2017-04-18 18:53:29 -0700 | [diff] [blame^] | 105 | p = argparse.ArgumentParser() |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 106 | p.add_argument('-O', '--project-override', action='append') |
| 107 | args, _ = p.parse_known_args(argv) |
| 108 | for override in args.project_override or (): |
| 109 | if override.startswith(PREFIX): |
| 110 | return override[len(PREFIX):] |
| 111 | return None |
| 112 | |
| 113 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 114 | def main(): |
| 115 | if '--verbose' in sys.argv: |
| 116 | logging.getLogger().setLevel(logging.INFO) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 117 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 118 | if REPO_ROOT is None or RECIPES_CFG is None: |
| 119 | logging.error( |
| 120 | 'In order to use this script, please copy it to your repo and ' |
| 121 | 'replace the REPO_ROOT and RECIPES_CFG values with approprite paths.') |
| 122 | sys.exit(1) |
| 123 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 124 | if sys.platform.startswith(('win', 'cygwin')): |
| 125 | git = 'git.bat' |
| 126 | else: |
| 127 | git = 'git' |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 128 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 129 | # Find the repository and config file to operate on. |
| 130 | repo_root = os.path.abspath( |
| 131 | os.path.join(os.path.dirname(__file__), REPO_ROOT)) |
| 132 | recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 133 | |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 134 | engine_url, engine_revision, engine_subpath, recipes_path = parse( |
| 135 | repo_root, recipes_cfg_path) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 136 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 137 | engine_path = find_engine_override(sys.argv[1:]) |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 138 | if not engine_path and engine_url.startswith('file://'): |
| 139 | engine_path = urlparse.urlparse(engine_url).path |
| 140 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 141 | if not engine_path: |
Robert Iannucci | 342977c | 2017-03-24 17:45:40 -0700 | [diff] [blame] | 142 | deps_path = os.path.join(recipes_path, '.recipe_deps') |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 143 | # Ensure that we have the recipe engine cloned. |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 144 | engine_root_path = os.path.join(deps_path, 'recipe_engine') |
| 145 | engine_path = os.path.join(engine_root_path, engine_subpath) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 146 | def ensure_engine(): |
| 147 | if not os.path.exists(deps_path): |
| 148 | os.makedirs(deps_path) |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 149 | if not os.path.exists(engine_root_path): |
| 150 | _subprocess_check_call([git, 'clone', engine_url, engine_root_path]) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 151 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 152 | needs_fetch = _subprocess_call( |
| 153 | [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision], |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 154 | cwd=engine_root_path, stdout=open(os.devnull, 'w')) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 155 | if needs_fetch: |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 156 | _subprocess_check_call([git, 'fetch'], cwd=engine_root_path) |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 157 | _subprocess_check_call( |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 158 | [git, 'checkout', '--quiet', engine_revision], cwd=engine_root_path) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 159 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 160 | try: |
| 161 | ensure_engine() |
| 162 | except subprocess.CalledProcessError: |
| 163 | logging.exception('ensure_engine failed') |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 164 | |
Robert Iannucci | e8b5085 | 2017-03-15 01:24:56 -0700 | [diff] [blame] | 165 | # Retry errors. |
| 166 | time.sleep(random.uniform(2,5)) |
| 167 | ensure_engine() |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 168 | |
| 169 | args = ['--package', recipes_cfg_path] + sys.argv[1:] |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 170 | return _subprocess_call([ |
| 171 | sys.executable, '-u', |
Robert Iannucci | 2ba659e | 2017-03-15 18:04:05 -0700 | [diff] [blame] | 172 | os.path.join(engine_path, 'recipes.py')] + args) |
Eric Boren | f171e16 | 2016-11-14 12:18:34 -0500 | [diff] [blame] | 173 | |
borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 174 | if __name__ == '__main__': |
| 175 | sys.exit(main()) |