borenet | 1ed2ae4 | 2016-07-26 11:52:17 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2016 The LUCI Authors. All rights reserved. |
| 3 | # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 | # that can be found in the LICENSE file. |
| 5 | """Bootstrap script to clone and forward to the recipe engine tool. |
| 6 | *********************************************************************** |
| 7 | ** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. ** |
| 8 | *********************************************************************** |
| 9 | This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py. |
| 10 | To fix bugs, fix in the github repo then copy it back to here and fix the |
| 11 | PER-REPO CONFIGURATION section to look like this one. |
| 12 | """ |
| 13 | import os |
| 14 | #### PER-REPO CONFIGURATION (editable) #### |
| 15 | |
| 16 | # The root of the repository relative to the directory of this file. |
| 17 | REPO_ROOT = os.path.join(os.pardir, os.pardir) |
| 18 | |
| 19 | # The path of the recipes.cfg file relative to the root of the repository. |
| 20 | RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg') |
| 21 | |
| 22 | #### END PER-REPO CONFIGURATION #### |
| 23 | BOOTSTRAP_VERSION = 1 |
| 24 | import ast |
| 25 | import logging |
| 26 | import random |
| 27 | import re |
| 28 | import subprocess |
| 29 | import sys |
| 30 | import time |
| 31 | import traceback |
| 32 | def parse_protobuf(fh): |
| 33 | """Parse the protobuf text format just well enough to understand recipes.cfg. |
| 34 | We don't use the protobuf library because we want to be as self-contained |
| 35 | as possible in this bootstrap, so it can be simply vendored into a client |
| 36 | repo. |
| 37 | We assume all fields are repeated since we don't have a proto spec to work |
| 38 | with. |
| 39 | Args: |
| 40 | fh: a filehandle containing the text format protobuf. |
| 41 | Returns: |
| 42 | A recursive dictionary of lists. |
| 43 | """ |
| 44 | def parse_atom(text): |
| 45 | if text == 'true': |
| 46 | return True |
| 47 | if text == 'false': |
| 48 | return False |
| 49 | return ast.literal_eval(text) |
| 50 | ret = {} |
| 51 | for line in fh: |
| 52 | line = line.strip() |
| 53 | m = re.match(r'(\w+)\s*:\s*(.*)', line) |
| 54 | if m: |
| 55 | ret.setdefault(m.group(1), []).append(parse_atom(m.group(2))) |
| 56 | continue |
| 57 | m = re.match(r'(\w+)\s*{', line) |
| 58 | if m: |
| 59 | subparse = parse_protobuf(fh) |
| 60 | ret.setdefault(m.group(1), []).append(subparse) |
| 61 | continue |
| 62 | if line == '}': |
| 63 | return ret |
| 64 | if line == '': |
| 65 | continue |
| 66 | raise ValueError('Could not understand line: <%s>' % line) |
| 67 | return ret |
| 68 | def get_unique(things): |
| 69 | if len(things) == 1: |
| 70 | return things[0] |
| 71 | elif len(things) == 0: |
| 72 | raise ValueError("Expected to get one thing, but dinna get none.") |
| 73 | else: |
| 74 | logging.warn('Expected to get one thing, but got a bunch: %s\n%s' % |
| 75 | (things, traceback.format_stack())) |
| 76 | return things[0] |
| 77 | def _subprocess_call(argv, **kwargs): |
| 78 | logging.info('Running %r', argv) |
| 79 | return subprocess.call(argv, **kwargs) |
| 80 | def _subprocess_check_call(argv, **kwargs): |
| 81 | logging.info('Running %r', argv) |
| 82 | subprocess.check_call(argv, **kwargs) |
| 83 | def main(): |
| 84 | if '--verbose' in sys.argv: |
| 85 | logging.getLogger().setLevel(logging.INFO) |
| 86 | if sys.platform.startswith(('win', 'cygwin')): |
| 87 | git = 'git.bat' |
| 88 | else: |
| 89 | git = 'git' |
| 90 | # Find the repository and config file to operate on. |
| 91 | repo_root = os.path.abspath( |
| 92 | os.path.join(os.path.dirname(__file__), REPO_ROOT)) |
| 93 | recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG) |
| 94 | with open(recipes_cfg_path, 'rU') as fh: |
| 95 | protobuf = parse_protobuf(fh) |
| 96 | engine_buf = get_unique([ |
| 97 | b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ]) |
| 98 | engine_url = get_unique(engine_buf['url']) |
| 99 | engine_revision = get_unique(engine_buf['revision']) |
| 100 | engine_subpath = (get_unique(engine_buf.get('path_override', [''])) |
| 101 | .replace('/', os.path.sep)) |
| 102 | recipes_path = os.path.join(repo_root, |
| 103 | get_unique(protobuf['recipes_path']).replace('/', os.path.sep)) |
| 104 | deps_path = os.path.join(recipes_path, '.recipe_deps') |
| 105 | engine_path = os.path.join(deps_path, 'recipe_engine') |
| 106 | # Ensure that we have the recipe engine cloned. |
| 107 | def ensure_engine(): |
| 108 | if not os.path.exists(deps_path): |
| 109 | os.makedirs(deps_path) |
| 110 | if not os.path.exists(engine_path): |
| 111 | _subprocess_check_call([git, 'clone', engine_url, engine_path]) |
| 112 | needs_fetch = _subprocess_call( |
| 113 | [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision], |
| 114 | cwd=engine_path, stdout=open(os.devnull, 'w')) |
| 115 | if needs_fetch: |
| 116 | _subprocess_check_call([git, 'fetch'], cwd=engine_path) |
| 117 | _subprocess_check_call( |
| 118 | [git, 'checkout', '--quiet', engine_revision], cwd=engine_path) |
| 119 | try: |
| 120 | ensure_engine() |
| 121 | except subprocess.CalledProcessError: |
| 122 | logging.exception('ensure_engine failed') |
| 123 | # Retry errors. |
| 124 | time.sleep(random.uniform(2,5)) |
| 125 | ensure_engine() |
| 126 | args = ['--package', recipes_cfg_path, |
| 127 | '--bootstrap-script', __file__] + sys.argv[1:] |
| 128 | return _subprocess_call([ |
| 129 | sys.executable, '-u', |
| 130 | os.path.join(engine_path, engine_subpath, 'recipes.py')] + args) |
| 131 | if __name__ == '__main__': |
| 132 | sys.exit(main()) |