Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright (c) 2016 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | """Module to automate the process of deploying to production. |
| 8 | |
| 9 | Example usage of this script: |
| 10 | 1. Update both autotest and chromite to the lastest commit that has passed |
| 11 | the test instance. |
| 12 | $ ./site_utils/automated_deploy.py |
| 13 | 2. Skip updating a repo, e.g. autotest |
| 14 | $ ./site_utils/automated_deploy.py --skip_autotest |
| 15 | 3. Update a given repo to a specific commit |
| 16 | $ ./site_utils/automated_deploy.py --autotest_hash='1234' |
| 17 | """ |
| 18 | |
| 19 | import argparse |
| 20 | import os |
| 21 | import re |
| 22 | import sys |
| 23 | import subprocess |
| 24 | |
| 25 | import common |
| 26 | from autotest_lib.client.common_lib import revision_control |
| 27 | from autotest_lib.site_utils.lib import infra |
| 28 | |
| 29 | AUTOTEST_DIR = common.autotest_dir |
| 30 | GIT_URL = {'autotest': |
| 31 | 'https://chromium.googlesource.com/chromiumos/third_party/autotest', |
| 32 | 'chromite': |
| 33 | 'https://chromium.googlesource.com/chromiumos/chromite'} |
| 34 | PROD_BRANCH = 'prod' |
| 35 | MASTER_AFE = 'cautotest' |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 36 | NOTIFY_GROUP = 'chromeos-infra-discuss@google.com' |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 37 | |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 38 | # CIPD packages whose prod refs should be updated. |
| 39 | _CIPD_PACKAGES = ( |
| 40 | 'chromiumos/infra/lucifer', |
Prathmesh Prabhu | 2f27423 | 2019-03-06 13:57:54 -0800 | [diff] [blame^] | 41 | 'chromiumos/infra/skylab/linux-amd64', |
Allen Li | 9faf4cb | 2018-07-10 10:45:56 -0700 | [diff] [blame] | 42 | 'chromiumos/infra/skylab-inventory', |
Allen Li | 146a361 | 2018-08-28 16:47:08 -0700 | [diff] [blame] | 43 | 'chromiumos/infra/skylab_swarming_worker/linux-amd64', |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 44 | ) |
| 45 | |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 46 | |
| 47 | class AutoDeployException(Exception): |
| 48 | """Raised when any deploy step fails.""" |
| 49 | |
| 50 | |
| 51 | def parse_arguments(): |
| 52 | """Parse command line arguments. |
| 53 | |
| 54 | @returns An argparse.Namespace populated with argument values. |
| 55 | """ |
| 56 | parser = argparse.ArgumentParser( |
| 57 | description=('Command to update prod branch for autotest, chromite ' |
| 58 | 'repos. Then deploy new changes to all lab servers.')) |
| 59 | parser.add_argument('--skip_autotest', action='store_true', default=False, |
| 60 | help='Skip updating autotest prod branch. Default is False.') |
| 61 | parser.add_argument('--skip_chromite', action='store_true', default=False, |
| 62 | help='Skip updating chromite prod branch. Default is False.') |
Shuqian Zhao | 67ee3b4 | 2017-12-07 11:12:18 -0800 | [diff] [blame] | 63 | parser.add_argument('--force_update', action='store_true', default=False, |
| 64 | help=('Force a deployment without updating both autotest and ' |
| 65 | 'chromite prod branch')) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 66 | parser.add_argument('--autotest_hash', type=str, default=None, |
| 67 | help='Update autotest prod branch to the given hash. If it is not' |
| 68 | ' specified, autotest prod branch will be rebased to ' |
| 69 | 'prod-next branch, which is the latest commit that has ' |
| 70 | 'passed our test instance.') |
| 71 | parser.add_argument('--chromite_hash', type=str, default=None, |
| 72 | help='Same as autotest_hash option.') |
| 73 | |
| 74 | results = parser.parse_args(sys.argv[1:]) |
| 75 | |
| 76 | # Verify the validity of the options. |
| 77 | if ((results.skip_autotest and results.autotest_hash) or |
| 78 | (results.skip_chromite and results.chromite_hash)): |
| 79 | parser.print_help() |
| 80 | print 'Cannot specify skip_* and *_hash options at the same time.' |
| 81 | sys.exit(1) |
Shuqian Zhao | 67ee3b4 | 2017-12-07 11:12:18 -0800 | [diff] [blame] | 82 | if results.force_update: |
| 83 | results.skip_autotest = True |
| 84 | results.skip_chromite = True |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 85 | return results |
| 86 | |
| 87 | |
| 88 | def clone_prod_branch(repo): |
| 89 | """Method to clone the prod branch for a given repo under /tmp/ dir. |
| 90 | |
| 91 | @param repo: Name of the git repo to be cloned. |
| 92 | |
| 93 | @returns path to the cloned repo. |
| 94 | @raises subprocess.CalledProcessError on a command failure. |
| 95 | @raised revision_control.GitCloneError when git clone fails. |
| 96 | """ |
| 97 | repo_dir = '/tmp/%s' % repo |
| 98 | print 'Cloning %s prod branch under %s' % (repo, repo_dir) |
| 99 | if os.path.exists(repo_dir): |
| 100 | infra.local_runner('rm -rf %s' % repo_dir) |
| 101 | git_repo = revision_control.GitRepo(repo_dir, GIT_URL[repo]) |
| 102 | git_repo.clone(remote_branch=PROD_BRANCH) |
| 103 | print 'Successfully cloned %s prod branch' % repo |
| 104 | return repo_dir |
| 105 | |
| 106 | |
| 107 | def update_prod_branch(repo, repo_dir, hash_to_rebase): |
| 108 | """Method to update the prod branch of the given repo to the given hash. |
| 109 | |
| 110 | @param repo: Name of the git repo to be updated. |
| 111 | @param repo_dir: path to the cloned repo. |
| 112 | @param hash_to_rebase: Hash to rebase the prod branch to. If it is None, |
| 113 | prod branch will rebase to prod-next branch. |
| 114 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 115 | @returns the range of the pushed commits as a string. E.g 123...345. If the |
| 116 | prod branch is already up-to-date, return None. |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 117 | @raises subprocess.CalledProcessError on a command failure. |
| 118 | """ |
| 119 | with infra.chdir(repo_dir): |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 120 | print 'Updating %s prod branch.' % repo |
| 121 | rebase_to = hash_to_rebase if hash_to_rebase else 'origin/prod-next' |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 122 | # Check whether prod branch is already up-to-date, which means there is |
| 123 | # no changes since last push. |
| 124 | print 'Detecting new changes since last push...' |
| 125 | diff = infra.local_runner('git log prod..%s --oneline' % rebase_to, |
| 126 | stream_output=True) |
| 127 | if diff: |
| 128 | print 'Find new changes, will update prod branch...' |
| 129 | infra.local_runner('git rebase %s prod' % rebase_to, |
| 130 | stream_output=True) |
| 131 | result = infra.local_runner('git push origin prod', |
| 132 | stream_output=True) |
| 133 | print 'Successfully pushed %s prod branch!\n' % repo |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 134 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 135 | # Get the pushed commit range, which is used to get pushed commits |
| 136 | # using git log E.g. 123..456, then run git log --oneline 123..456. |
| 137 | grep = re.search('(\w)*\.\.(\w)*', result) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 138 | |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 139 | if not grep: |
| 140 | raise AutoDeployException( |
| 141 | 'Fail to get pushed commits for repo %s from git log: %s' % |
| 142 | (repo, result)) |
| 143 | return grep.group(0) |
| 144 | else: |
| 145 | print 'No new %s changes found since last push.' % repo |
| 146 | return None |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 147 | |
| 148 | |
| 149 | def get_pushed_commits(repo, repo_dir, pushed_commits_range): |
| 150 | """Method to get the pushed commits. |
| 151 | |
| 152 | @param repo: Name of the updated git repo. |
| 153 | @param repo_dir: path to the cloned repo. |
| 154 | @param pushed_commits_range: The range of the pushed commits. E.g 123...345 |
| 155 | @return: the commits that are pushed to prod branch. The format likes this: |
| 156 | "git log --oneline A...B | grep autotest |
| 157 | A xxxx |
| 158 | B xxxx" |
| 159 | @raises subprocess.CalledProcessError on a command failure. |
| 160 | """ |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 161 | print 'Getting pushed CLs for %s repo.' % repo |
Shuqian Zhao | 673519b | 2017-05-05 15:13:25 -0700 | [diff] [blame] | 162 | if not pushed_commits_range: |
| 163 | return '\n%s:\nNo new changes since last push.' % repo |
| 164 | |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 165 | with infra.chdir(repo_dir): |
| 166 | get_commits_cmd = 'git log --oneline %s' % pushed_commits_range |
xixuan | 6d782dc | 2017-06-21 18:08:48 -0700 | [diff] [blame] | 167 | |
| 168 | pushed_commits = infra.local_runner( |
| 169 | get_commits_cmd, stream_output=True) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 170 | if repo == 'autotest': |
xixuan | 6d782dc | 2017-06-21 18:08:48 -0700 | [diff] [blame] | 171 | autotest_commits = '' |
| 172 | for cl in pushed_commits.splitlines(): |
| 173 | if 'autotest' in cl: |
| 174 | autotest_commits += '%s\n' % cl |
| 175 | |
| 176 | pushed_commits = autotest_commits |
| 177 | |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 178 | print 'Successfully got pushed CLs for %s repo!\n' % repo |
Aviv Keshet | 6dd1c3d | 2017-09-26 17:46:49 -0700 | [diff] [blame] | 179 | displayed_cmd = get_commits_cmd |
| 180 | if repo == 'autotest': |
| 181 | displayed_cmd += ' | grep autotest' |
| 182 | return '\n%s:\n%s\n%s\n' % (repo, displayed_cmd, pushed_commits) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 183 | |
| 184 | |
| 185 | def kick_off_deploy(): |
| 186 | """Method to kick off deploy script to deploy changes to lab servers. |
| 187 | |
| 188 | @raises subprocess.CalledProcessError on a repo command failure. |
| 189 | """ |
| 190 | print 'Start deploying changes to all lab servers...' |
| 191 | with infra.chdir(AUTOTEST_DIR): |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 192 | # Then kick off the deploy script. |
Aviv Keshet | fc59c14 | 2017-10-31 09:27:57 -0700 | [diff] [blame] | 193 | deploy_cmd = ('runlocalssh ./site_utils/deploy_server.py -x --afe=%s' % |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 194 | MASTER_AFE) |
Shuqian Zhao | 6ad127a | 2017-05-22 22:57:19 +0000 | [diff] [blame] | 195 | infra.local_runner(deploy_cmd, stream_output=True) |
Aviv Keshet | fc59c14 | 2017-10-31 09:27:57 -0700 | [diff] [blame] | 196 | print 'Successfully deployed changes to all lab servers.' |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 197 | |
| 198 | |
| 199 | def main(args): |
| 200 | """Main entry""" |
| 201 | options = parse_arguments() |
| 202 | repos = dict() |
| 203 | if not options.skip_autotest: |
| 204 | repos.update({'autotest': options.autotest_hash}) |
| 205 | if not options.skip_chromite: |
| 206 | repos.update({'chromite': options.chromite_hash}) |
| 207 | |
Daniel Erat | 590a499 | 2018-01-29 15:57:58 -0800 | [diff] [blame] | 208 | print 'Moving CIPD prod refs to prod-next' |
| 209 | for pkg in _CIPD_PACKAGES: |
| 210 | subprocess.check_call(['cipd', 'set-ref', pkg, '-version', 'prod-next', |
| 211 | '-ref', 'prod']) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 212 | try: |
| 213 | # update_log saves the git log of the updated repo. |
| 214 | update_log = '' |
| 215 | for repo, hash_to_rebase in repos.iteritems(): |
| 216 | repo_dir = clone_prod_branch(repo) |
| 217 | push_commits_range = update_prod_branch( |
| 218 | repo, repo_dir, hash_to_rebase) |
| 219 | update_log += get_pushed_commits(repo, repo_dir, push_commits_range) |
| 220 | |
| 221 | kick_off_deploy() |
| 222 | except revision_control.GitCloneError as e: |
| 223 | print 'Fail to clone prod branch. Error:\n%s\n' % e |
| 224 | raise |
| 225 | except subprocess.CalledProcessError as e: |
Shuqian Zhao | 1c3f246 | 2017-02-02 17:21:42 -0800 | [diff] [blame] | 226 | print ('Deploy fails when running a subprocess cmd :\n%s\n' |
| 227 | 'Below is the push log:\n%s\n' % (e.output, update_log)) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 228 | raise |
| 229 | except Exception as e: |
Shuqian Zhao | 1c3f246 | 2017-02-02 17:21:42 -0800 | [diff] [blame] | 230 | print 'Deploy fails with error:\n%s\nPush log:\n%s\n' % (e, update_log) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 231 | raise |
| 232 | |
| 233 | # When deploy succeeds, print the update_log. |
| 234 | print ('Deploy succeeds!!! Below is the push log of the updated repo:\n%s' |
Shuqian Zhao | a482c4a | 2016-11-21 18:49:41 -0800 | [diff] [blame] | 235 | 'Please email this to %s.'% (update_log, NOTIFY_GROUP)) |
Shuqian Zhao | ae2d078 | 2016-11-15 16:58:47 -0800 | [diff] [blame] | 236 | |
| 237 | |
| 238 | if __name__ == '__main__': |
| 239 | sys.exit(main(sys.argv)) |